Merge branch 'better'
This commit is contained in:
commit
47e57dd518
5
src/main/java/org/warp/jcwdb/AdvancedSaveable.java
Normal file
5
src/main/java/org/warp/jcwdb/AdvancedSaveable.java
Normal file
@ -0,0 +1,5 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
public interface AdvancedSaveable extends Saveable {
|
||||
public void save(boolean isEditFinished);
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class CacheIndexManager implements IndexManager {
|
||||
@ -32,12 +33,23 @@ public class CacheIndexManager implements IndexManager {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) {
|
||||
// TODO: implement
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> IndexDetails set(long index, DBDataOutput<T> writer) {
|
||||
// TODO: implement
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
|
||||
// TODO: implement
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(long index) {
|
||||
// TODO: implement
|
||||
|
||||
@ -8,9 +8,11 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
|
||||
|
||||
public class Cleaner {
|
||||
|
||||
public static final boolean DISABLE_CLEANER = false;
|
||||
public static final boolean ENABLE_CLEANER_LOGGING = false;
|
||||
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
|
||||
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
|
||||
private static final double NORMAL_REMOVED_ITEMS = 1000l;
|
||||
private static final double NORMAL_REMOVED_ITEMS = 2500l;
|
||||
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
|
||||
|
||||
private final Cleanable[] objectsToClean;
|
||||
@ -26,7 +28,9 @@ public class Cleaner {
|
||||
}
|
||||
|
||||
public void start() {
|
||||
this.cleanerThread.start();
|
||||
if (!DISABLE_CLEANER) {
|
||||
this.cleanerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -38,7 +42,7 @@ public class Cleaner {
|
||||
for (Cleanable cleanable : objectsToClean) {
|
||||
cleanedItems += cleanable.clean();
|
||||
}
|
||||
System.gc();
|
||||
//System.gc();
|
||||
return cleanedItems;
|
||||
}
|
||||
|
||||
@ -61,38 +65,38 @@ public class Cleaner {
|
||||
public void run() {
|
||||
while(!stopRequest) {
|
||||
try {
|
||||
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
|
||||
sleepFor(sleepInterval);
|
||||
final long time1 = System.currentTimeMillis();
|
||||
final double removedItems = clean();
|
||||
final long time2 = System.currentTimeMillis();
|
||||
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
|
||||
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
|
||||
|
||||
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
|
||||
if (removedItems > 0) {
|
||||
final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
|
||||
System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
|
||||
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO && removedItemsRatio >= REMOVED_ITEMS_RATIO) {
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
|
||||
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
|
||||
suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
|
||||
|
||||
double newSleepInterval = suggestedExecutionTimeByItemsCalculations;
|
||||
System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) {
|
||||
sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL;
|
||||
} else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) {
|
||||
sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
|
||||
} else {
|
||||
System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
sleepInterval = (int) newSleepInterval;
|
||||
}
|
||||
|
||||
|
||||
System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
|
||||
}catch (InterruptedException e) {
|
||||
|
||||
}
|
||||
|
||||
@ -35,4 +35,11 @@ public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T
|
||||
public long calculateHash(LightArrayList<T> value) {
|
||||
return value.internalList.hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DBLightArrayListParser{" +
|
||||
"db=" + db +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
|
||||
chunks.add(itm);
|
||||
chunkSizes.add(itm2);
|
||||
}
|
||||
return new LightBigList<T>(db, chunks, chunkSizes);
|
||||
return new LightBigList<>(db, chunks, chunkSizes);
|
||||
};
|
||||
}
|
||||
|
||||
@ -39,6 +39,6 @@ public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
|
||||
|
||||
@Override
|
||||
public long calculateHash(LightBigList<T> value) {
|
||||
return value.chunks.hashCode();
|
||||
return (((long)value.chunks.hashCode()) << 32) | (value.chunkSizes.hashCode() & 0xffffffffL);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,19 +1,19 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
public class DBStandardTypes {
|
||||
private static final int STD = 0xFFFFF000;
|
||||
public static final int BOOLEAN = STD| 0;
|
||||
public static final int BYTE = STD| 1;
|
||||
public static final int SHORT = STD| 2;
|
||||
public static final int CHAR = STD| 3;
|
||||
public static final int INTEGER = STD| 4;
|
||||
public static final int FLOAT = STD| 5;
|
||||
public static final int DOUBLE = STD| 6;
|
||||
public static final int STRING = STD| 7;
|
||||
public static final int BYTE_ARRAY = STD| 8;
|
||||
public static final int LIGHT_LIST_ARRAY = STD| 9;
|
||||
public static final int LIGHT_LIST_BIG = STD| 10;
|
||||
public static final int GENERIC_OBJECT = STD| 11;
|
||||
private static final int STD = 0xFFFFF000;
|
||||
public static final int BOOLEAN = STD| 0x000;
|
||||
public static final int BYTE = STD| 0x001;
|
||||
public static final int SHORT = STD| 0x002;
|
||||
public static final int CHAR = STD| 0x003;
|
||||
public static final int INTEGER = STD| 0x004;
|
||||
public static final int FLOAT = STD| 0x005;
|
||||
public static final int DOUBLE = STD| 0x006;
|
||||
public static final int STRING = STD| 0x007;
|
||||
public static final int BYTE_ARRAY = STD| 0x008;
|
||||
public static final int LIGHT_LIST_ARRAY = STD| 0x009;
|
||||
public static final int LIGHT_LIST_BIG = STD| 0x00A;
|
||||
public static final int GENERIC_OBJECT = STD| 0x00B;
|
||||
|
||||
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
|
||||
typesManager.registerType(String.class, STRING, new DBStringParser());
|
||||
|
||||
@ -10,7 +10,7 @@ import java.util.function.Function;
|
||||
* You must have only a maximum of 1 reference for each index
|
||||
* @param <T>
|
||||
*/
|
||||
public class EntryReference<T> implements Castable, Saveable {
|
||||
public class EntryReference<T> implements Castable, AdvancedSaveable {
|
||||
private final JCWDatabase.EntryReferenceTools db;
|
||||
private final long entryIndex;
|
||||
private final DBTypeParser<T> parser;
|
||||
@ -19,6 +19,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
private volatile boolean isHashCached;
|
||||
private volatile boolean loaded;
|
||||
private volatile boolean closed;
|
||||
private volatile boolean isFlushingAllowed;
|
||||
private final Object hashCacheLock = new Object();
|
||||
private final Object accessLock = new Object();
|
||||
private final Object closeLock = new Object();
|
||||
@ -66,14 +67,29 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
* Note that this method won't be called when closing without saving
|
||||
*/
|
||||
public void save() {
|
||||
this.save(false);
|
||||
}
|
||||
|
||||
public void save(boolean isEditFinished) {
|
||||
synchronized(accessLock) {
|
||||
if (loaded && !closed) {
|
||||
try {
|
||||
IndexDetails returnedDetails = db.write(entryIndex, parser.getWriter(value));
|
||||
if (value instanceof AdvancedSaveable) {
|
||||
((AdvancedSaveable)value).save(isEditFinished);
|
||||
} else if (value instanceof Saveable) {
|
||||
((Saveable)value).save();
|
||||
}
|
||||
IndexDetails returnedDetails = this.db.write(entryIndex, parser.getWriter(value));
|
||||
synchronized(hashCacheLock) {
|
||||
this.cachedHash = returnedDetails.getHash();
|
||||
this.isHashCached = true;
|
||||
}
|
||||
if (isEditFinished) {
|
||||
if (!isFlushingAllowed) {
|
||||
this.db.setFlushingAllowed(entryIndex, true);
|
||||
this.isFlushingAllowed = true;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
@ -90,7 +106,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
this.value = editFunction.apply(this.value, this);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -103,7 +119,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
this.value = editFunction.apply(this.value);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -116,7 +132,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
editFunction.accept(this.value, this);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -129,7 +145,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
editFunction.accept(this.value);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -145,7 +161,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(hashCacheLock) {
|
||||
this.isHashCached = false;
|
||||
}
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -174,6 +190,10 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(accessLock) {
|
||||
if (!loaded) {
|
||||
try {
|
||||
if (this.isFlushingAllowed) {
|
||||
this.db.setFlushingAllowed(entryIndex, false);
|
||||
this.isFlushingAllowed = false;
|
||||
}
|
||||
this.value = db.read(entryIndex, parser.getReader());
|
||||
this.loaded = true;
|
||||
} catch (IOException e) {
|
||||
@ -198,7 +218,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
return;
|
||||
}
|
||||
|
||||
save();
|
||||
save(true);
|
||||
|
||||
closed = true;
|
||||
}
|
||||
|
||||
@ -1,14 +1,18 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.Long2IntMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
|
||||
import it.unimi.dsi.fastutil.longs.*;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class FileAllocator implements AutoCloseable {
|
||||
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
|
||||
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 50000;
|
||||
|
||||
private final SeekableByteChannel dataFileChannel;
|
||||
private volatile long fileSize;
|
||||
@ -18,13 +22,19 @@ public class FileAllocator implements AutoCloseable {
|
||||
/**
|
||||
* index -> free space size
|
||||
*/
|
||||
private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
|
||||
private final Long2IntMap freeBytes = new Long2IntLinkedOpenHashMap();
|
||||
|
||||
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
|
||||
this.dataFileChannel = dataFileChannel;
|
||||
this.fileSize = this.dataFileChannel.size();
|
||||
}
|
||||
|
||||
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) throws IOException {
|
||||
this.dataFileChannel = dataFileChannel;
|
||||
this.fileSize = fileSize;
|
||||
this.freeBytes.putAll(freeBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: not implemented
|
||||
*
|
||||
@ -44,23 +54,27 @@ public class FileAllocator implements AutoCloseable {
|
||||
}
|
||||
|
||||
private long allocateIntoUnusedParts(int size) {
|
||||
ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
|
||||
long holeOffset = -1;
|
||||
int holeSize = 0;
|
||||
while (it.hasNext()) {
|
||||
Long2IntMap.Entry entry = it.next();
|
||||
int currentHoleSize = entry.getIntValue();
|
||||
Stream<Map.Entry<Long,Integer>> sorted =
|
||||
freeBytes.entrySet().stream()
|
||||
.sorted(Collections.reverseOrder(Map.Entry.comparingByValue()));
|
||||
final VariableWrapper<Long> holeOffset = new VariableWrapper<>(-1L);
|
||||
final VariableWrapper<Integer> holeSize = new VariableWrapper<>(0);
|
||||
sorted.anyMatch((entry) -> {
|
||||
int currentHoleSize = entry.getValue();
|
||||
if (currentHoleSize < size) {
|
||||
freeBytes.remove(holeOffset);
|
||||
if (holeSize > size) {
|
||||
freeBytes.put(holeOffset + size, holeSize - size);
|
||||
}
|
||||
break;
|
||||
return true;
|
||||
}
|
||||
holeOffset.var = entry.getKey();
|
||||
holeSize.var = currentHoleSize;
|
||||
return false;
|
||||
});
|
||||
if (holeOffset.var != -1L) {
|
||||
freeBytes.remove(holeOffset.var);
|
||||
if (holeSize.var > size) {
|
||||
freeBytes.put(holeOffset.var + size, holeSize.var - size);
|
||||
}
|
||||
holeOffset = entry.getLongKey();
|
||||
holeSize = currentHoleSize;
|
||||
}
|
||||
return holeOffset;
|
||||
return holeOffset.var;
|
||||
}
|
||||
|
||||
private long allocateToEnd(int size) {
|
||||
@ -103,13 +117,24 @@ public class FileAllocator implements AutoCloseable {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!addedToList) {
|
||||
if (!addedToList && length > 0) {
|
||||
freeBytes.put(startPosition, length);
|
||||
}
|
||||
}
|
||||
|
||||
if (startPosition + length >= fileSize) {
|
||||
fileSize = startPosition;
|
||||
}
|
||||
|
||||
// Remove the smallest hole in the file
|
||||
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
|
||||
freeBytes.remove(freeBytes.lastLongKey());
|
||||
Stream<Map.Entry<Long,Integer>> sorted =
|
||||
freeBytes.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByValue());
|
||||
Optional<Map.Entry<Long, Integer>> first = sorted.findFirst();
|
||||
if (first.isPresent()) {
|
||||
freeBytes.remove(first.get().getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package org.warp.jcwdb;
|
||||
import com.esotericsoftware.kryo.io.Input;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
import it.unimi.dsi.fastutil.longs.*;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectIterator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
@ -11,11 +12,10 @@ import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Iterator;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class FileIndexManager implements IndexManager {
|
||||
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
|
||||
private volatile long metadataFileChannelSize;
|
||||
private final FileAllocator fileAllocator;
|
||||
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
|
||||
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
|
||||
@ -34,13 +34,21 @@ public class FileIndexManager implements IndexManager {
|
||||
/**
|
||||
* Edit this using editIndex()
|
||||
*/
|
||||
private final LongSet dirtyLoadedIndices, removedIndices;
|
||||
private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
|
||||
private long firstAllocableIndex;
|
||||
|
||||
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
|
||||
loadedIndices = new Long2ObjectOpenHashMap<>();
|
||||
dirtyLoadedIndices = new LongOpenHashSet();
|
||||
removedIndices = new LongOpenHashSet();
|
||||
if (Cleaner.DISABLE_CLEANER) {
|
||||
loadedIndices = new Long2ObjectOpenHashMap<>();
|
||||
dirtyLoadedIndices = new LongOpenHashSet();
|
||||
flushingAllowedIndices = new LongOpenHashSet();
|
||||
removedIndices = new LongOpenHashSet();
|
||||
} else {
|
||||
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
|
||||
dirtyLoadedIndices = new LongLinkedOpenHashSet();
|
||||
flushingAllowedIndices = new LongLinkedOpenHashSet();
|
||||
removedIndices = new LongLinkedOpenHashSet();
|
||||
}
|
||||
if (Files.notExists(dataFile)) {
|
||||
Files.createFile(dataFile);
|
||||
}
|
||||
@ -49,13 +57,55 @@ public class FileIndexManager implements IndexManager {
|
||||
}
|
||||
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
fileAllocator = new FileAllocator(dataFileChannel);
|
||||
firstAllocableIndex = metadataFileChannel.size() / (long) IndexDetails.TOTAL_BYTES;
|
||||
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
|
||||
metadataFileChannelSize = metadataFileChannel.size();
|
||||
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
|
||||
if (firstAllocableIndex == 0) {
|
||||
firstAllocableIndex = 1;
|
||||
}
|
||||
}
|
||||
|
||||
private long getMetadataFileChannelSize() throws IOException {
|
||||
return metadataFileChannelSize;
|
||||
}
|
||||
|
||||
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
|
||||
Long2IntMap freeBytes = new Long2IntRBTreeMap();
|
||||
Long2IntMap usedBytes = new Long2IntRBTreeMap();
|
||||
long firstOffset = 0;
|
||||
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
|
||||
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
|
||||
if (indexDetails != null) {
|
||||
long offset = indexDetails.getOffset();
|
||||
usedBytes.put(offset, indexDetails.getSize());
|
||||
if (offset < firstOffset) {
|
||||
firstOffset = offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long previousEntryOffset = 0;
|
||||
long previousEntrySize = 0;
|
||||
ObjectIterator<Long2IntMap.Entry> it = usedBytes.long2IntEntrySet().iterator();
|
||||
while (it.hasNext()) {
|
||||
final Long2IntMap.Entry entry = it.next();
|
||||
final long entryOffset = entry.getLongKey();
|
||||
final long entrySize = entry.getIntValue();
|
||||
it.remove();
|
||||
|
||||
if (previousEntryOffset + previousEntrySize < entryOffset) {
|
||||
freeBytes.put(previousEntryOffset + previousEntrySize, (int) (entryOffset - (previousEntryOffset + previousEntrySize)));
|
||||
}
|
||||
|
||||
previousEntryOffset = entryOffset;
|
||||
previousEntrySize = entrySize;
|
||||
}
|
||||
|
||||
final long fileSize = previousEntryOffset + previousEntrySize;
|
||||
|
||||
return new FileAllocator(dataFileChannel, fileSize, freeBytes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T get(long index, DBReader<T> reader) throws IOException {
|
||||
checkClosed();
|
||||
@ -79,7 +129,7 @@ public class FileIndexManager implements IndexManager {
|
||||
public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException {
|
||||
checkClosed();
|
||||
final int dataSize = data.getSize();
|
||||
final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
|
||||
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
|
||||
if (indexDetails == null || indexDetails.getSize() < dataSize) {
|
||||
// Allocate new space
|
||||
IndexDetails newDetails = allocateAndWrite(index, data);
|
||||
@ -95,7 +145,7 @@ public class FileIndexManager implements IndexManager {
|
||||
fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize);
|
||||
}
|
||||
// Update index details
|
||||
editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
|
||||
indexDetails = editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
|
||||
// Write data
|
||||
writeExact(indexDetails, data);
|
||||
// Before returning, return IndexDetails
|
||||
@ -103,6 +153,16 @@ public class FileIndexManager implements IndexManager {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
|
||||
checkClosed();
|
||||
if (isUnloadingAllowed) {
|
||||
flushingAllowedIndices.add(index);
|
||||
} else {
|
||||
flushingAllowedIndices.remove(index);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> long add(DBDataOutput<T> data) throws IOException {
|
||||
checkClosed();
|
||||
@ -116,6 +176,19 @@ public class FileIndexManager implements IndexManager {
|
||||
return index;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> data) throws IOException {
|
||||
checkClosed();
|
||||
final int size = data.getSize();
|
||||
final long offset = fileAllocator.allocate(size);
|
||||
final int type = data.getType();
|
||||
final long hash = data.calculateHash();
|
||||
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
|
||||
final long index = createIndexMetadata(indexDetails);
|
||||
writeExact(indexDetails, data);
|
||||
return new FullIndexDetails(index, indexDetails);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the data at index.
|
||||
* The input size must be equal to the index size!
|
||||
@ -155,6 +228,7 @@ public class FileIndexManager implements IndexManager {
|
||||
}
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
dirtyLoadedIndices.remove(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
loadedIndices.remove(index);
|
||||
removedIndices.add(index);
|
||||
}
|
||||
@ -165,6 +239,7 @@ public class FileIndexManager implements IndexManager {
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
removedIndices.remove(index);
|
||||
dirtyLoadedIndices.remove(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
loadedIndices.remove(index);
|
||||
}
|
||||
// Update indices metadata
|
||||
@ -177,11 +252,14 @@ public class FileIndexManager implements IndexManager {
|
||||
if (dirtyLoadedIndices.contains(index)) {
|
||||
indexDetails = loadedIndices.get(index);
|
||||
dirtyLoadedIndices.remove(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
}
|
||||
}
|
||||
if (isDirty) {
|
||||
// Update indices metadata
|
||||
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
|
||||
long position = index * IndexDetails.TOTAL_BYTES;
|
||||
resizeMetadataFileChannel(position);
|
||||
SeekableByteChannel metadata = metadataFileChannel.position(position);
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
}
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
@ -239,9 +317,10 @@ public class FileIndexManager implements IndexManager {
|
||||
* @param details
|
||||
*/
|
||||
private void editIndex(long index, IndexDetails details) {
|
||||
synchronized (indicesMapsAccessLock) {// FIXXXX main3
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
loadedIndices.put(index, details);
|
||||
dirtyLoadedIndices.add(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
}
|
||||
}
|
||||
|
||||
@ -250,6 +329,7 @@ public class FileIndexManager implements IndexManager {
|
||||
long newIndex = firstAllocableIndex++;
|
||||
loadedIndices.put(newIndex, indexDetails);
|
||||
dirtyLoadedIndices.add(newIndex);
|
||||
flushingAllowedIndices.remove(newIndex);
|
||||
removedIndices.remove(newIndex);
|
||||
return newIndex;
|
||||
}
|
||||
@ -265,13 +345,25 @@ public class FileIndexManager implements IndexManager {
|
||||
|
||||
// Try to load the details from file
|
||||
final long metadataPosition = index * IndexDetails.TOTAL_BYTES;
|
||||
if (metadataPosition + IndexDetails.TOTAL_BYTES > metadataFileChannel.size()) {
|
||||
if (metadataPosition + IndexDetails.TOTAL_BYTES > getMetadataFileChannelSize()) {
|
||||
// Avoid underflow exception
|
||||
return null;
|
||||
}
|
||||
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
|
||||
IndexDetails indexDetails = readIndexDetailsAt(currentMetadataFileChannel);
|
||||
|
||||
if (indexDetails != null) {
|
||||
editIndex(index, indexDetails);
|
||||
return indexDetails;
|
||||
}
|
||||
|
||||
// No results found. Returning null
|
||||
return null;
|
||||
}
|
||||
|
||||
private IndexDetails readIndexDetailsAt(SeekableByteChannel currentMetadataFileChannel) throws IOException {
|
||||
IndexDetails indexDetails = null;
|
||||
synchronized (metadataByteBufferLock) {// FIXXXX main2
|
||||
synchronized (metadataByteBufferLock) {
|
||||
metadataByteBuffer.rewind();
|
||||
currentMetadataFileChannel.read(metadataByteBuffer);
|
||||
metadataByteBuffer.rewind();
|
||||
@ -287,14 +379,7 @@ public class FileIndexManager implements IndexManager {
|
||||
indexDetails = new IndexDetails(offset, size, type, hash);
|
||||
}
|
||||
}
|
||||
|
||||
if (indexDetails != null) {
|
||||
editIndex(index, indexDetails);
|
||||
return indexDetails;
|
||||
}
|
||||
|
||||
// No results found. Returning null
|
||||
return null;
|
||||
return indexDetails;
|
||||
}
|
||||
|
||||
private IndexDetails getIndexMetadata(long index) throws IOException {
|
||||
@ -318,7 +403,7 @@ public class FileIndexManager implements IndexManager {
|
||||
}
|
||||
|
||||
// Update indices metadata
|
||||
flushAllIndices();
|
||||
flushAllFlushableIndices();
|
||||
|
||||
// Remove removed indices
|
||||
removeRemovedIndices();
|
||||
@ -361,39 +446,56 @@ public class FileIndexManager implements IndexManager {
|
||||
@Override
|
||||
public long clean() {
|
||||
long cleaned = 0;
|
||||
long tim1 = System.currentTimeMillis();
|
||||
try {
|
||||
cleaned += flushAllIndices();
|
||||
cleaned += flushAllFlushableIndices();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
long tim2 = System.currentTimeMillis();
|
||||
try {
|
||||
cleaned += removeRemovedIndices();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
long tim3 = System.currentTimeMillis();
|
||||
cleaned += cleanExtraIndices();
|
||||
long tim4 = System.currentTimeMillis();
|
||||
if (Cleaner.ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] FileIndexManager CLEAN_TIME: " + (tim2-tim1) + "," + (tim3-tim2) + "," + (tim4-tim3));
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
private long flushAllIndices() throws IOException {
|
||||
private long flushAllFlushableIndices() throws IOException {
|
||||
long flushedIndices = 0;
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
long lastIndex = -2;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
for (long index : dirtyLoadedIndices) {
|
||||
IndexDetails indexDetails = loadedIndices.get(index);
|
||||
if (index - lastIndex != 1) {
|
||||
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
|
||||
if (!flushingAllowedIndices.contains(index)) {
|
||||
IndexDetails indexDetails = loadedIndices.get(index);
|
||||
long position = index * IndexDetails.TOTAL_BYTES;
|
||||
resizeMetadataFileChannel(position);
|
||||
if (index - lastIndex != 1) {
|
||||
metadata = metadata.position(position);
|
||||
}
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
lastIndex = index;
|
||||
flushedIndices++;
|
||||
}
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
lastIndex = index;
|
||||
flushedIndices++;
|
||||
}
|
||||
dirtyLoadedIndices.clear();
|
||||
dirtyLoadedIndices.addAll(flushingAllowedIndices);
|
||||
flushingAllowedIndices.clear();
|
||||
}
|
||||
return flushedIndices;
|
||||
}
|
||||
|
||||
private void resizeMetadataFileChannel(long position) {
|
||||
if (position + IndexDetails.TOTAL_BYTES > metadataFileChannelSize) {
|
||||
metadataFileChannelSize = position + IndexDetails.TOTAL_BYTES;
|
||||
}
|
||||
}
|
||||
|
||||
private long removeRemovedIndices() throws IOException {
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
|
||||
14
src/main/java/org/warp/jcwdb/FullIndexDetails.java
Normal file
14
src/main/java/org/warp/jcwdb/FullIndexDetails.java
Normal file
@ -0,0 +1,14 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
public class FullIndexDetails extends IndexDetails {
|
||||
private final long index;
|
||||
|
||||
public FullIndexDetails(long index, IndexDetails details) {
|
||||
super(details);
|
||||
this.index = index;
|
||||
}
|
||||
|
||||
public long getIndex() {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.BiPredicate;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public interface IndexManager extends Cleanable {
|
||||
@ -8,7 +10,9 @@ public interface IndexManager extends Cleanable {
|
||||
int getType(long index) throws IOException;
|
||||
long getHash(long index) throws IOException;
|
||||
<T> long add(DBDataOutput<T> writer) throws IOException;
|
||||
<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
|
||||
<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
|
||||
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
|
||||
void delete(long index) throws IOException;
|
||||
boolean has(long index);
|
||||
void close() throws IOException;
|
||||
|
||||
@ -1,12 +1,10 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
public final static long MAX_LOADED_INDICES = 10000;
|
||||
public final static long MAX_LOADED_INDICES = 1000;
|
||||
|
||||
private final TypesManager typesManager;
|
||||
private final MixedIndexDatabase indices;
|
||||
@ -36,7 +34,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
if (exists(0)) {
|
||||
return get(0);
|
||||
} else {
|
||||
LightList<T> newRoot = new LightArrayList<>(this);
|
||||
LightList<T> newRoot = new LightBigList<>(this);
|
||||
return set(0, newRoot);
|
||||
}
|
||||
}
|
||||
@ -63,8 +61,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
long index;
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
index = indices.add(typeParser.getWriter(value));
|
||||
hash = indices.getHash(index);
|
||||
FullIndexDetails fullIndexDetails = indices.addAndGetDetails(typeParser.getWriter(value));
|
||||
index = fullIndexDetails.getIndex();
|
||||
hash = fullIndexDetails.getHash();
|
||||
}
|
||||
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
|
||||
}
|
||||
@ -153,6 +152,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException {
|
||||
return indices.set(index, writer);
|
||||
}
|
||||
|
||||
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
|
||||
indices.setFlushingAllowed(index, isFlushingAllowed);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
@ -3,6 +3,7 @@ package org.warp.jcwdb;
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
|
||||
import java.io.IOError;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
@ -112,7 +113,7 @@ public class LightArrayList<T> implements LightList<T> {
|
||||
try {
|
||||
action.accept(db.get(index));
|
||||
} catch (IOException e) {
|
||||
throw (RuntimeException) new RuntimeException().initCause(e);
|
||||
throw new IOError(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -121,10 +122,11 @@ public class LightArrayList<T> implements LightList<T> {
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
final T[] elements = (T[]) new Objects[internalList.size()];
|
||||
final T[] elements = (T[]) new Object[internalList.size()];
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
try {
|
||||
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
T element = (T) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
elements[i] = element;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
@ -442,4 +444,12 @@ public class LightArrayList<T> implements LightList<T> {
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LightArrayList{" +
|
||||
"internalList=" + internalList +
|
||||
", db=" + db +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,18 +4,24 @@ import it.unimi.dsi.fastutil.ints.IntArrayList;
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
|
||||
import java.io.IOError;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
public class LightBigList<T> implements LightList<T> {
|
||||
public class LightBigList<T> implements LightList<T>, Saveable {
|
||||
|
||||
public static final int MAX_ELEMENTS_PER_CHUNK = 10000;
|
||||
public static final int MAX_ELEMENTS_PER_CHUNK = 200000;
|
||||
|
||||
public final LongArrayList chunks;
|
||||
public final IntArrayList chunkSizes;
|
||||
private final transient JCWDatabase db;
|
||||
private final JCWDatabase db;
|
||||
private LightArrayList<T> cachedChunk;
|
||||
private EntryReference<LightArrayList<T>> cachedChunkRef;
|
||||
private long cachedChunkIndex = -1;
|
||||
private int cachedChunkNumber = -1;
|
||||
private final Object cachedChunkLock = new Object();
|
||||
|
||||
/**
|
||||
* @param db Database reference
|
||||
@ -43,6 +49,23 @@ public class LightBigList<T> implements LightList<T> {
|
||||
this.db = db;
|
||||
this.chunks = chunks;
|
||||
this.chunkSizes = chunkSizes;
|
||||
if (this.chunks.size() > 0) {
|
||||
prepareAccessToChunk(0);
|
||||
}
|
||||
}
|
||||
|
||||
private void prepareAccessToChunk(int chunkNumber) {
|
||||
if (this.cachedChunkRef != null) {
|
||||
this.cachedChunkRef.save();
|
||||
}
|
||||
this.cachedChunkNumber = chunkNumber;
|
||||
this.cachedChunkIndex = this.chunks.getLong(chunkNumber);
|
||||
try {
|
||||
this.cachedChunkRef = db.get(this.cachedChunkIndex);
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
this.cachedChunk = this.cachedChunkRef.getValueReadOnly();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -53,16 +76,13 @@ public class LightBigList<T> implements LightList<T> {
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int chunkNumber = i;
|
||||
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
|
||||
try {
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
final EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
chunk.appendIndex(elementIndex);
|
||||
chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1);
|
||||
});
|
||||
synchronized (cachedChunkLock) {
|
||||
if (cachedChunkNumber != i) {
|
||||
prepareAccessToChunk(i);
|
||||
}
|
||||
cachedChunk.appendIndex(elementIndex);
|
||||
chunkSizes.set(chunkNumber, cachedChunk.size());
|
||||
return;
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -113,8 +133,8 @@ public class LightBigList<T> implements LightList<T> {
|
||||
if (o != null) {
|
||||
for (long chunkIndex : chunks) {
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightArrayList<T> chunk = chunkRef.getValueReadOnly();
|
||||
if (chunk.contains(o)) {
|
||||
return true;
|
||||
}
|
||||
@ -131,7 +151,8 @@ public class LightBigList<T> implements LightList<T> {
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public Iterator<T> iterator() {
|
||||
public Iterator<T> iterator()
|
||||
{
|
||||
throw new RuntimeException("iterator() isn't implemented!");
|
||||
}
|
||||
|
||||
@ -155,21 +176,50 @@ public class LightBigList<T> implements LightList<T> {
|
||||
@Override
|
||||
public void forEachReference(Consumer<? super EntryReference<T>> action) {
|
||||
Objects.requireNonNull(action);
|
||||
for (long chunkIndex : this.chunks) {
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
chunk.forEachReference(action);
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
synchronized (cachedChunkLock) {
|
||||
if (cachedChunkNumber != i) {
|
||||
prepareAccessToChunk(i);
|
||||
}
|
||||
cachedChunk.forEachReference(action);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* toArray() isn't implemented! DO NOT USE IT.
|
||||
* @return
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Deprecated
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
throw new RuntimeException("toArray() isn't implemented!");
|
||||
if (true) {
|
||||
throw new RuntimeException("toArray() isn't implemented!");
|
||||
}
|
||||
T[] result = (T[]) new Object[this.size()];
|
||||
|
||||
long currentOffset = 0;
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int currentChunkSize = chunkSizes.getInt(i);
|
||||
final long chunkStartOffset = currentOffset;
|
||||
currentOffset += currentChunkSize;
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
|
||||
try {
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightArrayList<T> chunk = chunkRef.getValueReadOnly();
|
||||
for (int i1 = 0; i1 < chunk.size(); i1++) {
|
||||
result[(int)(chunkStartOffset + i1)] = chunk.get(i);
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@ -230,7 +280,7 @@ public class LightBigList<T> implements LightList<T> {
|
||||
}
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
result.var = chunk.remove(relativeOffset);
|
||||
});
|
||||
@ -368,7 +418,7 @@ public class LightBigList<T> implements LightList<T> {
|
||||
}
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
chunk.set(relativeOffset, element);
|
||||
wrapper.var = element;
|
||||
@ -412,7 +462,7 @@ public class LightBigList<T> implements LightList<T> {
|
||||
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
|
||||
if (foundIndex >= 0) {
|
||||
return currentOffset + foundIndex;
|
||||
@ -441,7 +491,7 @@ public class LightBigList<T> implements LightList<T> {
|
||||
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
|
||||
if (foundIndex >= 0) {
|
||||
return currentOffset + foundIndex;
|
||||
@ -497,25 +547,35 @@ public class LightBigList<T> implements LightList<T> {
|
||||
@Override
|
||||
public boolean removeIf(Predicate<? super T> filter) {
|
||||
Objects.requireNonNull(filter);
|
||||
final VariableWrapper<Boolean> result = new VariableWrapper(false);
|
||||
boolean result = false;
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
try {
|
||||
final int chunkOffset = i;
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
boolean removed = chunk.removeIf(filter);
|
||||
if (removed) {
|
||||
result.var = true;
|
||||
chunkSizes.set(chunkOffset, chunk.size());
|
||||
}
|
||||
});
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
synchronized (cachedChunkLock) {
|
||||
if (cachedChunkNumber != i) {
|
||||
prepareAccessToChunk(i);
|
||||
}
|
||||
if (cachedChunk.removeIf(filter)) {
|
||||
result = true;
|
||||
chunkSizes.set(cachedChunkNumber, cachedChunk.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.var;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LightBigList{" +
|
||||
"chunks=" + chunks +
|
||||
", chunkSizes=" + chunkSizes +
|
||||
", db=" + db +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save() {
|
||||
if (this.cachedChunkRef != null) {
|
||||
this.cachedChunkRef.save();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,11 +1,7 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.Long2LongLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2LongMap;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class MixedIndexDatabase implements IndexManager {
|
||||
private final FileIndexManager fileIndices;
|
||||
@ -48,6 +44,11 @@ public class MixedIndexDatabase implements IndexManager {
|
||||
return fileIndices.add(writer);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException {
|
||||
return fileIndices.addAndGetDetails(writer);
|
||||
}
|
||||
@Override
|
||||
public <T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException {
|
||||
if (cacheIndices.has(index)) {
|
||||
@ -56,6 +57,14 @@ public class MixedIndexDatabase implements IndexManager {
|
||||
return fileIndices.set(index, writer);
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
|
||||
if (cacheIndices.has(index)) {
|
||||
cacheIndices.setFlushingAllowed(index, isFlushingAllowed);
|
||||
} else {
|
||||
fileIndices.setFlushingAllowed(index, isFlushingAllowed);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(long index) throws IOException {
|
||||
|
||||
@ -7,92 +7,91 @@ import org.warp.jcwdb.LightList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectList;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
public class App {
|
||||
static long time3;
|
||||
public static void main(String[] args) {
|
||||
public static void main(String[] args) throws IOException {
|
||||
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
|
||||
Files.delete(Paths.get(args[0]));
|
||||
Files.delete(Paths.get(args[1]));
|
||||
}
|
||||
System.out.println("Loading database...");
|
||||
long time0 = System.currentTimeMillis();
|
||||
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
|
||||
db.registerClass(StrangeAnimal.class, 0);
|
||||
try {
|
||||
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
|
||||
Files.delete(Paths.get(args[0]));
|
||||
Files.delete(Paths.get(args[1]));
|
||||
}
|
||||
System.out.println("Loading database...");
|
||||
long time0 = System.currentTimeMillis();
|
||||
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
|
||||
db.registerClass(StrangeAnimal.class, 0);
|
||||
try {
|
||||
long time01 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time01 - time0));
|
||||
System.out.println("Loading root...");
|
||||
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
|
||||
rootRef.editValue((root, saver) -> {
|
||||
long time1 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time1 - time01));
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Root:");
|
||||
// for (int i = 0; i < root.size(); i++) {
|
||||
// System.out.println(" - " + root.get(i));
|
||||
// }
|
||||
long prectime = System.currentTimeMillis();
|
||||
for (int i = 0; i < 20000/* 2000000 */; i++) {
|
||||
Animal animal = new StrangeAnimal(i % 40);
|
||||
root.add(animal);
|
||||
if (i > 0 && i % 200000 == 0) {
|
||||
long precprectime = prectime;
|
||||
prectime = System.currentTimeMillis();
|
||||
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)");
|
||||
}
|
||||
long time01 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time01 - time0));
|
||||
System.out.println("Loading root...");
|
||||
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
|
||||
rootRef.editValue((root, saver) -> {
|
||||
long time1 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time1 - time01));
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Root:");
|
||||
// for (int i = 0; i < root.size(); i++) {
|
||||
// System.out.println(" - " + root.get(i));
|
||||
// }
|
||||
long prectime = System.currentTimeMillis();
|
||||
for (int i = 0; i < 2000000/* 2000000 */; i++) {
|
||||
Animal animal = new StrangeAnimal(i % 40);
|
||||
root.addEntry(animal);
|
||||
if (i > 0 && i % 200000 == 0) {
|
||||
long precprectime = prectime;
|
||||
prectime = System.currentTimeMillis();
|
||||
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)" + " Total Time: " + (prectime - time1));
|
||||
}
|
||||
long time2 = System.currentTimeMillis();
|
||||
saver.save();
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Time elapsed: " + (time2 - time1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
long time2_0 = System.currentTimeMillis();
|
||||
System.out.println("Filtering strings...");
|
||||
//root.removeIf(Animal::hasFourLegs);
|
||||
long time2_1 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time2_1 - time2_0));
|
||||
ObjectList<Animal> results = new ObjectArrayList<>();
|
||||
|
||||
root.forEachReference((valueReference) -> {
|
||||
Animal value = valueReference.getValueReadOnly();
|
||||
if (Animal.hasFourLegs(value)) {
|
||||
results.add(value);
|
||||
}
|
||||
//System.out.println("val:" + value);
|
||||
});
|
||||
long time2_2 = System.currentTimeMillis();
|
||||
System.out.println("Matches: " + results.size());
|
||||
System.out.println("Time elapsed: " + (time2_2 - time2_1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
|
||||
long removedItems = 0;//db.clean();
|
||||
time3 = System.currentTimeMillis();
|
||||
System.out.println("Removed items: " + removedItems);
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Time elapsed: " + (time3 - time2_2));
|
||||
System.out.println("Saving database...");
|
||||
System.out.println("Root size: " + root.size());
|
||||
});
|
||||
db.close();
|
||||
long time4 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time4 - time3));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
} finally {
|
||||
if (db.isOpen()) {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
long time2 = System.currentTimeMillis();
|
||||
saver.save();
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Time elapsed: " + (time2 - time1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
long time2_0 = System.currentTimeMillis();
|
||||
System.out.println("Filtering strings...");
|
||||
long oldSize = root.size();
|
||||
root.removeIf(Animal::hasFourLegs);
|
||||
long time2_1 = System.currentTimeMillis();
|
||||
System.out.println("RemoveIf(x) removed items: " + (oldSize - root.size()));
|
||||
System.out.println("Time elapsed: " + (time2_1 - time2_0));
|
||||
ObjectList<Animal> results = new ObjectArrayList<>();
|
||||
|
||||
System.out.println("Retrieving items...");
|
||||
root.forEachReference((valueReference) -> {
|
||||
Animal value = valueReference.getValueReadOnly();
|
||||
if (Animal.hasFourLegs(value)) {
|
||||
results.add(value);
|
||||
}
|
||||
//System.out.println("val:" + value);
|
||||
});
|
||||
long time2_2 = System.currentTimeMillis();
|
||||
System.out.println("Matches: " + results.size());
|
||||
System.out.println("Time elapsed: " + (time2_2 - time2_1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
|
||||
db.clean();
|
||||
time3 = System.currentTimeMillis();
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Time elapsed: " + (time3 - time2_2));
|
||||
System.out.println("Saving database...");
|
||||
System.out.println("Root size: " + root.size());
|
||||
});
|
||||
db.close();
|
||||
long time4 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time4 - time3));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
} finally {
|
||||
if (db.isOpen()) {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,29 +1,18 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.warp.jcwdb.EntryReference;
|
||||
import org.warp.jcwdb.JCWDatabase;
|
||||
import org.warp.jcwdb.LightList;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
/**
|
||||
* Unit test for simple App.
|
||||
*/
|
||||
public class AppTest
|
||||
{
|
||||
/**
|
||||
* Rigorous Test :-)
|
||||
*/
|
||||
@Test
|
||||
public void shouldAnswerWithTrue()
|
||||
{
|
||||
assertTrue( true );
|
||||
}
|
||||
|
||||
public class AppTest {
|
||||
/**
|
||||
* Rigorous Test :-)
|
||||
*/
|
||||
@Test
|
||||
public void shouldAnswerWithTrue() {
|
||||
assertTrue(true);
|
||||
}
|
||||
}
|
||||
|
||||
51
src/test/java/org/warp/jcwdb/FileAllocatorTest.java
Normal file
51
src/test/java/org/warp/jcwdb/FileAllocatorTest.java
Normal file
@ -0,0 +1,51 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class FileAllocatorTest {
|
||||
|
||||
@Test
|
||||
public void shouldAllocateAtZero() throws IOException {
|
||||
Path tempFile = Files.createTempFile("", "");
|
||||
SeekableByteChannel byteCh = Files.newByteChannel(tempFile);
|
||||
FileAllocator allocator = new FileAllocator(byteCh);
|
||||
long offset1 = allocator.allocate(512);
|
||||
assertEquals(0, offset1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldAllocateAt512() throws IOException {
|
||||
Path tempFile = Files.createTempFile("", "");
|
||||
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
|
||||
byteCh.write(ByteBuffer.wrap(new byte[512]));
|
||||
FileAllocator allocator = new FileAllocator(byteCh);
|
||||
long offset1 = allocator.allocate(512);
|
||||
assertEquals(512, offset1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldAllocateUnusedSpace() throws IOException {
|
||||
Path tempFile = Files.createTempFile("", "");
|
||||
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
|
||||
FileAllocator allocator = new FileAllocator(byteCh);
|
||||
long offset1 = allocator.allocate(512);
|
||||
allocator.markFree(offset1, 512);
|
||||
long offset2 = allocator.allocate(128);
|
||||
long offset3 = allocator.allocate(512-128);
|
||||
long offset4 = allocator.allocate(128);
|
||||
assertEquals(0, offset2);
|
||||
assertEquals(128, offset3);
|
||||
assertEquals(512, offset4);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user