Fixed BigList and Cleaner
This commit is contained in:
parent
a4a981ad1c
commit
71a8148bdc
|
|
@ -0,0 +1,5 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
public interface AdvancedSaveable extends Saveable {
|
||||
public void save(boolean isEditFinished);
|
||||
}
|
||||
|
|
@ -45,6 +45,11 @@ public class CacheIndexManager implements IndexManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
|
||||
// TODO: implement
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(long index) {
|
||||
// TODO: implement
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
|
|||
|
||||
public class Cleaner {
|
||||
|
||||
public static final boolean DISABLE_CLEANER = false;
|
||||
public static final boolean ENABLE_CLEANER_LOGGING = false;
|
||||
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
|
||||
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
|
||||
private static final double NORMAL_REMOVED_ITEMS = 2500l;
|
||||
|
|
@ -26,7 +28,9 @@ public class Cleaner {
|
|||
}
|
||||
|
||||
public void start() {
|
||||
//this.cleanerThread.start();
|
||||
if (!DISABLE_CLEANER) {
|
||||
this.cleanerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -38,7 +42,7 @@ public class Cleaner {
|
|||
for (Cleanable cleanable : objectsToClean) {
|
||||
cleanedItems += cleanable.clean();
|
||||
}
|
||||
System.gc();
|
||||
//System.gc();
|
||||
return cleanedItems;
|
||||
}
|
||||
|
||||
|
|
@ -61,38 +65,38 @@ public class Cleaner {
|
|||
public void run() {
|
||||
while(!stopRequest) {
|
||||
try {
|
||||
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
|
||||
sleepFor(sleepInterval);
|
||||
final long time1 = System.currentTimeMillis();
|
||||
final double removedItems = clean();
|
||||
final long time2 = System.currentTimeMillis();
|
||||
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
|
||||
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
|
||||
|
||||
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
|
||||
if (removedItems > 0) {
|
||||
final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
|
||||
System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
|
||||
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
|
||||
suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
|
||||
|
||||
double newSleepInterval = suggestedExecutionTimeByItemsCalculations;
|
||||
System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) {
|
||||
sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL;
|
||||
} else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) {
|
||||
sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
|
||||
} else {
|
||||
System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
|
||||
sleepInterval = (int) newSleepInterval;
|
||||
}
|
||||
|
||||
|
||||
System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
|
||||
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
|
||||
}catch (InterruptedException e) {
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import java.util.function.Function;
|
|||
* You must have only a maximum of 1 reference for each index
|
||||
* @param <T>
|
||||
*/
|
||||
public class EntryReference<T> implements Castable, Saveable {
|
||||
public class EntryReference<T> implements Castable, AdvancedSaveable {
|
||||
private final JCWDatabase.EntryReferenceTools db;
|
||||
private final long entryIndex;
|
||||
private final DBTypeParser<T> parser;
|
||||
|
|
@ -19,6 +19,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
private volatile boolean isHashCached;
|
||||
private volatile boolean loaded;
|
||||
private volatile boolean closed;
|
||||
private volatile boolean isFlushingAllowed;
|
||||
private final Object hashCacheLock = new Object();
|
||||
private final Object accessLock = new Object();
|
||||
private final Object closeLock = new Object();
|
||||
|
|
@ -66,17 +67,29 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
* Note that this method won't be called when closing without saving
|
||||
*/
|
||||
public void save() {
|
||||
this.save(false);
|
||||
}
|
||||
|
||||
public void save(boolean isEditFinished) {
|
||||
synchronized(accessLock) {
|
||||
if (loaded && !closed) {
|
||||
try {
|
||||
if (value instanceof Saveable) {
|
||||
if (value instanceof AdvancedSaveable) {
|
||||
((AdvancedSaveable)value).save(isEditFinished);
|
||||
} else if (value instanceof Saveable) {
|
||||
((Saveable)value).save();
|
||||
}
|
||||
IndexDetails returnedDetails = db.write(entryIndex, parser.getWriter(value));
|
||||
IndexDetails returnedDetails = this.db.write(entryIndex, parser.getWriter(value));
|
||||
synchronized(hashCacheLock) {
|
||||
this.cachedHash = returnedDetails.getHash();
|
||||
this.isHashCached = true;
|
||||
}
|
||||
if (isEditFinished) {
|
||||
if (!isFlushingAllowed) {
|
||||
this.db.setFlushingAllowed(entryIndex, true);
|
||||
this.isFlushingAllowed = true;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
|
@ -93,7 +106,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
synchronized(accessLock) {
|
||||
load();
|
||||
this.value = editFunction.apply(this.value, this);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -106,7 +119,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
synchronized(accessLock) {
|
||||
load();
|
||||
this.value = editFunction.apply(this.value);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -119,7 +132,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
synchronized(accessLock) {
|
||||
load();
|
||||
editFunction.accept(this.value, this);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -132,7 +145,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
synchronized(accessLock) {
|
||||
load();
|
||||
editFunction.accept(this.value);
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -148,7 +161,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
synchronized(hashCacheLock) {
|
||||
this.isHashCached = false;
|
||||
}
|
||||
this.save();
|
||||
this.save(true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -177,6 +190,10 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
synchronized(accessLock) {
|
||||
if (!loaded) {
|
||||
try {
|
||||
if (this.isFlushingAllowed) {
|
||||
this.db.setFlushingAllowed(entryIndex, false);
|
||||
this.isFlushingAllowed = false;
|
||||
}
|
||||
this.value = db.read(entryIndex, parser.getReader());
|
||||
this.loaded = true;
|
||||
} catch (IOException e) {
|
||||
|
|
@ -201,7 +218,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
return;
|
||||
}
|
||||
|
||||
save();
|
||||
save(true);
|
||||
|
||||
closed = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,11 +12,10 @@ import java.nio.channels.SeekableByteChannel;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class FileIndexManager implements IndexManager {
|
||||
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
|
||||
private volatile long metadataFileChannelSize;
|
||||
private final FileAllocator fileAllocator;
|
||||
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
|
||||
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
|
||||
|
|
@ -35,13 +34,21 @@ public class FileIndexManager implements IndexManager {
|
|||
/**
|
||||
* Edit this using editIndex()
|
||||
*/
|
||||
private final LongSet dirtyLoadedIndices, removedIndices;
|
||||
private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
|
||||
private long firstAllocableIndex;
|
||||
|
||||
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
|
||||
loadedIndices = new Long2ObjectOpenHashMap<>();
|
||||
dirtyLoadedIndices = new LongOpenHashSet();
|
||||
removedIndices = new LongOpenHashSet();
|
||||
if (Cleaner.DISABLE_CLEANER) {
|
||||
loadedIndices = new Long2ObjectOpenHashMap<>();
|
||||
dirtyLoadedIndices = new LongOpenHashSet();
|
||||
flushingAllowedIndices = new LongOpenHashSet();
|
||||
removedIndices = new LongOpenHashSet();
|
||||
} else {
|
||||
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
|
||||
dirtyLoadedIndices = new LongLinkedOpenHashSet();
|
||||
flushingAllowedIndices = new LongLinkedOpenHashSet();
|
||||
removedIndices = new LongLinkedOpenHashSet();
|
||||
}
|
||||
if (Files.notExists(dataFile)) {
|
||||
Files.createFile(dataFile);
|
||||
}
|
||||
|
|
@ -51,17 +58,22 @@ public class FileIndexManager implements IndexManager {
|
|||
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
|
||||
firstAllocableIndex = metadataFileChannel.size() / (long) IndexDetails.TOTAL_BYTES;
|
||||
metadataFileChannelSize = metadataFileChannel.size();
|
||||
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
|
||||
if (firstAllocableIndex == 0) {
|
||||
firstAllocableIndex = 1;
|
||||
}
|
||||
}
|
||||
|
||||
private long getMetadataFileChannelSize() throws IOException {
|
||||
return metadataFileChannelSize;
|
||||
}
|
||||
|
||||
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
|
||||
Long2IntMap freeBytes = new Long2IntRBTreeMap();
|
||||
Long2IntMap usedBytes = new Long2IntRBTreeMap();
|
||||
long firstOffset = 0;
|
||||
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= metadataFileChannel.size()) {
|
||||
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
|
||||
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
|
||||
if (indexDetails != null) {
|
||||
long offset = indexDetails.getOffset();
|
||||
|
|
@ -117,7 +129,7 @@ public class FileIndexManager implements IndexManager {
|
|||
public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException {
|
||||
checkClosed();
|
||||
final int dataSize = data.getSize();
|
||||
final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
|
||||
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
|
||||
if (indexDetails == null || indexDetails.getSize() < dataSize) {
|
||||
// Allocate new space
|
||||
IndexDetails newDetails = allocateAndWrite(index, data);
|
||||
|
|
@ -133,7 +145,7 @@ public class FileIndexManager implements IndexManager {
|
|||
fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize);
|
||||
}
|
||||
// Update index details
|
||||
editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
|
||||
indexDetails = editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
|
||||
// Write data
|
||||
writeExact(indexDetails, data);
|
||||
// Before returning, return IndexDetails
|
||||
|
|
@ -141,6 +153,16 @@ public class FileIndexManager implements IndexManager {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
|
||||
checkClosed();
|
||||
if (isUnloadingAllowed) {
|
||||
flushingAllowedIndices.add(index);
|
||||
} else {
|
||||
flushingAllowedIndices.remove(index);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> long add(DBDataOutput<T> data) throws IOException {
|
||||
checkClosed();
|
||||
|
|
@ -206,6 +228,7 @@ public class FileIndexManager implements IndexManager {
|
|||
}
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
dirtyLoadedIndices.remove(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
loadedIndices.remove(index);
|
||||
removedIndices.add(index);
|
||||
}
|
||||
|
|
@ -216,6 +239,7 @@ public class FileIndexManager implements IndexManager {
|
|||
synchronized (indicesMapsAccessLock) {
|
||||
removedIndices.remove(index);
|
||||
dirtyLoadedIndices.remove(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
loadedIndices.remove(index);
|
||||
}
|
||||
// Update indices metadata
|
||||
|
|
@ -228,11 +252,14 @@ public class FileIndexManager implements IndexManager {
|
|||
if (dirtyLoadedIndices.contains(index)) {
|
||||
indexDetails = loadedIndices.get(index);
|
||||
dirtyLoadedIndices.remove(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
}
|
||||
}
|
||||
if (isDirty) {
|
||||
// Update indices metadata
|
||||
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
|
||||
long position = index * IndexDetails.TOTAL_BYTES;
|
||||
resizeMetadataFileChannel(position);
|
||||
SeekableByteChannel metadata = metadataFileChannel.position(position);
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
}
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
|
|
@ -293,6 +320,7 @@ public class FileIndexManager implements IndexManager {
|
|||
synchronized (indicesMapsAccessLock) {
|
||||
loadedIndices.put(index, details);
|
||||
dirtyLoadedIndices.add(index);
|
||||
flushingAllowedIndices.remove(index);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -301,6 +329,7 @@ public class FileIndexManager implements IndexManager {
|
|||
long newIndex = firstAllocableIndex++;
|
||||
loadedIndices.put(newIndex, indexDetails);
|
||||
dirtyLoadedIndices.add(newIndex);
|
||||
flushingAllowedIndices.remove(newIndex);
|
||||
removedIndices.remove(newIndex);
|
||||
return newIndex;
|
||||
}
|
||||
|
|
@ -316,7 +345,7 @@ public class FileIndexManager implements IndexManager {
|
|||
|
||||
// Try to load the details from file
|
||||
final long metadataPosition = index * IndexDetails.TOTAL_BYTES;
|
||||
if (metadataPosition + IndexDetails.TOTAL_BYTES > metadataFileChannel.size()) {
|
||||
if (metadataPosition + IndexDetails.TOTAL_BYTES > getMetadataFileChannelSize()) {
|
||||
// Avoid underflow exception
|
||||
return null;
|
||||
}
|
||||
|
|
@ -374,7 +403,7 @@ public class FileIndexManager implements IndexManager {
|
|||
}
|
||||
|
||||
// Update indices metadata
|
||||
flushAllIndices();
|
||||
flushAllFlushableIndices();
|
||||
|
||||
// Remove removed indices
|
||||
removeRemovedIndices();
|
||||
|
|
@ -417,39 +446,56 @@ public class FileIndexManager implements IndexManager {
|
|||
@Override
|
||||
public long clean() {
|
||||
long cleaned = 0;
|
||||
long tim1 = System.currentTimeMillis();
|
||||
try {
|
||||
cleaned += flushAllIndices();
|
||||
cleaned += flushAllFlushableIndices();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
long tim2 = System.currentTimeMillis();
|
||||
try {
|
||||
cleaned += removeRemovedIndices();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
long tim3 = System.currentTimeMillis();
|
||||
cleaned += cleanExtraIndices();
|
||||
long tim4 = System.currentTimeMillis();
|
||||
if (Cleaner.ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] FileIndexManager CLEAN_TIME: " + (tim2-tim1) + "," + (tim3-tim2) + "," + (tim4-tim3));
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
private long flushAllIndices() throws IOException {
|
||||
private long flushAllFlushableIndices() throws IOException {
|
||||
long flushedIndices = 0;
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
long lastIndex = -2;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
for (long index : dirtyLoadedIndices) {
|
||||
IndexDetails indexDetails = loadedIndices.get(index);
|
||||
if (index - lastIndex != 1) {
|
||||
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
|
||||
if (!flushingAllowedIndices.contains(index)) {
|
||||
IndexDetails indexDetails = loadedIndices.get(index);
|
||||
long position = index * IndexDetails.TOTAL_BYTES;
|
||||
resizeMetadataFileChannel(position);
|
||||
if (index - lastIndex != 1) {
|
||||
metadata = metadata.position(position);
|
||||
}
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
lastIndex = index;
|
||||
flushedIndices++;
|
||||
}
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
lastIndex = index;
|
||||
flushedIndices++;
|
||||
}
|
||||
dirtyLoadedIndices.clear();
|
||||
dirtyLoadedIndices.addAll(flushingAllowedIndices);
|
||||
flushingAllowedIndices.clear();
|
||||
}
|
||||
return flushedIndices;
|
||||
}
|
||||
|
||||
private void resizeMetadataFileChannel(long position) {
|
||||
if (position + IndexDetails.TOTAL_BYTES > metadataFileChannelSize) {
|
||||
metadataFileChannelSize = position + IndexDetails.TOTAL_BYTES;
|
||||
}
|
||||
}
|
||||
|
||||
private long removeRemovedIndices() throws IOException {
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ public interface IndexManager extends Cleanable {
|
|||
<T> long add(DBDataOutput<T> writer) throws IOException;
|
||||
<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
|
||||
<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
|
||||
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
|
||||
void delete(long index) throws IOException;
|
||||
boolean has(long index);
|
||||
void close() throws IOException;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
|
|
@ -154,6 +152,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
|||
public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException {
|
||||
return indices.set(index, writer);
|
||||
}
|
||||
|
||||
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
|
||||
indices.setFlushingAllowed(index, isFlushingAllowed);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package org.warp.jcwdb;
|
|||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
|
||||
import java.io.IOError;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
|
|
@ -112,7 +113,7 @@ public class LightArrayList<T> implements LightList<T> {
|
|||
try {
|
||||
action.accept(db.get(index));
|
||||
} catch (IOException e) {
|
||||
throw (RuntimeException) new RuntimeException().initCause(e);
|
||||
throw new IOError(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import it.unimi.dsi.fastutil.ints.IntArrayList;
|
|||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
|
||||
import java.io.IOError;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
|
|
@ -16,8 +17,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
public final LongArrayList chunks;
|
||||
public final IntArrayList chunkSizes;
|
||||
private final JCWDatabase db;
|
||||
private LightList<T> cachedChunk;
|
||||
private EntryReference<LightList<T>> cachedChunkRef;
|
||||
private LightArrayList<T> cachedChunk;
|
||||
private EntryReference<LightArrayList<T>> cachedChunkRef;
|
||||
private long cachedChunkIndex = -1;
|
||||
private int cachedChunkNumber = -1;
|
||||
private final Object cachedChunkLock = new Object();
|
||||
|
|
@ -132,8 +133,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
if (o != null) {
|
||||
for (long chunkIndex : chunks) {
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightArrayList<T> chunk = chunkRef.getValueReadOnly();
|
||||
if (chunk.contains(o)) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -175,13 +176,13 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
@Override
|
||||
public void forEachReference(Consumer<? super EntryReference<T>> action) {
|
||||
Objects.requireNonNull(action);
|
||||
for (long chunkIndex : this.chunks) {
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
chunk.forEachReference(action);
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
synchronized (cachedChunkLock) {
|
||||
if (cachedChunkNumber != i) {
|
||||
prepareAccessToChunk(i);
|
||||
}
|
||||
cachedChunk.forEachReference(action);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -194,6 +195,9 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
@Deprecated
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
if (true) {
|
||||
throw new RuntimeException("toArray() isn't implemented!");
|
||||
}
|
||||
T[] result = (T[]) new Object[this.size()];
|
||||
|
||||
long currentOffset = 0;
|
||||
|
|
@ -206,8 +210,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
final long chunkIndex = chunks.getLong(i);
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightArrayList<T> chunk = chunkRef.getValueReadOnly();
|
||||
for (int i1 = 0; i1 < chunk.size(); i1++) {
|
||||
result[(int)(chunkStartOffset + i1)] = chunk.get(i);
|
||||
}
|
||||
|
|
@ -276,7 +280,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
}
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
result.var = chunk.remove(relativeOffset);
|
||||
});
|
||||
|
|
@ -414,7 +418,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
}
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
chunk.set(relativeOffset, element);
|
||||
wrapper.var = element;
|
||||
|
|
@ -458,7 +462,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
|
||||
if (foundIndex >= 0) {
|
||||
return currentOffset + foundIndex;
|
||||
|
|
@ -487,7 +491,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
||||
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
|
||||
if (foundIndex >= 0) {
|
||||
return currentOffset + foundIndex;
|
||||
|
|
@ -543,26 +547,20 @@ public class LightBigList<T> implements LightList<T>, Saveable {
|
|||
@Override
|
||||
public boolean removeIf(Predicate<? super T> filter) {
|
||||
Objects.requireNonNull(filter);
|
||||
final VariableWrapper<Boolean> result = new VariableWrapper(false);
|
||||
boolean result = false;
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
try {
|
||||
final int chunkOffset = i;
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
boolean removed = chunk.removeIf(filter);
|
||||
if (removed) {
|
||||
result.var = true;
|
||||
chunkSizes.set(chunkOffset, chunk.size());
|
||||
}
|
||||
});
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
synchronized (cachedChunkLock) {
|
||||
if (cachedChunkNumber != i) {
|
||||
prepareAccessToChunk(i);
|
||||
}
|
||||
if (cachedChunk.removeIf(filter)) {
|
||||
result = true;
|
||||
chunkSizes.set(cachedChunkNumber, cachedChunk.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.var;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1,12 +1,7 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.Long2LongLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2LongMap;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class MixedIndexDatabase implements IndexManager {
|
||||
private final FileIndexManager fileIndices;
|
||||
|
|
@ -62,6 +57,14 @@ public class MixedIndexDatabase implements IndexManager {
|
|||
return fileIndices.set(index, writer);
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
|
||||
if (cacheIndices.has(index)) {
|
||||
cacheIndices.setFlushingAllowed(index, isFlushingAllowed);
|
||||
} else {
|
||||
fileIndices.setFlushingAllowed(index, isFlushingAllowed);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(long index) throws IOException {
|
||||
|
|
|
|||
|
|
@ -7,92 +7,91 @@ import org.warp.jcwdb.LightList;
|
|||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectList;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
public class App {
|
||||
static long time3;
|
||||
public static void main(String[] args) {
|
||||
public static void main(String[] args) throws IOException {
|
||||
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
|
||||
Files.delete(Paths.get(args[0]));
|
||||
Files.delete(Paths.get(args[1]));
|
||||
}
|
||||
System.out.println("Loading database...");
|
||||
long time0 = System.currentTimeMillis();
|
||||
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
|
||||
db.registerClass(StrangeAnimal.class, 0);
|
||||
try {
|
||||
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
|
||||
Files.delete(Paths.get(args[0]));
|
||||
Files.delete(Paths.get(args[1]));
|
||||
}
|
||||
System.out.println("Loading database...");
|
||||
long time0 = System.currentTimeMillis();
|
||||
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
|
||||
db.registerClass(StrangeAnimal.class, 0);
|
||||
try {
|
||||
long time01 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time01 - time0));
|
||||
System.out.println("Loading root...");
|
||||
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
|
||||
rootRef.editValue((root, saver) -> {
|
||||
long time1 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time1 - time01));
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Root:");
|
||||
// for (int i = 0; i < root.size(); i++) {
|
||||
// System.out.println(" - " + root.get(i));
|
||||
// }
|
||||
long prectime = System.currentTimeMillis();
|
||||
for (int i = 0; i < 20000000/* 2000000 */; i++) {
|
||||
Animal animal = new StrangeAnimal(i % 40);
|
||||
root.add(animal);
|
||||
if (i > 0 && i % 200000 == 0) {
|
||||
long precprectime = prectime;
|
||||
prectime = System.currentTimeMillis();
|
||||
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)");
|
||||
}
|
||||
long time01 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time01 - time0));
|
||||
System.out.println("Loading root...");
|
||||
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
|
||||
rootRef.editValue((root, saver) -> {
|
||||
long time1 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time1 - time01));
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Root:");
|
||||
// for (int i = 0; i < root.size(); i++) {
|
||||
// System.out.println(" - " + root.get(i));
|
||||
// }
|
||||
long prectime = System.currentTimeMillis();
|
||||
for (int i = 0; i < 2000000/* 2000000 */; i++) {
|
||||
Animal animal = new StrangeAnimal(i % 40);
|
||||
root.addEntry(animal);
|
||||
if (i > 0 && i % 200000 == 0) {
|
||||
long precprectime = prectime;
|
||||
prectime = System.currentTimeMillis();
|
||||
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)" + " Total Time: " + (prectime - time1));
|
||||
}
|
||||
long time2 = System.currentTimeMillis();
|
||||
saver.save();
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Time elapsed: " + (time2 - time1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
long time2_0 = System.currentTimeMillis();
|
||||
System.out.println("Filtering strings...");
|
||||
//root.removeIf(Animal::hasFourLegs);
|
||||
long time2_1 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time2_1 - time2_0));
|
||||
ObjectList<Animal> results = new ObjectArrayList<>();
|
||||
|
||||
root.forEachReference((valueReference) -> {
|
||||
Animal value = valueReference.getValueReadOnly();
|
||||
if (Animal.hasFourLegs(value)) {
|
||||
results.add(value);
|
||||
}
|
||||
//System.out.println("val:" + value);
|
||||
});
|
||||
long time2_2 = System.currentTimeMillis();
|
||||
System.out.println("Matches: " + results.size());
|
||||
System.out.println("Time elapsed: " + (time2_2 - time2_1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
|
||||
long removedItems = 0;//db.clean();
|
||||
time3 = System.currentTimeMillis();
|
||||
System.out.println("Removed items: " + removedItems);
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Time elapsed: " + (time3 - time2_2));
|
||||
System.out.println("Saving database...");
|
||||
System.out.println("Root size: " + root.size());
|
||||
});
|
||||
db.close();
|
||||
long time4 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time4 - time3));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
} finally {
|
||||
if (db.isOpen()) {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
long time2 = System.currentTimeMillis();
|
||||
saver.save();
|
||||
System.out.println("Root size: " + root.size());
|
||||
System.out.println("Time elapsed: " + (time2 - time1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
long time2_0 = System.currentTimeMillis();
|
||||
System.out.println("Filtering strings...");
|
||||
long oldSize = root.size();
|
||||
root.removeIf(Animal::hasFourLegs);
|
||||
long time2_1 = System.currentTimeMillis();
|
||||
System.out.println("RemoveIf(x) removed items: " + (oldSize - root.size()));
|
||||
System.out.println("Time elapsed: " + (time2_1 - time2_0));
|
||||
ObjectList<Animal> results = new ObjectArrayList<>();
|
||||
|
||||
System.out.println("Retrieving items...");
|
||||
root.forEachReference((valueReference) -> {
|
||||
Animal value = valueReference.getValueReadOnly();
|
||||
if (Animal.hasFourLegs(value)) {
|
||||
results.add(value);
|
||||
}
|
||||
//System.out.println("val:" + value);
|
||||
});
|
||||
long time2_2 = System.currentTimeMillis();
|
||||
System.out.println("Matches: " + results.size());
|
||||
System.out.println("Time elapsed: " + (time2_2 - time2_1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
|
||||
db.clean();
|
||||
time3 = System.currentTimeMillis();
|
||||
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
System.out.println("Time elapsed: " + (time3 - time2_2));
|
||||
System.out.println("Saving database...");
|
||||
System.out.println("Root size: " + root.size());
|
||||
});
|
||||
db.close();
|
||||
long time4 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time4 - time3));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
} finally {
|
||||
if (db.isOpen()) {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue