Fixed BigList and Cleaner

This commit is contained in:
Andrea Cavalli 2019-01-06 00:31:52 +01:00
parent a4a981ad1c
commit 71a8148bdc
11 changed files with 237 additions and 156 deletions

View File

@ -0,0 +1,5 @@
package org.warp.jcwdb;
public interface AdvancedSaveable extends Saveable {
public void save(boolean isEditFinished);
}

View File

@ -45,6 +45,11 @@ public class CacheIndexManager implements IndexManager {
return null; return null;
} }
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
// TODO: implement
}
@Override @Override
public void delete(long index) { public void delete(long index) {
// TODO: implement // TODO: implement

View File

@ -8,6 +8,8 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner { public class Cleaner {
public static final boolean DISABLE_CLEANER = false;
public static final boolean ENABLE_CLEANER_LOGGING = false;
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
private static final double NORMAL_REMOVED_ITEMS = 2500l; private static final double NORMAL_REMOVED_ITEMS = 2500l;
@ -26,7 +28,9 @@ public class Cleaner {
} }
public void start() { public void start() {
//this.cleanerThread.start(); if (!DISABLE_CLEANER) {
this.cleanerThread.start();
}
} }
/** /**
@ -38,7 +42,7 @@ public class Cleaner {
for (Cleanable cleanable : objectsToClean) { for (Cleanable cleanable : objectsToClean) {
cleanedItems += cleanable.clean(); cleanedItems += cleanable.clean();
} }
System.gc(); //System.gc();
return cleanedItems; return cleanedItems;
} }
@ -61,38 +65,38 @@ public class Cleaner {
public void run() { public void run() {
while(!stopRequest) { while(!stopRequest) {
try { try {
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms."); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
sleepFor(sleepInterval); sleepFor(sleepInterval);
final long time1 = System.currentTimeMillis(); final long time1 = System.currentTimeMillis();
final double removedItems = clean(); final double removedItems = clean();
final long time2 = System.currentTimeMillis(); final long time2 = System.currentTimeMillis();
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1)); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2; double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
if (removedItems > 0) { if (removedItems > 0) {
final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS; final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) { if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio; suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
} }
} }
System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms"); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
double newSleepInterval = suggestedExecutionTimeByItemsCalculations; double newSleepInterval = suggestedExecutionTimeByItemsCalculations;
System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms"); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) { if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) {
sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL; sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL;
} else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) { } else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) {
sleepInterval = (int) MINIMUM_SLEEP_INTERVAL; sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
} else { } else {
System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms"); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
sleepInterval = (int) newSleepInterval; sleepInterval = (int) newSleepInterval;
} }
System.out.println("[CLEANER] Cleaned " + removedItems + " items."); if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
}catch (InterruptedException e) { }catch (InterruptedException e) {
} }

View File

@ -10,7 +10,7 @@ import java.util.function.Function;
* You must have only a maximum of 1 reference for each index * You must have only a maximum of 1 reference for each index
* @param <T> * @param <T>
*/ */
public class EntryReference<T> implements Castable, Saveable { public class EntryReference<T> implements Castable, AdvancedSaveable {
private final JCWDatabase.EntryReferenceTools db; private final JCWDatabase.EntryReferenceTools db;
private final long entryIndex; private final long entryIndex;
private final DBTypeParser<T> parser; private final DBTypeParser<T> parser;
@ -19,6 +19,7 @@ public class EntryReference<T> implements Castable, Saveable {
private volatile boolean isHashCached; private volatile boolean isHashCached;
private volatile boolean loaded; private volatile boolean loaded;
private volatile boolean closed; private volatile boolean closed;
private volatile boolean isFlushingAllowed;
private final Object hashCacheLock = new Object(); private final Object hashCacheLock = new Object();
private final Object accessLock = new Object(); private final Object accessLock = new Object();
private final Object closeLock = new Object(); private final Object closeLock = new Object();
@ -66,17 +67,29 @@ public class EntryReference<T> implements Castable, Saveable {
* Note that this method won't be called when closing without saving * Note that this method won't be called when closing without saving
*/ */
public void save() { public void save() {
this.save(false);
}
public void save(boolean isEditFinished) {
synchronized(accessLock) { synchronized(accessLock) {
if (loaded && !closed) { if (loaded && !closed) {
try { try {
if (value instanceof Saveable) { if (value instanceof AdvancedSaveable) {
((AdvancedSaveable)value).save(isEditFinished);
} else if (value instanceof Saveable) {
((Saveable)value).save(); ((Saveable)value).save();
} }
IndexDetails returnedDetails = db.write(entryIndex, parser.getWriter(value)); IndexDetails returnedDetails = this.db.write(entryIndex, parser.getWriter(value));
synchronized(hashCacheLock) { synchronized(hashCacheLock) {
this.cachedHash = returnedDetails.getHash(); this.cachedHash = returnedDetails.getHash();
this.isHashCached = true; this.isHashCached = true;
} }
if (isEditFinished) {
if (!isFlushingAllowed) {
this.db.setFlushingAllowed(entryIndex, true);
this.isFlushingAllowed = true;
}
}
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -93,7 +106,7 @@ public class EntryReference<T> implements Castable, Saveable {
synchronized(accessLock) { synchronized(accessLock) {
load(); load();
this.value = editFunction.apply(this.value, this); this.value = editFunction.apply(this.value, this);
this.save(); this.save(true);
} }
} }
@ -106,7 +119,7 @@ public class EntryReference<T> implements Castable, Saveable {
synchronized(accessLock) { synchronized(accessLock) {
load(); load();
this.value = editFunction.apply(this.value); this.value = editFunction.apply(this.value);
this.save(); this.save(true);
} }
} }
@ -119,7 +132,7 @@ public class EntryReference<T> implements Castable, Saveable {
synchronized(accessLock) { synchronized(accessLock) {
load(); load();
editFunction.accept(this.value, this); editFunction.accept(this.value, this);
this.save(); this.save(true);
} }
} }
@ -132,7 +145,7 @@ public class EntryReference<T> implements Castable, Saveable {
synchronized(accessLock) { synchronized(accessLock) {
load(); load();
editFunction.accept(this.value); editFunction.accept(this.value);
this.save(); this.save(true);
} }
} }
@ -148,7 +161,7 @@ public class EntryReference<T> implements Castable, Saveable {
synchronized(hashCacheLock) { synchronized(hashCacheLock) {
this.isHashCached = false; this.isHashCached = false;
} }
this.save(); this.save(true);
} }
} }
@ -177,6 +190,10 @@ public class EntryReference<T> implements Castable, Saveable {
synchronized(accessLock) { synchronized(accessLock) {
if (!loaded) { if (!loaded) {
try { try {
if (this.isFlushingAllowed) {
this.db.setFlushingAllowed(entryIndex, false);
this.isFlushingAllowed = false;
}
this.value = db.read(entryIndex, parser.getReader()); this.value = db.read(entryIndex, parser.getReader());
this.loaded = true; this.loaded = true;
} catch (IOException e) { } catch (IOException e) {
@ -201,7 +218,7 @@ public class EntryReference<T> implements Castable, Saveable {
return; return;
} }
save(); save(true);
closed = true; closed = true;
} }

View File

@ -12,11 +12,10 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
public class FileIndexManager implements IndexManager { public class FileIndexManager implements IndexManager {
private final SeekableByteChannel dataFileChannel, metadataFileChannel; private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private volatile long metadataFileChannelSize;
private final FileAllocator fileAllocator; private final FileAllocator fileAllocator;
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES); private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES); private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
@ -35,13 +34,21 @@ public class FileIndexManager implements IndexManager {
/** /**
* Edit this using editIndex() * Edit this using editIndex()
*/ */
private final LongSet dirtyLoadedIndices, removedIndices; private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
private long firstAllocableIndex; private long firstAllocableIndex;
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException { public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
loadedIndices = new Long2ObjectOpenHashMap<>(); if (Cleaner.DISABLE_CLEANER) {
dirtyLoadedIndices = new LongOpenHashSet(); loadedIndices = new Long2ObjectOpenHashMap<>();
removedIndices = new LongOpenHashSet(); dirtyLoadedIndices = new LongOpenHashSet();
flushingAllowedIndices = new LongOpenHashSet();
removedIndices = new LongOpenHashSet();
} else {
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
dirtyLoadedIndices = new LongLinkedOpenHashSet();
flushingAllowedIndices = new LongLinkedOpenHashSet();
removedIndices = new LongLinkedOpenHashSet();
}
if (Files.notExists(dataFile)) { if (Files.notExists(dataFile)) {
Files.createFile(dataFile); Files.createFile(dataFile);
} }
@ -51,17 +58,22 @@ public class FileIndexManager implements IndexManager {
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE); dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE); metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0)); fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
firstAllocableIndex = metadataFileChannel.size() / (long) IndexDetails.TOTAL_BYTES; metadataFileChannelSize = metadataFileChannel.size();
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
if (firstAllocableIndex == 0) { if (firstAllocableIndex == 0) {
firstAllocableIndex = 1; firstAllocableIndex = 1;
} }
} }
private long getMetadataFileChannelSize() throws IOException {
return metadataFileChannelSize;
}
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException { private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
Long2IntMap freeBytes = new Long2IntRBTreeMap(); Long2IntMap freeBytes = new Long2IntRBTreeMap();
Long2IntMap usedBytes = new Long2IntRBTreeMap(); Long2IntMap usedBytes = new Long2IntRBTreeMap();
long firstOffset = 0; long firstOffset = 0;
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= metadataFileChannel.size()) { while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel); IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
if (indexDetails != null) { if (indexDetails != null) {
long offset = indexDetails.getOffset(); long offset = indexDetails.getOffset();
@ -117,7 +129,7 @@ public class FileIndexManager implements IndexManager {
public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException { public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException {
checkClosed(); checkClosed();
final int dataSize = data.getSize(); final int dataSize = data.getSize();
final IndexDetails indexDetails = getIndexMetadataUnsafe(index); IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < dataSize) { if (indexDetails == null || indexDetails.getSize() < dataSize) {
// Allocate new space // Allocate new space
IndexDetails newDetails = allocateAndWrite(index, data); IndexDetails newDetails = allocateAndWrite(index, data);
@ -133,7 +145,7 @@ public class FileIndexManager implements IndexManager {
fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize); fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize);
} }
// Update index details // Update index details
editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash()); indexDetails = editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
// Write data // Write data
writeExact(indexDetails, data); writeExact(indexDetails, data);
// Before returning, return IndexDetails // Before returning, return IndexDetails
@ -141,6 +153,16 @@ public class FileIndexManager implements IndexManager {
} }
} }
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
checkClosed();
if (isUnloadingAllowed) {
flushingAllowedIndices.add(index);
} else {
flushingAllowedIndices.remove(index);
}
}
@Override @Override
public <T> long add(DBDataOutput<T> data) throws IOException { public <T> long add(DBDataOutput<T> data) throws IOException {
checkClosed(); checkClosed();
@ -206,6 +228,7 @@ public class FileIndexManager implements IndexManager {
} }
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index); dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index); loadedIndices.remove(index);
removedIndices.add(index); removedIndices.add(index);
} }
@ -216,6 +239,7 @@ public class FileIndexManager implements IndexManager {
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {
removedIndices.remove(index); removedIndices.remove(index);
dirtyLoadedIndices.remove(index); dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index); loadedIndices.remove(index);
} }
// Update indices metadata // Update indices metadata
@ -228,11 +252,14 @@ public class FileIndexManager implements IndexManager {
if (dirtyLoadedIndices.contains(index)) { if (dirtyLoadedIndices.contains(index)) {
indexDetails = loadedIndices.get(index); indexDetails = loadedIndices.get(index);
dirtyLoadedIndices.remove(index); dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
} }
} }
if (isDirty) { if (isDirty) {
// Update indices metadata // Update indices metadata
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES); long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
SeekableByteChannel metadata = metadataFileChannel.position(position);
writeIndexDetails(metadata, indexDetails); writeIndexDetails(metadata, indexDetails);
} }
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {
@ -293,6 +320,7 @@ public class FileIndexManager implements IndexManager {
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {
loadedIndices.put(index, details); loadedIndices.put(index, details);
dirtyLoadedIndices.add(index); dirtyLoadedIndices.add(index);
flushingAllowedIndices.remove(index);
} }
} }
@ -301,6 +329,7 @@ public class FileIndexManager implements IndexManager {
long newIndex = firstAllocableIndex++; long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails); loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex); dirtyLoadedIndices.add(newIndex);
flushingAllowedIndices.remove(newIndex);
removedIndices.remove(newIndex); removedIndices.remove(newIndex);
return newIndex; return newIndex;
} }
@ -316,7 +345,7 @@ public class FileIndexManager implements IndexManager {
// Try to load the details from file // Try to load the details from file
final long metadataPosition = index * IndexDetails.TOTAL_BYTES; final long metadataPosition = index * IndexDetails.TOTAL_BYTES;
if (metadataPosition + IndexDetails.TOTAL_BYTES > metadataFileChannel.size()) { if (metadataPosition + IndexDetails.TOTAL_BYTES > getMetadataFileChannelSize()) {
// Avoid underflow exception // Avoid underflow exception
return null; return null;
} }
@ -374,7 +403,7 @@ public class FileIndexManager implements IndexManager {
} }
// Update indices metadata // Update indices metadata
flushAllIndices(); flushAllFlushableIndices();
// Remove removed indices // Remove removed indices
removeRemovedIndices(); removeRemovedIndices();
@ -417,39 +446,56 @@ public class FileIndexManager implements IndexManager {
@Override @Override
public long clean() { public long clean() {
long cleaned = 0; long cleaned = 0;
long tim1 = System.currentTimeMillis();
try { try {
cleaned += flushAllIndices(); cleaned += flushAllFlushableIndices();
} catch (IOException ex) { } catch (IOException ex) {
ex.printStackTrace(); ex.printStackTrace();
} }
long tim2 = System.currentTimeMillis();
try { try {
cleaned += removeRemovedIndices(); cleaned += removeRemovedIndices();
} catch (IOException ex) { } catch (IOException ex) {
ex.printStackTrace(); ex.printStackTrace();
} }
long tim3 = System.currentTimeMillis();
cleaned += cleanExtraIndices(); cleaned += cleanExtraIndices();
long tim4 = System.currentTimeMillis();
if (Cleaner.ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] FileIndexManager CLEAN_TIME: " + (tim2-tim1) + "," + (tim3-tim2) + "," + (tim4-tim3));
return cleaned; return cleaned;
} }
private long flushAllIndices() throws IOException { private long flushAllFlushableIndices() throws IOException {
long flushedIndices = 0; long flushedIndices = 0;
SeekableByteChannel metadata = metadataFileChannel; SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2; long lastIndex = -2;
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) { for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index); if (!flushingAllowedIndices.contains(index)) {
if (index - lastIndex != 1) { IndexDetails indexDetails = loadedIndices.get(index);
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
if (index - lastIndex != 1) {
metadata = metadata.position(position);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
} }
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
} }
dirtyLoadedIndices.clear(); dirtyLoadedIndices.clear();
dirtyLoadedIndices.addAll(flushingAllowedIndices);
flushingAllowedIndices.clear();
} }
return flushedIndices; return flushedIndices;
} }
private void resizeMetadataFileChannel(long position) {
if (position + IndexDetails.TOTAL_BYTES > metadataFileChannelSize) {
metadataFileChannelSize = position + IndexDetails.TOTAL_BYTES;
}
}
private long removeRemovedIndices() throws IOException { private long removeRemovedIndices() throws IOException {
SeekableByteChannel metadata = metadataFileChannel; SeekableByteChannel metadata = metadataFileChannel;
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {

View File

@ -12,6 +12,7 @@ public interface IndexManager extends Cleanable {
<T> long add(DBDataOutput<T> writer) throws IOException; <T> long add(DBDataOutput<T> writer) throws IOException;
<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException; <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException; <T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
void delete(long index) throws IOException; void delete(long index) throws IOException;
boolean has(long index); boolean has(long index);
void close() throws IOException; void close() throws IOException;

View File

@ -1,7 +1,5 @@
package org.warp.jcwdb; package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
@ -154,6 +152,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException { public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException {
return indices.set(index, writer); return indices.set(index, writer);
} }
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
indices.setFlushingAllowed(index, isFlushingAllowed);
}
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")

View File

@ -3,6 +3,7 @@ package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOError;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.function.Consumer; import java.util.function.Consumer;
@ -112,7 +113,7 @@ public class LightArrayList<T> implements LightList<T> {
try { try {
action.accept(db.get(index)); action.accept(db.get(index));
} catch (IOException e) { } catch (IOException e) {
throw (RuntimeException) new RuntimeException().initCause(e); throw new IOError(e);
} }
} }
} }

View File

@ -4,6 +4,7 @@ import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOError;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.function.Consumer; import java.util.function.Consumer;
@ -16,8 +17,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
public final LongArrayList chunks; public final LongArrayList chunks;
public final IntArrayList chunkSizes; public final IntArrayList chunkSizes;
private final JCWDatabase db; private final JCWDatabase db;
private LightList<T> cachedChunk; private LightArrayList<T> cachedChunk;
private EntryReference<LightList<T>> cachedChunkRef; private EntryReference<LightArrayList<T>> cachedChunkRef;
private long cachedChunkIndex = -1; private long cachedChunkIndex = -1;
private int cachedChunkNumber = -1; private int cachedChunkNumber = -1;
private final Object cachedChunkLock = new Object(); private final Object cachedChunkLock = new Object();
@ -132,8 +133,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
if (o != null) { if (o != null) {
for (long chunkIndex : chunks) { for (long chunkIndex : chunks) {
try { try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
LightList<T> chunk = chunkRef.getValueReadOnly(); LightArrayList<T> chunk = chunkRef.getValueReadOnly();
if (chunk.contains(o)) { if (chunk.contains(o)) {
return true; return true;
} }
@ -175,13 +176,13 @@ public class LightBigList<T> implements LightList<T>, Saveable {
@Override @Override
public void forEachReference(Consumer<? super EntryReference<T>> action) { public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action); Objects.requireNonNull(action);
for (long chunkIndex : this.chunks) { // Iterate through all chunks
try { for (int i = 0; i < chunks.size(); i++) {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); synchronized (cachedChunkLock) {
LightList<T> chunk = chunkRef.getValueReadOnly(); if (cachedChunkNumber != i) {
chunk.forEachReference(action); prepareAccessToChunk(i);
} catch (IOException ex) { }
throw (NullPointerException) new NullPointerException().initCause(ex); cachedChunk.forEachReference(action);
} }
} }
} }
@ -194,6 +195,9 @@ public class LightBigList<T> implements LightList<T>, Saveable {
@Deprecated @Deprecated
@Override @Override
public T[] toArray() { public T[] toArray() {
if (true) {
throw new RuntimeException("toArray() isn't implemented!");
}
T[] result = (T[]) new Object[this.size()]; T[] result = (T[]) new Object[this.size()];
long currentOffset = 0; long currentOffset = 0;
@ -206,8 +210,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
final long chunkIndex = chunks.getLong(i); final long chunkIndex = chunks.getLong(i);
try { try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
LightList<T> chunk = chunkRef.getValueReadOnly(); LightArrayList<T> chunk = chunkRef.getValueReadOnly();
for (int i1 = 0; i1 < chunk.size(); i1++) { for (int i1 = 0; i1 < chunk.size(); i1++) {
result[(int)(chunkStartOffset + i1)] = chunk.get(i); result[(int)(chunkStartOffset + i1)] = chunk.get(i);
} }
@ -276,7 +280,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
} }
try { try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> { chunkRef.editValue((chunk) -> {
result.var = chunk.remove(relativeOffset); result.var = chunk.remove(relativeOffset);
}); });
@ -414,7 +418,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
} }
try { try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> { chunkRef.editValue((chunk) -> {
chunk.set(relativeOffset, element); chunk.set(relativeOffset, element);
wrapper.var = element; wrapper.var = element;
@ -458,7 +462,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
// Get chunk index // Get chunk index
final long chunkIndex = chunks.getLong(i); final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref); final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
if (foundIndex >= 0) { if (foundIndex >= 0) {
return currentOffset + foundIndex; return currentOffset + foundIndex;
@ -487,7 +491,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
// Get chunk index // Get chunk index
final long chunkIndex = chunks.getLong(i); final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref); final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
if (foundIndex >= 0) { if (foundIndex >= 0) {
return currentOffset + foundIndex; return currentOffset + foundIndex;
@ -543,26 +547,20 @@ public class LightBigList<T> implements LightList<T>, Saveable {
@Override @Override
public boolean removeIf(Predicate<? super T> filter) { public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter); Objects.requireNonNull(filter);
final VariableWrapper<Boolean> result = new VariableWrapper(false); boolean result = false;
// Iterate through all chunks // Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) { for (int i = 0; i < chunks.size(); i++) {
try { synchronized (cachedChunkLock) {
final int chunkOffset = i; if (cachedChunkNumber != i) {
// Get chunk index prepareAccessToChunk(i);
final long chunkIndex = chunks.getLong(i); }
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex); if (cachedChunk.removeIf(filter)) {
chunkRef.editValue((chunk) -> { result = true;
boolean removed = chunk.removeIf(filter); chunkSizes.set(cachedChunkNumber, cachedChunk.size());
if (removed) { }
result.var = true;
chunkSizes.set(chunkOffset, chunk.size());
}
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
} }
} }
return result.var; return result;
} }
@Override @Override

View File

@ -1,12 +1,7 @@
package org.warp.jcwdb; package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.Long2LongLinkedOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2LongMap;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
public class MixedIndexDatabase implements IndexManager { public class MixedIndexDatabase implements IndexManager {
private final FileIndexManager fileIndices; private final FileIndexManager fileIndices;
@ -62,6 +57,14 @@ public class MixedIndexDatabase implements IndexManager {
return fileIndices.set(index, writer); return fileIndices.set(index, writer);
} }
} }
@Override
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
if (cacheIndices.has(index)) {
cacheIndices.setFlushingAllowed(index, isFlushingAllowed);
} else {
fileIndices.setFlushingAllowed(index, isFlushingAllowed);
}
}
@Override @Override
public void delete(long index) throws IOException { public void delete(long index) throws IOException {

View File

@ -7,92 +7,91 @@ import org.warp.jcwdb.LightList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.objects.ObjectList; import it.unimi.dsi.fastutil.objects.ObjectList;
import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.function.Predicate; import java.util.function.Predicate;
public class App { public class App {
static long time3; static long time3;
public static void main(String[] args) { public static void main(String[] args) throws IOException {
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
Files.delete(Paths.get(args[0]));
Files.delete(Paths.get(args[1]));
}
System.out.println("Loading database...");
long time0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
db.registerClass(StrangeAnimal.class, 0);
try { try {
if (args.length > 2 && Boolean.parseBoolean(args[2])) { long time01 = System.currentTimeMillis();
Files.delete(Paths.get(args[0])); System.out.println("Time elapsed: " + (time01 - time0));
Files.delete(Paths.get(args[1])); System.out.println("Loading root...");
} EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
System.out.println("Loading database..."); rootRef.editValue((root, saver) -> {
long time0 = System.currentTimeMillis(); long time1 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1])); System.out.println("Time elapsed: " + (time1 - time01));
db.registerClass(StrangeAnimal.class, 0); System.out.println("Root size: " + root.size());
try { System.out.println("Root:");
long time01 = System.currentTimeMillis(); // for (int i = 0; i < root.size(); i++) {
System.out.println("Time elapsed: " + (time01 - time0)); // System.out.println(" - " + root.get(i));
System.out.println("Loading root..."); // }
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class); long prectime = System.currentTimeMillis();
rootRef.editValue((root, saver) -> { for (int i = 0; i < 2000000/* 2000000 */; i++) {
long time1 = System.currentTimeMillis(); Animal animal = new StrangeAnimal(i % 40);
System.out.println("Time elapsed: " + (time1 - time01)); root.addEntry(animal);
System.out.println("Root size: " + root.size()); if (i > 0 && i % 200000 == 0) {
System.out.println("Root:"); long precprectime = prectime;
// for (int i = 0; i < root.size(); i++) { prectime = System.currentTimeMillis();
// System.out.println(" - " + root.get(i)); System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)" + " Total Time: " + (prectime - time1));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 20000000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40);
root.add(animal);
if (i > 0 && i % 200000 == 0) {
long precprectime = prectime;
prectime = System.currentTimeMillis();
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)");
}
} }
long time2 = System.currentTimeMillis();
saver.save();
System.out.println("Root size: " + root.size());
System.out.println("Time elapsed: " + (time2 - time1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
long time2_0 = System.currentTimeMillis();
System.out.println("Filtering strings...");
//root.removeIf(Animal::hasFourLegs);
long time2_1 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnly();
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
long removedItems = 0;//db.clean();
time3 = System.currentTimeMillis();
System.out.println("Removed items: " + removedItems);
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Time elapsed: " + (time3 - time2_2));
System.out.println("Saving database...");
System.out.println("Root size: " + root.size());
});
db.close();
long time4 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time4 - time3));
} catch (Exception ex) {
ex.printStackTrace();
} finally {
if (db.isOpen()) {
db.close();
} }
} long time2 = System.currentTimeMillis();
saver.save();
System.out.println("Root size: " + root.size());
System.out.println("Time elapsed: " + (time2 - time1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
long time2_0 = System.currentTimeMillis();
System.out.println("Filtering strings...");
long oldSize = root.size();
root.removeIf(Animal::hasFourLegs);
long time2_1 = System.currentTimeMillis();
System.out.println("RemoveIf(x) removed items: " + (oldSize - root.size()));
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
System.out.println("Retrieving items...");
root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnly();
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
db.clean();
time3 = System.currentTimeMillis();
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Time elapsed: " + (time3 - time2_2));
System.out.println("Saving database...");
System.out.println("Root size: " + root.size());
});
db.close();
long time4 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time4 - time3));
} catch (Exception ex) { } catch (Exception ex) {
ex.printStackTrace(); ex.printStackTrace();
} finally {
if (db.isOpen()) {
db.close();
}
} }
} }