Cleanable

This commit is contained in:
Cavallium 2018-11-27 17:47:19 +01:00
parent 0542c4c94a
commit 9db950fe1c
7 changed files with 232 additions and 38 deletions

View File

@ -0,0 +1,9 @@
package org.warp.jcwdb;
public interface Cleanable {
/**
* Clean the object
* @return the approximated number of cleaned items
*/
public long clean();
}

View File

@ -0,0 +1,53 @@
package org.warp.jcwdb;
import java.lang.ref.WeakReference;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner {
private Cleanable[] objectsToClean;
private Thread cleanerThread;
private int sleepInterval;
public Cleaner(Cleanable... objectsToClean) {
this.objectsToClean = objectsToClean;
}
public void start() {
this.cleanerThread = new Thread(new CleanLoop());
}
/**
* Clean
* @return number of removed items
*/
private long clean() {
long cleanedItems = 0;
for (Cleanable cleanable : objectsToClean) {
cleanedItems += cleanable.clean();
}
return cleanedItems;
}
public void stop() {
if (cleanerThread != null) {
cleanerThread.interrupt();
}
}
private class CleanLoop implements Runnable {
@Override
public void run() {
while(!cleanerThread.isInterrupted()) {
try {
Thread.sleep(sleepInterval);
System.out.println("Cleaned " + clean() + " items.");
} catch (InterruptedException e) {
}
}
}
}
}

View File

@ -7,8 +7,10 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.LongAVLTreeSet;
import it.unimi.dsi.fastutil.longs.LongSet;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
@ -25,6 +27,7 @@ public class FileIndexManager implements IndexManager {
private final Object closeLock = new Object();
private final Object metadataByteBufferLock = new Object();
private final Object maskByteBufferLock = new Object();
private final Object indicesMapsAccessLock = new Object();
/**
* Edit this using editIndex()
@ -132,9 +135,39 @@ public class FileIndexManager implements IndexManager {
if (indexDetails != null) {
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
dirtyLoadedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
synchronized(indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
}
}
public void flushAndUnload(long index) throws IOException {
if (removedIndices.contains(index)) {
synchronized(indicesMapsAccessLock) {
removedIndices.remove(index);
dirtyLoadedIndices.remove(index);
loadedIndices.remove(index);
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
if (dirtyLoadedIndices.contains(index)) {
synchronized(indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index);
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
IndexDetails indexDetails;
synchronized(indicesMapsAccessLock) {
indexDetails = loadedIndices.get(index);
}
writeIndexDetails(metadata, indexDetails);
}
synchronized(indicesMapsAccessLock) {
loadedIndices.remove(index);
}
}
@Override
@ -155,21 +188,28 @@ public class FileIndexManager implements IndexManager {
}
private void editIndex(long index, IndexDetails details) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
synchronized(indicesMapsAccessLock) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
}
}
private long createIndexMetadata(IndexDetails indexDetails) {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
removedIndices.remove(newIndex);
return newIndex;
synchronized(indicesMapsAccessLock) {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
removedIndices.remove(newIndex);
return newIndex;
}
}
private IndexDetails getIndexMetadataUnsafe(long index) throws IOException {
// Return index details if loaded
IndexDetails details = loadedIndices.getOrDefault(index, null);
IndexDetails details;
synchronized(indicesMapsAccessLock) {
details = loadedIndices.getOrDefault(index, null);
}
if (details != null) return details;
// Try to load the details from file
@ -221,39 +261,78 @@ public class FileIndexManager implements IndexManager {
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
synchronized(indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
}
synchronized (metadataByteBufferLock) {
metadataByteBuffer.rewind();
metadataByteBuffer.putInt(0);
metadataByteBuffer.putLong(indexDetails.getOffset());
metadataByteBuffer.putInt(indexDetails.getSize());
metadataByteBuffer.putInt(indexDetails.getType());
metadataByteBuffer.rewind();
metadata.write(metadataByteBuffer);
}
lastIndex = index;
}
// Remove removed indices
for (long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
synchronized (maskByteBufferLock) {
maskByteBuffer.rewind();
maskByteBuffer.putInt(IndexDetails.MASK_DELETED);
maskByteBuffer.rewind();
metadata.write(maskByteBuffer);
}
synchronized(indicesMapsAccessLock) {
for (long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
}
fileAllocator.close();
}
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
synchronized (metadataByteBufferLock) {
metadataByteBuffer.rewind();
metadataByteBuffer.putInt(0);
metadataByteBuffer.putLong(indexDetails.getOffset());
metadataByteBuffer.putInt(indexDetails.getSize());
metadataByteBuffer.putInt(indexDetails.getType());
metadataByteBuffer.rewind();
position.write(metadataByteBuffer);
}
}
private void eraseIndexDetails(SeekableByteChannel position) throws IOException {
synchronized (maskByteBufferLock) {
maskByteBuffer.rewind();
maskByteBuffer.putInt(IndexDetails.MASK_DELETED);
maskByteBuffer.rewind();
position.write(maskByteBuffer);
}
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
@Override
public long clean() {
return cleanExtraIndices();
}
private long cleanExtraIndices() {
long removedIndices = 0;
synchronized(indicesMapsAccessLock) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
long count = loadedIndices.size();
for (Entry<IndexDetails> loadedIndex : loadedIndices.long2ObjectEntrySet()) {
if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
break;
}
try {
flushAndUnload(loadedIndex.getLongKey());
} catch (IOException e) {
e.printStackTrace();
}
removedIndices++;
count--;
}
}
}
return removedIndices;
}
}

View File

@ -2,7 +2,7 @@ package org.warp.jcwdb;
import java.io.IOException;
public interface IndexManager {
public interface IndexManager extends Cleanable {
<T> T get(long index, DBReader<T> reader) throws IOException;
int getType(long index) throws IOException;
<T> long add(DBDataOutput<T> writer) throws IOException;

View File

@ -7,10 +7,15 @@ import java.util.ArrayList;
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class JCWDatabase implements AutoCloseable {
protected final TypesManager typesManager;
protected final MixedIndexDatabase indices;
public class JCWDatabase implements AutoCloseable, Cleanable {
public final static long MAX_LOADED_REFERENCES = 10;
public final static long MAX_LOADED_INDICES = 1000;
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
private final Cleaner databaseCleaner;
private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
private volatile boolean closed;
private final Object closeLock = new Object();
@ -28,6 +33,9 @@ public class JCWDatabase implements AutoCloseable {
e.printStackTrace();
}
}));
this.databaseCleaner = new Cleaner(this);
this.databaseCleaner.start();
}
public <T> EntryReference<LightList<T>> getRoot() throws IOException {
@ -140,4 +148,43 @@ public class JCWDatabase implements AutoCloseable {
throw new RuntimeException("Index Manager is closed.");
}
}
@Override
public long clean() {
return cleanEmptyReferences()
+ cleanExtraReferences()
+ indices.clean();
}
private long cleanEmptyReferences() {
long removed = 0;
synchronized(referencesAccessLock) {
for (Entry<WeakReference<EntryReference<?>>> entry : references.long2ObjectEntrySet()) {
if (entry.getValue().get() == null) {
references.remove(entry.getLongKey());
removed++;
}
}
}
return removed;
}
private long cleanExtraReferences() {
long removedReferences = 0;
synchronized(referencesAccessLock) {
if (references.size() > MAX_LOADED_REFERENCES) {
long count = 0;
for (Entry<WeakReference<EntryReference<?>>> entry : references.long2ObjectEntrySet()) {
if (count > MAX_LOADED_REFERENCES * 3l / 2l) {
references.remove(entry.getLongKey());
removedReferences++;
} else {
count++;
}
}
}
}
return removedReferences;
}
}

View File

@ -71,4 +71,10 @@ public class MixedIndexDatabase implements IndexManager {
this.cacheIndices.close();
this.fileIndices.close();
}
@Override
public long clean() {
return fileIndices.clean()
+ cacheIndices.clean();
}
}

View File

@ -27,7 +27,7 @@ public class App {
// System.out.println(" - " + root.get(i));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 2000000; i++) {
for (int i = 0; i < 2/*2000000*/; i++) {
root.add("Test " + i);
if (i > 0 && i % 200000 == 0) {
long precprectime = prectime;