Fixed cleaner crash

This commit is contained in:
Andrea Cavalli 2018-12-06 12:30:04 +01:00
parent aa5cf24add
commit b00cd8d6b5
5 changed files with 113 additions and 93 deletions

View File

@ -1,6 +1,8 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.channels.ClosedChannelException;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
@ -14,6 +16,7 @@ public class Cleaner {
private final Cleanable[] objectsToClean;
private final Thread cleanerThread;
private int sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
private volatile boolean stopRequest = false;
public Cleaner(Cleanable... objectsToClean) {
this.objectsToClean = objectsToClean;
@ -35,12 +38,13 @@ public class Cleaner {
for (Cleanable cleanable : objectsToClean) {
cleanedItems += cleanable.clean();
}
System.gc();
return cleanedItems;
}
public void stop() {
if (cleanerThread != null) {
cleanerThread.interrupt();
stopRequest = true;
while (cleanerThread.isAlive()) {
try {
Thread.sleep(100);
@ -55,7 +59,7 @@ public class Cleaner {
@Override
public void run() {
while(!cleanerThread.isInterrupted()) {
while(!stopRequest) {
try {
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
Thread.sleep(sleepInterval);
@ -87,6 +91,7 @@ public class Cleaner {
System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
}catch (InterruptedException e) {
}
}
}

View File

@ -79,8 +79,6 @@ public class EntryReference<T> implements Castable {
return;
}
db.removeEntryReference(entryIndex);
closed = true;
}
}

View File

@ -2,19 +2,15 @@ package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Iterator;
public class FileIndexManager implements IndexManager {
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
@ -103,6 +99,7 @@ public class FileIndexManager implements IndexManager {
/**
* Write the data at index.
* The input size must be equal to the index size!
*
* @param indexDetails
* @param data
* @throws IOException
@ -326,6 +323,7 @@ public class FileIndexManager implements IndexManager {
private long cleanExtraIndices() {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
long count = loadedIndices.size();
@ -335,16 +333,19 @@ public class FileIndexManager implements IndexManager {
if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
break;
}
try {
flushAndUnload(loadedIndex);
} catch (IOException e) {
e.printStackTrace();
}
toUnload.add(loadedIndex);
removedIndices++;
count--;
}
}
}
for (long index : toUnload.elements()) {
try {
flushAndUnload(index);
} catch (IOException e) {
e.printStackTrace();
}
}
return removedIndices;
}
}

View File

@ -13,8 +13,8 @@ import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
public class JCWDatabase implements AutoCloseable, Cleanable {
public final static long MAX_LOADED_REFERENCES = 10;
public final static long MAX_LOADED_INDICES = 1000;
public final static long MAX_LOADED_REFERENCES = 1000;
public final static long MAX_LOADED_INDICES = 10000;
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
@ -119,6 +119,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
}
}
public boolean isOpen() {
return !closed;
}
@Override
public void close() throws IOException {
if (closed) {

View File

@ -12,7 +12,8 @@ import java.nio.file.Paths;
import java.util.function.Predicate;
public class App {
public static void main(String[] args) throws Exception {
public static void main(String[] args) {
try {
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
Files.delete(Paths.get(args[0]));
Files.delete(Paths.get(args[1]));
@ -20,6 +21,7 @@ public class App {
System.out.println("Loading database...");
long time0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
try {
long time01 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time01 - time0));
System.out.println("Loading root...");
@ -33,7 +35,7 @@ public class App {
// System.out.println(" - " + root.get(i));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 2000000000/* 2000000 */; i++) {
for (int i = 0; i < 2000000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40);
root.add(animal);
if (i > 0 && i % 200000 == 0) {
@ -70,12 +72,22 @@ public class App {
long time3 = System.currentTimeMillis();
System.out.println("Removed items: " + removedItems);
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Time elapsed: " + (time3 - time2));
System.out.println("Time elapsed: " + (time3 - time2_2));
System.out.println("Saving database...");
System.out.println("Root size: " + root.size());
db.close();
long time3 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time3 - time2_2));
long time4 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time4 - time3));
} catch (Exception ex) {
ex.printStackTrace();
} finally {
if (db.isOpen()) {
db.close();
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
public static <T> Predicate<T> not(Predicate<T> t) {