Removed indices usage map

This commit is contained in:
Andrea Cavalli 2018-12-19 12:19:03 +01:00
parent 73ae46e7b4
commit 1d64d1565c
7 changed files with 107 additions and 38 deletions

View File

@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner {
private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
private static final double NORMAL_REMOVED_ITEMS = 1000l;
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
@ -63,7 +63,10 @@ public class Cleaner {
try {
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
sleepFor(sleepInterval);
final long time1 = System.currentTimeMillis();
final double removedItems = clean();
final long time2 = System.currentTimeMillis();
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);

View File

@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Iterator;
import java.util.function.Consumer;
public class FileIndexManager implements IndexManager {
@ -233,7 +234,7 @@ public class FileIndexManager implements IndexManager {
* @param details
*/
private void editIndex(long index, IndexDetails details) {
synchronized (indicesMapsAccessLock) {
synchronized (indicesMapsAccessLock) {// FIXXXX main3
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
}
@ -264,7 +265,8 @@ public class FileIndexManager implements IndexManager {
return null;
}
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
synchronized (metadataByteBufferLock) {
IndexDetails indexDetails = null;
synchronized (metadataByteBufferLock) {// FIXXXX main2
metadataByteBuffer.rewind();
currentMetadataFileChannel.read(metadataByteBuffer);
metadataByteBuffer.rewind();
@ -277,11 +279,14 @@ public class FileIndexManager implements IndexManager {
final int size = metadataByteBuffer.getInt();
final int type = metadataByteBuffer.getInt();
final long hash = metadataByteBuffer.getLong();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
indexDetails = new IndexDetails(offset, size, type, hash);
}
}
if (indexDetails != null) {
editIndex(index, indexDetails);
return indexDetails;
}
}
// No results found. Returning null
return null;
@ -308,31 +313,15 @@ public class FileIndexManager implements IndexManager {
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
}
}
flushAllIndices();
// Remove removed indices
synchronized (indicesMapsAccessLock) {
for (long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
}
removeRemovedIndices();
fileAllocator.close();
}
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
synchronized (metadataByteBufferLock) {
synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
final int size = indexDetails.getSize();
final int type = indexDetails.getType();
final long offset = indexDetails.getOffset();
@ -366,7 +355,51 @@ public class FileIndexManager implements IndexManager {
@Override
public long clean() {
return cleanExtraIndices();
long cleaned = 0;
try {
cleaned += flushAllIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
try {
cleaned += removeRemovedIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
cleaned += cleanExtraIndices();
return cleaned;
}
private long flushAllIndices() throws IOException {
long flushedIndices = 0;
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
}
dirtyLoadedIndices.clear();
}
return flushedIndices;
}
private long removeRemovedIndices() throws IOException {
SeekableByteChannel metadata = metadataFileChannel;
synchronized (indicesMapsAccessLock) {
long removed = this.removedIndices.size();
for (long index : this.removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
this.removedIndices.clear();
return removed;
}
}
private long cleanExtraIndices() {

View File

@ -18,7 +18,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
this.indices = new MixedIndexDatabase(dataFile, metadataFile);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
JCWDatabase.this.close();
@ -28,7 +28,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
}));
this.databaseCleaner = new Cleaner(this);
//this.databaseCleaner.start();
this.databaseCleaner.start();
}
public <T> EntryReference<LightList<T>> getRoot() throws IOException {

View File

@ -2,6 +2,7 @@ package org.warp.jcwdb;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
import it.unimi.dsi.fastutil.longs.LongArrayList;
@ -52,6 +53,7 @@ public class LightList<T> implements List<T> {
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>();
for (Long element : internalList) {
try {
@ -76,6 +78,30 @@ public class LightList<T> implements List<T> {
return elements.iterator();
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw (RuntimeException) new RuntimeException().initCause(e);
}
}
}
@SuppressWarnings("unchecked")
@Override

View File

@ -8,19 +8,16 @@ import java.nio.file.Path;
import java.util.function.Consumer;
public class MixedIndexDatabase implements IndexManager {
private final Long2LongMap mostAccessedIndices;
private final FileIndexManager fileIndices;
private final CacheIndexManager cacheIndices;
public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager();
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
incrementUsage(index);
if (cacheIndices.has(index)) {
return cacheIndices.get(index, reader);
} else {
@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager {
return cacheIndices.has(index) || fileIndices.has(index);
}
private void incrementUsage(long index) {
mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1);
}
@Override
public void close() throws IOException {
// TODO: move all cached indices to filesIndices before closing.

View File

@ -0,0 +1,12 @@
package org.warp.jcwdb;
import java.nio.channels.SeekableByteChannel;
public class VariableWrapper<T> {
public T var;
public VariableWrapper(T value) {
this.var = value;
}
}

View File

@ -37,7 +37,7 @@ public class App {
// System.out.println(" - " + root.get(i));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 20000/* 2000000 */; i++) {
for (int i = 0; i < 200000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40);
root.add(animal);
if (i > 0 && i % 200000 == 0) {
@ -59,13 +59,15 @@ public class App {
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
root.forEach((value) -> {
root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnly();
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");