Removed indices usage map

This commit is contained in:
Andrea Cavalli 2018-12-19 12:19:03 +01:00
parent 73ae46e7b4
commit 1d64d1565c
7 changed files with 107 additions and 38 deletions

View File

@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner { public class Cleaner {
private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
private static final double NORMAL_REMOVED_ITEMS = 1000l; private static final double NORMAL_REMOVED_ITEMS = 1000l;
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250% private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
@ -63,7 +63,10 @@ public class Cleaner {
try { try {
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms."); System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
sleepFor(sleepInterval); sleepFor(sleepInterval);
final long time1 = System.currentTimeMillis();
final double removedItems = clean(); final double removedItems = clean();
final long time2 = System.currentTimeMillis();
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2; double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems); System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);

View File

@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.util.Iterator;
import java.util.function.Consumer; import java.util.function.Consumer;
public class FileIndexManager implements IndexManager { public class FileIndexManager implements IndexManager {
@ -233,7 +234,7 @@ public class FileIndexManager implements IndexManager {
* @param details * @param details
*/ */
private void editIndex(long index, IndexDetails details) { private void editIndex(long index, IndexDetails details) {
synchronized (indicesMapsAccessLock) { synchronized (indicesMapsAccessLock) {// FIXXXX main3
loadedIndices.put(index, details); loadedIndices.put(index, details);
dirtyLoadedIndices.add(index); dirtyLoadedIndices.add(index);
} }
@ -264,7 +265,8 @@ public class FileIndexManager implements IndexManager {
return null; return null;
} }
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition); SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
synchronized (metadataByteBufferLock) { IndexDetails indexDetails = null;
synchronized (metadataByteBufferLock) {// FIXXXX main2
metadataByteBuffer.rewind(); metadataByteBuffer.rewind();
currentMetadataFileChannel.read(metadataByteBuffer); currentMetadataFileChannel.read(metadataByteBuffer);
metadataByteBuffer.rewind(); metadataByteBuffer.rewind();
@ -277,12 +279,15 @@ public class FileIndexManager implements IndexManager {
final int size = metadataByteBuffer.getInt(); final int size = metadataByteBuffer.getInt();
final int type = metadataByteBuffer.getInt(); final int type = metadataByteBuffer.getInt();
final long hash = metadataByteBuffer.getLong(); final long hash = metadataByteBuffer.getLong();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash); indexDetails = new IndexDetails(offset, size, type, hash);
editIndex(index, indexDetails);
return indexDetails;
} }
} }
if (indexDetails != null) {
editIndex(index, indexDetails);
return indexDetails;
}
// No results found. Returning null // No results found. Returning null
return null; return null;
} }
@ -308,31 +313,15 @@ public class FileIndexManager implements IndexManager {
} }
// Update indices metadata // Update indices metadata
SeekableByteChannel metadata = metadataFileChannel; flushAllIndices();
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
}
}
// Remove removed indices // Remove removed indices
synchronized (indicesMapsAccessLock) { removeRemovedIndices();
for (long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
}
fileAllocator.close(); fileAllocator.close();
} }
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException { private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
synchronized (metadataByteBufferLock) { synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
final int size = indexDetails.getSize(); final int size = indexDetails.getSize();
final int type = indexDetails.getType(); final int type = indexDetails.getType();
final long offset = indexDetails.getOffset(); final long offset = indexDetails.getOffset();
@ -366,7 +355,51 @@ public class FileIndexManager implements IndexManager {
@Override @Override
public long clean() { public long clean() {
return cleanExtraIndices(); long cleaned = 0;
try {
cleaned += flushAllIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
try {
cleaned += removeRemovedIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
cleaned += cleanExtraIndices();
return cleaned;
}
private long flushAllIndices() throws IOException {
long flushedIndices = 0;
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
}
dirtyLoadedIndices.clear();
}
return flushedIndices;
}
private long removeRemovedIndices() throws IOException {
SeekableByteChannel metadata = metadataFileChannel;
synchronized (indicesMapsAccessLock) {
long removed = this.removedIndices.size();
for (long index : this.removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
this.removedIndices.clear();
return removed;
}
} }
private long cleanExtraIndices() { private long cleanExtraIndices() {

View File

@ -18,7 +18,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException { public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this); this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile); this.indices = new MixedIndexDatabase(dataFile, metadataFile);
Runtime.getRuntime().addShutdownHook(new Thread(() -> { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try { try {
JCWDatabase.this.close(); JCWDatabase.this.close();
@ -28,7 +28,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
})); }));
this.databaseCleaner = new Cleaner(this); this.databaseCleaner = new Cleaner(this);
//this.databaseCleaner.start(); this.databaseCleaner.start();
} }
public <T> EntryReference<LightList<T>> getRoot() throws IOException { public <T> EntryReference<LightList<T>> getRoot() throws IOException {

View File

@ -2,6 +2,7 @@ package org.warp.jcwdb;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate; import java.util.function.Predicate;
import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList;
@ -52,6 +53,7 @@ public class LightList<T> implements List<T> {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
public Iterator<T> iterator() { public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>(); final ArrayList<T> elements = new ArrayList<>();
for (Long element : internalList) { for (Long element : internalList) {
try { try {
@ -76,6 +78,30 @@ public class LightList<T> implements List<T> {
return elements.iterator(); return elements.iterator();
} }
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw (RuntimeException) new RuntimeException().initCause(e);
}
}
}
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override

View File

@ -8,19 +8,16 @@ import java.nio.file.Path;
import java.util.function.Consumer; import java.util.function.Consumer;
public class MixedIndexDatabase implements IndexManager { public class MixedIndexDatabase implements IndexManager {
private final Long2LongMap mostAccessedIndices;
private final FileIndexManager fileIndices; private final FileIndexManager fileIndices;
private final CacheIndexManager cacheIndices; private final CacheIndexManager cacheIndices;
public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
this.fileIndices = new FileIndexManager(dataFile, metadataFile); this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager(); this.cacheIndices = new CacheIndexManager();
} }
@Override @Override
public <T> T get(long index, DBReader<T> reader) throws IOException { public <T> T get(long index, DBReader<T> reader) throws IOException {
incrementUsage(index);
if (cacheIndices.has(index)) { if (cacheIndices.has(index)) {
return cacheIndices.get(index, reader); return cacheIndices.get(index, reader);
} else { } else {
@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager {
return cacheIndices.has(index) || fileIndices.has(index); return cacheIndices.has(index) || fileIndices.has(index);
} }
private void incrementUsage(long index) {
mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1);
}
@Override @Override
public void close() throws IOException { public void close() throws IOException {
// TODO: move all cached indices to filesIndices before closing. // TODO: move all cached indices to filesIndices before closing.

View File

@ -0,0 +1,12 @@
package org.warp.jcwdb;
import java.nio.channels.SeekableByteChannel;
public class VariableWrapper<T> {
public T var;
public VariableWrapper(T value) {
this.var = value;
}
}

View File

@ -37,7 +37,7 @@ public class App {
// System.out.println(" - " + root.get(i)); // System.out.println(" - " + root.get(i));
// } // }
long prectime = System.currentTimeMillis(); long prectime = System.currentTimeMillis();
for (int i = 0; i < 20000/* 2000000 */; i++) { for (int i = 0; i < 200000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40); Animal animal = new StrangeAnimal(i % 40);
root.add(animal); root.add(animal);
if (i > 0 && i % 200000 == 0) { if (i > 0 && i % 200000 == 0) {
@ -59,13 +59,15 @@ public class App {
System.out.println("Time elapsed: " + (time2_1 - time2_0)); System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>(); ObjectList<Animal> results = new ObjectArrayList<>();
root.forEach((value) -> { root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnly();
if (Animal.hasFourLegs(value)) { if (Animal.hasFourLegs(value)) {
results.add(value); results.add(value);
} }
//System.out.println("val:" + value); //System.out.println("val:" + value);
}); });
long time2_2 = System.currentTimeMillis(); long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1)); System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: " System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");