diff --git a/src/main/java/org/warp/jcwdb/Cleaner.java b/src/main/java/org/warp/jcwdb/Cleaner.java index b71be1c..08ccf4d 100644 --- a/src/main/java/org/warp/jcwdb/Cleaner.java +++ b/src/main/java/org/warp/jcwdb/Cleaner.java @@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry; public class Cleaner { - private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes + private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second private static final double NORMAL_REMOVED_ITEMS = 1000l; private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250% @@ -63,7 +63,10 @@ public class Cleaner { try { System.out.println("[CLEANER] Waiting " + sleepInterval + "ms."); sleepFor(sleepInterval); + final long time1 = System.currentTimeMillis(); final double removedItems = clean(); + final long time2 = System.currentTimeMillis(); + System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1)); double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2; System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems); diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java index 2b2d14f..5ea06d9 100644 --- a/src/main/java/org/warp/jcwdb/FileIndexManager.java +++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java @@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.Iterator; import java.util.function.Consumer; public class FileIndexManager implements IndexManager { @@ -233,7 +234,7 @@ public class FileIndexManager implements IndexManager { * @param details */ private void editIndex(long index, IndexDetails details) { - synchronized (indicesMapsAccessLock) { + synchronized (indicesMapsAccessLock) {// FIXXXX main3 loadedIndices.put(index, details); dirtyLoadedIndices.add(index); } @@ -264,7 +265,8 @@ public class FileIndexManager implements IndexManager { return null; } SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition); - synchronized (metadataByteBufferLock) { + IndexDetails indexDetails = null; + synchronized (metadataByteBufferLock) {// FIXXXX main2 metadataByteBuffer.rewind(); currentMetadataFileChannel.read(metadataByteBuffer); metadataByteBuffer.rewind(); @@ -277,12 +279,15 @@ public class FileIndexManager implements IndexManager { final int size = metadataByteBuffer.getInt(); final int type = metadataByteBuffer.getInt(); final long hash = metadataByteBuffer.getLong(); - final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash); - editIndex(index, indexDetails); - return indexDetails; + indexDetails = new IndexDetails(offset, size, type, hash); } } + if (indexDetails != null) { + editIndex(index, indexDetails); + return indexDetails; + } + // No results found. Returning null return null; } @@ -308,31 +313,15 @@ public class FileIndexManager implements IndexManager { } // Update indices metadata - SeekableByteChannel metadata = metadataFileChannel; - long lastIndex = -2; - synchronized (indicesMapsAccessLock) { - for (long index : dirtyLoadedIndices) { - IndexDetails indexDetails = loadedIndices.get(index); - if (index - lastIndex != 1) { - metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); - } - writeIndexDetails(metadata, indexDetails); - lastIndex = index; - } - } + flushAllIndices(); // Remove removed indices - synchronized (indicesMapsAccessLock) { - for (long index : removedIndices) { - metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); - eraseIndexDetails(metadata); - } - } + removeRemovedIndices(); fileAllocator.close(); } private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException { - synchronized (metadataByteBufferLock) { + synchronized (metadataByteBufferLock) {// FIXXXX cleaner3 final int size = indexDetails.getSize(); final int type = indexDetails.getType(); final long offset = indexDetails.getOffset(); @@ -366,7 +355,51 @@ public class FileIndexManager implements IndexManager { @Override public long clean() { - return cleanExtraIndices(); + long cleaned = 0; + try { + cleaned += flushAllIndices(); + } catch (IOException ex) { + ex.printStackTrace(); + } + try { + cleaned += removeRemovedIndices(); + } catch (IOException ex) { + ex.printStackTrace(); + } + cleaned += cleanExtraIndices(); + return cleaned; + } + + private long flushAllIndices() throws IOException { + long flushedIndices = 0; + SeekableByteChannel metadata = metadataFileChannel; + long lastIndex = -2; + synchronized (indicesMapsAccessLock) { + for (long index : dirtyLoadedIndices) { + IndexDetails indexDetails = loadedIndices.get(index); + if (index - lastIndex != 1) { + metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); + } + writeIndexDetails(metadata, indexDetails); + lastIndex = index; + flushedIndices++; + } + dirtyLoadedIndices.clear(); + } + return flushedIndices; + } + + private long removeRemovedIndices() throws IOException { + SeekableByteChannel metadata = metadataFileChannel; + synchronized (indicesMapsAccessLock) { + long removed = this.removedIndices.size(); + for (long index : this.removedIndices) { + metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); + eraseIndexDetails(metadata); + } + this.removedIndices.clear(); + return removed; + } } private long cleanExtraIndices() { diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index 776ce7d..4d6a882 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -18,7 +18,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable { public JCWDatabase(Path dataFile, Path metadataFile) throws IOException { this.typesManager = new TypesManager(this); - this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile); + this.indices = new MixedIndexDatabase(dataFile, metadataFile); Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { JCWDatabase.this.close(); @@ -28,7 +28,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable { })); this.databaseCleaner = new Cleaner(this); - //this.databaseCleaner.start(); + this.databaseCleaner.start(); } public EntryReference> getRoot() throws IOException { diff --git a/src/main/java/org/warp/jcwdb/LightList.java b/src/main/java/org/warp/jcwdb/LightList.java index 0edddc7..2589dab 100644 --- a/src/main/java/org/warp/jcwdb/LightList.java +++ b/src/main/java/org/warp/jcwdb/LightList.java @@ -2,6 +2,7 @@ package org.warp.jcwdb; import java.io.IOException; import java.util.*; +import java.util.function.Consumer; import java.util.function.Predicate; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -52,6 +53,7 @@ public class LightList implements List { @SuppressWarnings("unchecked") @Override public Iterator iterator() { + System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!"); final ArrayList elements = new ArrayList<>(); for (Long element : internalList) { try { @@ -75,6 +77,30 @@ public class LightList implements List { } return elements.iterator(); } + + /** + * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY + * @param action + */ + @Deprecated + @Override + public void forEach(Consumer action) { + Objects.requireNonNull(action); + for (T t : this) { + action.accept(t); + } + } + + public void forEachReference(Consumer> action) { + Objects.requireNonNull(action); + for (long index : this.internalList) { + try { + action.accept(db.get(index)); + } catch (IOException e) { + throw (RuntimeException) new RuntimeException().initCause(e); + } + } + } @SuppressWarnings("unchecked") diff --git a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java index 3194293..3f4e255 100644 --- a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java +++ b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java @@ -8,19 +8,16 @@ import java.nio.file.Path; import java.util.function.Consumer; public class MixedIndexDatabase implements IndexManager { - private final Long2LongMap mostAccessedIndices; private final FileIndexManager fileIndices; private final CacheIndexManager cacheIndices; - public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { - this.mostAccessedIndices = new Long2LongLinkedOpenHashMap(); + public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException { this.fileIndices = new FileIndexManager(dataFile, metadataFile); this.cacheIndices = new CacheIndexManager(); } @Override public T get(long index, DBReader reader) throws IOException { - incrementUsage(index); if (cacheIndices.has(index)) { return cacheIndices.get(index, reader); } else { @@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager { return cacheIndices.has(index) || fileIndices.has(index); } - private void incrementUsage(long index) { - mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1); - } - @Override public void close() throws IOException { // TODO: move all cached indices to filesIndices before closing. diff --git a/src/main/java/org/warp/jcwdb/VariableWrapper.java b/src/main/java/org/warp/jcwdb/VariableWrapper.java new file mode 100644 index 0000000..0503674 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/VariableWrapper.java @@ -0,0 +1,12 @@ +package org.warp.jcwdb; + +import java.nio.channels.SeekableByteChannel; + +public class VariableWrapper { + + public T var; + + public VariableWrapper(T value) { + this.var = value; + } +} diff --git a/src/main/java/org/warp/jcwdb/exampleimpl/App.java b/src/main/java/org/warp/jcwdb/exampleimpl/App.java index 1413dcf..de5e5ab 100644 --- a/src/main/java/org/warp/jcwdb/exampleimpl/App.java +++ b/src/main/java/org/warp/jcwdb/exampleimpl/App.java @@ -37,7 +37,7 @@ public class App { // System.out.println(" - " + root.get(i)); // } long prectime = System.currentTimeMillis(); - for (int i = 0; i < 20000/* 2000000 */; i++) { + for (int i = 0; i < 200000/* 2000000 */; i++) { Animal animal = new StrangeAnimal(i % 40); root.add(animal); if (i > 0 && i % 200000 == 0) { @@ -59,13 +59,15 @@ public class App { System.out.println("Time elapsed: " + (time2_1 - time2_0)); ObjectList results = new ObjectArrayList<>(); - root.forEach((value) -> { + root.forEachReference((valueReference) -> { + Animal value = valueReference.getValueReadOnly(); if (Animal.hasFourLegs(value)) { results.add(value); } //System.out.println("val:" + value); }); long time2_2 = System.currentTimeMillis(); + System.out.println("Matches: " + results.size()); System.out.println("Time elapsed: " + (time2_2 - time2_1)); System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");