diff --git a/src/main/java/org/warp/jcwdb/CacheIndexManager.java b/src/main/java/org/warp/jcwdb/CacheIndexManager.java index 0044158..31c9cb9 100644 --- a/src/main/java/org/warp/jcwdb/CacheIndexManager.java +++ b/src/main/java/org/warp/jcwdb/CacheIndexManager.java @@ -3,10 +3,8 @@ package org.warp.jcwdb; import java.io.IOException; public class CacheIndexManager implements IndexManager { - private final TypesManager typesManager; - public CacheIndexManager(TypesManager typesManager) { - this.typesManager = typesManager; + public CacheIndexManager() { } @Override diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java index af9bd9d..de0e6db 100644 --- a/src/main/java/org/warp/jcwdb/FileIndexManager.java +++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java @@ -3,6 +3,11 @@ package org.warp.jcwdb; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.LongAVLTreeSet; +import it.unimi.dsi.fastutil.longs.LongSet; + import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.Channels; @@ -10,14 +15,8 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; public class FileIndexManager implements IndexManager { - private final TypesManager typesManager; private final SeekableByteChannel dataFileChannel, metadataFileChannel; private final FileAllocator fileAllocator; private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES); @@ -25,25 +24,24 @@ public class FileIndexManager implements IndexManager { private volatile boolean closed; private final Object closeLock = new Object(); private final Object metadataByteBufferLock = new Object(); - private final Object maskByteBufferBufferLock = new Object(); + private final Object maskByteBufferLock = new Object(); /** * Edit this using editIndex() * Get using getIndexMetadata() * This hashmap must contain all indices. */ - private final Map loadedIndices; + private final Long2ObjectMap loadedIndices; /** * Edit this using editIndex() */ - private final Set dirtyLoadedIndices, removedIndices; + private final LongSet dirtyLoadedIndices, removedIndices; private long firstAllocableIndex; - public FileIndexManager(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { - this.typesManager = typesManager; - loadedIndices = new HashMap<>(); - dirtyLoadedIndices = new HashSet<>(); - removedIndices = new HashSet<>(); + public FileIndexManager(Path dataFile, Path metadataFile) throws IOException { + loadedIndices = new Long2ObjectAVLTreeMap<>(); + dirtyLoadedIndices = new LongAVLTreeSet(); + removedIndices = new LongAVLTreeSet(); if (Files.notExists(dataFile)) { Files.createFile(dataFile); } @@ -162,7 +160,7 @@ public class FileIndexManager implements IndexManager { } private long createIndexMetadata(IndexDetails indexDetails) { - Long newIndex = firstAllocableIndex++; + long newIndex = firstAllocableIndex++; loadedIndices.put(newIndex, indexDetails); dirtyLoadedIndices.add(newIndex); removedIndices.remove(newIndex); @@ -223,7 +221,7 @@ public class FileIndexManager implements IndexManager { // Update indices metadata SeekableByteChannel metadata = metadataFileChannel; long lastIndex = -2; - for (Long index : dirtyLoadedIndices) { + for (long index : dirtyLoadedIndices) { IndexDetails indexDetails = loadedIndices.get(index); if (index - lastIndex != 1) { metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); @@ -241,9 +239,9 @@ public class FileIndexManager implements IndexManager { } // Remove removed indices - for (Long index : removedIndices) { + for (long index : removedIndices) { metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); - synchronized (maskByteBufferBufferLock) { + synchronized (maskByteBufferLock) { maskByteBuffer.rewind(); maskByteBuffer.putInt(IndexDetails.MASK_DELETED); maskByteBuffer.rewind(); diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index 6d3a226..0cff920 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -1,17 +1,17 @@ package org.warp.jcwdb; import java.io.IOException; +import java.lang.ref.WeakReference; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.WeakHashMap; + +import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; public class JCWDatabase implements AutoCloseable { protected final TypesManager typesManager; protected final MixedIndexDatabase indices; - private final WeakHashMap> references; + private final Long2ObjectMap>> references; private volatile boolean closed; private final Object closeLock = new Object(); private final Object indicesAccessLock = new Object(); @@ -20,7 +20,7 @@ public class JCWDatabase implements AutoCloseable { public JCWDatabase(Path dataFile, Path metadataFile) throws IOException { this.typesManager = new TypesManager(this); this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile); - this.references = new WeakHashMap<>(); + this.references = new Long2ObjectLinkedOpenHashMap<>(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { JCWDatabase.this.close(); @@ -43,15 +43,17 @@ public class JCWDatabase implements AutoCloseable { public EntryReference get(long index) throws IOException { checkClosed(); synchronized (referencesAccessLock) { - EntryReference ref = (EntryReference) this.references.getOrDefault(index, null); - if (ref == null) { + WeakReference> refRef = this.references.getOrDefault(index, null); + EntryReference ref; + if (refRef == null || (ref = (EntryReference) refRef.get()) == null) { int type; synchronized (indicesAccessLock) { type = this.indices.getType(index); } DBTypeParser typeParser = this.typesManager.get(type); ref = new EntryReference<>(this, index, typeParser); - this.references.put(index, ref); + refRef = new WeakReference>(ref); + this.references.put(index, refRef); } return ref; } @@ -67,7 +69,7 @@ public class JCWDatabase implements AutoCloseable { index = indices.add(typeParser.getWriter(value)); } ref = new EntryReference<>(this, index, typeParser, value); - this.references.put(index, ref); + this.references.put(index, new WeakReference>(ref)); return ref; } } @@ -95,7 +97,7 @@ public class JCWDatabase implements AutoCloseable { indices.set(index, typeParser.getWriter(value)); } ref = new EntryReference<>(this, index, typeParser); - this.references.put(index, ref); + this.references.put(index, new WeakReference>(ref)); return ref; } } @@ -120,8 +122,11 @@ public class JCWDatabase implements AutoCloseable { } synchronized (referencesAccessLock) { - for (Map.Entry> reference : references.entrySet()) { - reference.getValue().save(); + for (WeakReference> referenceRef : references.values()) { + EntryReference reference = referenceRef.get(); + if (reference != null) { + reference.close(); + } } } synchronized (indicesAccessLock) { diff --git a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java index c2e1d25..287fd0f 100644 --- a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java +++ b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java @@ -13,8 +13,8 @@ public class MixedIndexDatabase implements IndexManager { public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { this.mostAccessedIndices = new Long2LongLinkedOpenHashMap(); - this.fileIndices = new FileIndexManager(typesManager, dataFile, metadataFile); - this.cacheIndices = new CacheIndexManager(typesManager); + this.fileIndices = new FileIndexManager(dataFile, metadataFile); + this.cacheIndices = new CacheIndexManager(); } @Override diff --git a/src/main/java/org/warp/jcwdb/exampleimpl/App.java b/src/main/java/org/warp/jcwdb/exampleimpl/App.java index 58996a9..389c6eb 100644 --- a/src/main/java/org/warp/jcwdb/exampleimpl/App.java +++ b/src/main/java/org/warp/jcwdb/exampleimpl/App.java @@ -26,12 +26,19 @@ public class App { // for (int i = 0; i < root.size(); i++) { // System.out.println(" - " + root.get(i)); // } - for (int i = 0; i < 500000000; i++) { + long prectime = System.currentTimeMillis(); + for (int i = 0; i < 2000000; i++) { root.add("Test " + i); + if (i > 0 && i % 200000 == 0) { + long precprectime = prectime; + prectime = System.currentTimeMillis(); + System.out.println("Element "+i + " ("+(prectime-precprectime)+"ms)"); + } } long time2 = System.currentTimeMillis(); System.out.println("Root size: "+root.size()); System.out.println("Time elapsed: " + (time2 - time1)); + System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); System.out.println("Saving database..."); db.close(); long time3 = System.currentTimeMillis();