From 73ae46e7b42bf863201f993eb0241c6f93166980 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Sat, 15 Dec 2018 01:01:23 +0100 Subject: [PATCH] Removed reference storage --- .../org/warp/jcwdb/DBGenericObjectParser.java | 11 +- .../org/warp/jcwdb/DBTypedObjectParser.java | 5 + .../java/org/warp/jcwdb/FileIndexManager.java | 4 +- src/main/java/org/warp/jcwdb/JCWDatabase.java | 173 +++++------------- .../java/org/warp/jcwdb/TypesManager.java | 31 +++- .../java/org/warp/jcwdb/exampleimpl/App.java | 3 +- 6 files changed, 85 insertions(+), 142 deletions(-) create mode 100644 src/main/java/org/warp/jcwdb/DBTypedObjectParser.java diff --git a/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java b/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java index 58dd9ca..5e6f9ae 100644 --- a/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java +++ b/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java @@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output; import net.openhft.hashing.LongHashFunction; -public class DBGenericObjectParser extends DBTypeParserImpl { +public class DBGenericObjectParser extends DBTypeParserImpl implements DBTypedObjectParser { private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx(); private static final Kryo kryo = new Kryo(); static { @@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl { tmpO.close(); return hash; } + + + @Override + public void registerClass(Class clazz, int id) { + if (id >= Integer.MAX_VALUE - 100) { + throw new IndexOutOfBoundsException(); + } + kryo.register(clazz, id + 100); + } } diff --git a/src/main/java/org/warp/jcwdb/DBTypedObjectParser.java b/src/main/java/org/warp/jcwdb/DBTypedObjectParser.java new file mode 100644 index 0000000..5aa4353 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBTypedObjectParser.java @@ -0,0 +1,5 @@ +package org.warp.jcwdb; + +public interface DBTypedObjectParser extends DBTypeParser { + public void registerClass(Class clazz, int type); +} diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java index 4da900f..2b2d14f 100644 --- a/src/main/java/org/warp/jcwdb/FileIndexManager.java +++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java @@ -373,12 +373,12 @@ public class FileIndexManager implements IndexManager { long removedIndices = 0; LongArrayList toUnload = new LongArrayList(); synchronized (indicesMapsAccessLock) { - if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) { + if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) { long count = loadedIndices.size(); LongIterator it = loadedIndices.keySet().iterator(); while (it.hasNext()) { long loadedIndex = it.nextLong(); - if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) { + if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) { break; } toUnload.add(loadedIndex); diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index d1bcd72..776ce7d 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -1,36 +1,24 @@ package org.warp.jcwdb; -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.function.Consumer; - -import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry; import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.objects.ObjectIterator; + +import java.io.IOException; +import java.nio.file.Path; public class JCWDatabase implements AutoCloseable, Cleanable { - public final static long MAX_LOADED_REFERENCES = 1000; public final static long MAX_LOADED_INDICES = 10000; - + private final TypesManager typesManager; private final MixedIndexDatabase indices; private final Cleaner databaseCleaner; private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools(); - private final Long2ObjectMap>> references; private volatile boolean closed; private final Object closeLock = new Object(); private final Object indicesAccessLock = new Object(); - private final Object referencesAccessLock = new Object(); public JCWDatabase(Path dataFile, Path metadataFile) throws IOException { this.typesManager = new TypesManager(this); this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile); - this.references = new Long2ObjectLinkedOpenHashMap<>(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { JCWDatabase.this.close(); @@ -39,14 +27,14 @@ public class JCWDatabase implements AutoCloseable, Cleanable { } })); this.databaseCleaner = new Cleaner(this); - + //this.databaseCleaner.start(); } public EntryReference> getRoot() throws IOException { return getRoot(Object.class).cast(); } - + public EntryReference> getRoot(Class clazz) throws IOException { checkClosed(); if (exists(0)) { @@ -59,74 +47,64 @@ public class JCWDatabase implements AutoCloseable, Cleanable { public EntryReference get(long index) throws IOException { checkClosed(); - synchronized (referencesAccessLock) { - WeakReference> refRef = this.references.getOrDefault(index, null); - EntryReference ref; - if (refRef == null || (ref = (EntryReference) refRef.get()) == null) { - int type; - long hash; - synchronized (indicesAccessLock) { - type = this.indices.getType(index); - hash = this.indices.getHash(index); - } - DBTypeParser typeParser = this.typesManager.get(type); - ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser); - refRef = new WeakReference<>(ref); - this.references.put(index, refRef); - } - return ref; + int type; + long hash; + synchronized (indicesAccessLock) { + type = this.indices.getType(index); + hash = this.indices.getHash(index); } + DBTypeParser typeParser = this.typesManager.get(type); + return new EntryReference<>(entryReferenceTools, index, hash, typeParser); } protected EntryReference add(T value) throws IOException { checkClosed(); - synchronized (referencesAccessLock) { - EntryReference ref; - DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); - long index; - long hash; - synchronized (indicesAccessLock) { - index = indices.add(typeParser.getWriter(value)); - hash = indices.getHash(index); - } - ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value); - this.references.put(index, new WeakReference<>(ref)); - return ref; + DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); + long index; + long hash; + synchronized (indicesAccessLock) { + index = indices.add(typeParser.getWriter(value)); + hash = indices.getHash(index); } + return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value); } protected boolean exists(long index) { checkClosed(); - synchronized (referencesAccessLock) { - synchronized (indicesAccessLock) { - return this.references.containsKey(index) || this.indices.has(index); - } + synchronized (indicesAccessLock) { + return this.indices.has(index); } } protected EntryReference set(long index, T value) throws IOException { checkClosed(); - synchronized (referencesAccessLock) { - EntryReference ref; - if (exists(index)) { - ref = get(index); - ref.setValue(value); - return ref; - } else { - @SuppressWarnings("unchecked") - DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); - long hash; - synchronized (indicesAccessLock) { - IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value)); - hash = returnedDetails.getHash(); - } - ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser); - this.references.put(index, new WeakReference>(ref)); - return ref; + EntryReference ref; + if (exists(index)) { + ref = get(index); + ref.setValue(value); + return ref; + } else { + @SuppressWarnings("unchecked") + DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); + long hash; + synchronized (indicesAccessLock) { + IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value)); + hash = returnedDetails.getHash(); } + return new EntryReference<>(entryReferenceTools, index, hash, typeParser); } } + public void registerType(Class clazz, short type, DBTypeParser parser) { + final int addition = 0xEFFF8000; + int extendedType = addition | (type & 0x7FFF); + typesManager.registerType(clazz, extendedType, parser); + } + + public void registerClass(Class clazz, int type) { + typesManager.registerGenericClass(clazz, type); + } + public boolean isOpen() { return !closed; } @@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable { this.databaseCleaner.stop(); - synchronized (referencesAccessLock) { - ObjectIterator>> iterator = references.values().iterator(); - while (iterator.hasNext()) { - WeakReference> referenceRef = iterator.next(); - EntryReference reference = referenceRef.get(); - if (reference != null) { - reference.close(); - iterator.remove(); - } - - } - } synchronized (indicesAccessLock) { this.indices.close(); } @@ -171,56 +137,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable { @Override public long clean() { - long removedItems = cleanEmptyReferences() - + cleanExtraReferences() - + indices.clean(); + long removedItems = indices.clean(); return removedItems; } - - - private long cleanEmptyReferences() { - long removed = 0; - synchronized(referencesAccessLock) { - ObjectIterator>>> iterator = references.long2ObjectEntrySet().iterator(); - while (iterator.hasNext()) { - Entry>> entry = iterator.next(); - if (entry.getValue().get() == null) { - iterator.remove(); - removed++; - } - } - } - return removed; - } - - private long cleanExtraReferences() { - long removedReferences = 0; - synchronized(referencesAccessLock) { - if (references.size() > MAX_LOADED_REFERENCES) { - long count = 0; - ObjectIterator>>> iterator = references.long2ObjectEntrySet().iterator(); - while (iterator.hasNext()) { - Entry>> entry = iterator.next(); - if (count > MAX_LOADED_REFERENCES * 3l / 2l) { - WeakReference> weakRef = entry.getValue(); - EntryReference ref = weakRef.get(); - if (ref != null) { - try { - ref.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - iterator.remove(); - removedReferences++; - } else { - count++; - } - } - } - } - return removedReferences; - } public class EntryReferenceTools { private EntryReferenceTools() { diff --git a/src/main/java/org/warp/jcwdb/TypesManager.java b/src/main/java/org/warp/jcwdb/TypesManager.java index 239be89..6e4abde 100644 --- a/src/main/java/org/warp/jcwdb/TypesManager.java +++ b/src/main/java/org/warp/jcwdb/TypesManager.java @@ -1,26 +1,37 @@ package org.warp.jcwdb; -import java.util.Map; -import java.util.Map.Entry; -import java.util.HashMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; public class TypesManager { - private final Map> types; - private final Map, DBTypeParser> typesByClass; - private DBTypeParser fallbackParser; + private final Int2ObjectMap> types; + private final Object2ObjectMap, DBTypeParser> typesByClass; + private DBTypedObjectParser fallbackParser; public TypesManager(JCWDatabase db) { - types = new HashMap<>(); - typesByClass = new HashMap<>(); + types = new Int2ObjectOpenHashMap<>(); + typesByClass = new Object2ObjectOpenHashMap<>(); DBStandardTypes.registerStandardTypes(db, this); } - + public void registerType(Class clazz, int type, DBTypeParser parser) { this.types.put(type, parser); this.typesByClass.put(clazz, parser); } - public void registerTypeFallback(DBTypeParser parser) { + /** + * Use this method with the most used classes to save disk space. + * @param clazz + * @param id + * @param + */ + public void registerGenericClass(Class clazz, int id) { + this.fallbackParser.registerClass(clazz, id); + } + + public void registerTypeFallback(DBTypedObjectParser parser) { this.fallbackParser = parser; } diff --git a/src/main/java/org/warp/jcwdb/exampleimpl/App.java b/src/main/java/org/warp/jcwdb/exampleimpl/App.java index aaa9d2f..1413dcf 100644 --- a/src/main/java/org/warp/jcwdb/exampleimpl/App.java +++ b/src/main/java/org/warp/jcwdb/exampleimpl/App.java @@ -22,6 +22,7 @@ public class App { System.out.println("Loading database..."); long time0 = System.currentTimeMillis(); JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1])); + db.registerClass(StrangeAnimal.class, 0); try { long time01 = System.currentTimeMillis(); System.out.println("Time elapsed: " + (time01 - time0)); @@ -58,14 +59,12 @@ public class App { System.out.println("Time elapsed: " + (time2_1 - time2_0)); ObjectList results = new ObjectArrayList<>(); - /* root.forEach((value) -> { if (Animal.hasFourLegs(value)) { results.add(value); } //System.out.println("val:" + value); }); - */ long time2_2 = System.currentTimeMillis(); System.out.println("Time elapsed: " + (time2_2 - time2_1)); System.out.println("Used memory: "