From adfdb571d81e6c3adaf25ba1a6a1f7196603f2ea Mon Sep 17 00:00:00 2001 From: Cavallium Date: Tue, 11 Dec 2018 23:00:51 +0100 Subject: [PATCH] value reference private --- .../org/warp/jcwdb/CacheIndexManager.java | 10 +- src/main/java/org/warp/jcwdb/Cleaner.java | 21 ++- .../org/warp/jcwdb/DBLightListParser.java | 21 ++- .../java/org/warp/jcwdb/EntryReference.java | 125 +++++++++++++++--- .../java/org/warp/jcwdb/FileIndexManager.java | 32 +++-- .../java/org/warp/jcwdb/IndexManager.java | 4 +- src/main/java/org/warp/jcwdb/JCWDatabase.java | 33 +++-- src/main/java/org/warp/jcwdb/LightList.java | 118 ++++++++++------- .../org/warp/jcwdb/MixedIndexDatabase.java | 16 ++- src/main/java/org/warp/jcwdb/Saveable.java | 7 + .../java/org/warp/jcwdb/exampleimpl/App.java | 97 +++++++------- src/test/java/org/warp/jcwdb/AppTest.java | 10 +- 12 files changed, 333 insertions(+), 161 deletions(-) create mode 100644 src/main/java/org/warp/jcwdb/Saveable.java diff --git a/src/main/java/org/warp/jcwdb/CacheIndexManager.java b/src/main/java/org/warp/jcwdb/CacheIndexManager.java index 015defc..03e1681 100644 --- a/src/main/java/org/warp/jcwdb/CacheIndexManager.java +++ b/src/main/java/org/warp/jcwdb/CacheIndexManager.java @@ -1,6 +1,7 @@ package org.warp.jcwdb; import java.io.IOException; +import java.util.function.Consumer; public class CacheIndexManager implements IndexManager { @@ -19,6 +20,12 @@ public class CacheIndexManager implements IndexManager { return 0; } + @Override + public long getHash(long index) { + // TODO: implement + return 0; + } + @Override public long add(DBDataOutput writer) { // TODO: implement @@ -26,8 +33,9 @@ public class CacheIndexManager implements IndexManager { } @Override - public void set(long index, DBDataOutput writer) { + public IndexDetails set(long index, DBDataOutput writer) { // TODO: implement + return null; } @Override diff --git a/src/main/java/org/warp/jcwdb/Cleaner.java b/src/main/java/org/warp/jcwdb/Cleaner.java index 2a552be..b71be1c 100644 --- a/src/main/java/org/warp/jcwdb/Cleaner.java +++ b/src/main/java/org/warp/jcwdb/Cleaner.java @@ -62,15 +62,15 @@ public class Cleaner { while(!stopRequest) { try { System.out.println("[CLEANER] Waiting " + sleepInterval + "ms."); - Thread.sleep(sleepInterval); + sleepFor(sleepInterval); final double removedItems = clean(); - double suggestedExecutionTimeByItemsCalculations = sleepInterval; + double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2; System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems); if (removedItems > 0) { final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS; System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio); - if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio > REMOVED_ITEMS_RATIO) { + if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO && removedItemsRatio >= REMOVED_ITEMS_RATIO) { suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio; } } @@ -95,6 +95,21 @@ public class Cleaner { } } } + + private void sleepFor(int sleepInterval) throws InterruptedException { + int lastI = (int) Math.ceil(((double) sleepInterval) / 1000d); + for (int i = 0; i < lastI; i++) { + if (stopRequest) { + return; + } + if (i == lastI) { + Thread.sleep(sleepInterval % 1000); + } else { + Thread.sleep(lastI); + } + Thread.sleep(sleepInterval); + } + } } } diff --git a/src/main/java/org/warp/jcwdb/DBLightListParser.java b/src/main/java/org/warp/jcwdb/DBLightListParser.java index c5e1e0e..358ac27 100644 --- a/src/main/java/org/warp/jcwdb/DBLightListParser.java +++ b/src/main/java/org/warp/jcwdb/DBLightListParser.java @@ -4,42 +4,37 @@ import java.util.ArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; -public class DBLightListParser extends DBTypeParserImpl { +public class DBLightListParser extends DBTypeParserImpl> { private final JCWDatabase db; public DBLightListParser(JCWDatabase db) { this.db = db; } - public DBReader getReader() { + public DBReader> getReader() { return (i, size) -> { LongArrayList internalList = new LongArrayList(); - LongArrayList hashes = new LongArrayList(); - long max = size / (Long.BYTES * 2); + long max = size / Long.BYTES; for (int item = 0; item < max; item++){ long itm = i.readLong(); - long hash = i.readLong(); internalList.add(itm); - hashes.add(hash); } - return new LightList(db, internalList, hashes); + return new LightList(db, internalList); }; } - public DBDataOutput getWriter(final LightList value) { + public DBDataOutput> getWriter(final LightList value) { final int elementsCount = value.size(); return DBDataOutput.create((o) -> { LongArrayList list = value.internalList; - LongArrayList hashes = value.hashes; for (int i = 0; i < elementsCount; i++) { o.writeLong(list.getLong(i)); - o.writeLong(hashes.getLong(i)); } - }, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES * 2, calculateHash(value)); + }, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value)); } @Override - public long calculateHash(LightList value) { - return value.hashCodeLong(); + public long calculateHash(LightList value) { + return value.internalList.hashCode(); } } diff --git a/src/main/java/org/warp/jcwdb/EntryReference.java b/src/main/java/org/warp/jcwdb/EntryReference.java index 5ff818b..66481d4 100644 --- a/src/main/java/org/warp/jcwdb/EntryReference.java +++ b/src/main/java/org/warp/jcwdb/EntryReference.java @@ -1,31 +1,44 @@ package org.warp.jcwdb; import java.io.IOException; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; /** * You must have only a maximum of 1 reference for each index * @param */ -public class EntryReference implements Castable { +public class EntryReference implements Castable, Saveable { private final JCWDatabase.EntryReferenceTools db; private final long entryIndex; private final DBTypeParser parser; private T value; + private long cachedHash; + private volatile boolean isHashCached; + private volatile boolean loaded; private volatile boolean closed; + private final Object hashCacheLock = new Object(); + private final Object accessLock = new Object(); private final Object closeLock = new Object(); - public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, DBTypeParser parser) throws IOException { + public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, long hash, DBTypeParser parser) { + this.loaded = false; + this.isHashCached = false; this.db = db; this.entryIndex = entryId; this.parser = parser; - this.value = db.read(entryId, parser.getReader()); + this.value = null; } - public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, DBTypeParser parser, T value) { + public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, long hash, DBTypeParser parser, T value) { + this.loaded = true; + this.isHashCached = true; this.db = db; this.entryIndex = entryId; this.parser = parser; + this.cachedHash = hash; this.value = value; } @@ -38,44 +51,116 @@ public class EntryReference implements Castable { } public long calculateHash() { - return parser.calculateHash(value); + synchronized(accessLock) { + load(); + synchronized(hashCacheLock) { + if (isHashCached) { + return cachedHash; + } + } + return parser.calculateHash(this.value); + } } /** * Note that this method won't be called when closing without saving - * @throws IOException */ - public void save() throws IOException { - if (!closed) { - db.write(entryIndex, parser.getWriter(value)); + public void save() { + synchronized(accessLock) { + if (loaded && !closed) { + try { + IndexDetails returnedDetails = db.write(entryIndex, parser.getWriter(value)); + synchronized(hashCacheLock) { + this.cachedHash = returnedDetails.getHash(); + this.isHashCached = true; + } + } catch (IOException e) { + e.printStackTrace(); + } + } } } - public void editValue(Function editFunction) throws IOException { - this.value = editFunction.apply(this.value); - this.save(); + /** + * Reccomended way to edit the value + * @param editFunction + * @throws IOException + */ + public void editValue(BiFunction editFunction) throws IOException { + synchronized(accessLock) { + load(); + this.value = editFunction.apply(this.value, this); + this.save(); + } } /** - * Use editValue instead + * Reccomended way to edit the value + * @param editFunction + * @throws IOException + */ + public void editValue(BiConsumer editFunction) throws IOException { + synchronized(accessLock) { + load(); + editFunction.accept(this.value, this); + this.save(); + } + } + + /** + * Substitute the old value with a new one + * @param val + * @throws IOException + */ + public void setValue(T val) throws IOException { + synchronized(accessLock) { + this.loaded = true; + this.value = val; + synchronized(hashCacheLock) { + this.isHashCached = false; + } + this.save(); + } + } + + /** + * Use editValue instead. READ ONLY!! * @return */ @Deprecated() public T getValue() { - return this.value; + return getValueReadOnly(); } /** * DO NOT ATTEMPT TO MODIFY THE VALUE RETURNED * @return */ - public T getValueUnsafe() { - return this.value; + public T getValueReadOnly() { + synchronized(accessLock) { + load(); + return this.value; + } + } + private void load() { + synchronized(accessLock) { + if (!loaded) { + try { + this.value = db.read(entryIndex, parser.getReader()); + this.loaded = true; + } catch (IOException e) { + throw (NullPointerException) new NullPointerException(e.getLocalizedMessage()).initCause(e); + } + } + } + } + + @SuppressWarnings("unchecked") @Override - public T cast() { - return (T) this; + public U cast() { + return (U) this; } protected void close() throws IOException { @@ -105,4 +190,8 @@ public class EntryReference implements Castable { closed = true; } } + + public Object getAccessLock() { + return accessLock; + } } diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java index 4e98222..4da900f 100644 --- a/src/main/java/org/warp/jcwdb/FileIndexManager.java +++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java @@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.function.Consumer; public class FileIndexManager implements IndexManager { private final SeekableByteChannel dataFileChannel, metadataFileChannel; @@ -69,16 +70,21 @@ public class FileIndexManager implements IndexManager { } @Override - public void set(long index, DBDataOutput data) throws IOException { + public long getHash(long index) throws IOException { + return getIndexMetadata(index).getHash(); + } + + @Override + public IndexDetails set(long index, DBDataOutput data) throws IOException { checkClosed(); final int dataSize = data.getSize(); final IndexDetails indexDetails = getIndexMetadataUnsafe(index); if (indexDetails == null || indexDetails.getSize() < dataSize) { // Allocate new space - allocateAndWrite(index, data); + return allocateAndWrite(index, data); } else { // Check if size changed - if (indexDetails.getSize() > dataSize) { + if (dataSize < indexDetails.getSize()) { // Mark free the unused bytes fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize); } @@ -86,6 +92,8 @@ public class FileIndexManager implements IndexManager { editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash()); // Write data writeExact(indexDetails, data); + // Before returning, return IndexDetails + return indexDetails; } } @@ -122,13 +130,14 @@ public class FileIndexManager implements IndexManager { o.flush(); } - private void allocateAndWrite(final long index, DBDataOutput w) throws IOException { + private IndexDetails allocateAndWrite(final long index, DBDataOutput w) throws IOException { final int size = w.getSize(); final int type = w.getType(); final long hash = w.calculateHash(); final long offset = fileAllocator.allocate(size); IndexDetails details = editIndex(index, offset, size, type, hash); writeExact(details, w); + return details; } @Override @@ -156,16 +165,17 @@ public class FileIndexManager implements IndexManager { SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES); eraseIndexDetails(metadata); } - if (dirtyLoadedIndices.contains(index)) { - synchronized (indicesMapsAccessLock) { + boolean isDirty = false; + IndexDetails indexDetails = null; + synchronized (indicesMapsAccessLock) { + if (dirtyLoadedIndices.contains(index)) { + indexDetails = loadedIndices.get(index); dirtyLoadedIndices.remove(index); } + } + if (isDirty) { // Update indices metadata SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES); - IndexDetails indexDetails; - synchronized (indicesMapsAccessLock) { - indexDetails = loadedIndices.get(index); - } writeIndexDetails(metadata, indexDetails); } synchronized (indicesMapsAccessLock) { @@ -196,7 +206,7 @@ public class FileIndexManager implements IndexManager { */ private IndexDetails editIndex(long index, IndexDetails oldData, long offset, int size, int type, long hash) { if (oldData.getOffset() != offset || oldData.getSize() != size || oldData.getType() != type || oldData.getHash() != hash) { - editIndex(index, offset, size, type, hash); + return editIndex(index, offset, size, type, hash); } else { return oldData; } diff --git a/src/main/java/org/warp/jcwdb/IndexManager.java b/src/main/java/org/warp/jcwdb/IndexManager.java index 9c7ee80..f08b237 100644 --- a/src/main/java/org/warp/jcwdb/IndexManager.java +++ b/src/main/java/org/warp/jcwdb/IndexManager.java @@ -1,12 +1,14 @@ package org.warp.jcwdb; import java.io.IOException; +import java.util.function.Consumer; public interface IndexManager extends Cleanable { T get(long index, DBReader reader) throws IOException; int getType(long index) throws IOException; + long getHash(long index) throws IOException; long add(DBDataOutput writer) throws IOException; - void set(long index, DBDataOutput writer) throws IOException; + IndexDetails set(long index, DBDataOutput writer) throws IOException; void delete(long index) throws IOException; boolean has(long index); void close() throws IOException; diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index 75f749f..fff1453 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -5,6 +5,7 @@ import java.lang.ref.WeakReference; import java.nio.file.Path; import java.util.ArrayList; import java.util.Iterator; +import java.util.function.Consumer; import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; @@ -51,7 +52,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable { if (exists(0)) { return get(0); } else { - LightList newRoot = new LightList(this, new LongArrayList(), new LongArrayList()); + LightList newRoot = new LightList(this, new LongArrayList()); return set(0, newRoot); } } @@ -63,11 +64,13 @@ public class JCWDatabase implements AutoCloseable, Cleanable { EntryReference ref; if (refRef == null || (ref = (EntryReference) refRef.get()) == null) { int type; + long hash; synchronized (indicesAccessLock) { type = this.indices.getType(index); + hash = this.indices.getHash(index); } DBTypeParser typeParser = this.typesManager.get(type); - ref = new EntryReference<>(entryReferenceTools, index, typeParser); + ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser); refRef = new WeakReference<>(ref); this.references.put(index, refRef); } @@ -81,10 +84,12 @@ public class JCWDatabase implements AutoCloseable, Cleanable { EntryReference ref; DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); long index; + long hash; synchronized (indicesAccessLock) { index = indices.add(typeParser.getWriter(value)); + hash = indices.getHash(index); } - ref = new EntryReference<>(entryReferenceTools, index, typeParser, value); + ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value); this.references.put(index, new WeakReference<>(ref)); return ref; } @@ -105,14 +110,17 @@ public class JCWDatabase implements AutoCloseable, Cleanable { EntryReference ref; if (exists(index)) { ref = get(index); - ref.value = value; + ref.setValue(value); return ref; } else { + @SuppressWarnings("unchecked") DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); + long hash; synchronized (indicesAccessLock) { - indices.set(index, typeParser.getWriter(value)); + IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value)); + hash = returnedDetails.getHash(); } - ref = new EntryReference<>(entryReferenceTools, index, typeParser); + ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser); this.references.put(index, new WeakReference>(ref)); return ref; } @@ -194,6 +202,15 @@ public class JCWDatabase implements AutoCloseable, Cleanable { while (iterator.hasNext()) { Entry>> entry = iterator.next(); if (count > MAX_LOADED_REFERENCES * 3l / 2l) { + WeakReference> weakRef = entry.getValue(); + EntryReference ref = weakRef.get(); + if (ref != null) { + try { + ref.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } iterator.remove(); removedReferences++; } else { @@ -214,8 +231,8 @@ public class JCWDatabase implements AutoCloseable, Cleanable { return indices.get(index, reader); } - public void write(long index, DBDataOutput writer) throws IOException { - indices.set(index, writer); + public IndexDetails write(long index, DBDataOutput writer) throws IOException { + return indices.set(index, writer); } } diff --git a/src/main/java/org/warp/jcwdb/LightList.java b/src/main/java/org/warp/jcwdb/LightList.java index 862b38f..0edddc7 100644 --- a/src/main/java/org/warp/jcwdb/LightList.java +++ b/src/main/java/org/warp/jcwdb/LightList.java @@ -10,12 +10,10 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList; public class LightList implements List { public final LongArrayList internalList; - public final LongArrayList hashes; private final transient JCWDatabase db; - public LightList(JCWDatabase db, LongArrayList internalList, LongArrayList hashes) { + public LightList(JCWDatabase db, LongArrayList internalList) { this.internalList = internalList; - this.hashes = hashes; this.db = db; } @@ -36,7 +34,7 @@ public class LightList implements List { EntryReference ref = null; try { ref = db.get(element); - if (o.equals(ref.value)) { + if (o.equals(ref.getValueReadOnly())) { return true; } } catch (IOException e) { @@ -47,19 +45,37 @@ public class LightList implements List { return false; } + /** + * Use iteratorReferences() + */ + @Deprecated @SuppressWarnings("unchecked") @Override public Iterator iterator() { final ArrayList elements = new ArrayList<>(); for (Long element : internalList) { try { - elements.add((T) db.get(element).value); + elements.add((T) db.get(element).getValueReadOnly()); } catch (IOException e) { e.printStackTrace(); } } return elements.iterator(); } + + @SuppressWarnings("unchecked") + public Iterator> iteratorReferences() { + final ArrayList> elements = new ArrayList<>(); + for (Long element : internalList) { + try { + elements.add(db.get(element)); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements.iterator(); + } + @SuppressWarnings("unchecked") @Override @@ -67,7 +83,7 @@ public class LightList implements List { final T[] elements = (T[]) new Objects[internalList.size()]; for (int i = 0; i < elements.length; i++) { try { - elements[i] = (T) db.get(internalList.getLong(i)).value; + elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly(); } catch (IOException e) { e.printStackTrace(); } @@ -81,7 +97,7 @@ public class LightList implements List { final T1[] elements = (T1[]) new Objects[internalList.size()]; for (int i = 0; i < elements.length; i++) { try { - elements[i] = (T1) db.get(internalList.getLong(i)).value; + elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly(); } catch (IOException e) { e.printStackTrace(); } @@ -98,11 +114,10 @@ public class LightList implements List { public EntryReference addEntry(T o) { EntryReference ref = addToDatabase(o); if (internalList.add(ref.getIndex())) { - hashes.add(ref.getParser().calculateHash(ref.value)); + return ref; } else { return null; } - return ref; } @Override @@ -110,7 +125,6 @@ public class LightList implements List { int removeIndex = indexOf(o); if (removeIndex >= 0) { internalList.removeLong(removeIndex); - hashes.removeLong(removeIndex); return true; } return false; @@ -120,7 +134,6 @@ public class LightList implements List { int removeIndex = indexOfEntry(ref); if (removeIndex >= 0) { internalList.removeLong(removeIndex); - hashes.removeLong(removeIndex); return true; } return false; @@ -193,14 +206,30 @@ public class LightList implements List { @Override public void clear() { internalList.clear(); - hashes.clear(); + } + + /** + * Use getReference or getReadOnlyValue + */ + @Deprecated + @Override + public T get(int index) { + return getReadOnlyValue(index); } @SuppressWarnings("unchecked") - @Override - public T get(int index) { + public T getReadOnlyValue(int index) { try { - return (T) db.get(internalList.getLong(index)).value; + return (T) db.get(internalList.getLong(index)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + public EntryReference getReference(int index) { + try { + return db.get(internalList.getLong(index)).cast(); } catch (IOException e) { e.printStackTrace(); return null; @@ -212,10 +241,9 @@ public class LightList implements List { public T set(int index, T element) { EntryReference ref = addToDatabase(element); long oldIndex = internalList.set(index, ref.getIndex()); - hashes.set(index, ref.calculateHash()); try { ref.close(); - return ((EntryReference) (db.get(oldIndex))).value; + return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); } catch (IOException e) { throw (NullPointerException) new NullPointerException().initCause(e); } @@ -225,26 +253,26 @@ public class LightList implements List { public void add(int index, T element) { EntryReference ref = addToDatabase(element); internalList.add(index, ref.getIndex()); - hashes.add(index, ref.getParser().calculateHash(ref.value)); } @SuppressWarnings("unchecked") @Override public T remove(int index) { long oldIndex = internalList.removeLong(index); - hashes.removeLong(index); try { - return ((EntryReference) (db.get(oldIndex))).value; + return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); } catch (IOException e) { throw (NullPointerException) new NullPointerException().initCause(e); } } - + @Override public int indexOf(Object o) { EntryReference ref = addToDatabase(o); long objToRemoveHash = ref.calculateHash(); - + LongArrayList hashes = new LongArrayList(); + + for (int i = 0; i < hashes.size(); i++) { long hash = hashes.getLong(i); if (objToRemoveHash == hash) { @@ -261,18 +289,15 @@ public class LightList implements List { } public int indexOfEntry(EntryReference ref) { - long objToRemoveHash = ref.calculateHash(); - - for (int i = 0; i < hashes.size(); i++) { - long hash = hashes.getLong(i); - if (objToRemoveHash == hash) { - try { - if (ref.equals(db.get(internalList.getLong(i)))) { - return i; - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); + for (int i = 0; i < internalList.size(); i++) { + long index = internalList.getLong(i); + try { + EntryReference ref2 = db.get(index); + if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { + return i; } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); } } return -1; @@ -285,16 +310,17 @@ public class LightList implements List { int lastValue = -1; - for (int i = 0; i < hashes.size(); i++) { - long hash = hashes.getLong(i); - if (objToRemoveHash == hash) { - try { - if (ref.equals(db.get(internalList.getLong(i)))) { + for (int i = 0; i < internalList.size(); i++) { + long index2 = internalList.getLong(i); + try { + EntryReference ref2 = db.get(index2); + if (objToRemoveHash == ref2.calculateHash()) { + if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { lastValue = i; } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); } } return lastValue; @@ -336,19 +362,10 @@ public class LightList implements List { public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((hashes == null) ? 0 : hashes.hashCode()); result = prime * result + ((internalList == null) ? 0 : internalList.hashCode()); return result; } - public long hashCodeLong() { - final int prime = 31; - int result1 = prime + ((hashes == null) ? 0 : hashes.hashCode()); - int result2 = prime + ((internalList == null) ? 0 : internalList.hashCode()); - long result = (((long) result1) << 32) | (result2 & 0xffffffffL); - return result; - } - @SuppressWarnings("unchecked") @Override public boolean removeIf(Predicate filter) { @@ -357,13 +374,12 @@ public class LightList implements List { for (int i = 0; i < internalList.size(); ) { T obj; try { - obj = ((EntryReference) (db.get(internalList.getLong(i)).cast())).value; + obj = ((EntryReference) (db.get(internalList.getLong(i)).cast())).getValueReadOnly(); } catch (IOException e) { throw (NullPointerException) new NullPointerException().initCause(e); } if (filter.test(obj)) { internalList.removeLong(i); - hashes.removeLong(i); removed = true; } else { i++; diff --git a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java index 5e525d9..3194293 100644 --- a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java +++ b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java @@ -5,6 +5,7 @@ import it.unimi.dsi.fastutil.longs.Long2LongMap; import java.io.IOException; import java.nio.file.Path; +import java.util.function.Consumer; public class MixedIndexDatabase implements IndexManager { private final Long2LongMap mostAccessedIndices; @@ -36,17 +37,26 @@ public class MixedIndexDatabase implements IndexManager { } } + @Override + public long getHash(long index) throws IOException { + if (cacheIndices.has(index)) { + return cacheIndices.getHash(index); + } else { + return fileIndices.getHash(index); + } + } + @Override public long add(DBDataOutput writer) throws IOException { return fileIndices.add(writer); } @Override - public void set(long index, DBDataOutput writer) throws IOException { + public IndexDetails set(long index, DBDataOutput writer) throws IOException { if (cacheIndices.has(index)) { - cacheIndices.set(index, writer); + return cacheIndices.set(index, writer); } else { - fileIndices.set(index, writer); + return fileIndices.set(index, writer); } } diff --git a/src/main/java/org/warp/jcwdb/Saveable.java b/src/main/java/org/warp/jcwdb/Saveable.java new file mode 100644 index 0000000..385ee18 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/Saveable.java @@ -0,0 +1,7 @@ +package org.warp.jcwdb; + +import java.io.IOException; + +public interface Saveable { + public void save(); +} diff --git a/src/main/java/org/warp/jcwdb/exampleimpl/App.java b/src/main/java/org/warp/jcwdb/exampleimpl/App.java index c71eed4..243271c 100644 --- a/src/main/java/org/warp/jcwdb/exampleimpl/App.java +++ b/src/main/java/org/warp/jcwdb/exampleimpl/App.java @@ -12,6 +12,7 @@ import java.nio.file.Paths; import java.util.function.Predicate; public class App { + static long time3; public static void main(String[] args) { try { if (args.length > 2 && Boolean.parseBoolean(args[2])) { @@ -26,55 +27,57 @@ public class App { System.out.println("Time elapsed: " + (time01 - time0)); System.out.println("Loading root..."); EntryReference> rootRef = db.getRoot(Animal.class); - LightList root = rootRef.getValue(); - long time1 = System.currentTimeMillis(); - System.out.println("Time elapsed: " + (time1 - time01)); - System.out.println("Root size: " + root.size()); - System.out.println("Root:"); -// for (int i = 0; i < root.size(); i++) { -// System.out.println(" - " + root.get(i)); -// } - long prectime = System.currentTimeMillis(); - for (int i = 0; i < 2000000/* 2000000 */; i++) { - Animal animal = new StrangeAnimal(i % 40); - root.add(animal); - if (i > 0 && i % 200000 == 0) { - long precprectime = prectime; - prectime = System.currentTimeMillis(); - System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)"); + rootRef.editValue((root, saver) -> { + long time1 = System.currentTimeMillis(); + System.out.println("Time elapsed: " + (time1 - time01)); + System.out.println("Root size: " + root.size()); + System.out.println("Root:"); + // for (int i = 0; i < root.size(); i++) { + // System.out.println(" - " + root.get(i)); + // } + long prectime = System.currentTimeMillis(); + for (int i = 0; i < 20000/* 2000000 */; i++) { + Animal animal = new StrangeAnimal(i % 40); + root.add(animal); + if (i > 0 && i % 200000 == 0) { + long precprectime = prectime; + prectime = System.currentTimeMillis(); + System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)"); + } } - } - long time2 = System.currentTimeMillis(); - System.out.println("Root size: " + root.size()); - System.out.println("Time elapsed: " + (time2 - time1)); - System.out.println("Used memory: " - + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); - long time2_0 = System.currentTimeMillis(); - System.out.println("Filtering strings..."); - //root.removeIf(Animal::hasFourLegs); - long time2_1 = System.currentTimeMillis(); - System.out.println("Time elapsed: " + (time2_1 - time2_0)); - ObjectList results = new ObjectArrayList<>(); - - root.forEach((value) -> { - if (Animal.hasFourLegs(value)) { - results.add(value); - } - //System.out.println("val:" + value); + long time2 = System.currentTimeMillis(); + saver.save(); + System.out.println("Root size: " + root.size()); + System.out.println("Time elapsed: " + (time2 - time1)); + System.out.println("Used memory: " + + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); + long time2_0 = System.currentTimeMillis(); + System.out.println("Filtering strings..."); + //root.removeIf(Animal::hasFourLegs); + long time2_1 = System.currentTimeMillis(); + System.out.println("Time elapsed: " + (time2_1 - time2_0)); + ObjectList results = new ObjectArrayList<>(); + + root.forEach((value) -> { + if (Animal.hasFourLegs(value)) { + results.add(value); + } + //System.out.println("val:" + value); + }); + long time2_2 = System.currentTimeMillis(); + System.out.println("Time elapsed: " + (time2_2 - time2_1)); + System.out.println("Used memory: " + + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); + System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); + System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)..."); + long removedItems = db.clean(); + time3 = System.currentTimeMillis(); + System.out.println("Removed items: " + removedItems); + System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); + System.out.println("Time elapsed: " + (time3 - time2_2)); + System.out.println("Saving database..."); + System.out.println("Root size: " + root.size()); }); - long time2_2 = System.currentTimeMillis(); - System.out.println("Time elapsed: " + (time2_2 - time2_1)); - System.out.println("Used memory: " - + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); - System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); - System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)..."); - long removedItems = db.clean(); - long time3 = System.currentTimeMillis(); - System.out.println("Removed items: " + removedItems); - System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); - System.out.println("Time elapsed: " + (time3 - time2_2)); - System.out.println("Saving database..."); - System.out.println("Root size: " + root.size()); db.close(); long time4 = System.currentTimeMillis(); System.out.println("Time elapsed: " + (time4 - time3)); diff --git a/src/test/java/org/warp/jcwdb/AppTest.java b/src/test/java/org/warp/jcwdb/AppTest.java index b6b8aeb..187b8cd 100644 --- a/src/test/java/org/warp/jcwdb/AppTest.java +++ b/src/test/java/org/warp/jcwdb/AppTest.java @@ -41,14 +41,14 @@ public class AppTest JCWDatabase db = new JCWDatabase(path, idx); EntryReference> ref = db.getRoot(); - LightList root = ref.value; - root.add(customClass); + ref.editValue((root, saver) -> { + root.add(customClass); + }); db.close(); JCWDatabase db2 = new JCWDatabase(path, idx); EntryReference> ref2 = db2.getRoot(); - LightList root2 = ref2.value; - CustomClass customClass2 = root2.get(0); + CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0); assertTrue(customClass.equals(customClass2)); assertTrue(customClass.string.equals(customClass2.string)); db2.close(); @@ -99,7 +99,7 @@ public class AppTest Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx"); JCWDatabase db = new JCWDatabase(path, idx); EntryReference> ref = db.getRoot(); - LightList list1 = ref.value; + LightList list1 = ref.getValueReadOnly(); ArrayList list2 = new ArrayList(); String s = "a"; for (int i = 0; i < 10; i++) {