diff --git a/src/main/java/org/warp/jcwdb/Cleaner.java b/src/main/java/org/warp/jcwdb/Cleaner.java index b71be1c..08ccf4d 100644 --- a/src/main/java/org/warp/jcwdb/Cleaner.java +++ b/src/main/java/org/warp/jcwdb/Cleaner.java @@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry; public class Cleaner { - private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes + private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second private static final double NORMAL_REMOVED_ITEMS = 1000l; private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250% @@ -63,7 +63,10 @@ public class Cleaner { try { System.out.println("[CLEANER] Waiting " + sleepInterval + "ms."); sleepFor(sleepInterval); + final long time1 = System.currentTimeMillis(); final double removedItems = clean(); + final long time2 = System.currentTimeMillis(); + System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1)); double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2; System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems); diff --git a/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java b/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java index 58dd9ca..5e6f9ae 100644 --- a/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java +++ b/src/main/java/org/warp/jcwdb/DBGenericObjectParser.java @@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output; import net.openhft.hashing.LongHashFunction; -public class DBGenericObjectParser extends DBTypeParserImpl { +public class DBGenericObjectParser extends DBTypeParserImpl implements DBTypedObjectParser { private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx(); private static final Kryo kryo = new Kryo(); static { @@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl { tmpO.close(); return hash; } + + + @Override + public void registerClass(Class clazz, int id) { + if (id >= Integer.MAX_VALUE - 100) { + throw new IndexOutOfBoundsException(); + } + kryo.register(clazz, id + 100); + } } diff --git a/src/main/java/org/warp/jcwdb/DBLightListParser.java b/src/main/java/org/warp/jcwdb/DBLightArrayListParser.java similarity index 53% rename from src/main/java/org/warp/jcwdb/DBLightListParser.java rename to src/main/java/org/warp/jcwdb/DBLightArrayListParser.java index 358ac27..6a980fa 100644 --- a/src/main/java/org/warp/jcwdb/DBLightListParser.java +++ b/src/main/java/org/warp/jcwdb/DBLightArrayListParser.java @@ -1,40 +1,38 @@ package org.warp.jcwdb; -import java.util.ArrayList; - import it.unimi.dsi.fastutil.longs.LongArrayList; -public class DBLightListParser extends DBTypeParserImpl> { +public class DBLightArrayListParser extends DBTypeParserImpl> { private final JCWDatabase db; - public DBLightListParser(JCWDatabase db) { + public DBLightArrayListParser(JCWDatabase db) { this.db = db; } - - public DBReader> getReader() { + + public DBReader> getReader() { return (i, size) -> { LongArrayList internalList = new LongArrayList(); long max = size / Long.BYTES; - for (int item = 0; item < max; item++){ + for (int item = 0; item < max; item++) { long itm = i.readLong(); internalList.add(itm); } - return new LightList(db, internalList); + return new LightArrayList(db, internalList); }; } - public DBDataOutput> getWriter(final LightList value) { + public DBDataOutput> getWriter(final LightArrayList value) { final int elementsCount = value.size(); return DBDataOutput.create((o) -> { LongArrayList list = value.internalList; for (int i = 0; i < elementsCount; i++) { o.writeLong(list.getLong(i)); } - }, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value)); + }, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value)); } @Override - public long calculateHash(LightList value) { + public long calculateHash(LightArrayList value) { return value.internalList.hashCode(); } } diff --git a/src/main/java/org/warp/jcwdb/DBLightBigListParser.java b/src/main/java/org/warp/jcwdb/DBLightBigListParser.java new file mode 100644 index 0000000..876fd13 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBLightBigListParser.java @@ -0,0 +1,44 @@ +package org.warp.jcwdb; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; + +public class DBLightBigListParser extends DBTypeParserImpl> { + private final JCWDatabase db; + + public DBLightBigListParser(JCWDatabase db) { + this.db = db; + } + + public DBReader> getReader() { + return (i, size) -> { + LongArrayList chunks = new LongArrayList(); + IntArrayList chunkSizes = new IntArrayList(); + long max = size / (Long.BYTES + Integer.BYTES); + for (int item = 0; item < max; item++) { + long itm = i.readLong(); + int itm2 = i.readInt(); + chunks.add(itm); + chunkSizes.add(itm2); + } + return new LightBigList(db, chunks, chunkSizes); + }; + } + + public DBDataOutput> getWriter(final LightBigList value) { + final int elementsCount = value.chunksCount(); + return DBDataOutput.create((o) -> { + LongArrayList list = value.chunks; + IntArrayList list2 = value.chunkSizes; + for (int i = 0; i < elementsCount; i++) { + o.writeLong(list.getLong(i)); + o.writeInt(list2.getInt(i)); + } + }, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value)); + } + + @Override + public long calculateHash(LightBigList value) { + return value.chunks.hashCode(); + } +} diff --git a/src/main/java/org/warp/jcwdb/DBStandardTypes.java b/src/main/java/org/warp/jcwdb/DBStandardTypes.java index 23d8f3f..23c6796 100644 --- a/src/main/java/org/warp/jcwdb/DBStandardTypes.java +++ b/src/main/java/org/warp/jcwdb/DBStandardTypes.java @@ -11,12 +11,14 @@ public class DBStandardTypes { public static final int DOUBLE = STD| 6; public static final int STRING = STD| 7; public static final int BYTE_ARRAY = STD| 8; - public static final int LIGHT_LIST = STD| 9; - public static final int GENERIC_OBJECT = STD| 10; + public static final int LIGHT_LIST_ARRAY = STD| 9; + public static final int LIGHT_LIST_BIG = STD| 10; + public static final int GENERIC_OBJECT = STD| 11; public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) { typesManager.registerType(String.class, STRING, new DBStringParser()); - typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db)); + typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db)); + typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db)); typesManager.registerTypeFallback(new DBGenericObjectParser()); } } \ No newline at end of file diff --git a/src/main/java/org/warp/jcwdb/DBTypedObjectParser.java b/src/main/java/org/warp/jcwdb/DBTypedObjectParser.java new file mode 100644 index 0000000..5aa4353 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBTypedObjectParser.java @@ -0,0 +1,5 @@ +package org.warp.jcwdb; + +public interface DBTypedObjectParser extends DBTypeParser { + public void registerClass(Class clazz, int type); +} diff --git a/src/main/java/org/warp/jcwdb/EntryReference.java b/src/main/java/org/warp/jcwdb/EntryReference.java index 66481d4..977299c 100644 --- a/src/main/java/org/warp/jcwdb/EntryReference.java +++ b/src/main/java/org/warp/jcwdb/EntryReference.java @@ -86,7 +86,7 @@ public class EntryReference implements Castable, Saveable { * @param editFunction * @throws IOException */ - public void editValue(BiFunction editFunction) throws IOException { + public void editValue(BiFunction editFunction) { synchronized(accessLock) { load(); this.value = editFunction.apply(this.value, this); @@ -99,7 +99,20 @@ public class EntryReference implements Castable, Saveable { * @param editFunction * @throws IOException */ - public void editValue(BiConsumer editFunction) throws IOException { + public void editValue(Function editFunction) { + synchronized(accessLock) { + load(); + this.value = editFunction.apply(this.value); + this.save(); + } + } + + /** + * Reccomended way to edit the value + * @param editFunction + * @throws IOException + */ + public void editValue(BiConsumer editFunction) { synchronized(accessLock) { load(); editFunction.accept(this.value, this); @@ -107,12 +120,25 @@ public class EntryReference implements Castable, Saveable { } } + /** + * Reccomended way to edit the value + * @param editFunction + * @throws IOException + */ + public void editValue(Consumer editFunction) { + synchronized(accessLock) { + load(); + editFunction.accept(this.value); + this.save(); + } + } + /** * Substitute the old value with a new one * @param val * @throws IOException */ - public void setValue(T val) throws IOException { + public void setValue(T val) { synchronized(accessLock) { this.loaded = true; this.value = val; diff --git a/src/main/java/org/warp/jcwdb/FileAllocator.java b/src/main/java/org/warp/jcwdb/FileAllocator.java index 9fb9424..94eb771 100644 --- a/src/main/java/org/warp/jcwdb/FileAllocator.java +++ b/src/main/java/org/warp/jcwdb/FileAllocator.java @@ -1,62 +1,116 @@ package org.warp.jcwdb; -import com.esotericsoftware.kryo.io.Output; +import it.unimi.dsi.fastutil.longs.Long2IntMap; +import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap; +import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; -import java.util.*; public class FileAllocator implements AutoCloseable { + private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000; + private final SeekableByteChannel dataFileChannel; - private volatile long allocableOffset; + private volatile long fileSize; private volatile boolean closed; private final Object closeLock = new Object(); private final Object allocateLock = new Object(); - + /** + * index -> free space size + */ + private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b)); + public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException { this.dataFileChannel = dataFileChannel; - this.allocableOffset = this.dataFileChannel.size(); + this.fileSize = this.dataFileChannel.size(); } - + /** * TODO: not implemented + * * @param size * @return offset */ public long allocate(int size) { checkClosed(); synchronized (allocateLock) { - long allocatedOffset = allocableOffset; - allocableOffset += size; - return allocatedOffset; + long offset = allocateIntoUnusedParts(size); + if (offset == -1) { + return allocateToEnd(size); + } else { + return offset; + } } } - - public void close() throws IOException { - if (closed) { - return; - } - synchronized (closeLock) { - if (closed) { - return; - } - closed = true; - } + private long allocateIntoUnusedParts(int size) { + ObjectBidirectionalIterator it = freeBytes.long2IntEntrySet().iterator(); + long holeOffset = -1; + int holeSize = 0; + while (it.hasNext()) { + Long2IntMap.Entry entry = it.next(); + int currentHoleSize = entry.getIntValue(); + if (currentHoleSize < size) { + freeBytes.remove(holeOffset); + if (holeSize > size) { + freeBytes.put(holeOffset + size, holeSize - size); + } + break; + } + holeOffset = entry.getLongKey(); + holeSize = currentHoleSize; + } + return holeOffset; } - + + private long allocateToEnd(int size) { + long allocatedOffset = fileSize; + fileSize += size; + return allocatedOffset; + } + + + public void close() throws IOException { + if (closed) { + return; + } + synchronized (closeLock) { + if (closed) { + return; + } + closed = true; + } + } + /** * Frees the unused bytes + * * @param startPosition * @param length */ public void markFree(long startPosition, int length) { checkClosed(); - // TODO: advanced feature, not implemented. + + if (freeBytes.containsKey(startPosition + length)) { + int secondLength = freeBytes.remove(startPosition + length); + freeBytes.put(startPosition, length + secondLength); + } else { + boolean addedToList = false; + for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) { + if (entry.getLongKey() + entry.getIntValue() == startPosition) { + freeBytes.put(entry.getLongKey(), entry.getIntValue() + length); + addedToList = true; + break; + } + } + if (!addedToList) { + freeBytes.put(startPosition, length); + } + } + + if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) { + freeBytes.remove(freeBytes.lastLongKey()); + } } diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java index 4da900f..fe87653 100644 --- a/src/main/java/org/warp/jcwdb/FileIndexManager.java +++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java @@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.Iterator; import java.util.function.Consumer; public class FileIndexManager implements IndexManager { @@ -81,7 +82,12 @@ public class FileIndexManager implements IndexManager { final IndexDetails indexDetails = getIndexMetadataUnsafe(index); if (indexDetails == null || indexDetails.getSize() < dataSize) { // Allocate new space - return allocateAndWrite(index, data); + IndexDetails newDetails = allocateAndWrite(index, data); + if (indexDetails != null) { + // Mark free the old bytes + fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize()); + } + return newDetails; } else { // Check if size changed if (dataSize < indexDetails.getSize()) { @@ -233,7 +239,7 @@ public class FileIndexManager implements IndexManager { * @param details */ private void editIndex(long index, IndexDetails details) { - synchronized (indicesMapsAccessLock) { + synchronized (indicesMapsAccessLock) {// FIXXXX main3 loadedIndices.put(index, details); dirtyLoadedIndices.add(index); } @@ -264,7 +270,8 @@ public class FileIndexManager implements IndexManager { return null; } SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition); - synchronized (metadataByteBufferLock) { + IndexDetails indexDetails = null; + synchronized (metadataByteBufferLock) {// FIXXXX main2 metadataByteBuffer.rewind(); currentMetadataFileChannel.read(metadataByteBuffer); metadataByteBuffer.rewind(); @@ -277,12 +284,15 @@ public class FileIndexManager implements IndexManager { final int size = metadataByteBuffer.getInt(); final int type = metadataByteBuffer.getInt(); final long hash = metadataByteBuffer.getLong(); - final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash); - editIndex(index, indexDetails); - return indexDetails; + indexDetails = new IndexDetails(offset, size, type, hash); } } + if (indexDetails != null) { + editIndex(index, indexDetails); + return indexDetails; + } + // No results found. Returning null return null; } @@ -308,31 +318,15 @@ public class FileIndexManager implements IndexManager { } // Update indices metadata - SeekableByteChannel metadata = metadataFileChannel; - long lastIndex = -2; - synchronized (indicesMapsAccessLock) { - for (long index : dirtyLoadedIndices) { - IndexDetails indexDetails = loadedIndices.get(index); - if (index - lastIndex != 1) { - metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); - } - writeIndexDetails(metadata, indexDetails); - lastIndex = index; - } - } + flushAllIndices(); // Remove removed indices - synchronized (indicesMapsAccessLock) { - for (long index : removedIndices) { - metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); - eraseIndexDetails(metadata); - } - } + removeRemovedIndices(); fileAllocator.close(); } private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException { - synchronized (metadataByteBufferLock) { + synchronized (metadataByteBufferLock) {// FIXXXX cleaner3 final int size = indexDetails.getSize(); final int type = indexDetails.getType(); final long offset = indexDetails.getOffset(); @@ -366,19 +360,63 @@ public class FileIndexManager implements IndexManager { @Override public long clean() { - return cleanExtraIndices(); + long cleaned = 0; + try { + cleaned += flushAllIndices(); + } catch (IOException ex) { + ex.printStackTrace(); + } + try { + cleaned += removeRemovedIndices(); + } catch (IOException ex) { + ex.printStackTrace(); + } + cleaned += cleanExtraIndices(); + return cleaned; + } + + private long flushAllIndices() throws IOException { + long flushedIndices = 0; + SeekableByteChannel metadata = metadataFileChannel; + long lastIndex = -2; + synchronized (indicesMapsAccessLock) { + for (long index : dirtyLoadedIndices) { + IndexDetails indexDetails = loadedIndices.get(index); + if (index - lastIndex != 1) { + metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); + } + writeIndexDetails(metadata, indexDetails); + lastIndex = index; + flushedIndices++; + } + dirtyLoadedIndices.clear(); + } + return flushedIndices; + } + + private long removeRemovedIndices() throws IOException { + SeekableByteChannel metadata = metadataFileChannel; + synchronized (indicesMapsAccessLock) { + long removed = this.removedIndices.size(); + for (long index : this.removedIndices) { + metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); + eraseIndexDetails(metadata); + } + this.removedIndices.clear(); + return removed; + } } private long cleanExtraIndices() { long removedIndices = 0; LongArrayList toUnload = new LongArrayList(); synchronized (indicesMapsAccessLock) { - if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) { + if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) { long count = loadedIndices.size(); LongIterator it = loadedIndices.keySet().iterator(); while (it.hasNext()) { long loadedIndex = it.nextLong(); - if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) { + if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) { break; } toUnload.add(loadedIndex); diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index d1bcd72..49425ec 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -1,36 +1,24 @@ package org.warp.jcwdb; -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.function.Consumer; - -import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry; import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.objects.ObjectIterator; + +import java.io.IOException; +import java.nio.file.Path; public class JCWDatabase implements AutoCloseable, Cleanable { - public final static long MAX_LOADED_REFERENCES = 1000; public final static long MAX_LOADED_INDICES = 10000; - + private final TypesManager typesManager; private final MixedIndexDatabase indices; private final Cleaner databaseCleaner; private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools(); - private final Long2ObjectMap>> references; private volatile boolean closed; private final Object closeLock = new Object(); private final Object indicesAccessLock = new Object(); - private final Object referencesAccessLock = new Object(); public JCWDatabase(Path dataFile, Path metadataFile) throws IOException { this.typesManager = new TypesManager(this); - this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile); - this.references = new Long2ObjectLinkedOpenHashMap<>(); + this.indices = new MixedIndexDatabase(dataFile, metadataFile); Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { JCWDatabase.this.close(); @@ -39,94 +27,84 @@ public class JCWDatabase implements AutoCloseable, Cleanable { } })); this.databaseCleaner = new Cleaner(this); - - //this.databaseCleaner.start(); + + this.databaseCleaner.start(); } public EntryReference> getRoot() throws IOException { - return getRoot(Object.class).cast(); - } - - public EntryReference> getRoot(Class clazz) throws IOException { checkClosed(); if (exists(0)) { return get(0); } else { - LightList newRoot = new LightList(this, new LongArrayList()); + LightList newRoot = new LightArrayList<>(this); return set(0, newRoot); } } + public EntryReference> getRoot(Class clazz) throws IOException { + return getRoot().cast(); + } + public EntryReference get(long index) throws IOException { checkClosed(); - synchronized (referencesAccessLock) { - WeakReference> refRef = this.references.getOrDefault(index, null); - EntryReference ref; - if (refRef == null || (ref = (EntryReference) refRef.get()) == null) { - int type; - long hash; - synchronized (indicesAccessLock) { - type = this.indices.getType(index); - hash = this.indices.getHash(index); - } - DBTypeParser typeParser = this.typesManager.get(type); - ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser); - refRef = new WeakReference<>(ref); - this.references.put(index, refRef); - } - return ref; + int type; + long hash; + synchronized (indicesAccessLock) { + type = this.indices.getType(index); + hash = this.indices.getHash(index); } + DBTypeParser typeParser = this.typesManager.get(type); + return new EntryReference<>(entryReferenceTools, index, hash, typeParser); } protected EntryReference add(T value) throws IOException { checkClosed(); - synchronized (referencesAccessLock) { - EntryReference ref; - DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); - long index; - long hash; - synchronized (indicesAccessLock) { - index = indices.add(typeParser.getWriter(value)); - hash = indices.getHash(index); - } - ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value); - this.references.put(index, new WeakReference<>(ref)); - return ref; + DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); + long index; + long hash; + synchronized (indicesAccessLock) { + index = indices.add(typeParser.getWriter(value)); + hash = indices.getHash(index); } + return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value); } protected boolean exists(long index) { checkClosed(); - synchronized (referencesAccessLock) { - synchronized (indicesAccessLock) { - return this.references.containsKey(index) || this.indices.has(index); - } + synchronized (indicesAccessLock) { + return this.indices.has(index); } } protected EntryReference set(long index, T value) throws IOException { checkClosed(); - synchronized (referencesAccessLock) { - EntryReference ref; - if (exists(index)) { - ref = get(index); - ref.setValue(value); - return ref; - } else { - @SuppressWarnings("unchecked") - DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); - long hash; - synchronized (indicesAccessLock) { - IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value)); - hash = returnedDetails.getHash(); - } - ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser); - this.references.put(index, new WeakReference>(ref)); - return ref; + EntryReference ref; + if (exists(index)) { + ref = get(index); + ref.setValue(value); + return ref; + } else { + @SuppressWarnings("unchecked") + DBTypeParser typeParser = this.typesManager.get((Class) value.getClass()); + long hash; + synchronized (indicesAccessLock) { + IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value)); + hash = returnedDetails.getHash(); } + return new EntryReference<>(entryReferenceTools, index, hash, typeParser); } } + public void registerType(Class clazz, short type, DBTypeParser parser) { + final int addition = 0xEFFF8000; + int extendedType = addition | (type & 0x7FFF); + typesManager.registerType(clazz, extendedType, parser); + } + + public void registerClass(Class clazz, int type) { + typesManager.registerGenericClass(clazz, type); + } + public boolean isOpen() { return !closed; } @@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable { this.databaseCleaner.stop(); - synchronized (referencesAccessLock) { - ObjectIterator>> iterator = references.values().iterator(); - while (iterator.hasNext()) { - WeakReference> referenceRef = iterator.next(); - EntryReference reference = referenceRef.get(); - if (reference != null) { - reference.close(); - iterator.remove(); - } - - } - } synchronized (indicesAccessLock) { this.indices.close(); } @@ -171,56 +137,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable { @Override public long clean() { - long removedItems = cleanEmptyReferences() - + cleanExtraReferences() - + indices.clean(); + long removedItems = indices.clean(); return removedItems; } - - - private long cleanEmptyReferences() { - long removed = 0; - synchronized(referencesAccessLock) { - ObjectIterator>>> iterator = references.long2ObjectEntrySet().iterator(); - while (iterator.hasNext()) { - Entry>> entry = iterator.next(); - if (entry.getValue().get() == null) { - iterator.remove(); - removed++; - } - } - } - return removed; - } - - private long cleanExtraReferences() { - long removedReferences = 0; - synchronized(referencesAccessLock) { - if (references.size() > MAX_LOADED_REFERENCES) { - long count = 0; - ObjectIterator>>> iterator = references.long2ObjectEntrySet().iterator(); - while (iterator.hasNext()) { - Entry>> entry = iterator.next(); - if (count > MAX_LOADED_REFERENCES * 3l / 2l) { - WeakReference> weakRef = entry.getValue(); - EntryReference ref = weakRef.get(); - if (ref != null) { - try { - ref.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - iterator.remove(); - removedReferences++; - } else { - count++; - } - } - } - } - return removedReferences; - } public class EntryReferenceTools { private EntryReferenceTools() { diff --git a/src/main/java/org/warp/jcwdb/LightArrayList.java b/src/main/java/org/warp/jcwdb/LightArrayList.java new file mode 100644 index 0000000..8cbe1ef --- /dev/null +++ b/src/main/java/org/warp/jcwdb/LightArrayList.java @@ -0,0 +1,445 @@ +package org.warp.jcwdb; + +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; + +import java.io.IOException; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Predicate; + +public class LightArrayList implements LightList { + + public final LongArrayList internalList; + private final transient JCWDatabase db; + + /** + * @param db Database reference + */ + public LightArrayList(JCWDatabase db) { + this.db = db; + this.internalList = new LongArrayList(); + } + + /** + * @param db Database reference + * @param elements Elements to add + */ + public LightArrayList(JCWDatabase db, LongArrayList elements) { + this.db = db; + this.internalList = new LongArrayList(elements); + } + + @Override + public int size() { + return internalList.size(); + } + + @Override + public boolean isEmpty() { + return internalList.isEmpty(); + } + + @Override + public boolean contains(Object o) { + if (o != null) { + for (long element : internalList) { + EntryReference ref = null; + try { + ref = db.get(element); + if (o.equals(ref.getValueReadOnly())) { + return true; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + return false; + } + + /** + * Use iteratorReferences() + */ + @Deprecated + @SuppressWarnings("unchecked") + @Override + public Iterator iterator() { + System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!"); + final ArrayList elements = new ArrayList<>(); + for (long element : internalList) { + try { + elements.add((T) db.get(element).getValueReadOnly()); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements.iterator(); + } + + @SuppressWarnings("unchecked") + @Override + public Iterator> iteratorReferences() { + final ArrayList> elements = new ArrayList<>(); + for (long element : internalList) { + try { + elements.add(db.get(element)); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements.iterator(); + } + + /** + * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY + * + * @param action + */ + @Deprecated + @Override + public void forEach(Consumer action) { + Objects.requireNonNull(action); + for (T t : this) { + action.accept(t); + } + } + + @Override + public void forEachReference(Consumer> action) { + Objects.requireNonNull(action); + for (long index : this.internalList) { + try { + action.accept(db.get(index)); + } catch (IOException e) { + throw (RuntimeException) new RuntimeException().initCause(e); + } + } + } + + + @SuppressWarnings("unchecked") + @Override + public T[] toArray() { + final T[] elements = (T[]) new Objects[internalList.size()]; + for (int i = 0; i < elements.length; i++) { + try { + elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements; + } + + @SuppressWarnings("unchecked") + @Override + public T1[] toArray(T1[] a) { + final T1[] elements = (T1[]) new Objects[internalList.size()]; + for (int i = 0; i < elements.length; i++) { + try { + elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements; + } + + @Override + public boolean add(T o) { + EntryReference ref = addEntry(o); + return ref != null; + } + + @Override + public EntryReference addEntry(T o) { + EntryReference ref = addToDatabase(o); + if (internalList.add(ref.getIndex())) { + return ref; + } else { + return null; + } + } + + @Override + public boolean remove(Object o) { + int removeIndex = indexOf(o); + if (removeIndex >= 0) { + internalList.removeLong(removeIndex); + return true; + } + return false; + } + + @Override + public boolean remove(EntryReference ref) { + int removeIndex = indexOfEntry(ref); + if (removeIndex >= 0) { + internalList.removeLong(removeIndex); + return true; + } + return false; + } + + @Override + public boolean containsAll(Collection c) { + for (Object o : c) { + int objIndex = indexOf(o); + if (objIndex < 0) { + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= add((T) o); + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(int index, Collection c) { + boolean result = false; + int delta = 0; + for (Object o : c) { + add(index + delta, (T) o); + result = true; + delta++; + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= remove((T) o); + } + return result; + } + + @Override + public boolean retainAll(Collection c) { + boolean result = false; + LongArrayList collectionHashes = new LongArrayList(); + ObjectArrayList collection = new ObjectArrayList<>(); + collection.addAll(c); + for (Object o : c) { + collectionHashes.add(db.calculateHash(o)); + } + for (int i = 0; i < internalList.size(); i++) { + long hash = internalList.getLong(i); + int positionInCollection = collectionHashes.indexOf(hash); + if (positionInCollection == -1) { + remove(collection.get(positionInCollection)); + result = true; + } + } + return result; + } + + @Override + public void clear() { + internalList.clear(); + } + + /** + * Use getReference or getReadOnlyValue + */ + @Deprecated + @Override + public T get(int index) { + return getReadOnlyValue(index); + } + + @SuppressWarnings("unchecked") + public T getReadOnlyValue(int index) { + try { + return (T) db.get(internalList.getLong(index)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public EntryReference getReference(int index) { + try { + return db.get(internalList.getLong(index)).cast(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @SuppressWarnings("unchecked") + @Override + public T set(int index, T element) { + EntryReference ref = addToDatabase(element); + long oldIndex = internalList.set(index, ref.getIndex()); + try { + ref.close(); + return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + + @Override + public void add(int index, T element) { + EntryReference ref = addToDatabase(element); + internalList.add(index, ref.getIndex()); + } + + @SuppressWarnings("unchecked") + @Override + public T remove(int index) { + long oldIndex = internalList.removeLong(index); + try { + return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + + @Override + public int indexOf(Object o) { + EntryReference ref = addToDatabase(o); + long objToRemoveHash = ref.calculateHash(); + LongArrayList hashes = new LongArrayList(); + + + for (int i = 0; i < hashes.size(); i++) { + long hash = hashes.getLong(i); + if (objToRemoveHash == hash) { + try { + if (ref.equals(db.get(internalList.getLong(i)))) { + return i; + } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + } + return -1; + } + + @Override + public int indexOfEntry(EntryReference ref) { + for (int i = 0; i < internalList.size(); i++) { + long index = internalList.getLong(i); + try { + EntryReference ref2 = db.get(index); + if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { + return i; + } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + return -1; + } + + @Override + public void appendIndex(long elementIndex) { + internalList.add(elementIndex); + } + + @Override + public int lastIndexOf(Object o) { + EntryReference ref = addToDatabase(o).cast(); + return lastIndexOfEntry(ref); + } + + @Override + public int lastIndexOfEntry(EntryReference ref) { + long objToRemoveHash = ref.calculateHash(); + + int lastValue = -1; + + for (int i = 0; i < internalList.size(); i++) { + long index2 = internalList.getLong(i); + try { + EntryReference ref2 = db.get(index2); + if (objToRemoveHash == ref2.calculateHash()) { + if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { + lastValue = i; + } + } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + return lastValue; + } + + @Deprecated + @Override + public ListIterator listIterator() { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public ListIterator listIterator(int index) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public List subList(int fromIndex, int toIndex) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + private EntryReference addToDatabase(U obj) { + EntryReference ref; + try { + ref = db.add(obj); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + return ref; + } + + @Deprecated + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((internalList == null) ? 0 : internalList.hashCode()); + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeIf(Predicate filter) { + Objects.requireNonNull(filter); + boolean removed = false; + for (int i = 0; i < internalList.size(); ) { + T obj; + try { + obj = ((EntryReference) (db.get(internalList.getLong(i)).cast())).getValueReadOnly(); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + if (filter.test(obj)) { + internalList.removeLong(i); + removed = true; + } else { + i++; + } + } + return removed; + } +} diff --git a/src/main/java/org/warp/jcwdb/LightBigList.java b/src/main/java/org/warp/jcwdb/LightBigList.java new file mode 100644 index 0000000..53859ad --- /dev/null +++ b/src/main/java/org/warp/jcwdb/LightBigList.java @@ -0,0 +1,521 @@ +package org.warp.jcwdb; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; + +import java.io.IOException; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Predicate; + +public class LightBigList implements LightList { + + public static final int MAX_ELEMENTS_PER_CHUNK = 10000; + + public final LongArrayList chunks; + public final IntArrayList chunkSizes; + private final transient JCWDatabase db; + + /** + * @param db Database reference + */ + public LightBigList(JCWDatabase db) { + this.db = db; + this.chunks = new LongArrayList(); + this.chunkSizes = new IntArrayList(); + } + + /** + * @param db Database reference + * @param elements Elements to add + */ + public LightBigList(JCWDatabase db, LongArrayList elements) { + this.db = db; + this.chunks = new LongArrayList(); + this.chunkSizes = new IntArrayList(); + elements.forEach((long element) -> { + this.appendIndex(element); + }); + } + + public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) { + this.db = db; + this.chunks = chunks; + this.chunkSizes = chunkSizes; + } + + /** + * Append an index to the first free chunk + * @param elementIndex + */ + public void appendIndex(long elementIndex) { + for (int i = 0; i < chunks.size(); i++) { + final int chunkNumber = i; + if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) { + try { + final long chunkIndex = chunks.getLong(i); + final EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + chunk.appendIndex(elementIndex); + chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1); + }); + return; + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + } + LightList newChunk = new LightArrayList<>(db); + newChunk.appendIndex(elementIndex); + long newChunkIndex; + try { + newChunkIndex = db.add(newChunk).getIndex(); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + chunks.add(newChunkIndex); + chunkSizes.add(1); + } + + /** + * Get the elements count + * @return the size of the list + */ + @Override + public int size() { + int size = 0; + for (int chunkSize : this.chunkSizes) { + size += chunkSize; + } + return size; + } + + /** + * + * @return the count of chunks + */ + public int chunksCount() { + return this.chunkSizes.size(); + } + + /** + * Check if the list is empty + * @return true if the list is empty + */ + @Override + public boolean isEmpty() { + return this.size() <= 0; + } + + @Override + public boolean contains(Object o) { + if (o != null) { + for (long chunkIndex : chunks) { + try { + EntryReference> chunkRef = db.get(chunkIndex); + LightList chunk = chunkRef.getValueReadOnly(); + if (chunk.contains(o)) { + return true; + } + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + } + return false; + } + + /** + * Use iteratorReferences() + */ + @Deprecated + @Override + public Iterator iterator() { + throw new RuntimeException("iterator() isn't implemented!"); + } + + @Deprecated + @Override + public Iterator> iteratorReferences() { + throw new RuntimeException("iteratorReferences() isn't implemented!"); + } + + /** + * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY + * + * @param action + */ + @Deprecated + @Override + public void forEach(Consumer action) { + throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead"); + } + + @Override + public void forEachReference(Consumer> action) { + Objects.requireNonNull(action); + for (long chunkIndex : this.chunks) { + try { + EntryReference> chunkRef = db.get(chunkIndex); + LightList chunk = chunkRef.getValueReadOnly(); + chunk.forEachReference(action); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray() { + throw new RuntimeException("toArray() isn't implemented!"); + } + + @SuppressWarnings("unchecked") + @Override + public T1[] toArray(T1[] a) { + throw new RuntimeException("toArray() isn't implemented!"); + } + + /** + * Use addEntry(o) + * @param o + * @return + */ + @Deprecated + @Override + public boolean add(T o) { + EntryReference ref = addEntry(o); + return ref != null; + } + + @Override + public EntryReference addEntry(T o) { + EntryReference ref = addToDatabase(o); + appendIndex(ref.getIndex()); + return ref; + } + + @Override + public boolean remove(Object o) { + final int removeOffset = indexOf(o); + return removeAt(removeOffset) != null; + } + + @Override + public boolean remove(EntryReference ref) { + final int removeOffset = indexOfEntry(ref); + return removeAt(removeOffset) != null; + } + + private T removeAt(int removeOffset) { + final VariableWrapper result = new VariableWrapper<>(null); + long currentOffset = 0; + if (removeOffset >= 0) { + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + final int currentChunkSize = chunkSizes.getInt(i); + final long chunkStartOffset = currentOffset; + currentOffset += currentChunkSize; + // If the offset to remove is in the current chunk + if (currentOffset > removeOffset) { + // Get chunk index + final long chunkIndex = chunks.getLong(i); + // Get the offset relative to the current chunk + final int relativeOffset = (int) (removeOffset - chunkStartOffset); + + if (relativeOffset < 0) { + return null; + } + + try { + EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + result.var = chunk.remove(relativeOffset); + }); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + chunkSizes.set(removeOffset, currentChunkSize - 1); + break; + } + } + return result.var; + } + return null; + } + + @Override + public boolean containsAll(Collection c) { + for (Object o : c) { + int objIndex = indexOf(o); + if (objIndex < 0) { + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= add((T) o); + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(int index, Collection c) { + boolean result = false; + int delta = 0; + for (Object o : c) { + add(index + delta, (T) o); + result = true; + delta++; + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= remove((T) o); + } + return result; + } + + @Override + public boolean retainAll(Collection c) { + boolean result = false; + LongArrayList collectionHashes = new LongArrayList(); + ObjectArrayList collection = new ObjectArrayList<>(); + collection.addAll(c); + for (Object o : c) { + collectionHashes.add(db.calculateHash(o)); + } + for (int i = 0; i < chunks.size(); i++) { + long hash = chunks.getLong(i); + int positionInCollection = collectionHashes.indexOf(hash); + if (positionInCollection == -1) { + remove(collection.get(positionInCollection)); + result = true; + } + } + return result; + } + + @Override + public void clear() { + chunks.clear(); + chunkSizes.clear(); + } + + /** + * Use getReference or getReadOnlyValue + */ + @Deprecated + @Override + public T get(int index) { + return getReadOnlyValue(index); + } + + @SuppressWarnings("unchecked") + public T getReadOnlyValue(int index) { + try { + return (T) db.get(chunks.getLong(index)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public EntryReference getReference(int index) { + try { + return db.get(chunks.getLong(index)).cast(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @SuppressWarnings("unchecked") + @Override + public T set(int setOffset, final T element) { + long nextChunkOffset = 0; + VariableWrapper wrapper = new VariableWrapper<>(null); + if (setOffset >= 0) { + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + final int currentChunkSize = chunkSizes.getInt(i); + final long chunkStartOffset = nextChunkOffset; + nextChunkOffset += currentChunkSize; + // If the offset to remove is in the current chunk + if (nextChunkOffset > setOffset) { + // Get chunk index + final long chunkIndex = chunks.getLong(i); + // Get the offset relative to the current chunk + final int relativeOffset = (int) (setOffset - chunkStartOffset); + + if (relativeOffset < 0) { + throw new NullPointerException("Relative Offset < 0"); + } + + try { + EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + chunk.set(relativeOffset, element); + wrapper.var = element; + }); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + break; + } + } + return wrapper.var; + } + return null; + } + + @Override + public void add(int index, T element) { + throw new RuntimeException("add() isn't implemented!"); + } + + @SuppressWarnings("unchecked") + @Override + public T remove(int index) { + return this.removeAt(index); + } + + @Override + public int indexOf(Object o) { + EntryReference ref = addToDatabase(o).cast(); + return indexOfEntry(ref); + } + + @Override + public int indexOfEntry(EntryReference ref) { + int currentOffset = 0; + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + try { + final int currentChunkSize = chunkSizes.getInt(i); + // If the offset to remove is in the current chunk + + // Get chunk index + final long chunkIndex = chunks.getLong(i); + EntryReference> chunkRef = db.get(chunkIndex); + final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref); + if (foundIndex >= 0) { + return currentOffset + foundIndex; + } + currentOffset += currentChunkSize; + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + return -1; + } + + @Override + public int lastIndexOf(Object o) { + return lastIndexOfEntry(addToDatabase(o).cast()); + } + + @Override + public int lastIndexOfEntry(EntryReference ref) { + int currentOffset = 0; + // Iterate through all chunks + for (int i = chunks.size() - 1; i >= 0; i--) { + try { + final int currentChunkSize = chunkSizes.getInt(i); + // If the offset to remove is in the current chunk + + // Get chunk index + final long chunkIndex = chunks.getLong(i); + EntryReference> chunkRef = db.get(chunkIndex); + final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref); + if (foundIndex >= 0) { + return currentOffset + foundIndex; + } + currentOffset += currentChunkSize; + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + return -1; + } + + @Deprecated + @Override + public ListIterator listIterator() { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public ListIterator listIterator(int index) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public List subList(int fromIndex, int toIndex) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + private EntryReference addToDatabase(U obj) { + EntryReference ref; + try { + ref = db.add(obj); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + return ref; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((chunks == null) ? 0 : chunks.hashCode()); + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeIf(Predicate filter) { + Objects.requireNonNull(filter); + final VariableWrapper result = new VariableWrapper(false); + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + try { + final int chunkOffset = i; + // Get chunk index + final long chunkIndex = chunks.getLong(i); + EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + boolean removed = chunk.removeIf(filter); + if (removed) { + result.var = true; + chunkSizes.set(chunkOffset, chunk.size()); + } + }); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + return result.var; + } +} diff --git a/src/main/java/org/warp/jcwdb/LightList.java b/src/main/java/org/warp/jcwdb/LightList.java index 0edddc7..9c699ba 100644 --- a/src/main/java/org/warp/jcwdb/LightList.java +++ b/src/main/java/org/warp/jcwdb/LightList.java @@ -1,390 +1,24 @@ package org.warp.jcwdb; -import java.io.IOException; -import java.util.*; -import java.util.function.Predicate; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; -import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.objects.ObjectArrayList; +public interface LightList extends List { -public class LightList implements List { + Iterator> iteratorReferences(); - public final LongArrayList internalList; - private final transient JCWDatabase db; + void forEachReference(Consumer> action); - public LightList(JCWDatabase db, LongArrayList internalList) { - this.internalList = internalList; - this.db = db; - } + EntryReference addEntry(T o); - @Override - public int size() { - return internalList.size(); - } + boolean remove(EntryReference ref); - @Override - public boolean isEmpty() { - return internalList.isEmpty(); - } + EntryReference getReference(int index); - @Override - public boolean contains(Object o) { - if (o != null) { - for (Long element : internalList) { - EntryReference ref = null; - try { - ref = db.get(element); - if (o.equals(ref.getValueReadOnly())) { - return true; - } - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return false; - } + int indexOfEntry(EntryReference ref); - /** - * Use iteratorReferences() - */ - @Deprecated - @SuppressWarnings("unchecked") - @Override - public Iterator iterator() { - final ArrayList elements = new ArrayList<>(); - for (Long element : internalList) { - try { - elements.add((T) db.get(element).getValueReadOnly()); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements.iterator(); - } - - @SuppressWarnings("unchecked") - public Iterator> iteratorReferences() { - final ArrayList> elements = new ArrayList<>(); - for (Long element : internalList) { - try { - elements.add(db.get(element)); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements.iterator(); - } - + int lastIndexOfEntry(EntryReference ref); - @SuppressWarnings("unchecked") - @Override - public T[] toArray() { - final T[] elements = (T[]) new Objects[internalList.size()]; - for (int i = 0; i < elements.length; i++) { - try { - elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements; - } - - @SuppressWarnings("unchecked") - @Override - public T1[] toArray(T1[] a) { - final T1[] elements = (T1[]) new Objects[internalList.size()]; - for (int i = 0; i < elements.length; i++) { - try { - elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements; - } - - @Override - public boolean add(T o) { - EntryReference ref = addEntry(o); - return ref != null; - } - - public EntryReference addEntry(T o) { - EntryReference ref = addToDatabase(o); - if (internalList.add(ref.getIndex())) { - return ref; - } else { - return null; - } - } - - @Override - public boolean remove(Object o) { - int removeIndex = indexOf(o); - if (removeIndex >= 0) { - internalList.removeLong(removeIndex); - return true; - } - return false; - } - - public boolean remove(EntryReference ref) { - int removeIndex = indexOfEntry(ref); - if (removeIndex >= 0) { - internalList.removeLong(removeIndex); - return true; - } - return false; - } - - @Override - public boolean containsAll(Collection c) { - for (Object o : c) { - int objIndex = indexOf(o); - if (objIndex < 0) { - return false; - } - } - return true; - } - - @SuppressWarnings("unchecked") - @Override - public boolean addAll(Collection c) { - boolean result = false; - for (Object o : c) { - result |= add((T) o); - } - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean addAll(int index, Collection c) { - boolean result = false; - int delta = 0; - for (Object o : c) { - add(index + delta, (T) o); - result = true; - delta++; - } - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean removeAll(Collection c) { - boolean result = false; - for (Object o : c) { - result |= remove((T) o); - } - return result; - } - - @Override - public boolean retainAll(Collection c) { - boolean result = false; - LongArrayList collectionHashes = new LongArrayList(); - ObjectArrayList collection = new ObjectArrayList<>(); - collection.addAll(c); - for (Object o : c) { - collectionHashes.add(db.calculateHash(o)); - } - for (int i = 0; i < internalList.size(); i++) { - long hash = internalList.getLong(i); - int positionInCollection = collectionHashes.indexOf(hash); - if (positionInCollection == -1) { - remove(collection.get(positionInCollection)); - result = true; - } - } - return result; - } - - @Override - public void clear() { - internalList.clear(); - } - - /** - * Use getReference or getReadOnlyValue - */ - @Deprecated - @Override - public T get(int index) { - return getReadOnlyValue(index); - } - - @SuppressWarnings("unchecked") - public T getReadOnlyValue(int index) { - try { - return (T) db.get(internalList.getLong(index)).getValueReadOnly(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - public EntryReference getReference(int index) { - try { - return db.get(internalList.getLong(index)).cast(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - @SuppressWarnings("unchecked") - @Override - public T set(int index, T element) { - EntryReference ref = addToDatabase(element); - long oldIndex = internalList.set(index, ref.getIndex()); - try { - ref.close(); - return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - - @Override - public void add(int index, T element) { - EntryReference ref = addToDatabase(element); - internalList.add(index, ref.getIndex()); - } - - @SuppressWarnings("unchecked") - @Override - public T remove(int index) { - long oldIndex = internalList.removeLong(index); - try { - return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - - @Override - public int indexOf(Object o) { - EntryReference ref = addToDatabase(o); - long objToRemoveHash = ref.calculateHash(); - LongArrayList hashes = new LongArrayList(); - - - for (int i = 0; i < hashes.size(); i++) { - long hash = hashes.getLong(i); - if (objToRemoveHash == hash) { - try { - if (ref.equals(db.get(internalList.getLong(i)))) { - return i; - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - } - return -1; - } - - public int indexOfEntry(EntryReference ref) { - for (int i = 0; i < internalList.size(); i++) { - long index = internalList.getLong(i); - try { - EntryReference ref2 = db.get(index); - if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { - return i; - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - return -1; - } - - @Override - public int lastIndexOf(Object o) { - EntryReference ref = addToDatabase(o); - long objToRemoveHash = ref.calculateHash(); - - int lastValue = -1; - - for (int i = 0; i < internalList.size(); i++) { - long index2 = internalList.getLong(i); - try { - EntryReference ref2 = db.get(index2); - if (objToRemoveHash == ref2.calculateHash()) { - if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { - lastValue = i; - } - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - return lastValue; - } - - @Deprecated - @Override - public ListIterator listIterator() { - // TODO: implement - throw new RuntimeException("Not implemented!"); - } - - @Deprecated - @Override - public ListIterator listIterator(int index) { - // TODO: implement - throw new RuntimeException("Not implemented!"); - } - - @Deprecated - @Override - public List subList(int fromIndex, int toIndex) { - // TODO: implement - throw new RuntimeException("Not implemented!"); - } - - private EntryReference addToDatabase(U obj) { - EntryReference ref; - try { - ref = db.add(obj); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - return ref; - } - - @Deprecated - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((internalList == null) ? 0 : internalList.hashCode()); - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean removeIf(Predicate filter) { - Objects.requireNonNull(filter); - boolean removed = false; - for (int i = 0; i < internalList.size(); ) { - T obj; - try { - obj = ((EntryReference) (db.get(internalList.getLong(i)).cast())).getValueReadOnly(); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - if (filter.test(obj)) { - internalList.removeLong(i); - removed = true; - } else { - i++; - } - } - return removed; - } + void appendIndex(long elementIndex); } diff --git a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java index 3194293..3f4e255 100644 --- a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java +++ b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java @@ -8,19 +8,16 @@ import java.nio.file.Path; import java.util.function.Consumer; public class MixedIndexDatabase implements IndexManager { - private final Long2LongMap mostAccessedIndices; private final FileIndexManager fileIndices; private final CacheIndexManager cacheIndices; - public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { - this.mostAccessedIndices = new Long2LongLinkedOpenHashMap(); + public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException { this.fileIndices = new FileIndexManager(dataFile, metadataFile); this.cacheIndices = new CacheIndexManager(); } @Override public T get(long index, DBReader reader) throws IOException { - incrementUsage(index); if (cacheIndices.has(index)) { return cacheIndices.get(index, reader); } else { @@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager { return cacheIndices.has(index) || fileIndices.has(index); } - private void incrementUsage(long index) { - mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1); - } - @Override public void close() throws IOException { // TODO: move all cached indices to filesIndices before closing. diff --git a/src/main/java/org/warp/jcwdb/TypesManager.java b/src/main/java/org/warp/jcwdb/TypesManager.java index 239be89..6e4abde 100644 --- a/src/main/java/org/warp/jcwdb/TypesManager.java +++ b/src/main/java/org/warp/jcwdb/TypesManager.java @@ -1,26 +1,37 @@ package org.warp.jcwdb; -import java.util.Map; -import java.util.Map.Entry; -import java.util.HashMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; public class TypesManager { - private final Map> types; - private final Map, DBTypeParser> typesByClass; - private DBTypeParser fallbackParser; + private final Int2ObjectMap> types; + private final Object2ObjectMap, DBTypeParser> typesByClass; + private DBTypedObjectParser fallbackParser; public TypesManager(JCWDatabase db) { - types = new HashMap<>(); - typesByClass = new HashMap<>(); + types = new Int2ObjectOpenHashMap<>(); + typesByClass = new Object2ObjectOpenHashMap<>(); DBStandardTypes.registerStandardTypes(db, this); } - + public void registerType(Class clazz, int type, DBTypeParser parser) { this.types.put(type, parser); this.typesByClass.put(clazz, parser); } - public void registerTypeFallback(DBTypeParser parser) { + /** + * Use this method with the most used classes to save disk space. + * @param clazz + * @param id + * @param + */ + public void registerGenericClass(Class clazz, int id) { + this.fallbackParser.registerClass(clazz, id); + } + + public void registerTypeFallback(DBTypedObjectParser parser) { this.fallbackParser = parser; } diff --git a/src/main/java/org/warp/jcwdb/VariableWrapper.java b/src/main/java/org/warp/jcwdb/VariableWrapper.java new file mode 100644 index 0000000..0503674 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/VariableWrapper.java @@ -0,0 +1,12 @@ +package org.warp.jcwdb; + +import java.nio.channels.SeekableByteChannel; + +public class VariableWrapper { + + public T var; + + public VariableWrapper(T value) { + this.var = value; + } +} diff --git a/src/main/java/org/warp/jcwdb/exampleimpl/App.java b/src/main/java/org/warp/jcwdb/exampleimpl/App.java index aaa9d2f..4e0f70a 100644 --- a/src/main/java/org/warp/jcwdb/exampleimpl/App.java +++ b/src/main/java/org/warp/jcwdb/exampleimpl/App.java @@ -22,6 +22,7 @@ public class App { System.out.println("Loading database..."); long time0 = System.currentTimeMillis(); JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1])); + db.registerClass(StrangeAnimal.class, 0); try { long time01 = System.currentTimeMillis(); System.out.println("Time elapsed: " + (time01 - time0)); @@ -58,15 +59,15 @@ public class App { System.out.println("Time elapsed: " + (time2_1 - time2_0)); ObjectList results = new ObjectArrayList<>(); - /* - root.forEach((value) -> { + root.forEachReference((valueReference) -> { + Animal value = valueReference.getValueReadOnly(); if (Animal.hasFourLegs(value)) { results.add(value); } //System.out.println("val:" + value); }); - */ long time2_2 = System.currentTimeMillis(); + System.out.println("Matches: " + results.size()); System.out.println("Time elapsed: " + (time2_2 - time2_1)); System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB"); diff --git a/src/test/java/org/warp/jcwdb/AppTest.java b/src/test/java/org/warp/jcwdb/AppTest.java index 187b8cd..d632087 100644 --- a/src/test/java/org/warp/jcwdb/AppTest.java +++ b/src/test/java/org/warp/jcwdb/AppTest.java @@ -25,94 +25,5 @@ public class AppTest { assertTrue( true ); } - - /** - * - * @throws IOException - */ - @Test - public void shouldReadCustomClasses() throws IOException - { - CustomClass customClass = new CustomClass(); - customClass.primitive = 1; - customClass.string = "test"; - - Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx"); - - JCWDatabase db = new JCWDatabase(path, idx); - EntryReference> ref = db.getRoot(); - ref.editValue((root, saver) -> { - root.add(customClass); - }); - db.close(); - - JCWDatabase db2 = new JCWDatabase(path, idx); - EntryReference> ref2 = db2.getRoot(); - CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0); - assertTrue(customClass.equals(customClass2)); - assertTrue(customClass.string.equals(customClass2.string)); - db2.close(); - Files.delete(path); - Files.delete(idx); - } - - public static class CustomClass { - public String string; - public int primitive; - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + primitive; - result = prime * result + ((string == null) ? 0 : string.hashCode()); - return result; - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - CustomClass other = (CustomClass) obj; - if (primitive != other.primitive) - return false; - if (string == null) { - if (other.string != null) - return false; - } else if (!string.equals(other.string)) - return false; - return true; - } - - - } - - /** - * - * @throws IOException - */ - @Test - public void shouldActAsAnArrayListWithEqualObjects() throws IOException - { - Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx"); - JCWDatabase db = new JCWDatabase(path, idx); - EntryReference> ref = db.getRoot(); - LightList list1 = ref.getValueReadOnly(); - ArrayList list2 = new ArrayList(); - String s = "a"; - for (int i = 0; i < 10; i++) { - list1.add(s); - list2.add(s); - } - assertTrue(list1.size() == list2.size()); - for (int i = 0; i < 10; i++) { - assertTrue(list1.get(i) == list2.get(i)); - assertTrue(list2.get(i) == list1.get(i)); - } - db.close(); - Files.delete(path); - Files.delete(idx); - } + }