diff --git a/src/main/java/org/warp/jcwdb/DBLightListParser.java b/src/main/java/org/warp/jcwdb/DBLightArrayListParser.java similarity index 53% rename from src/main/java/org/warp/jcwdb/DBLightListParser.java rename to src/main/java/org/warp/jcwdb/DBLightArrayListParser.java index 358ac27..6a980fa 100644 --- a/src/main/java/org/warp/jcwdb/DBLightListParser.java +++ b/src/main/java/org/warp/jcwdb/DBLightArrayListParser.java @@ -1,40 +1,38 @@ package org.warp.jcwdb; -import java.util.ArrayList; - import it.unimi.dsi.fastutil.longs.LongArrayList; -public class DBLightListParser extends DBTypeParserImpl> { +public class DBLightArrayListParser extends DBTypeParserImpl> { private final JCWDatabase db; - public DBLightListParser(JCWDatabase db) { + public DBLightArrayListParser(JCWDatabase db) { this.db = db; } - - public DBReader> getReader() { + + public DBReader> getReader() { return (i, size) -> { LongArrayList internalList = new LongArrayList(); long max = size / Long.BYTES; - for (int item = 0; item < max; item++){ + for (int item = 0; item < max; item++) { long itm = i.readLong(); internalList.add(itm); } - return new LightList(db, internalList); + return new LightArrayList(db, internalList); }; } - public DBDataOutput> getWriter(final LightList value) { + public DBDataOutput> getWriter(final LightArrayList value) { final int elementsCount = value.size(); return DBDataOutput.create((o) -> { LongArrayList list = value.internalList; for (int i = 0; i < elementsCount; i++) { o.writeLong(list.getLong(i)); } - }, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value)); + }, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value)); } @Override - public long calculateHash(LightList value) { + public long calculateHash(LightArrayList value) { return value.internalList.hashCode(); } } diff --git a/src/main/java/org/warp/jcwdb/DBLightBigListParser.java b/src/main/java/org/warp/jcwdb/DBLightBigListParser.java new file mode 100644 index 0000000..876fd13 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBLightBigListParser.java @@ -0,0 +1,44 @@ +package org.warp.jcwdb; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; + +public class DBLightBigListParser extends DBTypeParserImpl> { + private final JCWDatabase db; + + public DBLightBigListParser(JCWDatabase db) { + this.db = db; + } + + public DBReader> getReader() { + return (i, size) -> { + LongArrayList chunks = new LongArrayList(); + IntArrayList chunkSizes = new IntArrayList(); + long max = size / (Long.BYTES + Integer.BYTES); + for (int item = 0; item < max; item++) { + long itm = i.readLong(); + int itm2 = i.readInt(); + chunks.add(itm); + chunkSizes.add(itm2); + } + return new LightBigList(db, chunks, chunkSizes); + }; + } + + public DBDataOutput> getWriter(final LightBigList value) { + final int elementsCount = value.chunksCount(); + return DBDataOutput.create((o) -> { + LongArrayList list = value.chunks; + IntArrayList list2 = value.chunkSizes; + for (int i = 0; i < elementsCount; i++) { + o.writeLong(list.getLong(i)); + o.writeInt(list2.getInt(i)); + } + }, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value)); + } + + @Override + public long calculateHash(LightBigList value) { + return value.chunks.hashCode(); + } +} diff --git a/src/main/java/org/warp/jcwdb/DBStandardTypes.java b/src/main/java/org/warp/jcwdb/DBStandardTypes.java index 23d8f3f..23c6796 100644 --- a/src/main/java/org/warp/jcwdb/DBStandardTypes.java +++ b/src/main/java/org/warp/jcwdb/DBStandardTypes.java @@ -11,12 +11,14 @@ public class DBStandardTypes { public static final int DOUBLE = STD| 6; public static final int STRING = STD| 7; public static final int BYTE_ARRAY = STD| 8; - public static final int LIGHT_LIST = STD| 9; - public static final int GENERIC_OBJECT = STD| 10; + public static final int LIGHT_LIST_ARRAY = STD| 9; + public static final int LIGHT_LIST_BIG = STD| 10; + public static final int GENERIC_OBJECT = STD| 11; public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) { typesManager.registerType(String.class, STRING, new DBStringParser()); - typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db)); + typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db)); + typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db)); typesManager.registerTypeFallback(new DBGenericObjectParser()); } } \ No newline at end of file diff --git a/src/main/java/org/warp/jcwdb/EntryReference.java b/src/main/java/org/warp/jcwdb/EntryReference.java index 66481d4..977299c 100644 --- a/src/main/java/org/warp/jcwdb/EntryReference.java +++ b/src/main/java/org/warp/jcwdb/EntryReference.java @@ -86,7 +86,7 @@ public class EntryReference implements Castable, Saveable { * @param editFunction * @throws IOException */ - public void editValue(BiFunction editFunction) throws IOException { + public void editValue(BiFunction editFunction) { synchronized(accessLock) { load(); this.value = editFunction.apply(this.value, this); @@ -99,7 +99,20 @@ public class EntryReference implements Castable, Saveable { * @param editFunction * @throws IOException */ - public void editValue(BiConsumer editFunction) throws IOException { + public void editValue(Function editFunction) { + synchronized(accessLock) { + load(); + this.value = editFunction.apply(this.value); + this.save(); + } + } + + /** + * Reccomended way to edit the value + * @param editFunction + * @throws IOException + */ + public void editValue(BiConsumer editFunction) { synchronized(accessLock) { load(); editFunction.accept(this.value, this); @@ -107,12 +120,25 @@ public class EntryReference implements Castable, Saveable { } } + /** + * Reccomended way to edit the value + * @param editFunction + * @throws IOException + */ + public void editValue(Consumer editFunction) { + synchronized(accessLock) { + load(); + editFunction.accept(this.value); + this.save(); + } + } + /** * Substitute the old value with a new one * @param val * @throws IOException */ - public void setValue(T val) throws IOException { + public void setValue(T val) { synchronized(accessLock) { this.loaded = true; this.value = val; diff --git a/src/main/java/org/warp/jcwdb/FileAllocator.java b/src/main/java/org/warp/jcwdb/FileAllocator.java index 9fb9424..4dcd34a 100644 --- a/src/main/java/org/warp/jcwdb/FileAllocator.java +++ b/src/main/java/org/warp/jcwdb/FileAllocator.java @@ -1,14 +1,7 @@ package org.warp.jcwdb; -import com.esotericsoftware.kryo.io.Output; - -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; -import java.util.*; public class FileAllocator implements AutoCloseable { private final SeekableByteChannel dataFileChannel; @@ -16,14 +9,15 @@ public class FileAllocator implements AutoCloseable { private volatile boolean closed; private final Object closeLock = new Object(); private final Object allocateLock = new Object(); - + public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException { this.dataFileChannel = dataFileChannel; this.allocableOffset = this.dataFileChannel.size(); } - + /** * TODO: not implemented + * * @param size * @return offset */ @@ -36,21 +30,22 @@ public class FileAllocator implements AutoCloseable { } } - + public void close() throws IOException { - if (closed) { - return; - } - synchronized (closeLock) { - if (closed) { - return; - } - closed = true; - } + if (closed) { + return; + } + synchronized (closeLock) { + if (closed) { + return; + } + closed = true; + } } - + /** * Frees the unused bytes + * * @param startPosition * @param length */ diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index 4d6a882..4fa8079 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -32,19 +32,19 @@ public class JCWDatabase implements AutoCloseable, Cleanable { } public EntryReference> getRoot() throws IOException { - return getRoot(Object.class).cast(); - } - - public EntryReference> getRoot(Class clazz) throws IOException { checkClosed(); if (exists(0)) { return get(0); } else { - LightList newRoot = new LightList(this, new LongArrayList()); + LightList newRoot = new LightBigList<>(this); return set(0, newRoot); } } + public EntryReference> getRoot(Class clazz) throws IOException { + return getRoot().cast(); + } + public EntryReference get(long index) throws IOException { checkClosed(); int type; diff --git a/src/main/java/org/warp/jcwdb/LightArrayList.java b/src/main/java/org/warp/jcwdb/LightArrayList.java new file mode 100644 index 0000000..8cbe1ef --- /dev/null +++ b/src/main/java/org/warp/jcwdb/LightArrayList.java @@ -0,0 +1,445 @@ +package org.warp.jcwdb; + +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; + +import java.io.IOException; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Predicate; + +public class LightArrayList implements LightList { + + public final LongArrayList internalList; + private final transient JCWDatabase db; + + /** + * @param db Database reference + */ + public LightArrayList(JCWDatabase db) { + this.db = db; + this.internalList = new LongArrayList(); + } + + /** + * @param db Database reference + * @param elements Elements to add + */ + public LightArrayList(JCWDatabase db, LongArrayList elements) { + this.db = db; + this.internalList = new LongArrayList(elements); + } + + @Override + public int size() { + return internalList.size(); + } + + @Override + public boolean isEmpty() { + return internalList.isEmpty(); + } + + @Override + public boolean contains(Object o) { + if (o != null) { + for (long element : internalList) { + EntryReference ref = null; + try { + ref = db.get(element); + if (o.equals(ref.getValueReadOnly())) { + return true; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + return false; + } + + /** + * Use iteratorReferences() + */ + @Deprecated + @SuppressWarnings("unchecked") + @Override + public Iterator iterator() { + System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!"); + final ArrayList elements = new ArrayList<>(); + for (long element : internalList) { + try { + elements.add((T) db.get(element).getValueReadOnly()); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements.iterator(); + } + + @SuppressWarnings("unchecked") + @Override + public Iterator> iteratorReferences() { + final ArrayList> elements = new ArrayList<>(); + for (long element : internalList) { + try { + elements.add(db.get(element)); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements.iterator(); + } + + /** + * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY + * + * @param action + */ + @Deprecated + @Override + public void forEach(Consumer action) { + Objects.requireNonNull(action); + for (T t : this) { + action.accept(t); + } + } + + @Override + public void forEachReference(Consumer> action) { + Objects.requireNonNull(action); + for (long index : this.internalList) { + try { + action.accept(db.get(index)); + } catch (IOException e) { + throw (RuntimeException) new RuntimeException().initCause(e); + } + } + } + + + @SuppressWarnings("unchecked") + @Override + public T[] toArray() { + final T[] elements = (T[]) new Objects[internalList.size()]; + for (int i = 0; i < elements.length; i++) { + try { + elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements; + } + + @SuppressWarnings("unchecked") + @Override + public T1[] toArray(T1[] a) { + final T1[] elements = (T1[]) new Objects[internalList.size()]; + for (int i = 0; i < elements.length; i++) { + try { + elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return elements; + } + + @Override + public boolean add(T o) { + EntryReference ref = addEntry(o); + return ref != null; + } + + @Override + public EntryReference addEntry(T o) { + EntryReference ref = addToDatabase(o); + if (internalList.add(ref.getIndex())) { + return ref; + } else { + return null; + } + } + + @Override + public boolean remove(Object o) { + int removeIndex = indexOf(o); + if (removeIndex >= 0) { + internalList.removeLong(removeIndex); + return true; + } + return false; + } + + @Override + public boolean remove(EntryReference ref) { + int removeIndex = indexOfEntry(ref); + if (removeIndex >= 0) { + internalList.removeLong(removeIndex); + return true; + } + return false; + } + + @Override + public boolean containsAll(Collection c) { + for (Object o : c) { + int objIndex = indexOf(o); + if (objIndex < 0) { + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= add((T) o); + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(int index, Collection c) { + boolean result = false; + int delta = 0; + for (Object o : c) { + add(index + delta, (T) o); + result = true; + delta++; + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= remove((T) o); + } + return result; + } + + @Override + public boolean retainAll(Collection c) { + boolean result = false; + LongArrayList collectionHashes = new LongArrayList(); + ObjectArrayList collection = new ObjectArrayList<>(); + collection.addAll(c); + for (Object o : c) { + collectionHashes.add(db.calculateHash(o)); + } + for (int i = 0; i < internalList.size(); i++) { + long hash = internalList.getLong(i); + int positionInCollection = collectionHashes.indexOf(hash); + if (positionInCollection == -1) { + remove(collection.get(positionInCollection)); + result = true; + } + } + return result; + } + + @Override + public void clear() { + internalList.clear(); + } + + /** + * Use getReference or getReadOnlyValue + */ + @Deprecated + @Override + public T get(int index) { + return getReadOnlyValue(index); + } + + @SuppressWarnings("unchecked") + public T getReadOnlyValue(int index) { + try { + return (T) db.get(internalList.getLong(index)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public EntryReference getReference(int index) { + try { + return db.get(internalList.getLong(index)).cast(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @SuppressWarnings("unchecked") + @Override + public T set(int index, T element) { + EntryReference ref = addToDatabase(element); + long oldIndex = internalList.set(index, ref.getIndex()); + try { + ref.close(); + return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + + @Override + public void add(int index, T element) { + EntryReference ref = addToDatabase(element); + internalList.add(index, ref.getIndex()); + } + + @SuppressWarnings("unchecked") + @Override + public T remove(int index) { + long oldIndex = internalList.removeLong(index); + try { + return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + + @Override + public int indexOf(Object o) { + EntryReference ref = addToDatabase(o); + long objToRemoveHash = ref.calculateHash(); + LongArrayList hashes = new LongArrayList(); + + + for (int i = 0; i < hashes.size(); i++) { + long hash = hashes.getLong(i); + if (objToRemoveHash == hash) { + try { + if (ref.equals(db.get(internalList.getLong(i)))) { + return i; + } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + } + return -1; + } + + @Override + public int indexOfEntry(EntryReference ref) { + for (int i = 0; i < internalList.size(); i++) { + long index = internalList.getLong(i); + try { + EntryReference ref2 = db.get(index); + if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { + return i; + } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + return -1; + } + + @Override + public void appendIndex(long elementIndex) { + internalList.add(elementIndex); + } + + @Override + public int lastIndexOf(Object o) { + EntryReference ref = addToDatabase(o).cast(); + return lastIndexOfEntry(ref); + } + + @Override + public int lastIndexOfEntry(EntryReference ref) { + long objToRemoveHash = ref.calculateHash(); + + int lastValue = -1; + + for (int i = 0; i < internalList.size(); i++) { + long index2 = internalList.getLong(i); + try { + EntryReference ref2 = db.get(index2); + if (objToRemoveHash == ref2.calculateHash()) { + if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { + lastValue = i; + } + } + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + } + return lastValue; + } + + @Deprecated + @Override + public ListIterator listIterator() { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public ListIterator listIterator(int index) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public List subList(int fromIndex, int toIndex) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + private EntryReference addToDatabase(U obj) { + EntryReference ref; + try { + ref = db.add(obj); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + return ref; + } + + @Deprecated + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((internalList == null) ? 0 : internalList.hashCode()); + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeIf(Predicate filter) { + Objects.requireNonNull(filter); + boolean removed = false; + for (int i = 0; i < internalList.size(); ) { + T obj; + try { + obj = ((EntryReference) (db.get(internalList.getLong(i)).cast())).getValueReadOnly(); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + if (filter.test(obj)) { + internalList.removeLong(i); + removed = true; + } else { + i++; + } + } + return removed; + } +} diff --git a/src/main/java/org/warp/jcwdb/LightBigList.java b/src/main/java/org/warp/jcwdb/LightBigList.java new file mode 100644 index 0000000..53859ad --- /dev/null +++ b/src/main/java/org/warp/jcwdb/LightBigList.java @@ -0,0 +1,521 @@ +package org.warp.jcwdb; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; + +import java.io.IOException; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Predicate; + +public class LightBigList implements LightList { + + public static final int MAX_ELEMENTS_PER_CHUNK = 10000; + + public final LongArrayList chunks; + public final IntArrayList chunkSizes; + private final transient JCWDatabase db; + + /** + * @param db Database reference + */ + public LightBigList(JCWDatabase db) { + this.db = db; + this.chunks = new LongArrayList(); + this.chunkSizes = new IntArrayList(); + } + + /** + * @param db Database reference + * @param elements Elements to add + */ + public LightBigList(JCWDatabase db, LongArrayList elements) { + this.db = db; + this.chunks = new LongArrayList(); + this.chunkSizes = new IntArrayList(); + elements.forEach((long element) -> { + this.appendIndex(element); + }); + } + + public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) { + this.db = db; + this.chunks = chunks; + this.chunkSizes = chunkSizes; + } + + /** + * Append an index to the first free chunk + * @param elementIndex + */ + public void appendIndex(long elementIndex) { + for (int i = 0; i < chunks.size(); i++) { + final int chunkNumber = i; + if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) { + try { + final long chunkIndex = chunks.getLong(i); + final EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + chunk.appendIndex(elementIndex); + chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1); + }); + return; + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + } + LightList newChunk = new LightArrayList<>(db); + newChunk.appendIndex(elementIndex); + long newChunkIndex; + try { + newChunkIndex = db.add(newChunk).getIndex(); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + chunks.add(newChunkIndex); + chunkSizes.add(1); + } + + /** + * Get the elements count + * @return the size of the list + */ + @Override + public int size() { + int size = 0; + for (int chunkSize : this.chunkSizes) { + size += chunkSize; + } + return size; + } + + /** + * + * @return the count of chunks + */ + public int chunksCount() { + return this.chunkSizes.size(); + } + + /** + * Check if the list is empty + * @return true if the list is empty + */ + @Override + public boolean isEmpty() { + return this.size() <= 0; + } + + @Override + public boolean contains(Object o) { + if (o != null) { + for (long chunkIndex : chunks) { + try { + EntryReference> chunkRef = db.get(chunkIndex); + LightList chunk = chunkRef.getValueReadOnly(); + if (chunk.contains(o)) { + return true; + } + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + } + return false; + } + + /** + * Use iteratorReferences() + */ + @Deprecated + @Override + public Iterator iterator() { + throw new RuntimeException("iterator() isn't implemented!"); + } + + @Deprecated + @Override + public Iterator> iteratorReferences() { + throw new RuntimeException("iteratorReferences() isn't implemented!"); + } + + /** + * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY + * + * @param action + */ + @Deprecated + @Override + public void forEach(Consumer action) { + throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead"); + } + + @Override + public void forEachReference(Consumer> action) { + Objects.requireNonNull(action); + for (long chunkIndex : this.chunks) { + try { + EntryReference> chunkRef = db.get(chunkIndex); + LightList chunk = chunkRef.getValueReadOnly(); + chunk.forEachReference(action); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray() { + throw new RuntimeException("toArray() isn't implemented!"); + } + + @SuppressWarnings("unchecked") + @Override + public T1[] toArray(T1[] a) { + throw new RuntimeException("toArray() isn't implemented!"); + } + + /** + * Use addEntry(o) + * @param o + * @return + */ + @Deprecated + @Override + public boolean add(T o) { + EntryReference ref = addEntry(o); + return ref != null; + } + + @Override + public EntryReference addEntry(T o) { + EntryReference ref = addToDatabase(o); + appendIndex(ref.getIndex()); + return ref; + } + + @Override + public boolean remove(Object o) { + final int removeOffset = indexOf(o); + return removeAt(removeOffset) != null; + } + + @Override + public boolean remove(EntryReference ref) { + final int removeOffset = indexOfEntry(ref); + return removeAt(removeOffset) != null; + } + + private T removeAt(int removeOffset) { + final VariableWrapper result = new VariableWrapper<>(null); + long currentOffset = 0; + if (removeOffset >= 0) { + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + final int currentChunkSize = chunkSizes.getInt(i); + final long chunkStartOffset = currentOffset; + currentOffset += currentChunkSize; + // If the offset to remove is in the current chunk + if (currentOffset > removeOffset) { + // Get chunk index + final long chunkIndex = chunks.getLong(i); + // Get the offset relative to the current chunk + final int relativeOffset = (int) (removeOffset - chunkStartOffset); + + if (relativeOffset < 0) { + return null; + } + + try { + EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + result.var = chunk.remove(relativeOffset); + }); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + chunkSizes.set(removeOffset, currentChunkSize - 1); + break; + } + } + return result.var; + } + return null; + } + + @Override + public boolean containsAll(Collection c) { + for (Object o : c) { + int objIndex = indexOf(o); + if (objIndex < 0) { + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= add((T) o); + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(int index, Collection c) { + boolean result = false; + int delta = 0; + for (Object o : c) { + add(index + delta, (T) o); + result = true; + delta++; + } + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeAll(Collection c) { + boolean result = false; + for (Object o : c) { + result |= remove((T) o); + } + return result; + } + + @Override + public boolean retainAll(Collection c) { + boolean result = false; + LongArrayList collectionHashes = new LongArrayList(); + ObjectArrayList collection = new ObjectArrayList<>(); + collection.addAll(c); + for (Object o : c) { + collectionHashes.add(db.calculateHash(o)); + } + for (int i = 0; i < chunks.size(); i++) { + long hash = chunks.getLong(i); + int positionInCollection = collectionHashes.indexOf(hash); + if (positionInCollection == -1) { + remove(collection.get(positionInCollection)); + result = true; + } + } + return result; + } + + @Override + public void clear() { + chunks.clear(); + chunkSizes.clear(); + } + + /** + * Use getReference or getReadOnlyValue + */ + @Deprecated + @Override + public T get(int index) { + return getReadOnlyValue(index); + } + + @SuppressWarnings("unchecked") + public T getReadOnlyValue(int index) { + try { + return (T) db.get(chunks.getLong(index)).getValueReadOnly(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public EntryReference getReference(int index) { + try { + return db.get(chunks.getLong(index)).cast(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @SuppressWarnings("unchecked") + @Override + public T set(int setOffset, final T element) { + long nextChunkOffset = 0; + VariableWrapper wrapper = new VariableWrapper<>(null); + if (setOffset >= 0) { + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + final int currentChunkSize = chunkSizes.getInt(i); + final long chunkStartOffset = nextChunkOffset; + nextChunkOffset += currentChunkSize; + // If the offset to remove is in the current chunk + if (nextChunkOffset > setOffset) { + // Get chunk index + final long chunkIndex = chunks.getLong(i); + // Get the offset relative to the current chunk + final int relativeOffset = (int) (setOffset - chunkStartOffset); + + if (relativeOffset < 0) { + throw new NullPointerException("Relative Offset < 0"); + } + + try { + EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + chunk.set(relativeOffset, element); + wrapper.var = element; + }); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + break; + } + } + return wrapper.var; + } + return null; + } + + @Override + public void add(int index, T element) { + throw new RuntimeException("add() isn't implemented!"); + } + + @SuppressWarnings("unchecked") + @Override + public T remove(int index) { + return this.removeAt(index); + } + + @Override + public int indexOf(Object o) { + EntryReference ref = addToDatabase(o).cast(); + return indexOfEntry(ref); + } + + @Override + public int indexOfEntry(EntryReference ref) { + int currentOffset = 0; + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + try { + final int currentChunkSize = chunkSizes.getInt(i); + // If the offset to remove is in the current chunk + + // Get chunk index + final long chunkIndex = chunks.getLong(i); + EntryReference> chunkRef = db.get(chunkIndex); + final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref); + if (foundIndex >= 0) { + return currentOffset + foundIndex; + } + currentOffset += currentChunkSize; + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + return -1; + } + + @Override + public int lastIndexOf(Object o) { + return lastIndexOfEntry(addToDatabase(o).cast()); + } + + @Override + public int lastIndexOfEntry(EntryReference ref) { + int currentOffset = 0; + // Iterate through all chunks + for (int i = chunks.size() - 1; i >= 0; i--) { + try { + final int currentChunkSize = chunkSizes.getInt(i); + // If the offset to remove is in the current chunk + + // Get chunk index + final long chunkIndex = chunks.getLong(i); + EntryReference> chunkRef = db.get(chunkIndex); + final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref); + if (foundIndex >= 0) { + return currentOffset + foundIndex; + } + currentOffset += currentChunkSize; + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + return -1; + } + + @Deprecated + @Override + public ListIterator listIterator() { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public ListIterator listIterator(int index) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + @Deprecated + @Override + public List subList(int fromIndex, int toIndex) { + // TODO: implement + throw new RuntimeException("Not implemented!"); + } + + private EntryReference addToDatabase(U obj) { + EntryReference ref; + try { + ref = db.add(obj); + } catch (IOException e) { + throw (NullPointerException) new NullPointerException().initCause(e); + } + return ref; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((chunks == null) ? 0 : chunks.hashCode()); + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean removeIf(Predicate filter) { + Objects.requireNonNull(filter); + final VariableWrapper result = new VariableWrapper(false); + // Iterate through all chunks + for (int i = 0; i < chunks.size(); i++) { + try { + final int chunkOffset = i; + // Get chunk index + final long chunkIndex = chunks.getLong(i); + EntryReference> chunkRef = db.get(chunkIndex); + chunkRef.editValue((chunk) -> { + boolean removed = chunk.removeIf(filter); + if (removed) { + result.var = true; + chunkSizes.set(chunkOffset, chunk.size()); + } + }); + } catch (IOException ex) { + throw (NullPointerException) new NullPointerException().initCause(ex); + } + } + return result.var; + } +} diff --git a/src/main/java/org/warp/jcwdb/LightList.java b/src/main/java/org/warp/jcwdb/LightList.java index 2589dab..9c699ba 100644 --- a/src/main/java/org/warp/jcwdb/LightList.java +++ b/src/main/java/org/warp/jcwdb/LightList.java @@ -1,416 +1,24 @@ package org.warp.jcwdb; -import java.io.IOException; -import java.util.*; +import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; -import java.util.function.Predicate; -import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.objects.ObjectArrayList; +public interface LightList extends List { -public class LightList implements List { + Iterator> iteratorReferences(); - public final LongArrayList internalList; - private final transient JCWDatabase db; + void forEachReference(Consumer> action); - public LightList(JCWDatabase db, LongArrayList internalList) { - this.internalList = internalList; - this.db = db; - } + EntryReference addEntry(T o); - @Override - public int size() { - return internalList.size(); - } + boolean remove(EntryReference ref); - @Override - public boolean isEmpty() { - return internalList.isEmpty(); - } + EntryReference getReference(int index); - @Override - public boolean contains(Object o) { - if (o != null) { - for (Long element : internalList) { - EntryReference ref = null; - try { - ref = db.get(element); - if (o.equals(ref.getValueReadOnly())) { - return true; - } - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return false; - } + int indexOfEntry(EntryReference ref); - /** - * Use iteratorReferences() - */ - @Deprecated - @SuppressWarnings("unchecked") - @Override - public Iterator iterator() { - System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!"); - final ArrayList elements = new ArrayList<>(); - for (Long element : internalList) { - try { - elements.add((T) db.get(element).getValueReadOnly()); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements.iterator(); - } - - @SuppressWarnings("unchecked") - public Iterator> iteratorReferences() { - final ArrayList> elements = new ArrayList<>(); - for (Long element : internalList) { - try { - elements.add(db.get(element)); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements.iterator(); - } + int lastIndexOfEntry(EntryReference ref); - /** - * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY - * @param action - */ - @Deprecated - @Override - public void forEach(Consumer action) { - Objects.requireNonNull(action); - for (T t : this) { - action.accept(t); - } - } - - public void forEachReference(Consumer> action) { - Objects.requireNonNull(action); - for (long index : this.internalList) { - try { - action.accept(db.get(index)); - } catch (IOException e) { - throw (RuntimeException) new RuntimeException().initCause(e); - } - } - } - - - @SuppressWarnings("unchecked") - @Override - public T[] toArray() { - final T[] elements = (T[]) new Objects[internalList.size()]; - for (int i = 0; i < elements.length; i++) { - try { - elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements; - } - - @SuppressWarnings("unchecked") - @Override - public T1[] toArray(T1[] a) { - final T1[] elements = (T1[]) new Objects[internalList.size()]; - for (int i = 0; i < elements.length; i++) { - try { - elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return elements; - } - - @Override - public boolean add(T o) { - EntryReference ref = addEntry(o); - return ref != null; - } - - public EntryReference addEntry(T o) { - EntryReference ref = addToDatabase(o); - if (internalList.add(ref.getIndex())) { - return ref; - } else { - return null; - } - } - - @Override - public boolean remove(Object o) { - int removeIndex = indexOf(o); - if (removeIndex >= 0) { - internalList.removeLong(removeIndex); - return true; - } - return false; - } - - public boolean remove(EntryReference ref) { - int removeIndex = indexOfEntry(ref); - if (removeIndex >= 0) { - internalList.removeLong(removeIndex); - return true; - } - return false; - } - - @Override - public boolean containsAll(Collection c) { - for (Object o : c) { - int objIndex = indexOf(o); - if (objIndex < 0) { - return false; - } - } - return true; - } - - @SuppressWarnings("unchecked") - @Override - public boolean addAll(Collection c) { - boolean result = false; - for (Object o : c) { - result |= add((T) o); - } - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean addAll(int index, Collection c) { - boolean result = false; - int delta = 0; - for (Object o : c) { - add(index + delta, (T) o); - result = true; - delta++; - } - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean removeAll(Collection c) { - boolean result = false; - for (Object o : c) { - result |= remove((T) o); - } - return result; - } - - @Override - public boolean retainAll(Collection c) { - boolean result = false; - LongArrayList collectionHashes = new LongArrayList(); - ObjectArrayList collection = new ObjectArrayList<>(); - collection.addAll(c); - for (Object o : c) { - collectionHashes.add(db.calculateHash(o)); - } - for (int i = 0; i < internalList.size(); i++) { - long hash = internalList.getLong(i); - int positionInCollection = collectionHashes.indexOf(hash); - if (positionInCollection == -1) { - remove(collection.get(positionInCollection)); - result = true; - } - } - return result; - } - - @Override - public void clear() { - internalList.clear(); - } - - /** - * Use getReference or getReadOnlyValue - */ - @Deprecated - @Override - public T get(int index) { - return getReadOnlyValue(index); - } - - @SuppressWarnings("unchecked") - public T getReadOnlyValue(int index) { - try { - return (T) db.get(internalList.getLong(index)).getValueReadOnly(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - public EntryReference getReference(int index) { - try { - return db.get(internalList.getLong(index)).cast(); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - @SuppressWarnings("unchecked") - @Override - public T set(int index, T element) { - EntryReference ref = addToDatabase(element); - long oldIndex = internalList.set(index, ref.getIndex()); - try { - ref.close(); - return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - - @Override - public void add(int index, T element) { - EntryReference ref = addToDatabase(element); - internalList.add(index, ref.getIndex()); - } - - @SuppressWarnings("unchecked") - @Override - public T remove(int index) { - long oldIndex = internalList.removeLong(index); - try { - return ((EntryReference) (db.get(oldIndex))).getValueReadOnly(); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - - @Override - public int indexOf(Object o) { - EntryReference ref = addToDatabase(o); - long objToRemoveHash = ref.calculateHash(); - LongArrayList hashes = new LongArrayList(); - - - for (int i = 0; i < hashes.size(); i++) { - long hash = hashes.getLong(i); - if (objToRemoveHash == hash) { - try { - if (ref.equals(db.get(internalList.getLong(i)))) { - return i; - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - } - return -1; - } - - public int indexOfEntry(EntryReference ref) { - for (int i = 0; i < internalList.size(); i++) { - long index = internalList.getLong(i); - try { - EntryReference ref2 = db.get(index); - if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { - return i; - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - return -1; - } - - @Override - public int lastIndexOf(Object o) { - EntryReference ref = addToDatabase(o); - long objToRemoveHash = ref.calculateHash(); - - int lastValue = -1; - - for (int i = 0; i < internalList.size(); i++) { - long index2 = internalList.getLong(i); - try { - EntryReference ref2 = db.get(index2); - if (objToRemoveHash == ref2.calculateHash()) { - if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) { - lastValue = i; - } - } - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - } - return lastValue; - } - - @Deprecated - @Override - public ListIterator listIterator() { - // TODO: implement - throw new RuntimeException("Not implemented!"); - } - - @Deprecated - @Override - public ListIterator listIterator(int index) { - // TODO: implement - throw new RuntimeException("Not implemented!"); - } - - @Deprecated - @Override - public List subList(int fromIndex, int toIndex) { - // TODO: implement - throw new RuntimeException("Not implemented!"); - } - - private EntryReference addToDatabase(U obj) { - EntryReference ref; - try { - ref = db.add(obj); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - return ref; - } - - @Deprecated - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((internalList == null) ? 0 : internalList.hashCode()); - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean removeIf(Predicate filter) { - Objects.requireNonNull(filter); - boolean removed = false; - for (int i = 0; i < internalList.size(); ) { - T obj; - try { - obj = ((EntryReference) (db.get(internalList.getLong(i)).cast())).getValueReadOnly(); - } catch (IOException e) { - throw (NullPointerException) new NullPointerException().initCause(e); - } - if (filter.test(obj)) { - internalList.removeLong(i); - removed = true; - } else { - i++; - } - } - return removed; - } + void appendIndex(long elementIndex); } diff --git a/src/test/java/org/warp/jcwdb/AppTest.java b/src/test/java/org/warp/jcwdb/AppTest.java index 187b8cd..d632087 100644 --- a/src/test/java/org/warp/jcwdb/AppTest.java +++ b/src/test/java/org/warp/jcwdb/AppTest.java @@ -25,94 +25,5 @@ public class AppTest { assertTrue( true ); } - - /** - * - * @throws IOException - */ - @Test - public void shouldReadCustomClasses() throws IOException - { - CustomClass customClass = new CustomClass(); - customClass.primitive = 1; - customClass.string = "test"; - - Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx"); - - JCWDatabase db = new JCWDatabase(path, idx); - EntryReference> ref = db.getRoot(); - ref.editValue((root, saver) -> { - root.add(customClass); - }); - db.close(); - - JCWDatabase db2 = new JCWDatabase(path, idx); - EntryReference> ref2 = db2.getRoot(); - CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0); - assertTrue(customClass.equals(customClass2)); - assertTrue(customClass.string.equals(customClass2.string)); - db2.close(); - Files.delete(path); - Files.delete(idx); - } - - public static class CustomClass { - public String string; - public int primitive; - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + primitive; - result = prime * result + ((string == null) ? 0 : string.hashCode()); - return result; - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - CustomClass other = (CustomClass) obj; - if (primitive != other.primitive) - return false; - if (string == null) { - if (other.string != null) - return false; - } else if (!string.equals(other.string)) - return false; - return true; - } - - - } - - /** - * - * @throws IOException - */ - @Test - public void shouldActAsAnArrayListWithEqualObjects() throws IOException - { - Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx"); - JCWDatabase db = new JCWDatabase(path, idx); - EntryReference> ref = db.getRoot(); - LightList list1 = ref.getValueReadOnly(); - ArrayList list2 = new ArrayList(); - String s = "a"; - for (int i = 0; i < 10; i++) { - list1.add(s); - list2.add(s); - } - assertTrue(list1.size() == list2.size()); - for (int i = 0; i < 10; i++) { - assertTrue(list1.get(i) == list2.get(i)); - assertTrue(list2.get(i) == list1.get(i)); - } - db.close(); - Files.delete(path); - Files.delete(idx); - } + }