andreacavalli/strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Chunked list

Browse Source
This commit is contained in:
Andrea Cavalli 2018-12-20 00:16:16 +01:00
parent 1d64d1565c
commit f9372a0aaf
10 changed files with 1085 additions and 535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/org/warp/jcwdb/DBLightListParser.java → src/main/java/org/warp/jcwdb/DBLightArrayListParser.java
View File

@ -1,40 +1,38 @@
package org.warp.jcwdb;
import java.util.ArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightListParser<T> extends DBTypeParserImpl<LightList<T>> {
public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T>> {
private final JCWDatabase db;
public DBLightListParser(JCWDatabase db) {
public DBLightArrayListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightList<T>> getReader() {
public DBReader<LightArrayList<T>> getReader() {
return (i, size) -> {
LongArrayList internalList = new LongArrayList();
long max = size / Long.BYTES;
for (int item = 0; item < max; item++){
for (int item = 0; item < max; item++) {
long itm = i.readLong();
internalList.add(itm);
}
return new LightList<T>(db, internalList);
return new LightArrayList<T>(db, internalList);
};
}
public DBDataOutput<LightList<T>> getWriter(final LightList<T> value) {
public DBDataOutput<LightArrayList<T>> getWriter(final LightArrayList<T> value) {
final int elementsCount = value.size();
return DBDataOutput.create((o) -> {
LongArrayList list = value.internalList;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
}
}, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value));
}, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value));
}
@Override
public long calculateHash(LightList<T> value) {
public long calculateHash(LightArrayList<T> value) {
return value.internalList.hashCode();
}
}

44
src/main/java/org/warp/jcwdb/DBLightBigListParser.java Normal file
View File

@ -0,0 +1,44 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
private final JCWDatabase db;
public DBLightBigListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightBigList<T>> getReader() {
return (i, size) -> {
LongArrayList chunks = new LongArrayList();
IntArrayList chunkSizes = new IntArrayList();
long max = size / (Long.BYTES + Integer.BYTES);
for (int item = 0; item < max; item++) {
long itm = i.readLong();
int itm2 = i.readInt();
chunks.add(itm);
chunkSizes.add(itm2);
}
return new LightBigList<T>(db, chunks, chunkSizes);
};
}
public DBDataOutput<LightBigList<T>> getWriter(final LightBigList<T> value) {
final int elementsCount = value.chunksCount();
return DBDataOutput.create((o) -> {
LongArrayList list = value.chunks;
IntArrayList list2 = value.chunkSizes;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
o.writeInt(list2.getInt(i));
}
}, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value));
}
@Override
public long calculateHash(LightBigList<T> value) {
return value.chunks.hashCode();
}
}

8
src/main/java/org/warp/jcwdb/DBStandardTypes.java
View File

@ -11,12 +11,14 @@ public class DBStandardTypes {
public static final int DOUBLE = STD| 6;
public static final int STRING = STD| 7;
public static final int BYTE_ARRAY = STD| 8;
public static final int LIGHT_LIST = STD| 9;
public static final int GENERIC_OBJECT = STD| 10;
public static final int LIGHT_LIST_ARRAY = STD| 9;
public static final int LIGHT_LIST_BIG = STD| 10;
public static final int GENERIC_OBJECT = STD| 11;
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
typesManager.registerType(String.class, STRING, new DBStringParser());
typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db));
typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db));
typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db));
typesManager.registerTypeFallback(new DBGenericObjectParser());
}
}

32
src/main/java/org/warp/jcwdb/EntryReference.java
View File

@ -86,7 +86,7 @@ public class EntryReference<T> implements Castable, Saveable {
* @param editFunction
* @throws IOException
*/
public void editValue(BiFunction<T, Saveable, T> editFunction) throws IOException {
public void editValue(BiFunction<T, Saveable, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value, this);
@ -99,7 +99,20 @@ public class EntryReference<T> implements Castable, Saveable {
* @param editFunction
* @throws IOException
*/
public void editValue(BiConsumer<T, Saveable> editFunction) throws IOException {
public void editValue(Function<T, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value);
this.save();
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(BiConsumer<T, Saveable> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value, this);
@ -107,12 +120,25 @@ public class EntryReference<T> implements Castable, Saveable {
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(Consumer<T> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value);
this.save();
}
}
/**
* Substitute the old value with a new one
* @param val
* @throws IOException
*/
public void setValue(T val) throws IOException {
public void setValue(T val) {
synchronized(accessLock) {
this.loaded = true;
this.value = val;

35
src/main/java/org/warp/jcwdb/FileAllocator.java
View File

@ -1,14 +1,7 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Output;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.util.*;
public class FileAllocator implements AutoCloseable {
private final SeekableByteChannel dataFileChannel;
@ -16,14 +9,15 @@ public class FileAllocator implements AutoCloseable {
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object allocateLock = new Object();
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
this.dataFileChannel = dataFileChannel;
this.allocableOffset = this.dataFileChannel.size();
}
/**
* TODO: not implemented
*
* @param size
* @return offset
*/
@ -36,21 +30,22 @@ public class FileAllocator implements AutoCloseable {
}
}
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
/**
* Frees the unused bytes
*
* @param startPosition
* @param length
*/

10
src/main/java/org/warp/jcwdb/JCWDatabase.java
View File

@ -32,19 +32,19 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
}
public <T> EntryReference<LightList<T>> getRoot() throws IOException {
return getRoot(Object.class).cast();
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
checkClosed();
if (exists(0)) {
return get(0);
} else {
LightList<T> newRoot = new LightList<T>(this, new LongArrayList());
LightList<T> newRoot = new LightBigList<>(this);
return set(0, newRoot);
}
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
return getRoot().cast();
}
public <T> EntryReference<T> get(long index) throws IOException {
checkClosed();
int type;

445
src/main/java/org/warp/jcwdb/LightArrayList.java Normal file
View File

@ -0,0 +1,445 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
public class LightArrayList<T> implements LightList<T> {
public final LongArrayList internalList;
private final transient JCWDatabase db;
/**
* @param db Database reference
*/
public LightArrayList(JCWDatabase db) {
this.db = db;
this.internalList = new LongArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightArrayList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.internalList = new LongArrayList(elements);
}
@Override
public int size() {
return internalList.size();
}
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long element : internalList) {
EntryReference<T> ref = null;
try {
ref = db.get(element);
if (o.equals(ref.getValueReadOnly())) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add((T) db.get(element).getValueReadOnly());
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@SuppressWarnings("unchecked")
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add(db.get(element));
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw (RuntimeException) new RuntimeException().initCause(e);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
final T[] elements = (T[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
if (internalList.add(ref.getIndex())) {
return ref;
} else {
return null;
}
}
@Override
public boolean remove(Object o) {
int removeIndex = indexOf(o);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean remove(EntryReference<T> ref) {
int removeIndex = indexOfEntry(ref);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < internalList.size(); i++) {
long hash = internalList.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
internalList.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(internalList.getLong(index)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
try {
return db.get(internalList.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
long oldIndex = internalList.set(index, ref.getIndex());
try {
ref.close();
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public void add(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
internalList.add(index, ref.getIndex());
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
long oldIndex = internalList.removeLong(index);
try {
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public int indexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
LongArrayList hashes = new LongArrayList();
for (int i = 0; i < hashes.size(); i++) {
long hash = hashes.getLong(i);
if (objToRemoveHash == hash) {
try {
if (ref.equals(db.get(internalList.getLong(i)))) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
}
return -1;
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
for (int i = 0; i < internalList.size(); i++) {
long index = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index);
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return -1;
}
@Override
public void appendIndex(long elementIndex) {
internalList.add(elementIndex);
}
@Override
public int lastIndexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return lastIndexOfEntry(ref);
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
long objToRemoveHash = ref.calculateHash();
int lastValue = -1;
for (int i = 0; i < internalList.size(); i++) {
long index2 = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index2);
if (objToRemoveHash == ref2.calculateHash()) {
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
lastValue = i;
}
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return lastValue;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Deprecated
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean removed = false;
for (int i = 0; i < internalList.size(); ) {
T obj;
try {
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
if (filter.test(obj)) {
internalList.removeLong(i);
removed = true;
} else {
i++;
}
}
return removed;
}
}

521
src/main/java/org/warp/jcwdb/LightBigList.java Normal file
View File

@ -0,0 +1,521 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
public class LightBigList<T> implements LightList<T> {
public static final int MAX_ELEMENTS_PER_CHUNK = 10000;
public final LongArrayList chunks;
public final IntArrayList chunkSizes;
private final transient JCWDatabase db;
/**
* @param db Database reference
*/
public LightBigList(JCWDatabase db) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightBigList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
elements.forEach((long element) -> {
this.appendIndex(element);
});
}
public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
this.db = db;
this.chunks = chunks;
this.chunkSizes = chunkSizes;
}
/**
* Append an index to the first free chunk
* @param elementIndex
*/
public void appendIndex(long elementIndex) {
for (int i = 0; i < chunks.size(); i++) {
final int chunkNumber = i;
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
try {
final long chunkIndex = chunks.getLong(i);
final EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
chunk.appendIndex(elementIndex);
chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1);
});
return;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
LightList<T> newChunk = new LightArrayList<>(db);
newChunk.appendIndex(elementIndex);
long newChunkIndex;
try {
newChunkIndex = db.add(newChunk).getIndex();
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunks.add(newChunkIndex);
chunkSizes.add(1);
}
/**
* Get the elements count
* @return the size of the list
*/
@Override
public int size() {
int size = 0;
for (int chunkSize : this.chunkSizes) {
size += chunkSize;
}
return size;
}
/**
*
* @return the count of chunks
*/
public int chunksCount() {
return this.chunkSizes.size();
}
/**
* Check if the list is empty
* @return true if the list is empty
*/
@Override
public boolean isEmpty() {
return this.size() <= 0;
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long chunkIndex : chunks) {
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
LightList<T> chunk = chunkRef.getValueReadOnly();
if (chunk.contains(o)) {
return true;
}
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@Override
public Iterator<T> iterator() {
throw new RuntimeException("iterator() isn't implemented!");
}
@Deprecated
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
throw new RuntimeException("iteratorReferences() isn't implemented!");
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long chunkIndex : this.chunks) {
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
LightList<T> chunk = chunkRef.getValueReadOnly();
chunk.forEachReference(action);
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
throw new RuntimeException("toArray() isn't implemented!");
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
throw new RuntimeException("toArray() isn't implemented!");
}
/**
* Use addEntry(o)
* @param o
* @return
*/
@Deprecated
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
appendIndex(ref.getIndex());
return ref;
}
@Override
public boolean remove(Object o) {
final int removeOffset = indexOf(o);
return removeAt(removeOffset) != null;
}
@Override
public boolean remove(EntryReference<T> ref) {
final int removeOffset = indexOfEntry(ref);
return removeAt(removeOffset) != null;
}
private T removeAt(int removeOffset) {
final VariableWrapper<T> result = new VariableWrapper<>(null);
long currentOffset = 0;
if (removeOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = currentOffset;
currentOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (currentOffset > removeOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (removeOffset - chunkStartOffset);
if (relativeOffset < 0) {
return null;
}
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
result.var = chunk.remove(relativeOffset);
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunkSizes.set(removeOffset, currentChunkSize - 1);
break;
}
}
return result.var;
}
return null;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < chunks.size(); i++) {
long hash = chunks.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
chunks.clear();
chunkSizes.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(chunks.getLong(index)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
try {
return db.get(chunks.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int setOffset, final T element) {
long nextChunkOffset = 0;
VariableWrapper<T> wrapper = new VariableWrapper<>(null);
if (setOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = nextChunkOffset;
nextChunkOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (nextChunkOffset > setOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (setOffset - chunkStartOffset);
if (relativeOffset < 0) {
throw new NullPointerException("Relative Offset < 0");
}
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
chunk.set(relativeOffset, element);
wrapper.var = element;
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
break;
}
}
return wrapper.var;
}
return null;
}
@Override
public void add(int index, T element) {
throw new RuntimeException("add() isn't implemented!");
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
return this.removeAt(index);
}
@Override
public int indexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return indexOfEntry(ref);
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Override
public int lastIndexOf(Object o) {
return lastIndexOfEntry(addToDatabase(o).cast());
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = chunks.size() - 1; i >= 0; i--) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
final VariableWrapper<Boolean> result = new VariableWrapper(false);
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
try {
final int chunkOffset = i;
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
boolean removed = chunk.removeIf(filter);
if (removed) {
result.var = true;
chunkSizes.set(chunkOffset, chunk.size());
}
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return result.var;
}
}

414
src/main/java/org/warp/jcwdb/LightList.java
View File

@ -1,416 +1,24 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.*;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Predicate;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
public interface LightList<T> extends List<T> {
public class LightList<T> implements List<T> {
Iterator<EntryReference<T>> iteratorReferences();
public final LongArrayList internalList;
private final transient JCWDatabase db;
void forEachReference(Consumer<? super EntryReference<T>> action);
public LightList(JCWDatabase db, LongArrayList internalList) {
this.internalList = internalList;
this.db = db;
}
EntryReference<T> addEntry(T o);
@Override
public int size() {
return internalList.size();
}
boolean remove(EntryReference<T> ref);
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
EntryReference<T> getReference(int index);
@Override
public boolean contains(Object o) {
if (o != null) {
for (Long element : internalList) {
EntryReference<T> ref = null;
try {
ref = db.get(element);
if (o.equals(ref.getValueReadOnly())) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
int indexOfEntry(EntryReference<T> ref);
/**
* Use iteratorReferences()
*/
@Deprecated
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>();
for (Long element : internalList) {
try {
elements.add((T) db.get(element).getValueReadOnly());
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@SuppressWarnings("unchecked")
public Iterator<EntryReference<T>> iteratorReferences() {
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
for (Long element : internalList) {
try {
elements.add(db.get(element));
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
int lastIndexOfEntry(EntryReference<T> ref);
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw (RuntimeException) new RuntimeException().initCause(e);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
final T[] elements = (T[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
if (internalList.add(ref.getIndex())) {
return ref;
} else {
return null;
}
}
@Override
public boolean remove(Object o) {
int removeIndex = indexOf(o);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
public boolean remove(EntryReference<T> ref) {
int removeIndex = indexOfEntry(ref);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < internalList.size(); i++) {
long hash = internalList.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
internalList.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(internalList.getLong(index)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public EntryReference<T> getReference(int index) {
try {
return db.get(internalList.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
long oldIndex = internalList.set(index, ref.getIndex());
try {
ref.close();
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public void add(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
internalList.add(index, ref.getIndex());
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
long oldIndex = internalList.removeLong(index);
try {
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public int indexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
LongArrayList hashes = new LongArrayList();
for (int i = 0; i < hashes.size(); i++) {
long hash = hashes.getLong(i);
if (objToRemoveHash == hash) {
try {
if (ref.equals(db.get(internalList.getLong(i)))) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
}
return -1;
}
public int indexOfEntry(EntryReference<T> ref) {
for (int i = 0; i < internalList.size(); i++) {
long index = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index);
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return -1;
}
@Override
public int lastIndexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
int lastValue = -1;
for (int i = 0; i < internalList.size(); i++) {
long index2 = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index2);
if (objToRemoveHash == ref2.calculateHash()) {
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
lastValue = i;
}
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return lastValue;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Deprecated
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean removed = false;
for (int i = 0; i < internalList.size(); ) {
T obj;
try {
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
if (filter.test(obj)) {
internalList.removeLong(i);
removed = true;
} else {
i++;
}
}
return removed;
}
void appendIndex(long elementIndex);
}

91
src/test/java/org/warp/jcwdb/AppTest.java
View File

@ -25,94 +25,5 @@ public class AppTest
{
assertTrue( true );
}
/**
*
* @throws IOException
*/
@Test
public void shouldReadCustomClasses() throws IOException
{
CustomClass customClass = new CustomClass();
customClass.primitive = 1;
customClass.string = "test";
Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
JCWDatabase db = new JCWDatabase(path, idx);
EntryReference<LightList<CustomClass>> ref = db.getRoot();
ref.editValue((root, saver) -> {
root.add(customClass);
});
db.close();
JCWDatabase db2 = new JCWDatabase(path, idx);
EntryReference<LightList<CustomClass>> ref2 = db2.getRoot();
CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0);
assertTrue(customClass.equals(customClass2));
assertTrue(customClass.string.equals(customClass2.string));
db2.close();
Files.delete(path);
Files.delete(idx);
}
public static class CustomClass {
public String string;
public int primitive;
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + primitive;
result = prime * result + ((string == null) ? 0 : string.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
CustomClass other = (CustomClass) obj;
if (primitive != other.primitive)
return false;
if (string == null) {
if (other.string != null)
return false;
} else if (!string.equals(other.string))
return false;
return true;
}
}
/**
*
* @throws IOException
*/
@Test
public void shouldActAsAnArrayListWithEqualObjects() throws IOException
{
Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
JCWDatabase db = new JCWDatabase(path, idx);
EntryReference<LightList<String>> ref = db.getRoot();
LightList<String> list1 = ref.getValueReadOnly();
ArrayList<String> list2 = new ArrayList<String>();
String s = "a";
for (int i = 0; i < 10; i++) {
list1.add(s);
list2.add(s);
}
assertTrue(list1.size() == list2.size());
for (int i = 0; i < 10; i++) {
assertTrue(list1.get(i) == list2.get(i));
assertTrue(list2.get(i) == list1.get(i));
}
db.close();
Files.delete(path);
Files.delete(idx);
}
}