582 lines
14 KiB
Java
582 lines
14 KiB
Java
package org.warp.jcwdb;
|
|
|
|
import it.unimi.dsi.fastutil.ints.IntArrayList;
|
|
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
|
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
|
|
|
import java.io.IOError;
|
|
import java.io.IOException;
|
|
import java.util.*;
|
|
import java.util.function.Consumer;
|
|
import java.util.function.Predicate;
|
|
|
|
public class LightBigList<T> implements LightList<T>, Saveable {
|
|
|
|
public static final int MAX_ELEMENTS_PER_CHUNK = 200000;
|
|
|
|
public final LongArrayList chunks;
|
|
public final IntArrayList chunkSizes;
|
|
private final JCWDatabase db;
|
|
private LightArrayList<T> cachedChunk;
|
|
private EntryReference<LightArrayList<T>> cachedChunkRef;
|
|
private long cachedChunkIndex = -1;
|
|
private int cachedChunkNumber = -1;
|
|
private final Object cachedChunkLock = new Object();
|
|
|
|
/**
|
|
* @param db Database reference
|
|
*/
|
|
public LightBigList(JCWDatabase db) {
|
|
this.db = db;
|
|
this.chunks = new LongArrayList();
|
|
this.chunkSizes = new IntArrayList();
|
|
}
|
|
|
|
/**
|
|
* @param db Database reference
|
|
* @param elements Elements to add
|
|
*/
|
|
public LightBigList(JCWDatabase db, LongArrayList elements) {
|
|
this.db = db;
|
|
this.chunks = new LongArrayList();
|
|
this.chunkSizes = new IntArrayList();
|
|
elements.forEach((long element) -> {
|
|
this.appendIndex(element);
|
|
});
|
|
}
|
|
|
|
public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
|
|
this.db = db;
|
|
this.chunks = chunks;
|
|
this.chunkSizes = chunkSizes;
|
|
if (this.chunks.size() > 0) {
|
|
prepareAccessToChunk(0);
|
|
}
|
|
}
|
|
|
|
private void prepareAccessToChunk(int chunkNumber) {
|
|
if (this.cachedChunkRef != null) {
|
|
this.cachedChunkRef.save();
|
|
}
|
|
this.cachedChunkNumber = chunkNumber;
|
|
this.cachedChunkIndex = this.chunks.getLong(chunkNumber);
|
|
try {
|
|
this.cachedChunkRef = db.get(this.cachedChunkIndex);
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
this.cachedChunk = this.cachedChunkRef.getValueReadOnly();
|
|
}
|
|
|
|
/**
|
|
* Append an index to the first free chunk
|
|
* @param elementIndex
|
|
*/
|
|
public void appendIndex(long elementIndex) {
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
final int chunkNumber = i;
|
|
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
|
|
synchronized (cachedChunkLock) {
|
|
if (cachedChunkNumber != i) {
|
|
prepareAccessToChunk(i);
|
|
}
|
|
cachedChunk.appendIndex(elementIndex);
|
|
chunkSizes.set(chunkNumber, cachedChunk.size());
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
LightList<T> newChunk = new LightArrayList<>(db);
|
|
newChunk.appendIndex(elementIndex);
|
|
long newChunkIndex;
|
|
try {
|
|
newChunkIndex = db.add(newChunk).getIndex();
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
chunks.add(newChunkIndex);
|
|
chunkSizes.add(1);
|
|
}
|
|
|
|
/**
|
|
* Get the elements count
|
|
* @return the size of the list
|
|
*/
|
|
@Override
|
|
public int size() {
|
|
int size = 0;
|
|
for (int chunkSize : this.chunkSizes) {
|
|
size += chunkSize;
|
|
}
|
|
return size;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @return the count of chunks
|
|
*/
|
|
public int chunksCount() {
|
|
return this.chunkSizes.size();
|
|
}
|
|
|
|
/**
|
|
* Check if the list is empty
|
|
* @return true if the list is empty
|
|
*/
|
|
@Override
|
|
public boolean isEmpty() {
|
|
return this.size() <= 0;
|
|
}
|
|
|
|
@Override
|
|
public boolean contains(Object o) {
|
|
if (o != null) {
|
|
for (long chunkIndex : chunks) {
|
|
try {
|
|
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
|
LightArrayList<T> chunk = chunkRef.getValueReadOnly();
|
|
if (chunk.contains(o)) {
|
|
return true;
|
|
}
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Use iteratorReferences()
|
|
*/
|
|
@Deprecated
|
|
@Override
|
|
public Iterator<T> iterator()
|
|
{
|
|
throw new RuntimeException("iterator() isn't implemented!");
|
|
}
|
|
|
|
@Deprecated
|
|
@Override
|
|
public Iterator<EntryReference<T>> iteratorReferences() {
|
|
throw new RuntimeException("iteratorReferences() isn't implemented!");
|
|
}
|
|
|
|
/**
|
|
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
|
|
*
|
|
* @param action
|
|
*/
|
|
@Deprecated
|
|
@Override
|
|
public void forEach(Consumer<? super T> action) {
|
|
throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
|
|
}
|
|
|
|
@Override
|
|
public void forEachReference(Consumer<? super EntryReference<T>> action) {
|
|
Objects.requireNonNull(action);
|
|
// Iterate through all chunks
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
synchronized (cachedChunkLock) {
|
|
if (cachedChunkNumber != i) {
|
|
prepareAccessToChunk(i);
|
|
}
|
|
cachedChunk.forEachReference(action);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* toArray() isn't implemented! DO NOT USE IT.
|
|
* @return
|
|
*/
|
|
@SuppressWarnings("unchecked")
|
|
@Deprecated
|
|
@Override
|
|
public T[] toArray() {
|
|
if (true) {
|
|
throw new RuntimeException("toArray() isn't implemented!");
|
|
}
|
|
T[] result = (T[]) new Object[this.size()];
|
|
|
|
long currentOffset = 0;
|
|
// Iterate through all chunks
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
final int currentChunkSize = chunkSizes.getInt(i);
|
|
final long chunkStartOffset = currentOffset;
|
|
currentOffset += currentChunkSize;
|
|
// Get chunk index
|
|
final long chunkIndex = chunks.getLong(i);
|
|
|
|
try {
|
|
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
|
LightArrayList<T> chunk = chunkRef.getValueReadOnly();
|
|
for (int i1 = 0; i1 < chunk.size(); i1++) {
|
|
result[(int)(chunkStartOffset + i1)] = chunk.get(i);
|
|
}
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public <T1> T1[] toArray(T1[] a) {
|
|
throw new RuntimeException("toArray() isn't implemented!");
|
|
}
|
|
|
|
/**
|
|
* Use addEntry(o)
|
|
* @param o
|
|
* @return
|
|
*/
|
|
@Deprecated
|
|
@Override
|
|
public boolean add(T o) {
|
|
EntryReference<T> ref = addEntry(o);
|
|
return ref != null;
|
|
}
|
|
|
|
@Override
|
|
public EntryReference<T> addEntry(T o) {
|
|
EntryReference<T> ref = addToDatabase(o);
|
|
appendIndex(ref.getIndex());
|
|
return ref;
|
|
}
|
|
|
|
@Override
|
|
public boolean remove(Object o) {
|
|
final int removeOffset = indexOf(o);
|
|
return removeAt(removeOffset) != null;
|
|
}
|
|
|
|
@Override
|
|
public boolean remove(EntryReference<T> ref) {
|
|
final int removeOffset = indexOfEntry(ref);
|
|
return removeAt(removeOffset) != null;
|
|
}
|
|
|
|
private T removeAt(int removeOffset) {
|
|
final VariableWrapper<T> result = new VariableWrapper<>(null);
|
|
long currentOffset = 0;
|
|
if (removeOffset >= 0) {
|
|
// Iterate through all chunks
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
final int currentChunkSize = chunkSizes.getInt(i);
|
|
final long chunkStartOffset = currentOffset;
|
|
currentOffset += currentChunkSize;
|
|
// If the offset to remove is in the current chunk
|
|
if (currentOffset > removeOffset) {
|
|
// Get chunk index
|
|
final long chunkIndex = chunks.getLong(i);
|
|
// Get the offset relative to the current chunk
|
|
final int relativeOffset = (int) (removeOffset - chunkStartOffset);
|
|
|
|
if (relativeOffset < 0) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
|
chunkRef.editValue((chunk) -> {
|
|
result.var = chunk.remove(relativeOffset);
|
|
});
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
chunkSizes.set(removeOffset, currentChunkSize - 1);
|
|
break;
|
|
}
|
|
}
|
|
return result.var;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
@Override
|
|
public boolean containsAll(Collection<?> c) {
|
|
for (Object o : c) {
|
|
int objIndex = indexOf(o);
|
|
if (objIndex < 0) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public boolean addAll(Collection<? extends T> c) {
|
|
boolean result = false;
|
|
for (Object o : c) {
|
|
result |= add((T) o);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public boolean addAll(int index, Collection<? extends T> c) {
|
|
boolean result = false;
|
|
int delta = 0;
|
|
for (Object o : c) {
|
|
add(index + delta, (T) o);
|
|
result = true;
|
|
delta++;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public boolean removeAll(Collection<?> c) {
|
|
boolean result = false;
|
|
for (Object o : c) {
|
|
result |= remove((T) o);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
@Override
|
|
public boolean retainAll(Collection<?> c) {
|
|
boolean result = false;
|
|
LongArrayList collectionHashes = new LongArrayList();
|
|
ObjectArrayList<Object> collection = new ObjectArrayList<>();
|
|
collection.addAll(c);
|
|
for (Object o : c) {
|
|
collectionHashes.add(db.calculateHash(o));
|
|
}
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
long hash = chunks.getLong(i);
|
|
int positionInCollection = collectionHashes.indexOf(hash);
|
|
if (positionInCollection == -1) {
|
|
remove(collection.get(positionInCollection));
|
|
result = true;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
@Override
|
|
public void clear() {
|
|
chunks.clear();
|
|
chunkSizes.clear();
|
|
}
|
|
|
|
/**
|
|
* Use getReference or getReadOnlyValue
|
|
*/
|
|
@Deprecated
|
|
@Override
|
|
public T get(int index) {
|
|
return getReadOnlyValue(index);
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
public T getReadOnlyValue(int index) {
|
|
try {
|
|
return (T) db.get(chunks.getLong(index)).getValueReadOnly();
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
return null;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public EntryReference<T> getReference(int index) {
|
|
try {
|
|
return db.get(chunks.getLong(index)).cast();
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
return null;
|
|
}
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public T set(int setOffset, final T element) {
|
|
long nextChunkOffset = 0;
|
|
VariableWrapper<T> wrapper = new VariableWrapper<>(null);
|
|
if (setOffset >= 0) {
|
|
// Iterate through all chunks
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
final int currentChunkSize = chunkSizes.getInt(i);
|
|
final long chunkStartOffset = nextChunkOffset;
|
|
nextChunkOffset += currentChunkSize;
|
|
// If the offset to remove is in the current chunk
|
|
if (nextChunkOffset > setOffset) {
|
|
// Get chunk index
|
|
final long chunkIndex = chunks.getLong(i);
|
|
// Get the offset relative to the current chunk
|
|
final int relativeOffset = (int) (setOffset - chunkStartOffset);
|
|
|
|
if (relativeOffset < 0) {
|
|
throw new NullPointerException("Relative Offset < 0");
|
|
}
|
|
|
|
try {
|
|
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
|
chunkRef.editValue((chunk) -> {
|
|
chunk.set(relativeOffset, element);
|
|
wrapper.var = element;
|
|
});
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
return wrapper.var;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
@Override
|
|
public void add(int index, T element) {
|
|
throw new RuntimeException("add() isn't implemented!");
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public T remove(int index) {
|
|
return this.removeAt(index);
|
|
}
|
|
|
|
@Override
|
|
public int indexOf(Object o) {
|
|
EntryReference<T> ref = addToDatabase(o).cast();
|
|
return indexOfEntry(ref);
|
|
}
|
|
|
|
@Override
|
|
public int indexOfEntry(EntryReference<T> ref) {
|
|
int currentOffset = 0;
|
|
// Iterate through all chunks
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
try {
|
|
final int currentChunkSize = chunkSizes.getInt(i);
|
|
// If the offset to remove is in the current chunk
|
|
|
|
// Get chunk index
|
|
final long chunkIndex = chunks.getLong(i);
|
|
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
|
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
|
|
if (foundIndex >= 0) {
|
|
return currentOffset + foundIndex;
|
|
}
|
|
currentOffset += currentChunkSize;
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
@Override
|
|
public int lastIndexOf(Object o) {
|
|
return lastIndexOfEntry(addToDatabase(o).cast());
|
|
}
|
|
|
|
@Override
|
|
public int lastIndexOfEntry(EntryReference<T> ref) {
|
|
int currentOffset = 0;
|
|
// Iterate through all chunks
|
|
for (int i = chunks.size() - 1; i >= 0; i--) {
|
|
try {
|
|
final int currentChunkSize = chunkSizes.getInt(i);
|
|
// If the offset to remove is in the current chunk
|
|
|
|
// Get chunk index
|
|
final long chunkIndex = chunks.getLong(i);
|
|
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
|
|
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
|
|
if (foundIndex >= 0) {
|
|
return currentOffset + foundIndex;
|
|
}
|
|
currentOffset += currentChunkSize;
|
|
} catch (IOException ex) {
|
|
throw (NullPointerException) new NullPointerException().initCause(ex);
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
@Deprecated
|
|
@Override
|
|
public ListIterator<T> listIterator() {
|
|
// TODO: implement
|
|
throw new RuntimeException("Not implemented!");
|
|
}
|
|
|
|
@Deprecated
|
|
@Override
|
|
public ListIterator<T> listIterator(int index) {
|
|
// TODO: implement
|
|
throw new RuntimeException("Not implemented!");
|
|
}
|
|
|
|
@Deprecated
|
|
@Override
|
|
public List<T> subList(int fromIndex, int toIndex) {
|
|
// TODO: implement
|
|
throw new RuntimeException("Not implemented!");
|
|
}
|
|
|
|
private <U> EntryReference<U> addToDatabase(U obj) {
|
|
EntryReference<U> ref;
|
|
try {
|
|
ref = db.add(obj);
|
|
} catch (IOException e) {
|
|
throw (NullPointerException) new NullPointerException().initCause(e);
|
|
}
|
|
return ref;
|
|
}
|
|
|
|
@Override
|
|
public int hashCode() {
|
|
final int prime = 31;
|
|
int result = 1;
|
|
result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
|
|
return result;
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
@Override
|
|
public boolean removeIf(Predicate<? super T> filter) {
|
|
Objects.requireNonNull(filter);
|
|
boolean result = false;
|
|
// Iterate through all chunks
|
|
for (int i = 0; i < chunks.size(); i++) {
|
|
synchronized (cachedChunkLock) {
|
|
if (cachedChunkNumber != i) {
|
|
prepareAccessToChunk(i);
|
|
}
|
|
if (cachedChunk.removeIf(filter)) {
|
|
result = true;
|
|
chunkSizes.set(cachedChunkNumber, cachedChunk.size());
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
@Override
|
|
public String toString() {
|
|
return "LightBigList{" +
|
|
"chunks=" + chunks +
|
|
", chunkSizes=" + chunkSizes +
|
|
", db=" + db +
|
|
'}';
|
|
}
|
|
|
|
@Override
|
|
public void save() {
|
|
if (this.cachedChunkRef != null) {
|
|
this.cachedChunkRef.save();
|
|
}
|
|
}
|
|
}
|