package org.warp.jcwdb; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import java.io.IOError; import java.io.IOException; import java.util.*; import java.util.function.Consumer; import java.util.function.Predicate; public class LightBigList implements LightList, Saveable { public static final int MAX_ELEMENTS_PER_CHUNK = 200000; public final LongArrayList chunks; public final IntArrayList chunkSizes; private final JCWDatabase db; private LightArrayList cachedChunk; private EntryReference> cachedChunkRef; private long cachedChunkIndex = -1; private int cachedChunkNumber = -1; private final Object cachedChunkLock = new Object(); /** * @param db Database reference */ public LightBigList(JCWDatabase db) { this.db = db; this.chunks = new LongArrayList(); this.chunkSizes = new IntArrayList(); } /** * @param db Database reference * @param elements Elements to add */ public LightBigList(JCWDatabase db, LongArrayList elements) { this.db = db; this.chunks = new LongArrayList(); this.chunkSizes = new IntArrayList(); elements.forEach((long element) -> { this.appendIndex(element); }); } public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) { this.db = db; this.chunks = chunks; this.chunkSizes = chunkSizes; if (this.chunks.size() > 0) { prepareAccessToChunk(0); } } private void prepareAccessToChunk(int chunkNumber) { if (this.cachedChunkRef != null) { this.cachedChunkRef.save(); } this.cachedChunkNumber = chunkNumber; this.cachedChunkIndex = this.chunks.getLong(chunkNumber); try { this.cachedChunkRef = db.get(this.cachedChunkIndex); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } this.cachedChunk = this.cachedChunkRef.getValueReadOnly(); } /** * Append an index to the first free chunk * @param elementIndex */ public void appendIndex(long elementIndex) { for (int i = 0; i < chunks.size(); i++) { final int chunkNumber = i; if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) { synchronized (cachedChunkLock) { if (cachedChunkNumber != i) { prepareAccessToChunk(i); } cachedChunk.appendIndex(elementIndex); chunkSizes.set(chunkNumber, cachedChunk.size()); return; } } } LightList newChunk = new LightArrayList<>(db); newChunk.appendIndex(elementIndex); long newChunkIndex; try { newChunkIndex = db.add(newChunk).getIndex(); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } chunks.add(newChunkIndex); chunkSizes.add(1); } /** * Get the elements count * @return the size of the list */ @Override public int size() { int size = 0; for (int chunkSize : this.chunkSizes) { size += chunkSize; } return size; } /** * * @return the count of chunks */ public int chunksCount() { return this.chunkSizes.size(); } /** * Check if the list is empty * @return true if the list is empty */ @Override public boolean isEmpty() { return this.size() <= 0; } @Override public boolean contains(Object o) { if (o != null) { for (long chunkIndex : chunks) { try { EntryReference> chunkRef = db.get(chunkIndex); LightArrayList chunk = chunkRef.getValueReadOnly(); if (chunk.contains(o)) { return true; } } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } } return false; } /** * Use iteratorReferences() */ @Deprecated @Override public Iterator iterator() { throw new RuntimeException("iterator() isn't implemented!"); } @Deprecated @Override public Iterator> iteratorReferences() { throw new RuntimeException("iteratorReferences() isn't implemented!"); } /** * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY * * @param action */ @Deprecated @Override public void forEach(Consumer action) { throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead"); } @Override public void forEachReference(Consumer> action) { Objects.requireNonNull(action); // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { synchronized (cachedChunkLock) { if (cachedChunkNumber != i) { prepareAccessToChunk(i); } cachedChunk.forEachReference(action); } } } /** * toArray() isn't implemented! DO NOT USE IT. * @return */ @SuppressWarnings("unchecked") @Deprecated @Override public T[] toArray() { if (true) { throw new RuntimeException("toArray() isn't implemented!"); } T[] result = (T[]) new Object[this.size()]; long currentOffset = 0; // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { final int currentChunkSize = chunkSizes.getInt(i); final long chunkStartOffset = currentOffset; currentOffset += currentChunkSize; // Get chunk index final long chunkIndex = chunks.getLong(i); try { EntryReference> chunkRef = db.get(chunkIndex); LightArrayList chunk = chunkRef.getValueReadOnly(); for (int i1 = 0; i1 < chunk.size(); i1++) { result[(int)(chunkStartOffset + i1)] = chunk.get(i); } } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } return result; } @SuppressWarnings("unchecked") @Override public T1[] toArray(T1[] a) { throw new RuntimeException("toArray() isn't implemented!"); } /** * Use addEntry(o) * @param o * @return */ @Deprecated @Override public boolean add(T o) { EntryReference ref = addEntry(o); return ref != null; } @Override public EntryReference addEntry(T o) { EntryReference ref = addToDatabase(o); appendIndex(ref.getIndex()); return ref; } @Override public boolean remove(Object o) { final int removeOffset = indexOf(o); return removeAt(removeOffset) != null; } @Override public boolean remove(EntryReference ref) { final int removeOffset = indexOfEntry(ref); return removeAt(removeOffset) != null; } private T removeAt(int removeOffset) { final VariableWrapper result = new VariableWrapper<>(null); long currentOffset = 0; if (removeOffset >= 0) { // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { final int currentChunkSize = chunkSizes.getInt(i); final long chunkStartOffset = currentOffset; currentOffset += currentChunkSize; // If the offset to remove is in the current chunk if (currentOffset > removeOffset) { // Get chunk index final long chunkIndex = chunks.getLong(i); // Get the offset relative to the current chunk final int relativeOffset = (int) (removeOffset - chunkStartOffset); if (relativeOffset < 0) { return null; } try { EntryReference> chunkRef = db.get(chunkIndex); chunkRef.editValue((chunk) -> { result.var = chunk.remove(relativeOffset); }); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } chunkSizes.set(removeOffset, currentChunkSize - 1); break; } } return result.var; } return null; } @Override public boolean containsAll(Collection c) { for (Object o : c) { int objIndex = indexOf(o); if (objIndex < 0) { return false; } } return true; } @SuppressWarnings("unchecked") @Override public boolean addAll(Collection c) { boolean result = false; for (Object o : c) { result |= add((T) o); } return result; } @SuppressWarnings("unchecked") @Override public boolean addAll(int index, Collection c) { boolean result = false; int delta = 0; for (Object o : c) { add(index + delta, (T) o); result = true; delta++; } return result; } @SuppressWarnings("unchecked") @Override public boolean removeAll(Collection c) { boolean result = false; for (Object o : c) { result |= remove((T) o); } return result; } @Override public boolean retainAll(Collection c) { boolean result = false; LongArrayList collectionHashes = new LongArrayList(); ObjectArrayList collection = new ObjectArrayList<>(); collection.addAll(c); for (Object o : c) { collectionHashes.add(db.calculateHash(o)); } for (int i = 0; i < chunks.size(); i++) { long hash = chunks.getLong(i); int positionInCollection = collectionHashes.indexOf(hash); if (positionInCollection == -1) { remove(collection.get(positionInCollection)); result = true; } } return result; } @Override public void clear() { chunks.clear(); chunkSizes.clear(); } /** * Use getReference or getReadOnlyValue */ @Deprecated @Override public T get(int index) { return getReadOnlyValue(index); } @SuppressWarnings("unchecked") public T getReadOnlyValue(int index) { try { return (T) db.get(chunks.getLong(index)).getValueReadOnly(); } catch (IOException e) { e.printStackTrace(); return null; } } @Override public EntryReference getReference(int index) { try { return db.get(chunks.getLong(index)).cast(); } catch (IOException e) { e.printStackTrace(); return null; } } @SuppressWarnings("unchecked") @Override public T set(int setOffset, final T element) { long nextChunkOffset = 0; VariableWrapper wrapper = new VariableWrapper<>(null); if (setOffset >= 0) { // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { final int currentChunkSize = chunkSizes.getInt(i); final long chunkStartOffset = nextChunkOffset; nextChunkOffset += currentChunkSize; // If the offset to remove is in the current chunk if (nextChunkOffset > setOffset) { // Get chunk index final long chunkIndex = chunks.getLong(i); // Get the offset relative to the current chunk final int relativeOffset = (int) (setOffset - chunkStartOffset); if (relativeOffset < 0) { throw new NullPointerException("Relative Offset < 0"); } try { EntryReference> chunkRef = db.get(chunkIndex); chunkRef.editValue((chunk) -> { chunk.set(relativeOffset, element); wrapper.var = element; }); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } break; } } return wrapper.var; } return null; } @Override public void add(int index, T element) { throw new RuntimeException("add() isn't implemented!"); } @SuppressWarnings("unchecked") @Override public T remove(int index) { return this.removeAt(index); } @Override public int indexOf(Object o) { EntryReference ref = addToDatabase(o).cast(); return indexOfEntry(ref); } @Override public int indexOfEntry(EntryReference ref) { int currentOffset = 0; // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { try { final int currentChunkSize = chunkSizes.getInt(i); // If the offset to remove is in the current chunk // Get chunk index final long chunkIndex = chunks.getLong(i); EntryReference> chunkRef = db.get(chunkIndex); final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref); if (foundIndex >= 0) { return currentOffset + foundIndex; } currentOffset += currentChunkSize; } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } return -1; } @Override public int lastIndexOf(Object o) { return lastIndexOfEntry(addToDatabase(o).cast()); } @Override public int lastIndexOfEntry(EntryReference ref) { int currentOffset = 0; // Iterate through all chunks for (int i = chunks.size() - 1; i >= 0; i--) { try { final int currentChunkSize = chunkSizes.getInt(i); // If the offset to remove is in the current chunk // Get chunk index final long chunkIndex = chunks.getLong(i); EntryReference> chunkRef = db.get(chunkIndex); final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref); if (foundIndex >= 0) { return currentOffset + foundIndex; } currentOffset += currentChunkSize; } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } return -1; } @Deprecated @Override public ListIterator listIterator() { // TODO: implement throw new RuntimeException("Not implemented!"); } @Deprecated @Override public ListIterator listIterator(int index) { // TODO: implement throw new RuntimeException("Not implemented!"); } @Deprecated @Override public List subList(int fromIndex, int toIndex) { // TODO: implement throw new RuntimeException("Not implemented!"); } private EntryReference addToDatabase(U obj) { EntryReference ref; try { ref = db.add(obj); } catch (IOException e) { throw (NullPointerException) new NullPointerException().initCause(e); } return ref; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((chunks == null) ? 0 : chunks.hashCode()); return result; } @SuppressWarnings("unchecked") @Override public boolean removeIf(Predicate filter) { Objects.requireNonNull(filter); boolean result = false; // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { synchronized (cachedChunkLock) { if (cachedChunkNumber != i) { prepareAccessToChunk(i); } if (cachedChunk.removeIf(filter)) { result = true; chunkSizes.set(cachedChunkNumber, cachedChunk.size()); } } } return result; } @Override public String toString() { return "LightBigList{" + "chunks=" + chunks + ", chunkSizes=" + chunkSizes + ", db=" + db + '}'; } @Override public void save() { if (this.cachedChunkRef != null) { this.cachedChunkRef.save(); } } }