package org.warp.jcwdb; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import java.io.IOException; import java.util.*; import java.util.function.Consumer; import java.util.function.Predicate; public class LightBigList implements LightList { public static final int MAX_ELEMENTS_PER_CHUNK = 10000; public final LongArrayList chunks; public final IntArrayList chunkSizes; private final transient JCWDatabase db; /** * @param db Database reference */ public LightBigList(JCWDatabase db) { this.db = db; this.chunks = new LongArrayList(); this.chunkSizes = new IntArrayList(); } /** * @param db Database reference * @param elements Elements to add */ public LightBigList(JCWDatabase db, LongArrayList elements) { this.db = db; this.chunks = new LongArrayList(); this.chunkSizes = new IntArrayList(); elements.forEach((long element) -> { this.appendIndex(element); }); } public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) { this.db = db; this.chunks = chunks; this.chunkSizes = chunkSizes; } /** * Append an index to the first free chunk * @param elementIndex */ public void appendIndex(long elementIndex) { for (int i = 0; i < chunks.size(); i++) { final int chunkNumber = i; if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) { try { final long chunkIndex = chunks.getLong(i); final EntryReference> chunkRef = db.get(chunkIndex); chunkRef.editValue((chunk) -> { chunk.appendIndex(elementIndex); chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1); }); return; } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } } LightList newChunk = new LightArrayList<>(db); newChunk.appendIndex(elementIndex); long newChunkIndex; try { newChunkIndex = db.add(newChunk).getIndex(); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } chunks.add(newChunkIndex); chunkSizes.add(1); } /** * Get the elements count * @return the size of the list */ @Override public int size() { int size = 0; for (int chunkSize : this.chunkSizes) { size += chunkSize; } return size; } /** * * @return the count of chunks */ public int chunksCount() { return this.chunkSizes.size(); } /** * Check if the list is empty * @return true if the list is empty */ @Override public boolean isEmpty() { return this.size() <= 0; } @Override public boolean contains(Object o) { if (o != null) { for (long chunkIndex : chunks) { try { EntryReference> chunkRef = db.get(chunkIndex); LightList chunk = chunkRef.getValueReadOnly(); if (chunk.contains(o)) { return true; } } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } } return false; } /** * Use iteratorReferences() */ @Deprecated @Override public Iterator iterator() { throw new RuntimeException("iterator() isn't implemented!"); } @Deprecated @Override public Iterator> iteratorReferences() { throw new RuntimeException("iteratorReferences() isn't implemented!"); } /** * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY * * @param action */ @Deprecated @Override public void forEach(Consumer action) { throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead"); } @Override public void forEachReference(Consumer> action) { Objects.requireNonNull(action); for (long chunkIndex : this.chunks) { try { EntryReference> chunkRef = db.get(chunkIndex); LightList chunk = chunkRef.getValueReadOnly(); chunk.forEachReference(action); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } } @SuppressWarnings("unchecked") @Override public T[] toArray() { throw new RuntimeException("toArray() isn't implemented!"); } @SuppressWarnings("unchecked") @Override public T1[] toArray(T1[] a) { throw new RuntimeException("toArray() isn't implemented!"); } /** * Use addEntry(o) * @param o * @return */ @Deprecated @Override public boolean add(T o) { EntryReference ref = addEntry(o); return ref != null; } @Override public EntryReference addEntry(T o) { EntryReference ref = addToDatabase(o); appendIndex(ref.getIndex()); return ref; } @Override public boolean remove(Object o) { final int removeOffset = indexOf(o); return removeAt(removeOffset) != null; } @Override public boolean remove(EntryReference ref) { final int removeOffset = indexOfEntry(ref); return removeAt(removeOffset) != null; } private T removeAt(int removeOffset) { final VariableWrapper result = new VariableWrapper<>(null); long currentOffset = 0; if (removeOffset >= 0) { // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { final int currentChunkSize = chunkSizes.getInt(i); final long chunkStartOffset = currentOffset; currentOffset += currentChunkSize; // If the offset to remove is in the current chunk if (currentOffset > removeOffset) { // Get chunk index final long chunkIndex = chunks.getLong(i); // Get the offset relative to the current chunk final int relativeOffset = (int) (removeOffset - chunkStartOffset); if (relativeOffset < 0) { return null; } try { EntryReference> chunkRef = db.get(chunkIndex); chunkRef.editValue((chunk) -> { result.var = chunk.remove(relativeOffset); }); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } chunkSizes.set(removeOffset, currentChunkSize - 1); break; } } return result.var; } return null; } @Override public boolean containsAll(Collection c) { for (Object o : c) { int objIndex = indexOf(o); if (objIndex < 0) { return false; } } return true; } @SuppressWarnings("unchecked") @Override public boolean addAll(Collection c) { boolean result = false; for (Object o : c) { result |= add((T) o); } return result; } @SuppressWarnings("unchecked") @Override public boolean addAll(int index, Collection c) { boolean result = false; int delta = 0; for (Object o : c) { add(index + delta, (T) o); result = true; delta++; } return result; } @SuppressWarnings("unchecked") @Override public boolean removeAll(Collection c) { boolean result = false; for (Object o : c) { result |= remove((T) o); } return result; } @Override public boolean retainAll(Collection c) { boolean result = false; LongArrayList collectionHashes = new LongArrayList(); ObjectArrayList collection = new ObjectArrayList<>(); collection.addAll(c); for (Object o : c) { collectionHashes.add(db.calculateHash(o)); } for (int i = 0; i < chunks.size(); i++) { long hash = chunks.getLong(i); int positionInCollection = collectionHashes.indexOf(hash); if (positionInCollection == -1) { remove(collection.get(positionInCollection)); result = true; } } return result; } @Override public void clear() { chunks.clear(); chunkSizes.clear(); } /** * Use getReference or getReadOnlyValue */ @Deprecated @Override public T get(int index) { return getReadOnlyValue(index); } @SuppressWarnings("unchecked") public T getReadOnlyValue(int index) { try { return (T) db.get(chunks.getLong(index)).getValueReadOnly(); } catch (IOException e) { e.printStackTrace(); return null; } } @Override public EntryReference getReference(int index) { try { return db.get(chunks.getLong(index)).cast(); } catch (IOException e) { e.printStackTrace(); return null; } } @SuppressWarnings("unchecked") @Override public T set(int setOffset, final T element) { long nextChunkOffset = 0; VariableWrapper wrapper = new VariableWrapper<>(null); if (setOffset >= 0) { // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { final int currentChunkSize = chunkSizes.getInt(i); final long chunkStartOffset = nextChunkOffset; nextChunkOffset += currentChunkSize; // If the offset to remove is in the current chunk if (nextChunkOffset > setOffset) { // Get chunk index final long chunkIndex = chunks.getLong(i); // Get the offset relative to the current chunk final int relativeOffset = (int) (setOffset - chunkStartOffset); if (relativeOffset < 0) { throw new NullPointerException("Relative Offset < 0"); } try { EntryReference> chunkRef = db.get(chunkIndex); chunkRef.editValue((chunk) -> { chunk.set(relativeOffset, element); wrapper.var = element; }); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } break; } } return wrapper.var; } return null; } @Override public void add(int index, T element) { throw new RuntimeException("add() isn't implemented!"); } @SuppressWarnings("unchecked") @Override public T remove(int index) { return this.removeAt(index); } @Override public int indexOf(Object o) { EntryReference ref = addToDatabase(o).cast(); return indexOfEntry(ref); } @Override public int indexOfEntry(EntryReference ref) { int currentOffset = 0; // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { try { final int currentChunkSize = chunkSizes.getInt(i); // If the offset to remove is in the current chunk // Get chunk index final long chunkIndex = chunks.getLong(i); EntryReference> chunkRef = db.get(chunkIndex); final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref); if (foundIndex >= 0) { return currentOffset + foundIndex; } currentOffset += currentChunkSize; } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } return -1; } @Override public int lastIndexOf(Object o) { return lastIndexOfEntry(addToDatabase(o).cast()); } @Override public int lastIndexOfEntry(EntryReference ref) { int currentOffset = 0; // Iterate through all chunks for (int i = chunks.size() - 1; i >= 0; i--) { try { final int currentChunkSize = chunkSizes.getInt(i); // If the offset to remove is in the current chunk // Get chunk index final long chunkIndex = chunks.getLong(i); EntryReference> chunkRef = db.get(chunkIndex); final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref); if (foundIndex >= 0) { return currentOffset + foundIndex; } currentOffset += currentChunkSize; } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } return -1; } @Deprecated @Override public ListIterator listIterator() { // TODO: implement throw new RuntimeException("Not implemented!"); } @Deprecated @Override public ListIterator listIterator(int index) { // TODO: implement throw new RuntimeException("Not implemented!"); } @Deprecated @Override public List subList(int fromIndex, int toIndex) { // TODO: implement throw new RuntimeException("Not implemented!"); } private EntryReference addToDatabase(U obj) { EntryReference ref; try { ref = db.add(obj); } catch (IOException e) { throw (NullPointerException) new NullPointerException().initCause(e); } return ref; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((chunks == null) ? 0 : chunks.hashCode()); return result; } @SuppressWarnings("unchecked") @Override public boolean removeIf(Predicate filter) { Objects.requireNonNull(filter); final VariableWrapper result = new VariableWrapper(false); // Iterate through all chunks for (int i = 0; i < chunks.size(); i++) { try { final int chunkOffset = i; // Get chunk index final long chunkIndex = chunks.getLong(i); EntryReference> chunkRef = db.get(chunkIndex); chunkRef.editValue((chunk) -> { boolean removed = chunk.removeIf(filter); if (removed) { result.var = true; chunkSizes.set(chunkOffset, chunk.size()); } }); } catch (IOException ex) { throw (NullPointerException) new NullPointerException().initCause(ex); } } return result.var; } }