Merge branch 'better'
This commit is contained in:
commit
a12decf796
|
|
@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
|
|||
|
||||
public class Cleaner {
|
||||
|
||||
private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes
|
||||
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
|
||||
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
|
||||
private static final double NORMAL_REMOVED_ITEMS = 1000l;
|
||||
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
|
||||
|
|
@ -63,7 +63,10 @@ public class Cleaner {
|
|||
try {
|
||||
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
|
||||
sleepFor(sleepInterval);
|
||||
final long time1 = System.currentTimeMillis();
|
||||
final double removedItems = clean();
|
||||
final long time2 = System.currentTimeMillis();
|
||||
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
|
||||
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
|
||||
|
||||
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output;
|
|||
|
||||
import net.openhft.hashing.LongHashFunction;
|
||||
|
||||
public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
|
||||
public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
|
||||
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
|
||||
private static final Kryo kryo = new Kryo();
|
||||
static {
|
||||
|
|
@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
|
|||
tmpO.close();
|
||||
return hash;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public <U> void registerClass(Class<U> clazz, int id) {
|
||||
if (id >= Integer.MAX_VALUE - 100) {
|
||||
throw new IndexOutOfBoundsException();
|
||||
}
|
||||
kryo.register(clazz, id + 100);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,38 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
|
||||
public class DBLightListParser<T> extends DBTypeParserImpl<LightList<T>> {
|
||||
public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T>> {
|
||||
private final JCWDatabase db;
|
||||
|
||||
public DBLightListParser(JCWDatabase db) {
|
||||
public DBLightArrayListParser(JCWDatabase db) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
public DBReader<LightList<T>> getReader() {
|
||||
|
||||
public DBReader<LightArrayList<T>> getReader() {
|
||||
return (i, size) -> {
|
||||
LongArrayList internalList = new LongArrayList();
|
||||
long max = size / Long.BYTES;
|
||||
for (int item = 0; item < max; item++){
|
||||
for (int item = 0; item < max; item++) {
|
||||
long itm = i.readLong();
|
||||
internalList.add(itm);
|
||||
}
|
||||
return new LightList<T>(db, internalList);
|
||||
return new LightArrayList<T>(db, internalList);
|
||||
};
|
||||
}
|
||||
|
||||
public DBDataOutput<LightList<T>> getWriter(final LightList<T> value) {
|
||||
public DBDataOutput<LightArrayList<T>> getWriter(final LightArrayList<T> value) {
|
||||
final int elementsCount = value.size();
|
||||
return DBDataOutput.create((o) -> {
|
||||
LongArrayList list = value.internalList;
|
||||
for (int i = 0; i < elementsCount; i++) {
|
||||
o.writeLong(list.getLong(i));
|
||||
}
|
||||
}, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value));
|
||||
}, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long calculateHash(LightList<T> value) {
|
||||
public long calculateHash(LightArrayList<T> value) {
|
||||
return value.internalList.hashCode();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.ints.IntArrayList;
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
|
||||
public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
|
||||
private final JCWDatabase db;
|
||||
|
||||
public DBLightBigListParser(JCWDatabase db) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
public DBReader<LightBigList<T>> getReader() {
|
||||
return (i, size) -> {
|
||||
LongArrayList chunks = new LongArrayList();
|
||||
IntArrayList chunkSizes = new IntArrayList();
|
||||
long max = size / (Long.BYTES + Integer.BYTES);
|
||||
for (int item = 0; item < max; item++) {
|
||||
long itm = i.readLong();
|
||||
int itm2 = i.readInt();
|
||||
chunks.add(itm);
|
||||
chunkSizes.add(itm2);
|
||||
}
|
||||
return new LightBigList<T>(db, chunks, chunkSizes);
|
||||
};
|
||||
}
|
||||
|
||||
public DBDataOutput<LightBigList<T>> getWriter(final LightBigList<T> value) {
|
||||
final int elementsCount = value.chunksCount();
|
||||
return DBDataOutput.create((o) -> {
|
||||
LongArrayList list = value.chunks;
|
||||
IntArrayList list2 = value.chunkSizes;
|
||||
for (int i = 0; i < elementsCount; i++) {
|
||||
o.writeLong(list.getLong(i));
|
||||
o.writeInt(list2.getInt(i));
|
||||
}
|
||||
}, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long calculateHash(LightBigList<T> value) {
|
||||
return value.chunks.hashCode();
|
||||
}
|
||||
}
|
||||
|
|
@ -11,12 +11,14 @@ public class DBStandardTypes {
|
|||
public static final int DOUBLE = STD| 6;
|
||||
public static final int STRING = STD| 7;
|
||||
public static final int BYTE_ARRAY = STD| 8;
|
||||
public static final int LIGHT_LIST = STD| 9;
|
||||
public static final int GENERIC_OBJECT = STD| 10;
|
||||
public static final int LIGHT_LIST_ARRAY = STD| 9;
|
||||
public static final int LIGHT_LIST_BIG = STD| 10;
|
||||
public static final int GENERIC_OBJECT = STD| 11;
|
||||
|
||||
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
|
||||
typesManager.registerType(String.class, STRING, new DBStringParser());
|
||||
typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db));
|
||||
typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db));
|
||||
typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db));
|
||||
typesManager.registerTypeFallback(new DBGenericObjectParser());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
|
||||
public <U> void registerClass(Class<U> clazz, int type);
|
||||
}
|
||||
|
|
@ -86,7 +86,7 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
* @param editFunction
|
||||
* @throws IOException
|
||||
*/
|
||||
public void editValue(BiFunction<T, Saveable, T> editFunction) throws IOException {
|
||||
public void editValue(BiFunction<T, Saveable, T> editFunction) {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
this.value = editFunction.apply(this.value, this);
|
||||
|
|
@ -99,7 +99,20 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
* @param editFunction
|
||||
* @throws IOException
|
||||
*/
|
||||
public void editValue(BiConsumer<T, Saveable> editFunction) throws IOException {
|
||||
public void editValue(Function<T, T> editFunction) {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
this.value = editFunction.apply(this.value);
|
||||
this.save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reccomended way to edit the value
|
||||
* @param editFunction
|
||||
* @throws IOException
|
||||
*/
|
||||
public void editValue(BiConsumer<T, Saveable> editFunction) {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
editFunction.accept(this.value, this);
|
||||
|
|
@ -107,12 +120,25 @@ public class EntryReference<T> implements Castable, Saveable {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reccomended way to edit the value
|
||||
* @param editFunction
|
||||
* @throws IOException
|
||||
*/
|
||||
public void editValue(Consumer<T> editFunction) {
|
||||
synchronized(accessLock) {
|
||||
load();
|
||||
editFunction.accept(this.value);
|
||||
this.save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Substitute the old value with a new one
|
||||
* @param val
|
||||
* @throws IOException
|
||||
*/
|
||||
public void setValue(T val) throws IOException {
|
||||
public void setValue(T val) {
|
||||
synchronized(accessLock) {
|
||||
this.loaded = true;
|
||||
this.value = val;
|
||||
|
|
|
|||
|
|
@ -1,62 +1,116 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
import it.unimi.dsi.fastutil.longs.Long2IntMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.util.*;
|
||||
|
||||
public class FileAllocator implements AutoCloseable {
|
||||
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
|
||||
|
||||
private final SeekableByteChannel dataFileChannel;
|
||||
private volatile long allocableOffset;
|
||||
private volatile long fileSize;
|
||||
private volatile boolean closed;
|
||||
private final Object closeLock = new Object();
|
||||
private final Object allocateLock = new Object();
|
||||
|
||||
/**
|
||||
* index -> free space size
|
||||
*/
|
||||
private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
|
||||
|
||||
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
|
||||
this.dataFileChannel = dataFileChannel;
|
||||
this.allocableOffset = this.dataFileChannel.size();
|
||||
this.fileSize = this.dataFileChannel.size();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* TODO: not implemented
|
||||
*
|
||||
* @param size
|
||||
* @return offset
|
||||
*/
|
||||
public long allocate(int size) {
|
||||
checkClosed();
|
||||
synchronized (allocateLock) {
|
||||
long allocatedOffset = allocableOffset;
|
||||
allocableOffset += size;
|
||||
return allocatedOffset;
|
||||
long offset = allocateIntoUnusedParts(size);
|
||||
if (offset == -1) {
|
||||
return allocateToEnd(size);
|
||||
} else {
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void close() throws IOException {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
synchronized (closeLock) {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
}
|
||||
private long allocateIntoUnusedParts(int size) {
|
||||
ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
|
||||
long holeOffset = -1;
|
||||
int holeSize = 0;
|
||||
while (it.hasNext()) {
|
||||
Long2IntMap.Entry entry = it.next();
|
||||
int currentHoleSize = entry.getIntValue();
|
||||
if (currentHoleSize < size) {
|
||||
freeBytes.remove(holeOffset);
|
||||
if (holeSize > size) {
|
||||
freeBytes.put(holeOffset + size, holeSize - size);
|
||||
}
|
||||
break;
|
||||
}
|
||||
holeOffset = entry.getLongKey();
|
||||
holeSize = currentHoleSize;
|
||||
}
|
||||
return holeOffset;
|
||||
}
|
||||
|
||||
|
||||
private long allocateToEnd(int size) {
|
||||
long allocatedOffset = fileSize;
|
||||
fileSize += size;
|
||||
return allocatedOffset;
|
||||
}
|
||||
|
||||
|
||||
public void close() throws IOException {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
synchronized (closeLock) {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Frees the unused bytes
|
||||
*
|
||||
* @param startPosition
|
||||
* @param length
|
||||
*/
|
||||
public void markFree(long startPosition, int length) {
|
||||
checkClosed();
|
||||
// TODO: advanced feature, not implemented.
|
||||
|
||||
if (freeBytes.containsKey(startPosition + length)) {
|
||||
int secondLength = freeBytes.remove(startPosition + length);
|
||||
freeBytes.put(startPosition, length + secondLength);
|
||||
} else {
|
||||
boolean addedToList = false;
|
||||
for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
|
||||
if (entry.getLongKey() + entry.getIntValue() == startPosition) {
|
||||
freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
|
||||
addedToList = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!addedToList) {
|
||||
freeBytes.put(startPosition, length);
|
||||
}
|
||||
}
|
||||
|
||||
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
|
||||
freeBytes.remove(freeBytes.lastLongKey());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Iterator;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class FileIndexManager implements IndexManager {
|
||||
|
|
@ -81,7 +82,12 @@ public class FileIndexManager implements IndexManager {
|
|||
final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
|
||||
if (indexDetails == null || indexDetails.getSize() < dataSize) {
|
||||
// Allocate new space
|
||||
return allocateAndWrite(index, data);
|
||||
IndexDetails newDetails = allocateAndWrite(index, data);
|
||||
if (indexDetails != null) {
|
||||
// Mark free the old bytes
|
||||
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
|
||||
}
|
||||
return newDetails;
|
||||
} else {
|
||||
// Check if size changed
|
||||
if (dataSize < indexDetails.getSize()) {
|
||||
|
|
@ -233,7 +239,7 @@ public class FileIndexManager implements IndexManager {
|
|||
* @param details
|
||||
*/
|
||||
private void editIndex(long index, IndexDetails details) {
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
synchronized (indicesMapsAccessLock) {// FIXXXX main3
|
||||
loadedIndices.put(index, details);
|
||||
dirtyLoadedIndices.add(index);
|
||||
}
|
||||
|
|
@ -264,7 +270,8 @@ public class FileIndexManager implements IndexManager {
|
|||
return null;
|
||||
}
|
||||
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
|
||||
synchronized (metadataByteBufferLock) {
|
||||
IndexDetails indexDetails = null;
|
||||
synchronized (metadataByteBufferLock) {// FIXXXX main2
|
||||
metadataByteBuffer.rewind();
|
||||
currentMetadataFileChannel.read(metadataByteBuffer);
|
||||
metadataByteBuffer.rewind();
|
||||
|
|
@ -277,12 +284,15 @@ public class FileIndexManager implements IndexManager {
|
|||
final int size = metadataByteBuffer.getInt();
|
||||
final int type = metadataByteBuffer.getInt();
|
||||
final long hash = metadataByteBuffer.getLong();
|
||||
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
|
||||
editIndex(index, indexDetails);
|
||||
return indexDetails;
|
||||
indexDetails = new IndexDetails(offset, size, type, hash);
|
||||
}
|
||||
}
|
||||
|
||||
if (indexDetails != null) {
|
||||
editIndex(index, indexDetails);
|
||||
return indexDetails;
|
||||
}
|
||||
|
||||
// No results found. Returning null
|
||||
return null;
|
||||
}
|
||||
|
|
@ -308,31 +318,15 @@ public class FileIndexManager implements IndexManager {
|
|||
}
|
||||
|
||||
// Update indices metadata
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
long lastIndex = -2;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
for (long index : dirtyLoadedIndices) {
|
||||
IndexDetails indexDetails = loadedIndices.get(index);
|
||||
if (index - lastIndex != 1) {
|
||||
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
|
||||
}
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
lastIndex = index;
|
||||
}
|
||||
}
|
||||
flushAllIndices();
|
||||
|
||||
// Remove removed indices
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
for (long index : removedIndices) {
|
||||
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
|
||||
eraseIndexDetails(metadata);
|
||||
}
|
||||
}
|
||||
removeRemovedIndices();
|
||||
fileAllocator.close();
|
||||
}
|
||||
|
||||
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
|
||||
synchronized (metadataByteBufferLock) {
|
||||
synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
|
||||
final int size = indexDetails.getSize();
|
||||
final int type = indexDetails.getType();
|
||||
final long offset = indexDetails.getOffset();
|
||||
|
|
@ -366,19 +360,63 @@ public class FileIndexManager implements IndexManager {
|
|||
|
||||
@Override
|
||||
public long clean() {
|
||||
return cleanExtraIndices();
|
||||
long cleaned = 0;
|
||||
try {
|
||||
cleaned += flushAllIndices();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
try {
|
||||
cleaned += removeRemovedIndices();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
cleaned += cleanExtraIndices();
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
private long flushAllIndices() throws IOException {
|
||||
long flushedIndices = 0;
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
long lastIndex = -2;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
for (long index : dirtyLoadedIndices) {
|
||||
IndexDetails indexDetails = loadedIndices.get(index);
|
||||
if (index - lastIndex != 1) {
|
||||
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
|
||||
}
|
||||
writeIndexDetails(metadata, indexDetails);
|
||||
lastIndex = index;
|
||||
flushedIndices++;
|
||||
}
|
||||
dirtyLoadedIndices.clear();
|
||||
}
|
||||
return flushedIndices;
|
||||
}
|
||||
|
||||
private long removeRemovedIndices() throws IOException {
|
||||
SeekableByteChannel metadata = metadataFileChannel;
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
long removed = this.removedIndices.size();
|
||||
for (long index : this.removedIndices) {
|
||||
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
|
||||
eraseIndexDetails(metadata);
|
||||
}
|
||||
this.removedIndices.clear();
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
||||
private long cleanExtraIndices() {
|
||||
long removedIndices = 0;
|
||||
LongArrayList toUnload = new LongArrayList();
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
|
||||
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
|
||||
long count = loadedIndices.size();
|
||||
LongIterator it = loadedIndices.keySet().iterator();
|
||||
while (it.hasNext()) {
|
||||
long loadedIndex = it.nextLong();
|
||||
if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
|
||||
if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
|
||||
break;
|
||||
}
|
||||
toUnload.add(loadedIndex);
|
||||
|
|
|
|||
|
|
@ -1,36 +1,24 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.ref.WeakReference;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectIterator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
public final static long MAX_LOADED_REFERENCES = 1000;
|
||||
public final static long MAX_LOADED_INDICES = 10000;
|
||||
|
||||
|
||||
private final TypesManager typesManager;
|
||||
private final MixedIndexDatabase indices;
|
||||
private final Cleaner databaseCleaner;
|
||||
private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
|
||||
private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
|
||||
private volatile boolean closed;
|
||||
private final Object closeLock = new Object();
|
||||
private final Object indicesAccessLock = new Object();
|
||||
private final Object referencesAccessLock = new Object();
|
||||
|
||||
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
|
||||
this.typesManager = new TypesManager(this);
|
||||
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
|
||||
this.references = new Long2ObjectLinkedOpenHashMap<>();
|
||||
this.indices = new MixedIndexDatabase(dataFile, metadataFile);
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
||||
try {
|
||||
JCWDatabase.this.close();
|
||||
|
|
@ -39,94 +27,84 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
|||
}
|
||||
}));
|
||||
this.databaseCleaner = new Cleaner(this);
|
||||
|
||||
//this.databaseCleaner.start();
|
||||
|
||||
this.databaseCleaner.start();
|
||||
}
|
||||
|
||||
public <T> EntryReference<LightList<T>> getRoot() throws IOException {
|
||||
return getRoot(Object.class).cast();
|
||||
}
|
||||
|
||||
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
|
||||
checkClosed();
|
||||
if (exists(0)) {
|
||||
return get(0);
|
||||
} else {
|
||||
LightList<T> newRoot = new LightList<T>(this, new LongArrayList());
|
||||
LightList<T> newRoot = new LightArrayList<>(this);
|
||||
return set(0, newRoot);
|
||||
}
|
||||
}
|
||||
|
||||
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
|
||||
return getRoot().cast();
|
||||
}
|
||||
|
||||
public <T> EntryReference<T> get(long index) throws IOException {
|
||||
checkClosed();
|
||||
synchronized (referencesAccessLock) {
|
||||
WeakReference<EntryReference<?>> refRef = this.references.getOrDefault(index, null);
|
||||
EntryReference<T> ref;
|
||||
if (refRef == null || (ref = (EntryReference<T>) refRef.get()) == null) {
|
||||
int type;
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
type = this.indices.getType(index);
|
||||
hash = this.indices.getHash(index);
|
||||
}
|
||||
DBTypeParser<T> typeParser = this.typesManager.get(type);
|
||||
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
|
||||
refRef = new WeakReference<>(ref);
|
||||
this.references.put(index, refRef);
|
||||
}
|
||||
return ref;
|
||||
int type;
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
type = this.indices.getType(index);
|
||||
hash = this.indices.getHash(index);
|
||||
}
|
||||
DBTypeParser<T> typeParser = this.typesManager.get(type);
|
||||
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
|
||||
}
|
||||
|
||||
protected <T> EntryReference<T> add(T value) throws IOException {
|
||||
checkClosed();
|
||||
synchronized (referencesAccessLock) {
|
||||
EntryReference<T> ref;
|
||||
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
|
||||
long index;
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
index = indices.add(typeParser.getWriter(value));
|
||||
hash = indices.getHash(index);
|
||||
}
|
||||
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
|
||||
this.references.put(index, new WeakReference<>(ref));
|
||||
return ref;
|
||||
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
|
||||
long index;
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
index = indices.add(typeParser.getWriter(value));
|
||||
hash = indices.getHash(index);
|
||||
}
|
||||
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
|
||||
}
|
||||
|
||||
protected boolean exists(long index) {
|
||||
checkClosed();
|
||||
synchronized (referencesAccessLock) {
|
||||
synchronized (indicesAccessLock) {
|
||||
return this.references.containsKey(index) || this.indices.has(index);
|
||||
}
|
||||
synchronized (indicesAccessLock) {
|
||||
return this.indices.has(index);
|
||||
}
|
||||
}
|
||||
|
||||
protected <T> EntryReference<T> set(long index, T value) throws IOException {
|
||||
checkClosed();
|
||||
synchronized (referencesAccessLock) {
|
||||
EntryReference<T> ref;
|
||||
if (exists(index)) {
|
||||
ref = get(index);
|
||||
ref.setValue(value);
|
||||
return ref;
|
||||
} else {
|
||||
@SuppressWarnings("unchecked")
|
||||
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
|
||||
hash = returnedDetails.getHash();
|
||||
}
|
||||
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
|
||||
this.references.put(index, new WeakReference<EntryReference<?>>(ref));
|
||||
return ref;
|
||||
EntryReference<T> ref;
|
||||
if (exists(index)) {
|
||||
ref = get(index);
|
||||
ref.setValue(value);
|
||||
return ref;
|
||||
} else {
|
||||
@SuppressWarnings("unchecked")
|
||||
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
|
||||
hash = returnedDetails.getHash();
|
||||
}
|
||||
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
|
||||
}
|
||||
}
|
||||
|
||||
public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
|
||||
final int addition = 0xEFFF8000;
|
||||
int extendedType = addition | (type & 0x7FFF);
|
||||
typesManager.registerType(clazz, extendedType, parser);
|
||||
}
|
||||
|
||||
public <U> void registerClass(Class<U> clazz, int type) {
|
||||
typesManager.registerGenericClass(clazz, type);
|
||||
}
|
||||
|
||||
public boolean isOpen() {
|
||||
return !closed;
|
||||
}
|
||||
|
|
@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
|||
|
||||
this.databaseCleaner.stop();
|
||||
|
||||
synchronized (referencesAccessLock) {
|
||||
ObjectIterator<WeakReference<EntryReference<?>>> iterator = references.values().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
WeakReference<EntryReference<?>> referenceRef = iterator.next();
|
||||
EntryReference<?> reference = referenceRef.get();
|
||||
if (reference != null) {
|
||||
reference.close();
|
||||
iterator.remove();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
synchronized (indicesAccessLock) {
|
||||
this.indices.close();
|
||||
}
|
||||
|
|
@ -171,56 +137,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
|||
|
||||
@Override
|
||||
public long clean() {
|
||||
long removedItems = cleanEmptyReferences()
|
||||
+ cleanExtraReferences()
|
||||
+ indices.clean();
|
||||
long removedItems = indices.clean();
|
||||
return removedItems;
|
||||
}
|
||||
|
||||
|
||||
private long cleanEmptyReferences() {
|
||||
long removed = 0;
|
||||
synchronized(referencesAccessLock) {
|
||||
ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
|
||||
if (entry.getValue().get() == null) {
|
||||
iterator.remove();
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
private long cleanExtraReferences() {
|
||||
long removedReferences = 0;
|
||||
synchronized(referencesAccessLock) {
|
||||
if (references.size() > MAX_LOADED_REFERENCES) {
|
||||
long count = 0;
|
||||
ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
|
||||
if (count > MAX_LOADED_REFERENCES * 3l / 2l) {
|
||||
WeakReference<EntryReference<?>> weakRef = entry.getValue();
|
||||
EntryReference<?> ref = weakRef.get();
|
||||
if (ref != null) {
|
||||
try {
|
||||
ref.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
iterator.remove();
|
||||
removedReferences++;
|
||||
} else {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return removedReferences;
|
||||
}
|
||||
|
||||
public class EntryReferenceTools {
|
||||
private EntryReferenceTools() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,445 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
public class LightArrayList<T> implements LightList<T> {
|
||||
|
||||
public final LongArrayList internalList;
|
||||
private final transient JCWDatabase db;
|
||||
|
||||
/**
|
||||
* @param db Database reference
|
||||
*/
|
||||
public LightArrayList(JCWDatabase db) {
|
||||
this.db = db;
|
||||
this.internalList = new LongArrayList();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param db Database reference
|
||||
* @param elements Elements to add
|
||||
*/
|
||||
public LightArrayList(JCWDatabase db, LongArrayList elements) {
|
||||
this.db = db;
|
||||
this.internalList = new LongArrayList(elements);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
return internalList.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return internalList.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean contains(Object o) {
|
||||
if (o != null) {
|
||||
for (long element : internalList) {
|
||||
EntryReference<T> ref = null;
|
||||
try {
|
||||
ref = db.get(element);
|
||||
if (o.equals(ref.getValueReadOnly())) {
|
||||
return true;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use iteratorReferences()
|
||||
*/
|
||||
@Deprecated
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public Iterator<T> iterator() {
|
||||
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
|
||||
final ArrayList<T> elements = new ArrayList<>();
|
||||
for (long element : internalList) {
|
||||
try {
|
||||
elements.add((T) db.get(element).getValueReadOnly());
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements.iterator();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public Iterator<EntryReference<T>> iteratorReferences() {
|
||||
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
|
||||
for (long element : internalList) {
|
||||
try {
|
||||
elements.add(db.get(element));
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements.iterator();
|
||||
}
|
||||
|
||||
/**
|
||||
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
|
||||
*
|
||||
* @param action
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public void forEach(Consumer<? super T> action) {
|
||||
Objects.requireNonNull(action);
|
||||
for (T t : this) {
|
||||
action.accept(t);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void forEachReference(Consumer<? super EntryReference<T>> action) {
|
||||
Objects.requireNonNull(action);
|
||||
for (long index : this.internalList) {
|
||||
try {
|
||||
action.accept(db.get(index));
|
||||
} catch (IOException e) {
|
||||
throw (RuntimeException) new RuntimeException().initCause(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
final T[] elements = (T[]) new Objects[internalList.size()];
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
try {
|
||||
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public <T1> T1[] toArray(T1[] a) {
|
||||
final T1[] elements = (T1[]) new Objects[internalList.size()];
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
try {
|
||||
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean add(T o) {
|
||||
EntryReference<T> ref = addEntry(o);
|
||||
return ref != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryReference<T> addEntry(T o) {
|
||||
EntryReference<T> ref = addToDatabase(o);
|
||||
if (internalList.add(ref.getIndex())) {
|
||||
return ref;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(Object o) {
|
||||
int removeIndex = indexOf(o);
|
||||
if (removeIndex >= 0) {
|
||||
internalList.removeLong(removeIndex);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(EntryReference<T> ref) {
|
||||
int removeIndex = indexOfEntry(ref);
|
||||
if (removeIndex >= 0) {
|
||||
internalList.removeLong(removeIndex);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsAll(Collection<?> c) {
|
||||
for (Object o : c) {
|
||||
int objIndex = indexOf(o);
|
||||
if (objIndex < 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean addAll(Collection<? extends T> c) {
|
||||
boolean result = false;
|
||||
for (Object o : c) {
|
||||
result |= add((T) o);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean addAll(int index, Collection<? extends T> c) {
|
||||
boolean result = false;
|
||||
int delta = 0;
|
||||
for (Object o : c) {
|
||||
add(index + delta, (T) o);
|
||||
result = true;
|
||||
delta++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean removeAll(Collection<?> c) {
|
||||
boolean result = false;
|
||||
for (Object o : c) {
|
||||
result |= remove((T) o);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean retainAll(Collection<?> c) {
|
||||
boolean result = false;
|
||||
LongArrayList collectionHashes = new LongArrayList();
|
||||
ObjectArrayList<Object> collection = new ObjectArrayList<>();
|
||||
collection.addAll(c);
|
||||
for (Object o : c) {
|
||||
collectionHashes.add(db.calculateHash(o));
|
||||
}
|
||||
for (int i = 0; i < internalList.size(); i++) {
|
||||
long hash = internalList.getLong(i);
|
||||
int positionInCollection = collectionHashes.indexOf(hash);
|
||||
if (positionInCollection == -1) {
|
||||
remove(collection.get(positionInCollection));
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
internalList.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Use getReference or getReadOnlyValue
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public T get(int index) {
|
||||
return getReadOnlyValue(index);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public T getReadOnlyValue(int index) {
|
||||
try {
|
||||
return (T) db.get(internalList.getLong(index)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryReference<T> getReference(int index) {
|
||||
try {
|
||||
return db.get(internalList.getLong(index)).cast();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T set(int index, T element) {
|
||||
EntryReference<T> ref = addToDatabase(element);
|
||||
long oldIndex = internalList.set(index, ref.getIndex());
|
||||
try {
|
||||
ref.close();
|
||||
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(int index, T element) {
|
||||
EntryReference<T> ref = addToDatabase(element);
|
||||
internalList.add(index, ref.getIndex());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T remove(int index) {
|
||||
long oldIndex = internalList.removeLong(index);
|
||||
try {
|
||||
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int indexOf(Object o) {
|
||||
EntryReference<?> ref = addToDatabase(o);
|
||||
long objToRemoveHash = ref.calculateHash();
|
||||
LongArrayList hashes = new LongArrayList();
|
||||
|
||||
|
||||
for (int i = 0; i < hashes.size(); i++) {
|
||||
long hash = hashes.getLong(i);
|
||||
if (objToRemoveHash == hash) {
|
||||
try {
|
||||
if (ref.equals(db.get(internalList.getLong(i)))) {
|
||||
return i;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int indexOfEntry(EntryReference<T> ref) {
|
||||
for (int i = 0; i < internalList.size(); i++) {
|
||||
long index = internalList.getLong(i);
|
||||
try {
|
||||
EntryReference<?> ref2 = db.get(index);
|
||||
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
|
||||
return i;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendIndex(long elementIndex) {
|
||||
internalList.add(elementIndex);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lastIndexOf(Object o) {
|
||||
EntryReference<T> ref = addToDatabase(o).cast();
|
||||
return lastIndexOfEntry(ref);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lastIndexOfEntry(EntryReference<T> ref) {
|
||||
long objToRemoveHash = ref.calculateHash();
|
||||
|
||||
int lastValue = -1;
|
||||
|
||||
for (int i = 0; i < internalList.size(); i++) {
|
||||
long index2 = internalList.getLong(i);
|
||||
try {
|
||||
EntryReference<?> ref2 = db.get(index2);
|
||||
if (objToRemoveHash == ref2.calculateHash()) {
|
||||
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
|
||||
lastValue = i;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public ListIterator<T> listIterator() {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public ListIterator<T> listIterator(int index) {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public List<T> subList(int fromIndex, int toIndex) {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
private <U> EntryReference<U> addToDatabase(U obj) {
|
||||
EntryReference<U> ref;
|
||||
try {
|
||||
ref = db.add(obj);
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
return ref;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean removeIf(Predicate<? super T> filter) {
|
||||
Objects.requireNonNull(filter);
|
||||
boolean removed = false;
|
||||
for (int i = 0; i < internalList.size(); ) {
|
||||
T obj;
|
||||
try {
|
||||
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
if (filter.test(obj)) {
|
||||
internalList.removeLong(i);
|
||||
removed = true;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,521 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.ints.IntArrayList;
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
public class LightBigList<T> implements LightList<T> {
|
||||
|
||||
public static final int MAX_ELEMENTS_PER_CHUNK = 10000;
|
||||
|
||||
public final LongArrayList chunks;
|
||||
public final IntArrayList chunkSizes;
|
||||
private final transient JCWDatabase db;
|
||||
|
||||
/**
|
||||
* @param db Database reference
|
||||
*/
|
||||
public LightBigList(JCWDatabase db) {
|
||||
this.db = db;
|
||||
this.chunks = new LongArrayList();
|
||||
this.chunkSizes = new IntArrayList();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param db Database reference
|
||||
* @param elements Elements to add
|
||||
*/
|
||||
public LightBigList(JCWDatabase db, LongArrayList elements) {
|
||||
this.db = db;
|
||||
this.chunks = new LongArrayList();
|
||||
this.chunkSizes = new IntArrayList();
|
||||
elements.forEach((long element) -> {
|
||||
this.appendIndex(element);
|
||||
});
|
||||
}
|
||||
|
||||
public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
|
||||
this.db = db;
|
||||
this.chunks = chunks;
|
||||
this.chunkSizes = chunkSizes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append an index to the first free chunk
|
||||
* @param elementIndex
|
||||
*/
|
||||
public void appendIndex(long elementIndex) {
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int chunkNumber = i;
|
||||
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
|
||||
try {
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
final EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
chunk.appendIndex(elementIndex);
|
||||
chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1);
|
||||
});
|
||||
return;
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
LightList<T> newChunk = new LightArrayList<>(db);
|
||||
newChunk.appendIndex(elementIndex);
|
||||
long newChunkIndex;
|
||||
try {
|
||||
newChunkIndex = db.add(newChunk).getIndex();
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
chunks.add(newChunkIndex);
|
||||
chunkSizes.add(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the elements count
|
||||
* @return the size of the list
|
||||
*/
|
||||
@Override
|
||||
public int size() {
|
||||
int size = 0;
|
||||
for (int chunkSize : this.chunkSizes) {
|
||||
size += chunkSize;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return the count of chunks
|
||||
*/
|
||||
public int chunksCount() {
|
||||
return this.chunkSizes.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the list is empty
|
||||
* @return true if the list is empty
|
||||
*/
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return this.size() <= 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean contains(Object o) {
|
||||
if (o != null) {
|
||||
for (long chunkIndex : chunks) {
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
if (chunk.contains(o)) {
|
||||
return true;
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use iteratorReferences()
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public Iterator<T> iterator() {
|
||||
throw new RuntimeException("iterator() isn't implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public Iterator<EntryReference<T>> iteratorReferences() {
|
||||
throw new RuntimeException("iteratorReferences() isn't implemented!");
|
||||
}
|
||||
|
||||
/**
|
||||
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
|
||||
*
|
||||
* @param action
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public void forEach(Consumer<? super T> action) {
|
||||
throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void forEachReference(Consumer<? super EntryReference<T>> action) {
|
||||
Objects.requireNonNull(action);
|
||||
for (long chunkIndex : this.chunks) {
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
chunk.forEachReference(action);
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
throw new RuntimeException("toArray() isn't implemented!");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public <T1> T1[] toArray(T1[] a) {
|
||||
throw new RuntimeException("toArray() isn't implemented!");
|
||||
}
|
||||
|
||||
/**
|
||||
* Use addEntry(o)
|
||||
* @param o
|
||||
* @return
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public boolean add(T o) {
|
||||
EntryReference<T> ref = addEntry(o);
|
||||
return ref != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryReference<T> addEntry(T o) {
|
||||
EntryReference<T> ref = addToDatabase(o);
|
||||
appendIndex(ref.getIndex());
|
||||
return ref;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(Object o) {
|
||||
final int removeOffset = indexOf(o);
|
||||
return removeAt(removeOffset) != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(EntryReference<T> ref) {
|
||||
final int removeOffset = indexOfEntry(ref);
|
||||
return removeAt(removeOffset) != null;
|
||||
}
|
||||
|
||||
private T removeAt(int removeOffset) {
|
||||
final VariableWrapper<T> result = new VariableWrapper<>(null);
|
||||
long currentOffset = 0;
|
||||
if (removeOffset >= 0) {
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int currentChunkSize = chunkSizes.getInt(i);
|
||||
final long chunkStartOffset = currentOffset;
|
||||
currentOffset += currentChunkSize;
|
||||
// If the offset to remove is in the current chunk
|
||||
if (currentOffset > removeOffset) {
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
// Get the offset relative to the current chunk
|
||||
final int relativeOffset = (int) (removeOffset - chunkStartOffset);
|
||||
|
||||
if (relativeOffset < 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
result.var = chunk.remove(relativeOffset);
|
||||
});
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
chunkSizes.set(removeOffset, currentChunkSize - 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result.var;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsAll(Collection<?> c) {
|
||||
for (Object o : c) {
|
||||
int objIndex = indexOf(o);
|
||||
if (objIndex < 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean addAll(Collection<? extends T> c) {
|
||||
boolean result = false;
|
||||
for (Object o : c) {
|
||||
result |= add((T) o);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean addAll(int index, Collection<? extends T> c) {
|
||||
boolean result = false;
|
||||
int delta = 0;
|
||||
for (Object o : c) {
|
||||
add(index + delta, (T) o);
|
||||
result = true;
|
||||
delta++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean removeAll(Collection<?> c) {
|
||||
boolean result = false;
|
||||
for (Object o : c) {
|
||||
result |= remove((T) o);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean retainAll(Collection<?> c) {
|
||||
boolean result = false;
|
||||
LongArrayList collectionHashes = new LongArrayList();
|
||||
ObjectArrayList<Object> collection = new ObjectArrayList<>();
|
||||
collection.addAll(c);
|
||||
for (Object o : c) {
|
||||
collectionHashes.add(db.calculateHash(o));
|
||||
}
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
long hash = chunks.getLong(i);
|
||||
int positionInCollection = collectionHashes.indexOf(hash);
|
||||
if (positionInCollection == -1) {
|
||||
remove(collection.get(positionInCollection));
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
chunks.clear();
|
||||
chunkSizes.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Use getReference or getReadOnlyValue
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public T get(int index) {
|
||||
return getReadOnlyValue(index);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public T getReadOnlyValue(int index) {
|
||||
try {
|
||||
return (T) db.get(chunks.getLong(index)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryReference<T> getReference(int index) {
|
||||
try {
|
||||
return db.get(chunks.getLong(index)).cast();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T set(int setOffset, final T element) {
|
||||
long nextChunkOffset = 0;
|
||||
VariableWrapper<T> wrapper = new VariableWrapper<>(null);
|
||||
if (setOffset >= 0) {
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int currentChunkSize = chunkSizes.getInt(i);
|
||||
final long chunkStartOffset = nextChunkOffset;
|
||||
nextChunkOffset += currentChunkSize;
|
||||
// If the offset to remove is in the current chunk
|
||||
if (nextChunkOffset > setOffset) {
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
// Get the offset relative to the current chunk
|
||||
final int relativeOffset = (int) (setOffset - chunkStartOffset);
|
||||
|
||||
if (relativeOffset < 0) {
|
||||
throw new NullPointerException("Relative Offset < 0");
|
||||
}
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
chunk.set(relativeOffset, element);
|
||||
wrapper.var = element;
|
||||
});
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return wrapper.var;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(int index, T element) {
|
||||
throw new RuntimeException("add() isn't implemented!");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T remove(int index) {
|
||||
return this.removeAt(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int indexOf(Object o) {
|
||||
EntryReference<T> ref = addToDatabase(o).cast();
|
||||
return indexOfEntry(ref);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int indexOfEntry(EntryReference<T> ref) {
|
||||
int currentOffset = 0;
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
try {
|
||||
final int currentChunkSize = chunkSizes.getInt(i);
|
||||
// If the offset to remove is in the current chunk
|
||||
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
|
||||
if (foundIndex >= 0) {
|
||||
return currentOffset + foundIndex;
|
||||
}
|
||||
currentOffset += currentChunkSize;
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lastIndexOf(Object o) {
|
||||
return lastIndexOfEntry(addToDatabase(o).cast());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lastIndexOfEntry(EntryReference<T> ref) {
|
||||
int currentOffset = 0;
|
||||
// Iterate through all chunks
|
||||
for (int i = chunks.size() - 1; i >= 0; i--) {
|
||||
try {
|
||||
final int currentChunkSize = chunkSizes.getInt(i);
|
||||
// If the offset to remove is in the current chunk
|
||||
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
|
||||
if (foundIndex >= 0) {
|
||||
return currentOffset + foundIndex;
|
||||
}
|
||||
currentOffset += currentChunkSize;
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public ListIterator<T> listIterator() {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public ListIterator<T> listIterator(int index) {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public List<T> subList(int fromIndex, int toIndex) {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
private <U> EntryReference<U> addToDatabase(U obj) {
|
||||
EntryReference<U> ref;
|
||||
try {
|
||||
ref = db.add(obj);
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
return ref;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean removeIf(Predicate<? super T> filter) {
|
||||
Objects.requireNonNull(filter);
|
||||
final VariableWrapper<Boolean> result = new VariableWrapper(false);
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
try {
|
||||
final int chunkOffset = i;
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
boolean removed = chunk.removeIf(filter);
|
||||
if (removed) {
|
||||
result.var = true;
|
||||
chunkSizes.set(chunkOffset, chunk.size());
|
||||
}
|
||||
});
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
return result.var;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,390 +1,24 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.LongArrayList;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
public interface LightList<T> extends List<T> {
|
||||
|
||||
public class LightList<T> implements List<T> {
|
||||
Iterator<EntryReference<T>> iteratorReferences();
|
||||
|
||||
public final LongArrayList internalList;
|
||||
private final transient JCWDatabase db;
|
||||
void forEachReference(Consumer<? super EntryReference<T>> action);
|
||||
|
||||
public LightList(JCWDatabase db, LongArrayList internalList) {
|
||||
this.internalList = internalList;
|
||||
this.db = db;
|
||||
}
|
||||
EntryReference<T> addEntry(T o);
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
return internalList.size();
|
||||
}
|
||||
boolean remove(EntryReference<T> ref);
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return internalList.isEmpty();
|
||||
}
|
||||
EntryReference<T> getReference(int index);
|
||||
|
||||
@Override
|
||||
public boolean contains(Object o) {
|
||||
if (o != null) {
|
||||
for (Long element : internalList) {
|
||||
EntryReference<T> ref = null;
|
||||
try {
|
||||
ref = db.get(element);
|
||||
if (o.equals(ref.getValueReadOnly())) {
|
||||
return true;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
int indexOfEntry(EntryReference<T> ref);
|
||||
|
||||
/**
|
||||
* Use iteratorReferences()
|
||||
*/
|
||||
@Deprecated
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public Iterator<T> iterator() {
|
||||
final ArrayList<T> elements = new ArrayList<>();
|
||||
for (Long element : internalList) {
|
||||
try {
|
||||
elements.add((T) db.get(element).getValueReadOnly());
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements.iterator();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public Iterator<EntryReference<T>> iteratorReferences() {
|
||||
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
|
||||
for (Long element : internalList) {
|
||||
try {
|
||||
elements.add(db.get(element));
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements.iterator();
|
||||
}
|
||||
|
||||
int lastIndexOfEntry(EntryReference<T> ref);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
final T[] elements = (T[]) new Objects[internalList.size()];
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
try {
|
||||
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public <T1> T1[] toArray(T1[] a) {
|
||||
final T1[] elements = (T1[]) new Objects[internalList.size()];
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
try {
|
||||
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean add(T o) {
|
||||
EntryReference<T> ref = addEntry(o);
|
||||
return ref != null;
|
||||
}
|
||||
|
||||
public EntryReference<T> addEntry(T o) {
|
||||
EntryReference<T> ref = addToDatabase(o);
|
||||
if (internalList.add(ref.getIndex())) {
|
||||
return ref;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(Object o) {
|
||||
int removeIndex = indexOf(o);
|
||||
if (removeIndex >= 0) {
|
||||
internalList.removeLong(removeIndex);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean remove(EntryReference<T> ref) {
|
||||
int removeIndex = indexOfEntry(ref);
|
||||
if (removeIndex >= 0) {
|
||||
internalList.removeLong(removeIndex);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsAll(Collection<?> c) {
|
||||
for (Object o : c) {
|
||||
int objIndex = indexOf(o);
|
||||
if (objIndex < 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean addAll(Collection<? extends T> c) {
|
||||
boolean result = false;
|
||||
for (Object o : c) {
|
||||
result |= add((T) o);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean addAll(int index, Collection<? extends T> c) {
|
||||
boolean result = false;
|
||||
int delta = 0;
|
||||
for (Object o : c) {
|
||||
add(index + delta, (T) o);
|
||||
result = true;
|
||||
delta++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean removeAll(Collection<?> c) {
|
||||
boolean result = false;
|
||||
for (Object o : c) {
|
||||
result |= remove((T) o);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean retainAll(Collection<?> c) {
|
||||
boolean result = false;
|
||||
LongArrayList collectionHashes = new LongArrayList();
|
||||
ObjectArrayList<Object> collection = new ObjectArrayList<>();
|
||||
collection.addAll(c);
|
||||
for (Object o : c) {
|
||||
collectionHashes.add(db.calculateHash(o));
|
||||
}
|
||||
for (int i = 0; i < internalList.size(); i++) {
|
||||
long hash = internalList.getLong(i);
|
||||
int positionInCollection = collectionHashes.indexOf(hash);
|
||||
if (positionInCollection == -1) {
|
||||
remove(collection.get(positionInCollection));
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
internalList.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Use getReference or getReadOnlyValue
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public T get(int index) {
|
||||
return getReadOnlyValue(index);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public T getReadOnlyValue(int index) {
|
||||
try {
|
||||
return (T) db.get(internalList.getLong(index)).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public EntryReference<T> getReference(int index) {
|
||||
try {
|
||||
return db.get(internalList.getLong(index)).cast();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T set(int index, T element) {
|
||||
EntryReference<T> ref = addToDatabase(element);
|
||||
long oldIndex = internalList.set(index, ref.getIndex());
|
||||
try {
|
||||
ref.close();
|
||||
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(int index, T element) {
|
||||
EntryReference<T> ref = addToDatabase(element);
|
||||
internalList.add(index, ref.getIndex());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T remove(int index) {
|
||||
long oldIndex = internalList.removeLong(index);
|
||||
try {
|
||||
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int indexOf(Object o) {
|
||||
EntryReference<?> ref = addToDatabase(o);
|
||||
long objToRemoveHash = ref.calculateHash();
|
||||
LongArrayList hashes = new LongArrayList();
|
||||
|
||||
|
||||
for (int i = 0; i < hashes.size(); i++) {
|
||||
long hash = hashes.getLong(i);
|
||||
if (objToRemoveHash == hash) {
|
||||
try {
|
||||
if (ref.equals(db.get(internalList.getLong(i)))) {
|
||||
return i;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public int indexOfEntry(EntryReference<T> ref) {
|
||||
for (int i = 0; i < internalList.size(); i++) {
|
||||
long index = internalList.getLong(i);
|
||||
try {
|
||||
EntryReference<?> ref2 = db.get(index);
|
||||
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
|
||||
return i;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lastIndexOf(Object o) {
|
||||
EntryReference<?> ref = addToDatabase(o);
|
||||
long objToRemoveHash = ref.calculateHash();
|
||||
|
||||
int lastValue = -1;
|
||||
|
||||
for (int i = 0; i < internalList.size(); i++) {
|
||||
long index2 = internalList.getLong(i);
|
||||
try {
|
||||
EntryReference<?> ref2 = db.get(index2);
|
||||
if (objToRemoveHash == ref2.calculateHash()) {
|
||||
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
|
||||
lastValue = i;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
}
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public ListIterator<T> listIterator() {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public ListIterator<T> listIterator(int index) {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public List<T> subList(int fromIndex, int toIndex) {
|
||||
// TODO: implement
|
||||
throw new RuntimeException("Not implemented!");
|
||||
}
|
||||
|
||||
private <U> EntryReference<U> addToDatabase(U obj) {
|
||||
EntryReference<U> ref;
|
||||
try {
|
||||
ref = db.add(obj);
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
return ref;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public boolean removeIf(Predicate<? super T> filter) {
|
||||
Objects.requireNonNull(filter);
|
||||
boolean removed = false;
|
||||
for (int i = 0; i < internalList.size(); ) {
|
||||
T obj;
|
||||
try {
|
||||
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
|
||||
} catch (IOException e) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(e);
|
||||
}
|
||||
if (filter.test(obj)) {
|
||||
internalList.removeLong(i);
|
||||
removed = true;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
void appendIndex(long elementIndex);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,19 +8,16 @@ import java.nio.file.Path;
|
|||
import java.util.function.Consumer;
|
||||
|
||||
public class MixedIndexDatabase implements IndexManager {
|
||||
private final Long2LongMap mostAccessedIndices;
|
||||
private final FileIndexManager fileIndices;
|
||||
private final CacheIndexManager cacheIndices;
|
||||
|
||||
public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
|
||||
this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
|
||||
public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
|
||||
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
|
||||
this.cacheIndices = new CacheIndexManager();
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T get(long index, DBReader<T> reader) throws IOException {
|
||||
incrementUsage(index);
|
||||
if (cacheIndices.has(index)) {
|
||||
return cacheIndices.get(index, reader);
|
||||
} else {
|
||||
|
|
@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager {
|
|||
return cacheIndices.has(index) || fileIndices.has(index);
|
||||
}
|
||||
|
||||
private void incrementUsage(long index) {
|
||||
mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
// TODO: move all cached indices to filesIndices before closing.
|
||||
|
|
|
|||
|
|
@ -1,26 +1,37 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.HashMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||
|
||||
public class TypesManager {
|
||||
private final Map<Integer, DBTypeParser<?>> types;
|
||||
private final Map<Class<?>, DBTypeParser<?>> typesByClass;
|
||||
private DBTypeParser<?> fallbackParser;
|
||||
private final Int2ObjectMap<DBTypeParser<?>> types;
|
||||
private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
|
||||
private DBTypedObjectParser<?> fallbackParser;
|
||||
|
||||
public TypesManager(JCWDatabase db) {
|
||||
types = new HashMap<>();
|
||||
typesByClass = new HashMap<>();
|
||||
types = new Int2ObjectOpenHashMap<>();
|
||||
typesByClass = new Object2ObjectOpenHashMap<>();
|
||||
DBStandardTypes.registerStandardTypes(db, this);
|
||||
}
|
||||
|
||||
|
||||
public <T> void registerType(Class<T> clazz, int type, DBTypeParser<T> parser) {
|
||||
this.types.put(type, parser);
|
||||
this.typesByClass.put(clazz, parser);
|
||||
}
|
||||
|
||||
public void registerTypeFallback(DBTypeParser<?> parser) {
|
||||
/**
|
||||
* Use this method with the most used classes to save disk space.
|
||||
* @param clazz
|
||||
* @param id
|
||||
* @param <T>
|
||||
*/
|
||||
public <T> void registerGenericClass(Class<T> clazz, int id) {
|
||||
this.fallbackParser.registerClass(clazz, id);
|
||||
}
|
||||
|
||||
public void registerTypeFallback(DBTypedObjectParser<?> parser) {
|
||||
this.fallbackParser = parser;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
package org.warp.jcwdb;
|
||||
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
|
||||
public class VariableWrapper<T> {
|
||||
|
||||
public T var;
|
||||
|
||||
public VariableWrapper(T value) {
|
||||
this.var = value;
|
||||
}
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ public class App {
|
|||
System.out.println("Loading database...");
|
||||
long time0 = System.currentTimeMillis();
|
||||
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
|
||||
db.registerClass(StrangeAnimal.class, 0);
|
||||
try {
|
||||
long time01 = System.currentTimeMillis();
|
||||
System.out.println("Time elapsed: " + (time01 - time0));
|
||||
|
|
@ -58,15 +59,15 @@ public class App {
|
|||
System.out.println("Time elapsed: " + (time2_1 - time2_0));
|
||||
ObjectList<Animal> results = new ObjectArrayList<>();
|
||||
|
||||
/*
|
||||
root.forEach((value) -> {
|
||||
root.forEachReference((valueReference) -> {
|
||||
Animal value = valueReference.getValueReadOnly();
|
||||
if (Animal.hasFourLegs(value)) {
|
||||
results.add(value);
|
||||
}
|
||||
//System.out.println("val:" + value);
|
||||
});
|
||||
*/
|
||||
long time2_2 = System.currentTimeMillis();
|
||||
System.out.println("Matches: " + results.size());
|
||||
System.out.println("Time elapsed: " + (time2_2 - time2_1));
|
||||
System.out.println("Used memory: "
|
||||
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
|
||||
|
|
|
|||
|
|
@ -25,94 +25,5 @@ public class AppTest
|
|||
{
|
||||
assertTrue( true );
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
@Test
|
||||
public void shouldReadCustomClasses() throws IOException
|
||||
{
|
||||
CustomClass customClass = new CustomClass();
|
||||
customClass.primitive = 1;
|
||||
customClass.string = "test";
|
||||
|
||||
Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
|
||||
|
||||
JCWDatabase db = new JCWDatabase(path, idx);
|
||||
EntryReference<LightList<CustomClass>> ref = db.getRoot();
|
||||
ref.editValue((root, saver) -> {
|
||||
root.add(customClass);
|
||||
});
|
||||
db.close();
|
||||
|
||||
JCWDatabase db2 = new JCWDatabase(path, idx);
|
||||
EntryReference<LightList<CustomClass>> ref2 = db2.getRoot();
|
||||
CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0);
|
||||
assertTrue(customClass.equals(customClass2));
|
||||
assertTrue(customClass.string.equals(customClass2.string));
|
||||
db2.close();
|
||||
Files.delete(path);
|
||||
Files.delete(idx);
|
||||
}
|
||||
|
||||
public static class CustomClass {
|
||||
public String string;
|
||||
public int primitive;
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + primitive;
|
||||
result = prime * result + ((string == null) ? 0 : string.hashCode());
|
||||
return result;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
CustomClass other = (CustomClass) obj;
|
||||
if (primitive != other.primitive)
|
||||
return false;
|
||||
if (string == null) {
|
||||
if (other.string != null)
|
||||
return false;
|
||||
} else if (!string.equals(other.string))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
@Test
|
||||
public void shouldActAsAnArrayListWithEqualObjects() throws IOException
|
||||
{
|
||||
Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
|
||||
JCWDatabase db = new JCWDatabase(path, idx);
|
||||
EntryReference<LightList<String>> ref = db.getRoot();
|
||||
LightList<String> list1 = ref.getValueReadOnly();
|
||||
ArrayList<String> list2 = new ArrayList<String>();
|
||||
String s = "a";
|
||||
for (int i = 0; i < 10; i++) {
|
||||
list1.add(s);
|
||||
list2.add(s);
|
||||
}
|
||||
assertTrue(list1.size() == list2.size());
|
||||
for (int i = 0; i < 10; i++) {
|
||||
assertTrue(list1.get(i) == list2.get(i));
|
||||
assertTrue(list2.get(i) == list1.get(i));
|
||||
}
|
||||
db.close();
|
||||
Files.delete(path);
|
||||
Files.delete(idx);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue