Merge branch 'better'

This commit is contained in:
Andrea Cavalli 2018-12-20 01:12:54 +01:00
commit a12decf796
18 changed files with 1321 additions and 695 deletions

View File

@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner {
private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
private static final double NORMAL_REMOVED_ITEMS = 1000l;
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
@ -63,7 +63,10 @@ public class Cleaner {
try {
System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
sleepFor(sleepInterval);
final long time1 = System.currentTimeMillis();
final double removedItems = clean();
final long time2 = System.currentTimeMillis();
System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);

View File

@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output;
import net.openhft.hashing.LongHashFunction;
public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
private static final Kryo kryo = new Kryo();
static {
@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
tmpO.close();
return hash;
}
@Override
public <U> void registerClass(Class<U> clazz, int id) {
if (id >= Integer.MAX_VALUE - 100) {
throw new IndexOutOfBoundsException();
}
kryo.register(clazz, id + 100);
}
}

View File

@ -1,40 +1,38 @@
package org.warp.jcwdb;
import java.util.ArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightListParser<T> extends DBTypeParserImpl<LightList<T>> {
public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T>> {
private final JCWDatabase db;
public DBLightListParser(JCWDatabase db) {
public DBLightArrayListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightList<T>> getReader() {
public DBReader<LightArrayList<T>> getReader() {
return (i, size) -> {
LongArrayList internalList = new LongArrayList();
long max = size / Long.BYTES;
for (int item = 0; item < max; item++){
for (int item = 0; item < max; item++) {
long itm = i.readLong();
internalList.add(itm);
}
return new LightList<T>(db, internalList);
return new LightArrayList<T>(db, internalList);
};
}
public DBDataOutput<LightList<T>> getWriter(final LightList<T> value) {
public DBDataOutput<LightArrayList<T>> getWriter(final LightArrayList<T> value) {
final int elementsCount = value.size();
return DBDataOutput.create((o) -> {
LongArrayList list = value.internalList;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
}
}, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value));
}, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value));
}
@Override
public long calculateHash(LightList<T> value) {
public long calculateHash(LightArrayList<T> value) {
return value.internalList.hashCode();
}
}

View File

@ -0,0 +1,44 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
private final JCWDatabase db;
public DBLightBigListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightBigList<T>> getReader() {
return (i, size) -> {
LongArrayList chunks = new LongArrayList();
IntArrayList chunkSizes = new IntArrayList();
long max = size / (Long.BYTES + Integer.BYTES);
for (int item = 0; item < max; item++) {
long itm = i.readLong();
int itm2 = i.readInt();
chunks.add(itm);
chunkSizes.add(itm2);
}
return new LightBigList<T>(db, chunks, chunkSizes);
};
}
public DBDataOutput<LightBigList<T>> getWriter(final LightBigList<T> value) {
final int elementsCount = value.chunksCount();
return DBDataOutput.create((o) -> {
LongArrayList list = value.chunks;
IntArrayList list2 = value.chunkSizes;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
o.writeInt(list2.getInt(i));
}
}, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value));
}
@Override
public long calculateHash(LightBigList<T> value) {
return value.chunks.hashCode();
}
}

View File

@ -11,12 +11,14 @@ public class DBStandardTypes {
public static final int DOUBLE = STD| 6;
public static final int STRING = STD| 7;
public static final int BYTE_ARRAY = STD| 8;
public static final int LIGHT_LIST = STD| 9;
public static final int GENERIC_OBJECT = STD| 10;
public static final int LIGHT_LIST_ARRAY = STD| 9;
public static final int LIGHT_LIST_BIG = STD| 10;
public static final int GENERIC_OBJECT = STD| 11;
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
typesManager.registerType(String.class, STRING, new DBStringParser());
typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db));
typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db));
typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db));
typesManager.registerTypeFallback(new DBGenericObjectParser());
}
}

View File

@ -0,0 +1,5 @@
package org.warp.jcwdb;
public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
public <U> void registerClass(Class<U> clazz, int type);
}

View File

@ -86,7 +86,7 @@ public class EntryReference<T> implements Castable, Saveable {
* @param editFunction
* @throws IOException
*/
public void editValue(BiFunction<T, Saveable, T> editFunction) throws IOException {
public void editValue(BiFunction<T, Saveable, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value, this);
@ -99,7 +99,20 @@ public class EntryReference<T> implements Castable, Saveable {
* @param editFunction
* @throws IOException
*/
public void editValue(BiConsumer<T, Saveable> editFunction) throws IOException {
public void editValue(Function<T, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value);
this.save();
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(BiConsumer<T, Saveable> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value, this);
@ -107,12 +120,25 @@ public class EntryReference<T> implements Castable, Saveable {
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(Consumer<T> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value);
this.save();
}
}
/**
* Substitute the old value with a new one
* @param val
* @throws IOException
*/
public void setValue(T val) throws IOException {
public void setValue(T val) {
synchronized(accessLock) {
this.loaded = true;
this.value = val;

View File

@ -1,62 +1,116 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.Long2IntMap;
import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.util.*;
public class FileAllocator implements AutoCloseable {
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
private final SeekableByteChannel dataFileChannel;
private volatile long allocableOffset;
private volatile long fileSize;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object allocateLock = new Object();
/**
* index -> free space size
*/
private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
this.dataFileChannel = dataFileChannel;
this.allocableOffset = this.dataFileChannel.size();
this.fileSize = this.dataFileChannel.size();
}
/**
* TODO: not implemented
*
* @param size
* @return offset
*/
public long allocate(int size) {
checkClosed();
synchronized (allocateLock) {
long allocatedOffset = allocableOffset;
allocableOffset += size;
return allocatedOffset;
long offset = allocateIntoUnusedParts(size);
if (offset == -1) {
return allocateToEnd(size);
} else {
return offset;
}
}
}
private long allocateIntoUnusedParts(int size) {
ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
long holeOffset = -1;
int holeSize = 0;
while (it.hasNext()) {
Long2IntMap.Entry entry = it.next();
int currentHoleSize = entry.getIntValue();
if (currentHoleSize < size) {
freeBytes.remove(holeOffset);
if (holeSize > size) {
freeBytes.put(holeOffset + size, holeSize - size);
}
break;
}
holeOffset = entry.getLongKey();
holeSize = currentHoleSize;
}
return holeOffset;
}
private long allocateToEnd(int size) {
long allocatedOffset = fileSize;
fileSize += size;
return allocatedOffset;
}
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
/**
* Frees the unused bytes
*
* @param startPosition
* @param length
*/
public void markFree(long startPosition, int length) {
checkClosed();
// TODO: advanced feature, not implemented.
if (freeBytes.containsKey(startPosition + length)) {
int secondLength = freeBytes.remove(startPosition + length);
freeBytes.put(startPosition, length + secondLength);
} else {
boolean addedToList = false;
for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
if (entry.getLongKey() + entry.getIntValue() == startPosition) {
freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
addedToList = true;
break;
}
}
if (!addedToList) {
freeBytes.put(startPosition, length);
}
}
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
freeBytes.remove(freeBytes.lastLongKey());
}
}

View File

@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Iterator;
import java.util.function.Consumer;
public class FileIndexManager implements IndexManager {
@ -81,7 +82,12 @@ public class FileIndexManager implements IndexManager {
final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < dataSize) {
// Allocate new space
return allocateAndWrite(index, data);
IndexDetails newDetails = allocateAndWrite(index, data);
if (indexDetails != null) {
// Mark free the old bytes
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
return newDetails;
} else {
// Check if size changed
if (dataSize < indexDetails.getSize()) {
@ -233,7 +239,7 @@ public class FileIndexManager implements IndexManager {
* @param details
*/
private void editIndex(long index, IndexDetails details) {
synchronized (indicesMapsAccessLock) {
synchronized (indicesMapsAccessLock) {// FIXXXX main3
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
}
@ -264,7 +270,8 @@ public class FileIndexManager implements IndexManager {
return null;
}
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
synchronized (metadataByteBufferLock) {
IndexDetails indexDetails = null;
synchronized (metadataByteBufferLock) {// FIXXXX main2
metadataByteBuffer.rewind();
currentMetadataFileChannel.read(metadataByteBuffer);
metadataByteBuffer.rewind();
@ -277,12 +284,15 @@ public class FileIndexManager implements IndexManager {
final int size = metadataByteBuffer.getInt();
final int type = metadataByteBuffer.getInt();
final long hash = metadataByteBuffer.getLong();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
editIndex(index, indexDetails);
return indexDetails;
indexDetails = new IndexDetails(offset, size, type, hash);
}
}
if (indexDetails != null) {
editIndex(index, indexDetails);
return indexDetails;
}
// No results found. Returning null
return null;
}
@ -308,31 +318,15 @@ public class FileIndexManager implements IndexManager {
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
}
}
flushAllIndices();
// Remove removed indices
synchronized (indicesMapsAccessLock) {
for (long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
}
removeRemovedIndices();
fileAllocator.close();
}
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
synchronized (metadataByteBufferLock) {
synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
final int size = indexDetails.getSize();
final int type = indexDetails.getType();
final long offset = indexDetails.getOffset();
@ -366,19 +360,63 @@ public class FileIndexManager implements IndexManager {
@Override
public long clean() {
return cleanExtraIndices();
long cleaned = 0;
try {
cleaned += flushAllIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
try {
cleaned += removeRemovedIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
cleaned += cleanExtraIndices();
return cleaned;
}
private long flushAllIndices() throws IOException {
long flushedIndices = 0;
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
}
dirtyLoadedIndices.clear();
}
return flushedIndices;
}
private long removeRemovedIndices() throws IOException {
SeekableByteChannel metadata = metadataFileChannel;
synchronized (indicesMapsAccessLock) {
long removed = this.removedIndices.size();
for (long index : this.removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
this.removedIndices.clear();
return removed;
}
}
private long cleanExtraIndices() {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
long count = loadedIndices.size();
LongIterator it = loadedIndices.keySet().iterator();
while (it.hasNext()) {
long loadedIndex = it.nextLong();
if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
break;
}
toUnload.add(loadedIndex);

View File

@ -1,36 +1,24 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.function.Consumer;
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import java.io.IOException;
import java.nio.file.Path;
public class JCWDatabase implements AutoCloseable, Cleanable {
public final static long MAX_LOADED_REFERENCES = 1000;
public final static long MAX_LOADED_INDICES = 10000;
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
private final Cleaner databaseCleaner;
private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object indicesAccessLock = new Object();
private final Object referencesAccessLock = new Object();
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
this.references = new Long2ObjectLinkedOpenHashMap<>();
this.indices = new MixedIndexDatabase(dataFile, metadataFile);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
JCWDatabase.this.close();
@ -40,93 +28,83 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
}));
this.databaseCleaner = new Cleaner(this);
//this.databaseCleaner.start();
this.databaseCleaner.start();
}
public <T> EntryReference<LightList<T>> getRoot() throws IOException {
return getRoot(Object.class).cast();
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
checkClosed();
if (exists(0)) {
return get(0);
} else {
LightList<T> newRoot = new LightList<T>(this, new LongArrayList());
LightList<T> newRoot = new LightArrayList<>(this);
return set(0, newRoot);
}
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
return getRoot().cast();
}
public <T> EntryReference<T> get(long index) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
WeakReference<EntryReference<?>> refRef = this.references.getOrDefault(index, null);
EntryReference<T> ref;
if (refRef == null || (ref = (EntryReference<T>) refRef.get()) == null) {
int type;
long hash;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
hash = this.indices.getHash(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
refRef = new WeakReference<>(ref);
this.references.put(index, refRef);
}
return ref;
int type;
long hash;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
hash = this.indices.getHash(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
protected <T> EntryReference<T> add(T value) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
EntryReference<T> ref;
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long index;
long hash;
synchronized (indicesAccessLock) {
index = indices.add(typeParser.getWriter(value));
hash = indices.getHash(index);
}
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
this.references.put(index, new WeakReference<>(ref));
return ref;
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long index;
long hash;
synchronized (indicesAccessLock) {
index = indices.add(typeParser.getWriter(value));
hash = indices.getHash(index);
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
}
protected boolean exists(long index) {
checkClosed();
synchronized (referencesAccessLock) {
synchronized (indicesAccessLock) {
return this.references.containsKey(index) || this.indices.has(index);
}
synchronized (indicesAccessLock) {
return this.indices.has(index);
}
}
protected <T> EntryReference<T> set(long index, T value) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
EntryReference<T> ref;
if (exists(index)) {
ref = get(index);
ref.setValue(value);
return ref;
} else {
@SuppressWarnings("unchecked")
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long hash;
synchronized (indicesAccessLock) {
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
hash = returnedDetails.getHash();
}
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
this.references.put(index, new WeakReference<EntryReference<?>>(ref));
return ref;
EntryReference<T> ref;
if (exists(index)) {
ref = get(index);
ref.setValue(value);
return ref;
} else {
@SuppressWarnings("unchecked")
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long hash;
synchronized (indicesAccessLock) {
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
hash = returnedDetails.getHash();
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
}
public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
final int addition = 0xEFFF8000;
int extendedType = addition | (type & 0x7FFF);
typesManager.registerType(clazz, extendedType, parser);
}
public <U> void registerClass(Class<U> clazz, int type) {
typesManager.registerGenericClass(clazz, type);
}
public boolean isOpen() {
return !closed;
}
@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
this.databaseCleaner.stop();
synchronized (referencesAccessLock) {
ObjectIterator<WeakReference<EntryReference<?>>> iterator = references.values().iterator();
while (iterator.hasNext()) {
WeakReference<EntryReference<?>> referenceRef = iterator.next();
EntryReference<?> reference = referenceRef.get();
if (reference != null) {
reference.close();
iterator.remove();
}
}
}
synchronized (indicesAccessLock) {
this.indices.close();
}
@ -171,57 +137,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
@Override
public long clean() {
long removedItems = cleanEmptyReferences()
+ cleanExtraReferences()
+ indices.clean();
long removedItems = indices.clean();
return removedItems;
}
private long cleanEmptyReferences() {
long removed = 0;
synchronized(referencesAccessLock) {
ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
while (iterator.hasNext()) {
Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
if (entry.getValue().get() == null) {
iterator.remove();
removed++;
}
}
}
return removed;
}
private long cleanExtraReferences() {
long removedReferences = 0;
synchronized(referencesAccessLock) {
if (references.size() > MAX_LOADED_REFERENCES) {
long count = 0;
ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
while (iterator.hasNext()) {
Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
if (count > MAX_LOADED_REFERENCES * 3l / 2l) {
WeakReference<EntryReference<?>> weakRef = entry.getValue();
EntryReference<?> ref = weakRef.get();
if (ref != null) {
try {
ref.close();
} catch (IOException e) {
e.printStackTrace();
}
}
iterator.remove();
removedReferences++;
} else {
count++;
}
}
}
}
return removedReferences;
}
public class EntryReferenceTools {
private EntryReferenceTools() {

View File

@ -0,0 +1,445 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
public class LightArrayList<T> implements LightList<T> {
public final LongArrayList internalList;
private final transient JCWDatabase db;
/**
* @param db Database reference
*/
public LightArrayList(JCWDatabase db) {
this.db = db;
this.internalList = new LongArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightArrayList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.internalList = new LongArrayList(elements);
}
@Override
public int size() {
return internalList.size();
}
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long element : internalList) {
EntryReference<T> ref = null;
try {
ref = db.get(element);
if (o.equals(ref.getValueReadOnly())) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add((T) db.get(element).getValueReadOnly());
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@SuppressWarnings("unchecked")
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add(db.get(element));
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw (RuntimeException) new RuntimeException().initCause(e);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
final T[] elements = (T[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
if (internalList.add(ref.getIndex())) {
return ref;
} else {
return null;
}
}
@Override
public boolean remove(Object o) {
int removeIndex = indexOf(o);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean remove(EntryReference<T> ref) {
int removeIndex = indexOfEntry(ref);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < internalList.size(); i++) {
long hash = internalList.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
internalList.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(internalList.getLong(index)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
try {
return db.get(internalList.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
long oldIndex = internalList.set(index, ref.getIndex());
try {
ref.close();
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public void add(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
internalList.add(index, ref.getIndex());
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
long oldIndex = internalList.removeLong(index);
try {
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public int indexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
LongArrayList hashes = new LongArrayList();
for (int i = 0; i < hashes.size(); i++) {
long hash = hashes.getLong(i);
if (objToRemoveHash == hash) {
try {
if (ref.equals(db.get(internalList.getLong(i)))) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
}
return -1;
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
for (int i = 0; i < internalList.size(); i++) {
long index = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index);
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return -1;
}
@Override
public void appendIndex(long elementIndex) {
internalList.add(elementIndex);
}
@Override
public int lastIndexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return lastIndexOfEntry(ref);
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
long objToRemoveHash = ref.calculateHash();
int lastValue = -1;
for (int i = 0; i < internalList.size(); i++) {
long index2 = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index2);
if (objToRemoveHash == ref2.calculateHash()) {
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
lastValue = i;
}
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return lastValue;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Deprecated
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean removed = false;
for (int i = 0; i < internalList.size(); ) {
T obj;
try {
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
if (filter.test(obj)) {
internalList.removeLong(i);
removed = true;
} else {
i++;
}
}
return removed;
}
}

View File

@ -0,0 +1,521 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
public class LightBigList<T> implements LightList<T> {
public static final int MAX_ELEMENTS_PER_CHUNK = 10000;
public final LongArrayList chunks;
public final IntArrayList chunkSizes;
private final transient JCWDatabase db;
/**
* @param db Database reference
*/
public LightBigList(JCWDatabase db) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightBigList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
elements.forEach((long element) -> {
this.appendIndex(element);
});
}
public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
this.db = db;
this.chunks = chunks;
this.chunkSizes = chunkSizes;
}
/**
* Append an index to the first free chunk
* @param elementIndex
*/
public void appendIndex(long elementIndex) {
for (int i = 0; i < chunks.size(); i++) {
final int chunkNumber = i;
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
try {
final long chunkIndex = chunks.getLong(i);
final EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
chunk.appendIndex(elementIndex);
chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1);
});
return;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
LightList<T> newChunk = new LightArrayList<>(db);
newChunk.appendIndex(elementIndex);
long newChunkIndex;
try {
newChunkIndex = db.add(newChunk).getIndex();
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunks.add(newChunkIndex);
chunkSizes.add(1);
}
/**
* Get the elements count
* @return the size of the list
*/
@Override
public int size() {
int size = 0;
for (int chunkSize : this.chunkSizes) {
size += chunkSize;
}
return size;
}
/**
*
* @return the count of chunks
*/
public int chunksCount() {
return this.chunkSizes.size();
}
/**
* Check if the list is empty
* @return true if the list is empty
*/
@Override
public boolean isEmpty() {
return this.size() <= 0;
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long chunkIndex : chunks) {
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
LightList<T> chunk = chunkRef.getValueReadOnly();
if (chunk.contains(o)) {
return true;
}
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@Override
public Iterator<T> iterator() {
throw new RuntimeException("iterator() isn't implemented!");
}
@Deprecated
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
throw new RuntimeException("iteratorReferences() isn't implemented!");
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long chunkIndex : this.chunks) {
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
LightList<T> chunk = chunkRef.getValueReadOnly();
chunk.forEachReference(action);
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
throw new RuntimeException("toArray() isn't implemented!");
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
throw new RuntimeException("toArray() isn't implemented!");
}
/**
* Use addEntry(o)
* @param o
* @return
*/
@Deprecated
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
appendIndex(ref.getIndex());
return ref;
}
@Override
public boolean remove(Object o) {
final int removeOffset = indexOf(o);
return removeAt(removeOffset) != null;
}
@Override
public boolean remove(EntryReference<T> ref) {
final int removeOffset = indexOfEntry(ref);
return removeAt(removeOffset) != null;
}
private T removeAt(int removeOffset) {
final VariableWrapper<T> result = new VariableWrapper<>(null);
long currentOffset = 0;
if (removeOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = currentOffset;
currentOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (currentOffset > removeOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (removeOffset - chunkStartOffset);
if (relativeOffset < 0) {
return null;
}
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
result.var = chunk.remove(relativeOffset);
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunkSizes.set(removeOffset, currentChunkSize - 1);
break;
}
}
return result.var;
}
return null;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < chunks.size(); i++) {
long hash = chunks.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
chunks.clear();
chunkSizes.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(chunks.getLong(index)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
try {
return db.get(chunks.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int setOffset, final T element) {
long nextChunkOffset = 0;
VariableWrapper<T> wrapper = new VariableWrapper<>(null);
if (setOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = nextChunkOffset;
nextChunkOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (nextChunkOffset > setOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (setOffset - chunkStartOffset);
if (relativeOffset < 0) {
throw new NullPointerException("Relative Offset < 0");
}
try {
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
chunk.set(relativeOffset, element);
wrapper.var = element;
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
break;
}
}
return wrapper.var;
}
return null;
}
@Override
public void add(int index, T element) {
throw new RuntimeException("add() isn't implemented!");
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
return this.removeAt(index);
}
@Override
public int indexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return indexOfEntry(ref);
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Override
public int lastIndexOf(Object o) {
return lastIndexOfEntry(addToDatabase(o).cast());
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = chunks.size() - 1; i >= 0; i--) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
final VariableWrapper<Boolean> result = new VariableWrapper(false);
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
try {
final int chunkOffset = i;
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
boolean removed = chunk.removeIf(filter);
if (removed) {
result.var = true;
chunkSizes.set(chunkOffset, chunk.size());
}
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return result.var;
}
}

View File

@ -1,390 +1,24 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.*;
import java.util.function.Predicate;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
public interface LightList<T> extends List<T> {
public class LightList<T> implements List<T> {
Iterator<EntryReference<T>> iteratorReferences();
public final LongArrayList internalList;
private final transient JCWDatabase db;
void forEachReference(Consumer<? super EntryReference<T>> action);
public LightList(JCWDatabase db, LongArrayList internalList) {
this.internalList = internalList;
this.db = db;
}
EntryReference<T> addEntry(T o);
@Override
public int size() {
return internalList.size();
}
boolean remove(EntryReference<T> ref);
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
EntryReference<T> getReference(int index);
@Override
public boolean contains(Object o) {
if (o != null) {
for (Long element : internalList) {
EntryReference<T> ref = null;
try {
ref = db.get(element);
if (o.equals(ref.getValueReadOnly())) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
int indexOfEntry(EntryReference<T> ref);
/**
* Use iteratorReferences()
*/
@Deprecated
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
final ArrayList<T> elements = new ArrayList<>();
for (Long element : internalList) {
try {
elements.add((T) db.get(element).getValueReadOnly());
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
int lastIndexOfEntry(EntryReference<T> ref);
@SuppressWarnings("unchecked")
public Iterator<EntryReference<T>> iteratorReferences() {
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
for (Long element : internalList) {
try {
elements.add(db.get(element));
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
final T[] elements = (T[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
if (internalList.add(ref.getIndex())) {
return ref;
} else {
return null;
}
}
@Override
public boolean remove(Object o) {
int removeIndex = indexOf(o);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
public boolean remove(EntryReference<T> ref) {
int removeIndex = indexOfEntry(ref);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < internalList.size(); i++) {
long hash = internalList.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
internalList.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(internalList.getLong(index)).getValueReadOnly();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public EntryReference<T> getReference(int index) {
try {
return db.get(internalList.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
long oldIndex = internalList.set(index, ref.getIndex());
try {
ref.close();
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public void add(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
internalList.add(index, ref.getIndex());
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
long oldIndex = internalList.removeLong(index);
try {
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public int indexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
LongArrayList hashes = new LongArrayList();
for (int i = 0; i < hashes.size(); i++) {
long hash = hashes.getLong(i);
if (objToRemoveHash == hash) {
try {
if (ref.equals(db.get(internalList.getLong(i)))) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
}
return -1;
}
public int indexOfEntry(EntryReference<T> ref) {
for (int i = 0; i < internalList.size(); i++) {
long index = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index);
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return -1;
}
@Override
public int lastIndexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
int lastValue = -1;
for (int i = 0; i < internalList.size(); i++) {
long index2 = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index2);
if (objToRemoveHash == ref2.calculateHash()) {
if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
lastValue = i;
}
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return lastValue;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Deprecated
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean removed = false;
for (int i = 0; i < internalList.size(); ) {
T obj;
try {
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
if (filter.test(obj)) {
internalList.removeLong(i);
removed = true;
} else {
i++;
}
}
return removed;
}
void appendIndex(long elementIndex);
}

View File

@ -8,19 +8,16 @@ import java.nio.file.Path;
import java.util.function.Consumer;
public class MixedIndexDatabase implements IndexManager {
private final Long2LongMap mostAccessedIndices;
private final FileIndexManager fileIndices;
private final CacheIndexManager cacheIndices;
public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager();
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
incrementUsage(index);
if (cacheIndices.has(index)) {
return cacheIndices.get(index, reader);
} else {
@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager {
return cacheIndices.has(index) || fileIndices.has(index);
}
private void incrementUsage(long index) {
mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1);
}
@Override
public void close() throws IOException {
// TODO: move all cached indices to filesIndices before closing.

View File

@ -1,17 +1,18 @@
package org.warp.jcwdb;
import java.util.Map;
import java.util.Map.Entry;
import java.util.HashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
public class TypesManager {
private final Map<Integer, DBTypeParser<?>> types;
private final Map<Class<?>, DBTypeParser<?>> typesByClass;
private DBTypeParser<?> fallbackParser;
private final Int2ObjectMap<DBTypeParser<?>> types;
private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
private DBTypedObjectParser<?> fallbackParser;
public TypesManager(JCWDatabase db) {
types = new HashMap<>();
typesByClass = new HashMap<>();
types = new Int2ObjectOpenHashMap<>();
typesByClass = new Object2ObjectOpenHashMap<>();
DBStandardTypes.registerStandardTypes(db, this);
}
@ -20,7 +21,17 @@ public class TypesManager {
this.typesByClass.put(clazz, parser);
}
public void registerTypeFallback(DBTypeParser<?> parser) {
/**
* Use this method with the most used classes to save disk space.
* @param clazz
* @param id
* @param <T>
*/
public <T> void registerGenericClass(Class<T> clazz, int id) {
this.fallbackParser.registerClass(clazz, id);
}
public void registerTypeFallback(DBTypedObjectParser<?> parser) {
this.fallbackParser = parser;
}

View File

@ -0,0 +1,12 @@
package org.warp.jcwdb;
import java.nio.channels.SeekableByteChannel;
public class VariableWrapper<T> {
public T var;
public VariableWrapper(T value) {
this.var = value;
}
}

View File

@ -22,6 +22,7 @@ public class App {
System.out.println("Loading database...");
long time0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
db.registerClass(StrangeAnimal.class, 0);
try {
long time01 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time01 - time0));
@ -58,15 +59,15 @@ public class App {
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
/*
root.forEach((value) -> {
root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnly();
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
*/
long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");

View File

@ -26,93 +26,4 @@ public class AppTest
assertTrue( true );
}
/**
*
* @throws IOException
*/
@Test
public void shouldReadCustomClasses() throws IOException
{
CustomClass customClass = new CustomClass();
customClass.primitive = 1;
customClass.string = "test";
Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
JCWDatabase db = new JCWDatabase(path, idx);
EntryReference<LightList<CustomClass>> ref = db.getRoot();
ref.editValue((root, saver) -> {
root.add(customClass);
});
db.close();
JCWDatabase db2 = new JCWDatabase(path, idx);
EntryReference<LightList<CustomClass>> ref2 = db2.getRoot();
CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0);
assertTrue(customClass.equals(customClass2));
assertTrue(customClass.string.equals(customClass2.string));
db2.close();
Files.delete(path);
Files.delete(idx);
}
public static class CustomClass {
public String string;
public int primitive;
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + primitive;
result = prime * result + ((string == null) ? 0 : string.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
CustomClass other = (CustomClass) obj;
if (primitive != other.primitive)
return false;
if (string == null) {
if (other.string != null)
return false;
} else if (!string.equals(other.string))
return false;
return true;
}
}
/**
*
* @throws IOException
*/
@Test
public void shouldActAsAnArrayListWithEqualObjects() throws IOException
{
Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
JCWDatabase db = new JCWDatabase(path, idx);
EntryReference<LightList<String>> ref = db.getRoot();
LightList<String> list1 = ref.getValueReadOnly();
ArrayList<String> list2 = new ArrayList<String>();
String s = "a";
for (int i = 0; i < 10; i++) {
list1.add(s);
list2.add(s);
}
assertTrue(list1.size() == list2.size());
for (int i = 0; i < 10; i++) {
assertTrue(list1.get(i) == list2.get(i));
assertTrue(list2.get(i) == list1.get(i));
}
db.close();
Files.delete(path);
Files.delete(idx);
}
}