Chunked list works
This commit is contained in:
parent
e4fbd613db
commit
a4a981ad1c
@ -1,6 +1,7 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class CacheIndexManager implements IndexManager {
|
||||
@ -32,6 +33,12 @@ public class CacheIndexManager implements IndexManager {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) {
|
||||
// TODO: implement
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> IndexDetails set(long index, DBDataOutput<T> writer) {
|
||||
// TODO: implement
|
||||
|
@ -10,7 +10,7 @@ public class Cleaner {
|
||||
|
||||
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
|
||||
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
|
||||
private static final double NORMAL_REMOVED_ITEMS = 1000l;
|
||||
private static final double NORMAL_REMOVED_ITEMS = 2500l;
|
||||
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
|
||||
|
||||
private final Cleanable[] objectsToClean;
|
||||
@ -26,7 +26,7 @@ public class Cleaner {
|
||||
}
|
||||
|
||||
public void start() {
|
||||
this.cleanerThread.start();
|
||||
//this.cleanerThread.start();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -73,7 +73,7 @@ public class Cleaner {
|
||||
if (removedItems > 0) {
|
||||
final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
|
||||
System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
|
||||
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO && removedItemsRatio >= REMOVED_ITEMS_RATIO) {
|
||||
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
|
||||
suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
|
||||
}
|
||||
}
|
||||
|
@ -35,4 +35,11 @@ public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T
|
||||
public long calculateHash(LightArrayList<T> value) {
|
||||
return value.internalList.hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DBLightArrayListParser{" +
|
||||
"db=" + db +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
@ -21,7 +21,7 @@ public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
|
||||
chunks.add(itm);
|
||||
chunkSizes.add(itm2);
|
||||
}
|
||||
return new LightBigList<T>(db, chunks, chunkSizes);
|
||||
return new LightBigList<>(db, chunks, chunkSizes);
|
||||
};
|
||||
}
|
||||
|
||||
@ -39,6 +39,6 @@ public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
|
||||
|
||||
@Override
|
||||
public long calculateHash(LightBigList<T> value) {
|
||||
return value.chunks.hashCode();
|
||||
return (((long)value.chunks.hashCode()) << 32) | (value.chunkSizes.hashCode() & 0xffffffffL);
|
||||
}
|
||||
}
|
||||
|
@ -2,18 +2,18 @@ package org.warp.jcwdb;
|
||||
|
||||
public class DBStandardTypes {
|
||||
private static final int STD = 0xFFFFF000;
|
||||
public static final int BOOLEAN = STD| 0;
|
||||
public static final int BYTE = STD| 1;
|
||||
public static final int SHORT = STD| 2;
|
||||
public static final int CHAR = STD| 3;
|
||||
public static final int INTEGER = STD| 4;
|
||||
public static final int FLOAT = STD| 5;
|
||||
public static final int DOUBLE = STD| 6;
|
||||
public static final int STRING = STD| 7;
|
||||
public static final int BYTE_ARRAY = STD| 8;
|
||||
public static final int LIGHT_LIST_ARRAY = STD| 9;
|
||||
public static final int LIGHT_LIST_BIG = STD| 10;
|
||||
public static final int GENERIC_OBJECT = STD| 11;
|
||||
public static final int BOOLEAN = STD| 0x000;
|
||||
public static final int BYTE = STD| 0x001;
|
||||
public static final int SHORT = STD| 0x002;
|
||||
public static final int CHAR = STD| 0x003;
|
||||
public static final int INTEGER = STD| 0x004;
|
||||
public static final int FLOAT = STD| 0x005;
|
||||
public static final int DOUBLE = STD| 0x006;
|
||||
public static final int STRING = STD| 0x007;
|
||||
public static final int BYTE_ARRAY = STD| 0x008;
|
||||
public static final int LIGHT_LIST_ARRAY = STD| 0x009;
|
||||
public static final int LIGHT_LIST_BIG = STD| 0x00A;
|
||||
public static final int GENERIC_OBJECT = STD| 0x00B;
|
||||
|
||||
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
|
||||
typesManager.registerType(String.class, STRING, new DBStringParser());
|
||||
|
@ -69,6 +69,9 @@ public class EntryReference<T> implements Castable, Saveable {
|
||||
synchronized(accessLock) {
|
||||
if (loaded && !closed) {
|
||||
try {
|
||||
if (value instanceof Saveable) {
|
||||
((Saveable)value).save();
|
||||
}
|
||||
IndexDetails returnedDetails = db.write(entryIndex, parser.getWriter(value));
|
||||
synchronized(hashCacheLock) {
|
||||
this.cachedHash = returnedDetails.getHash();
|
||||
|
@ -1,14 +1,18 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import it.unimi.dsi.fastutil.longs.Long2IntMap;
|
||||
import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
|
||||
import it.unimi.dsi.fastutil.longs.*;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class FileAllocator implements AutoCloseable {
|
||||
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
|
||||
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 50000;
|
||||
|
||||
private final SeekableByteChannel dataFileChannel;
|
||||
private volatile long fileSize;
|
||||
@ -18,13 +22,19 @@ public class FileAllocator implements AutoCloseable {
|
||||
/**
|
||||
* index -> free space size
|
||||
*/
|
||||
private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
|
||||
private final Long2IntMap freeBytes = new Long2IntLinkedOpenHashMap();
|
||||
|
||||
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
|
||||
this.dataFileChannel = dataFileChannel;
|
||||
this.fileSize = this.dataFileChannel.size();
|
||||
}
|
||||
|
||||
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) throws IOException {
|
||||
this.dataFileChannel = dataFileChannel;
|
||||
this.fileSize = fileSize;
|
||||
this.freeBytes.putAll(freeBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: not implemented
|
||||
*
|
||||
@ -44,23 +54,27 @@ public class FileAllocator implements AutoCloseable {
|
||||
}
|
||||
|
||||
private long allocateIntoUnusedParts(int size) {
|
||||
ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
|
||||
long holeOffset = -1;
|
||||
int holeSize = 0;
|
||||
while (it.hasNext()) {
|
||||
Long2IntMap.Entry entry = it.next();
|
||||
int currentHoleSize = entry.getIntValue();
|
||||
Stream<Map.Entry<Long,Integer>> sorted =
|
||||
freeBytes.entrySet().stream()
|
||||
.sorted(Collections.reverseOrder(Map.Entry.comparingByValue()));
|
||||
final VariableWrapper<Long> holeOffset = new VariableWrapper<>(-1L);
|
||||
final VariableWrapper<Integer> holeSize = new VariableWrapper<>(0);
|
||||
sorted.anyMatch((entry) -> {
|
||||
int currentHoleSize = entry.getValue();
|
||||
if (currentHoleSize < size) {
|
||||
freeBytes.remove(holeOffset);
|
||||
if (holeSize > size) {
|
||||
freeBytes.put(holeOffset + size, holeSize - size);
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
holeOffset.var = entry.getKey();
|
||||
holeSize.var = currentHoleSize;
|
||||
return false;
|
||||
});
|
||||
if (holeOffset.var != -1L) {
|
||||
freeBytes.remove(holeOffset.var);
|
||||
if (holeSize.var > size) {
|
||||
freeBytes.put(holeOffset.var + size, holeSize.var - size);
|
||||
}
|
||||
holeOffset = entry.getLongKey();
|
||||
holeSize = currentHoleSize;
|
||||
}
|
||||
return holeOffset;
|
||||
return holeOffset.var;
|
||||
}
|
||||
|
||||
private long allocateToEnd(int size) {
|
||||
@ -103,13 +117,24 @@ public class FileAllocator implements AutoCloseable {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!addedToList) {
|
||||
if (!addedToList && length > 0) {
|
||||
freeBytes.put(startPosition, length);
|
||||
}
|
||||
}
|
||||
|
||||
if (startPosition + length >= fileSize) {
|
||||
fileSize = startPosition;
|
||||
}
|
||||
|
||||
// Remove the smallest hole in the file
|
||||
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
|
||||
freeBytes.remove(freeBytes.lastLongKey());
|
||||
Stream<Map.Entry<Long,Integer>> sorted =
|
||||
freeBytes.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByValue());
|
||||
Optional<Map.Entry<Long, Integer>> first = sorted.findFirst();
|
||||
if (first.isPresent()) {
|
||||
freeBytes.remove(first.get().getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3,6 +3,7 @@ package org.warp.jcwdb;
|
||||
import com.esotericsoftware.kryo.io.Input;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
import it.unimi.dsi.fastutil.longs.*;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectIterator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
@ -11,7 +12,7 @@ import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Iterator;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class FileIndexManager implements IndexManager {
|
||||
@ -49,13 +50,50 @@ public class FileIndexManager implements IndexManager {
|
||||
}
|
||||
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
fileAllocator = new FileAllocator(dataFileChannel);
|
||||
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
|
||||
firstAllocableIndex = metadataFileChannel.size() / (long) IndexDetails.TOTAL_BYTES;
|
||||
if (firstAllocableIndex == 0) {
|
||||
firstAllocableIndex = 1;
|
||||
}
|
||||
}
|
||||
|
||||
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
|
||||
Long2IntMap freeBytes = new Long2IntRBTreeMap();
|
||||
Long2IntMap usedBytes = new Long2IntRBTreeMap();
|
||||
long firstOffset = 0;
|
||||
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= metadataFileChannel.size()) {
|
||||
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
|
||||
if (indexDetails != null) {
|
||||
long offset = indexDetails.getOffset();
|
||||
usedBytes.put(offset, indexDetails.getSize());
|
||||
if (offset < firstOffset) {
|
||||
firstOffset = offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long previousEntryOffset = 0;
|
||||
long previousEntrySize = 0;
|
||||
ObjectIterator<Long2IntMap.Entry> it = usedBytes.long2IntEntrySet().iterator();
|
||||
while (it.hasNext()) {
|
||||
final Long2IntMap.Entry entry = it.next();
|
||||
final long entryOffset = entry.getLongKey();
|
||||
final long entrySize = entry.getIntValue();
|
||||
it.remove();
|
||||
|
||||
if (previousEntryOffset + previousEntrySize < entryOffset) {
|
||||
freeBytes.put(previousEntryOffset + previousEntrySize, (int) (entryOffset - (previousEntryOffset + previousEntrySize)));
|
||||
}
|
||||
|
||||
previousEntryOffset = entryOffset;
|
||||
previousEntrySize = entrySize;
|
||||
}
|
||||
|
||||
final long fileSize = previousEntryOffset + previousEntrySize;
|
||||
|
||||
return new FileAllocator(dataFileChannel, fileSize, freeBytes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T get(long index, DBReader<T> reader) throws IOException {
|
||||
checkClosed();
|
||||
@ -116,6 +154,19 @@ public class FileIndexManager implements IndexManager {
|
||||
return index;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> data) throws IOException {
|
||||
checkClosed();
|
||||
final int size = data.getSize();
|
||||
final long offset = fileAllocator.allocate(size);
|
||||
final int type = data.getType();
|
||||
final long hash = data.calculateHash();
|
||||
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
|
||||
final long index = createIndexMetadata(indexDetails);
|
||||
writeExact(indexDetails, data);
|
||||
return new FullIndexDetails(index, indexDetails);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the data at index.
|
||||
* The input size must be equal to the index size!
|
||||
@ -239,7 +290,7 @@ public class FileIndexManager implements IndexManager {
|
||||
* @param details
|
||||
*/
|
||||
private void editIndex(long index, IndexDetails details) {
|
||||
synchronized (indicesMapsAccessLock) {// FIXXXX main3
|
||||
synchronized (indicesMapsAccessLock) {
|
||||
loadedIndices.put(index, details);
|
||||
dirtyLoadedIndices.add(index);
|
||||
}
|
||||
@ -270,8 +321,20 @@ public class FileIndexManager implements IndexManager {
|
||||
return null;
|
||||
}
|
||||
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
|
||||
IndexDetails indexDetails = readIndexDetailsAt(currentMetadataFileChannel);
|
||||
|
||||
if (indexDetails != null) {
|
||||
editIndex(index, indexDetails);
|
||||
return indexDetails;
|
||||
}
|
||||
|
||||
// No results found. Returning null
|
||||
return null;
|
||||
}
|
||||
|
||||
private IndexDetails readIndexDetailsAt(SeekableByteChannel currentMetadataFileChannel) throws IOException {
|
||||
IndexDetails indexDetails = null;
|
||||
synchronized (metadataByteBufferLock) {// FIXXXX main2
|
||||
synchronized (metadataByteBufferLock) {
|
||||
metadataByteBuffer.rewind();
|
||||
currentMetadataFileChannel.read(metadataByteBuffer);
|
||||
metadataByteBuffer.rewind();
|
||||
@ -287,16 +350,9 @@ public class FileIndexManager implements IndexManager {
|
||||
indexDetails = new IndexDetails(offset, size, type, hash);
|
||||
}
|
||||
}
|
||||
|
||||
if (indexDetails != null) {
|
||||
editIndex(index, indexDetails);
|
||||
return indexDetails;
|
||||
}
|
||||
|
||||
// No results found. Returning null
|
||||
return null;
|
||||
}
|
||||
|
||||
private IndexDetails getIndexMetadata(long index) throws IOException {
|
||||
IndexDetails details = getIndexMetadataUnsafe(index);
|
||||
if (details == null)
|
||||
|
14
src/main/java/org/warp/jcwdb/FullIndexDetails.java
Normal file
14
src/main/java/org/warp/jcwdb/FullIndexDetails.java
Normal file
@ -0,0 +1,14 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
public class FullIndexDetails extends IndexDetails {
|
||||
private final long index;
|
||||
|
||||
public FullIndexDetails(long index, IndexDetails details) {
|
||||
super(details);
|
||||
this.index = index;
|
||||
}
|
||||
|
||||
public long getIndex() {
|
||||
return index;
|
||||
}
|
||||
}
|
@ -1,6 +1,8 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.BiPredicate;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public interface IndexManager extends Cleanable {
|
||||
@ -8,6 +10,7 @@ public interface IndexManager extends Cleanable {
|
||||
int getType(long index) throws IOException;
|
||||
long getHash(long index) throws IOException;
|
||||
<T> long add(DBDataOutput<T> writer) throws IOException;
|
||||
<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
|
||||
<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
|
||||
void delete(long index) throws IOException;
|
||||
boolean has(long index);
|
||||
|
@ -6,7 +6,7 @@ import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
public final static long MAX_LOADED_INDICES = 10000;
|
||||
public final static long MAX_LOADED_INDICES = 1000;
|
||||
|
||||
private final TypesManager typesManager;
|
||||
private final MixedIndexDatabase indices;
|
||||
@ -36,7 +36,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
if (exists(0)) {
|
||||
return get(0);
|
||||
} else {
|
||||
LightList<T> newRoot = new LightArrayList<>(this);
|
||||
LightList<T> newRoot = new LightBigList<>(this);
|
||||
return set(0, newRoot);
|
||||
}
|
||||
}
|
||||
@ -63,8 +63,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
|
||||
long index;
|
||||
long hash;
|
||||
synchronized (indicesAccessLock) {
|
||||
index = indices.add(typeParser.getWriter(value));
|
||||
hash = indices.getHash(index);
|
||||
FullIndexDetails fullIndexDetails = indices.addAndGetDetails(typeParser.getWriter(value));
|
||||
index = fullIndexDetails.getIndex();
|
||||
hash = fullIndexDetails.getHash();
|
||||
}
|
||||
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
|
||||
}
|
||||
|
@ -121,10 +121,11 @@ public class LightArrayList<T> implements LightList<T> {
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
final T[] elements = (T[]) new Objects[internalList.size()];
|
||||
final T[] elements = (T[]) new Object[internalList.size()];
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
try {
|
||||
elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
T element = (T) db.get(internalList.getLong(i)).getValueReadOnly();
|
||||
elements[i] = element;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
@ -442,4 +443,12 @@ public class LightArrayList<T> implements LightList<T> {
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LightArrayList{" +
|
||||
"internalList=" + internalList +
|
||||
", db=" + db +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
@ -9,13 +9,18 @@ import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
public class LightBigList<T> implements LightList<T> {
|
||||
public class LightBigList<T> implements LightList<T>, Saveable {
|
||||
|
||||
public static final int MAX_ELEMENTS_PER_CHUNK = 10000;
|
||||
public static final int MAX_ELEMENTS_PER_CHUNK = 200000;
|
||||
|
||||
public final LongArrayList chunks;
|
||||
public final IntArrayList chunkSizes;
|
||||
private final transient JCWDatabase db;
|
||||
private final JCWDatabase db;
|
||||
private LightList<T> cachedChunk;
|
||||
private EntryReference<LightList<T>> cachedChunkRef;
|
||||
private long cachedChunkIndex = -1;
|
||||
private int cachedChunkNumber = -1;
|
||||
private final Object cachedChunkLock = new Object();
|
||||
|
||||
/**
|
||||
* @param db Database reference
|
||||
@ -43,6 +48,23 @@ public class LightBigList<T> implements LightList<T> {
|
||||
this.db = db;
|
||||
this.chunks = chunks;
|
||||
this.chunkSizes = chunkSizes;
|
||||
if (this.chunks.size() > 0) {
|
||||
prepareAccessToChunk(0);
|
||||
}
|
||||
}
|
||||
|
||||
private void prepareAccessToChunk(int chunkNumber) {
|
||||
if (this.cachedChunkRef != null) {
|
||||
this.cachedChunkRef.save();
|
||||
}
|
||||
this.cachedChunkNumber = chunkNumber;
|
||||
this.cachedChunkIndex = this.chunks.getLong(chunkNumber);
|
||||
try {
|
||||
this.cachedChunkRef = db.get(this.cachedChunkIndex);
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
this.cachedChunk = this.cachedChunkRef.getValueReadOnly();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -53,16 +75,13 @@ public class LightBigList<T> implements LightList<T> {
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int chunkNumber = i;
|
||||
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
|
||||
try {
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
final EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
chunkRef.editValue((chunk) -> {
|
||||
chunk.appendIndex(elementIndex);
|
||||
chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1);
|
||||
});
|
||||
synchronized (cachedChunkLock) {
|
||||
if (cachedChunkNumber != i) {
|
||||
prepareAccessToChunk(i);
|
||||
}
|
||||
cachedChunk.appendIndex(elementIndex);
|
||||
chunkSizes.set(chunkNumber, cachedChunk.size());
|
||||
return;
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -131,7 +150,8 @@ public class LightBigList<T> implements LightList<T> {
|
||||
*/
|
||||
@Deprecated
|
||||
@Override
|
||||
public Iterator<T> iterator() {
|
||||
public Iterator<T> iterator()
|
||||
{
|
||||
throw new RuntimeException("iterator() isn't implemented!");
|
||||
}
|
||||
|
||||
@ -166,10 +186,36 @@ public class LightBigList<T> implements LightList<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* toArray() isn't implemented! DO NOT USE IT.
|
||||
* @return
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Deprecated
|
||||
@Override
|
||||
public T[] toArray() {
|
||||
throw new RuntimeException("toArray() isn't implemented!");
|
||||
T[] result = (T[]) new Object[this.size()];
|
||||
|
||||
long currentOffset = 0;
|
||||
// Iterate through all chunks
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
final int currentChunkSize = chunkSizes.getInt(i);
|
||||
final long chunkStartOffset = currentOffset;
|
||||
currentOffset += currentChunkSize;
|
||||
// Get chunk index
|
||||
final long chunkIndex = chunks.getLong(i);
|
||||
|
||||
try {
|
||||
EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
|
||||
LightList<T> chunk = chunkRef.getValueReadOnly();
|
||||
for (int i1 = 0; i1 < chunk.size(); i1++) {
|
||||
result[(int)(chunkStartOffset + i1)] = chunk.get(i);
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
throw (NullPointerException) new NullPointerException().initCause(ex);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@ -518,4 +564,20 @@ public class LightBigList<T> implements LightList<T> {
|
||||
}
|
||||
return result.var;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LightBigList{" +
|
||||
"chunks=" + chunks +
|
||||
", chunkSizes=" + chunkSizes +
|
||||
", db=" + db +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save() {
|
||||
if (this.cachedChunkRef != null) {
|
||||
this.cachedChunkRef.save();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -5,6 +5,7 @@ import it.unimi.dsi.fastutil.longs.Long2LongMap;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class MixedIndexDatabase implements IndexManager {
|
||||
@ -48,6 +49,11 @@ public class MixedIndexDatabase implements IndexManager {
|
||||
return fileIndices.add(writer);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException {
|
||||
return fileIndices.addAndGetDetails(writer);
|
||||
}
|
||||
@Override
|
||||
public <T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException {
|
||||
if (cacheIndices.has(index)) {
|
||||
|
@ -37,7 +37,7 @@ public class App {
|
||||
// System.out.println(" - " + root.get(i));
|
||||
// }
|
||||
long prectime = System.currentTimeMillis();
|
||||
for (int i = 0; i < 20000/* 2000000 */; i++) {
|
||||
for (int i = 0; i < 20000000/* 2000000 */; i++) {
|
||||
Animal animal = new StrangeAnimal(i % 40);
|
||||
root.add(animal);
|
||||
if (i > 0 && i % 200000 == 0) {
|
||||
|
@ -1,29 +1,18 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.warp.jcwdb.EntryReference;
|
||||
import org.warp.jcwdb.JCWDatabase;
|
||||
import org.warp.jcwdb.LightList;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
/**
|
||||
* Unit test for simple App.
|
||||
*/
|
||||
public class AppTest
|
||||
{
|
||||
public class AppTest {
|
||||
/**
|
||||
* Rigorous Test :-)
|
||||
*/
|
||||
@Test
|
||||
public void shouldAnswerWithTrue()
|
||||
{
|
||||
public void shouldAnswerWithTrue() {
|
||||
assertTrue(true);
|
||||
}
|
||||
|
||||
}
|
||||
|
51
src/test/java/org/warp/jcwdb/FileAllocatorTest.java
Normal file
51
src/test/java/org/warp/jcwdb/FileAllocatorTest.java
Normal file
@ -0,0 +1,51 @@
|
||||
package org.warp.jcwdb;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class FileAllocatorTest {
|
||||
|
||||
@Test
|
||||
public void shouldAllocateAtZero() throws IOException {
|
||||
Path tempFile = Files.createTempFile("", "");
|
||||
SeekableByteChannel byteCh = Files.newByteChannel(tempFile);
|
||||
FileAllocator allocator = new FileAllocator(byteCh);
|
||||
long offset1 = allocator.allocate(512);
|
||||
assertEquals(0, offset1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldAllocateAt512() throws IOException {
|
||||
Path tempFile = Files.createTempFile("", "");
|
||||
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
|
||||
byteCh.write(ByteBuffer.wrap(new byte[512]));
|
||||
FileAllocator allocator = new FileAllocator(byteCh);
|
||||
long offset1 = allocator.allocate(512);
|
||||
assertEquals(512, offset1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldAllocateUnusedSpace() throws IOException {
|
||||
Path tempFile = Files.createTempFile("", "");
|
||||
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
|
||||
FileAllocator allocator = new FileAllocator(byteCh);
|
||||
long offset1 = allocator.allocate(512);
|
||||
allocator.markFree(offset1, 512);
|
||||
long offset2 = allocator.allocate(128);
|
||||
long offset3 = allocator.allocate(512-128);
|
||||
long offset4 = allocator.allocate(128);
|
||||
assertEquals(0, offset2);
|
||||
assertEquals(128, offset3);
|
||||
assertEquals(512, offset4);
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user