Finished implementing FileIndexManager

This commit is contained in:
Andrea Cavalli 2018-11-21 01:02:25 +01:00
parent 487840a80a
commit d9da3274c2
17 changed files with 493 additions and 85 deletions

View File

@ -11,33 +11,40 @@ public class CacheIndexManager implements IndexManager {
@Override
public <T> T get(long index, DBReader<T> reader) {
// TODO: implement
return null;
}
@Override
public int getType(long index) throws IOException {
// TODO Auto-generated method stub
public int getType(long index) {
// TODO: implement
return 0;
}
@Override
public <T> long add(DBDataOutput<T> writer) {
// TODO: implement
return 0;
}
@Override
public <T> void set(long index, DBDataOutput<T> writer) {
// TODO: implement
}
@Override
public void delete(long index) {
// TODO: implement
}
@Override
public boolean has(long index) {
// TODO: implement
return false;
}
@Override
public void close() throws IOException {
// TODO Auto-generated method stub
public void close() {
// TODO: implement
}
}

View File

@ -1,5 +1,5 @@
package org.warp.jcwdb;
public interface Castable {
public <T> T cast();
<T> T cast();
}

View File

@ -1,11 +1,11 @@
package org.warp.jcwdb;
public interface DBDataOutput<T> {
public int getSize();
public int getType();
public DBWriter<T> getWriter();
int getSize();
int getType();
DBWriter<T> getWriter();
public static <T> DBDataOutput<T> create(DBWriter<T> writer, int type, int size) {
static <T> DBDataOutput<T> create(DBWriter<T> writer, int type, int size) {
return new DBDataOutput<T>() {
@Override

View File

@ -0,0 +1,36 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Output;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
public class DBLightListParser extends DBTypeParserImpl<LightList> {
private static final Kryo kryo = new Kryo();
private final JCWDatabase db;
public DBLightListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightList> getReader() {
return (i) -> {
ArrayList<Long> internalList = (ArrayList<Long>) kryo.readClassAndObject(i);
return new LightList(db, internalList);
};
}
public DBDataOutput<LightList> getWriter(final LightList value) {
// TODO: optimize by writing longs directly, to make length determinable without writing to memory the output.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Output out = new Output(baos);
kryo.writeClassAndObject(out, value.internalList);
out.close();
byte[] b = baos.toByteArray();
return DBDataOutput.create((o) -> {
o.write(b);
}, DBStandardTypes.STRING, b.length);
}
}

View File

@ -3,5 +3,5 @@ package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
public interface DBReader<T> {
public T read(Input i);
T read(Input i);
}

View File

@ -1,18 +1,20 @@
package org.warp.jcwdb;
public class DBStandardTypes {
private static final int STD = 0xFFFFF000;
public static final int BOOLEAN = STD| 0;
public static final int BYTE = STD| 1;
public static final int SHORT = STD| 2;
public static final int CHAR = STD| 3;
public static final int INTEGER = STD| 4;
public static final int FLOAT = STD| 5;
public static final int DOUBLE = STD| 6;
public static final int STRING = STD| 7;
private static final int STD = 0xFFFFF000;
public static final int BOOLEAN = STD| 0;
public static final int BYTE = STD| 1;
public static final int SHORT = STD| 2;
public static final int CHAR = STD| 3;
public static final int INTEGER = STD| 4;
public static final int FLOAT = STD| 5;
public static final int DOUBLE = STD| 6;
public static final int STRING = STD| 7;
public static final int BYTE_ARRAY = STD| 8;
public static final int LIGHT_LIST = STD| 9;
public static void registerStandardTypes(TypesManager typesManager) {
typesManager.registerType(STRING, new DBStringParser());
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
typesManager.registerType(String.class, STRING, new DBStringParser());
typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db));
}
}

View File

@ -1,6 +1,6 @@
package org.warp.jcwdb;
public interface DBTypeParser<T> extends Castable {
public DBReader<T> getReader();
public DBDataOutput<T> getWriter(final T value);
DBReader<T> getReader();
DBDataOutput<T> getWriter(final T value);
}

View File

@ -3,5 +3,5 @@ package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Output;
public interface DBWriter<T> {
public void write(Output o);
void write(Output o);
}

View File

@ -2,20 +2,58 @@ package org.warp.jcwdb;
import java.io.IOException;
public class EntryReference<T> {
/**
* You must have only a maximum of 1 reference for each index
* @param <T>
*/
public class EntryReference<T> implements Castable, AutoCloseable {
private final JCWDatabase db;
private final long entryId;
private final long entryIndex;
private final DBTypeParser<T> parser;
public T value;
private volatile boolean closed;
private final Object closeLock = new Object();
public EntryReference(JCWDatabase db, long entryId, DBTypeParser<T> parser) throws IOException {
this.db = db;
this.entryId = entryId;
this.entryIndex = entryId;
this.parser = parser;
this.value = db.getIndexManager().get(entryId, parser.getReader());
this.value = db.indices.get(entryId, parser.getReader());
}
public DBTypeParser<T> getParser() {
return parser;
}
public long getIndex() {
return entryIndex;
}
public void save() throws IOException {
db.getIndexManager().set(entryId, parser.getWriter(value));
if (!closed) {
db.indices.set(entryIndex, parser.getWriter(value));
}
}
@Override
public <T> T cast() {
return (T) this;
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
db.removeEntryReference(entryIndex);
save();
closed = true;
}
}
}

View File

@ -10,7 +10,7 @@ import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.util.*;
public class FileAllocator {
public class FileAllocator implements AutoCloseable {
private final SeekableByteChannel dataFileChannel;
private volatile long allocableOffset;
private volatile boolean closed;
@ -24,7 +24,6 @@ public class FileAllocator {
/**
* TODO: not implemented
* @param size
* @param type
* @return offset
*/
public long allocate(int size) {

View File

@ -3,8 +3,6 @@ package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
@ -33,15 +31,18 @@ public class FileIndexManager implements IndexManager {
/**
* Edit this using editIndex()
*/
private final Set<Long> dirtyLoadedIndices;
private final Set<Long> dirtyLoadedIndices, removedIndices;
private long firstAllocableIndex;
public FileIndexManager(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.typesManager = typesManager;
loadedIndices = new HashMap<>();
dirtyLoadedIndices = new HashSet<>();
removedIndices = new HashSet<>();
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.CREATE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.CREATE);
fileAllocator = new FileAllocator(dataFileChannel);
firstAllocableIndex = metadataFileChannel.size() / (long) IndexDetails.TOTAL_BYTES;
}
@Override
@ -62,7 +63,7 @@ public class FileIndexManager implements IndexManager {
@Override
public <T> void set(long index, DBDataOutput<T> data) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < data.getSize()) {
allocateAndWrite(index, data);
} else {
@ -70,13 +71,43 @@ public class FileIndexManager implements IndexManager {
editIndex(index, indexDetails.getOffset(), data.getSize(), indexDetails.getType());
fileAllocator.markFree(indexDetails.getOffset()+data.getSize(), data.getSize());
}
write(index, indexDetails, data);
writeExact(indexDetails, data);
}
}
private void write(final long index, final IndexDetails indexDetails, DBDataOutput<?> data) throws IOException {
@Override
public <T> long add(DBDataOutput<T> data) throws IOException {
checkClosed();
final int size = data.getSize();
final long offset = fileAllocator.allocate(size);
final int type = data.getType();
final IndexDetails indexDetails = new IndexDetails(offset, size, type);
final long index = createIndexMetadata(indexDetails);
writeExact(indexDetails, data);
return index;
}
private long createIndexMetadata(IndexDetails indexDetails) {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
removedIndices.remove(newIndex);
return newIndex;
}
/**
* Write the data at index.
* The input size must be equal to the index size!
* @param indexDetails
* @param data
* @throws IOException
*/
private void writeExact(final IndexDetails indexDetails, DBDataOutput<?> data) throws IOException {
if (indexDetails.getSize() != data.getSize()) {
throw new IOException("Unable to write " + data.getSize() + " in a space of " + indexDetails.getSize());
}
final long offset = indexDetails.getOffset();
final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset)));
data.getWriter().write(o);
o.close();
@ -87,19 +118,30 @@ public class FileIndexManager implements IndexManager {
final int type = w.getType();
final long offset = fileAllocator.allocate(size);
IndexDetails details = editIndex(index, offset, size, type);
write(index, details, w);
writeExact(details, w);
}
@Override
public void delete(long index) {
public void delete(long index) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails != null) {
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
dirtyLoadedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
}
@Override
public boolean has(long index) {
checkClosed();
return false;
try {
return getIndexMetadataUnsafe(index) != null;
} catch (IOException ex) {
ex.printStackTrace();
return false;
}
}
private IndexDetails editIndex(long index, long offset, int size, int type) {
@ -108,14 +150,30 @@ public class FileIndexManager implements IndexManager {
return indexDetails;
}
private IndexDetails getIndexMetadataUnsafe(long index) {
private IndexDetails getIndexMetadataUnsafe(long index) throws IOException {
// Return index details if loaded
IndexDetails details = loadedIndices.getOrDefault(index, null);
if (details != null) return details;
// TODO: implement index loading from file
// Try to load the details from file
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
ByteBuffer currentMetadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
currentMetadataFileChannel.read(currentMetadataByteBuffer);
currentMetadataByteBuffer.flip();
// If it's not deleted continue
if ((currentMetadataByteBuffer.get() & IndexDetails.MASK_DELETED) == 0) {
final long offset = currentMetadataByteBuffer.getLong();
final int size = currentMetadataByteBuffer.getInt();
final int type = currentMetadataByteBuffer.getInt();
final IndexDetails indexDetails = new IndexDetails(offset, size, type);
editIndex(index, indexDetails);
return indexDetails;
}
// No results found. Returning null
return null;
}
private IndexDetails getIndexMetadata(long index) throws IOException {
IndexDetails details = getIndexMetadataUnsafe(index);
if (details == null)
@ -141,6 +199,7 @@ public class FileIndexManager implements IndexManager {
closed = true;
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel;
ByteBuffer metadataEntryBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
long lastIndex = -2;
@ -149,12 +208,23 @@ public class FileIndexManager implements IndexManager {
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
metadataEntryBuffer.put((byte) 0);
metadataEntryBuffer.putLong(indexDetails.getOffset());
metadataEntryBuffer.putInt(indexDetails.getSize());
metadataEntryBuffer.putInt(indexDetails.getType());
metadataEntryBuffer.flip();
metadata.write(metadataEntryBuffer);
lastIndex = index;
}
// Remove removed indices
ByteBuffer updatedMaskBuffer = ByteBuffer.allocateDirect(1);
for (Long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
updatedMaskBuffer.put(IndexDetails.MASK_DELETED);
updatedMaskBuffer.flip();
metadata.write(updatedMaskBuffer);
}
fileAllocator.close();
}

View File

@ -3,10 +3,15 @@ package org.warp.jcwdb;
import java.util.Objects;
public class IndexDetails {
/**
* The bitmask is used to determine if an index has been deleted
*/
public static final int BITMASK = 1; // 1 byte
public static final int OFFSET_BYTES = Long.SIZE;
public static final int DATA_SIZE_BYTES = Integer.SIZE;
public static final int TYPE_BYTES = Integer.SIZE;
public static final int TOTAL_BYTES = OFFSET_BYTES + DATA_SIZE_BYTES + TYPE_BYTES;
public static final int TOTAL_BYTES = BITMASK + OFFSET_BYTES + DATA_SIZE_BYTES + TYPE_BYTES;
public static final byte MASK_DELETED = 0b00000001;
private final long offset;
private final int size;
private final int type;

View File

@ -3,10 +3,11 @@ package org.warp.jcwdb;
import java.io.IOException;
public interface IndexManager {
public <T> T get(long index, DBReader<T> reader) throws IOException;
<T> T get(long index, DBReader<T> reader) throws IOException;
int getType(long index) throws IOException;
public <T> void set(long index, DBDataOutput<T> writer) throws IOException;
public void delete(long index) throws IOException;
public boolean has(long index);
public void close() throws IOException;
<T> long add(DBDataOutput<T> writer) throws IOException;
<T> void set(long index, DBDataOutput<T> writer) throws IOException;
void delete(long index) throws IOException;
boolean has(long index);
void close() throws IOException;
}

View File

@ -2,30 +2,64 @@ package org.warp.jcwdb;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
import java.util.WeakHashMap;
public class JCWDatabase {
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
public class JCWDatabase implements AutoCloseable {
protected final TypesManager typesManager;
protected final MixedIndexDatabase indices;
private final WeakHashMap<Long, EntryReference<?>> references;
private final WeakHashMap<Object, EntryReference<?>> referencesByObject;
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager();
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
this.references = new WeakHashMap<>();
this.referencesByObject = new WeakHashMap<>();
}
public <T> EntryReference<T> getRoot() {
// Get type of index 0
int type = this.indices.getType(0);
// Get the parser
DBTypeParser<T> parser = this.typesManager.get(type);
// Return the reference
public <T> EntryReference<T> getRoot() throws IOException {
try {
return new EntryReference<T>(this, 0, parser);
return get(0);
} catch (IOException e) {
throw new RuntimeException("Can't load root!", e);
throw new IOException("Can't load root!", e);
}
}
public IndexManager getIndexManager() {
return indices;
public <T> EntryReference<T> get(long index) throws IOException {
EntryReference<T> ref = (EntryReference<T>) this.references.getOrDefault(index, null);
if (ref == null) {
int type = this.indices.getType(index);
DBTypeParser<T> typeParser = this.typesManager.get(type);
ref = new EntryReference<>(this, index, typeParser);
this.references.put(index, ref);
this.referencesByObject.put(ref.value, ref);
}
return ref;
}
public <T> EntryReference<T> add(T o) throws IOException {
EntryReference<T> ref = (EntryReference<T>) referencesByObject.getOrDefault(o, null);
if (ref != null) {
return ref;
}
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) o.getClass());
long index = indices.add(typeParser.getWriter(o));
ref = new EntryReference<>(this, index, typeParser);
this.references.put(index, ref);
this.referencesByObject.put(o, ref);
return ref;
}
protected void removeEntryReference(long index) {
this.references.remove(index);
}
@Override
public void close() throws Exception {
for (Map.Entry<Long, EntryReference<?>> reference : references.entrySet()) {
reference.getValue().close();
}
this.indices.close();
}
}

View File

@ -0,0 +1,193 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.*;
public class LightList<T> implements List<T> {
public final ArrayList<Long> internalList;
private final transient JCWDatabase db;
public LightList(JCWDatabase db, ArrayList<Long> internalList) {
this.internalList = internalList;
this.db = db;
}
@Override
public int size() {
return internalList.size();
}
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (Long element : internalList) {
try {
EntryReference<T> ref = db.get(element);
if (o.equals(ref.value)) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
@Override
public Iterator<T> iterator() {
final ArrayList<T> elements = new ArrayList<>();
for (Long element : internalList) {
try {
elements.add((T) db.get(element).value);
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@Override
public T[] toArray() {
final T[] elements = (T[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T) db.get(internalList.get(i)).value;
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.get(i)).value;
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref;
try {
ref = db.add(o);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return internalList.add(ref.getIndex());
}
@Override
public boolean remove(Object o) {
EntryReference<T> ref;
try {
ref = db.add((T) o);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return internalList.remove(ref.getIndex());
}
@Override
public boolean containsAll(Collection<?> c) {
// TODO: implement
return false;
}
@Override
public boolean addAll(Collection<? extends T> c) {
// TODO: implement
return false;
}
@Override
public boolean addAll(int index, Collection<? extends T> c) {
// TODO: implement
return false;
}
@Override
public boolean removeAll(Collection<?> c) {
// TODO: implement
return false;
}
@Override
public boolean retainAll(Collection<?> c) {
// TODO: implement
return false;
}
@Override
public void clear() {
// TODO: implement
}
@Override
public T get(int index) {
// TODO: implement
return null;
}
@Override
public T set(int index, T element) {
// TODO: implement
return null;
}
@Override
public void add(int index, T element) {
// TODO: implement
}
@Override
public T remove(int index) {
// TODO: implement
return null;
}
@Override
public int indexOf(Object o) {
// TODO: implement
return 0;
}
@Override
public int lastIndexOf(Object o) {
// TODO: implement
return 0;
}
@Override
public ListIterator<T> listIterator() {
// TODO: implement
return null;
}
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
return null;
}
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
return null;
}
}

View File

@ -20,30 +20,46 @@ public class MixedIndexDatabase implements IndexManager {
}
@Override
public <T> T get(long index, DBReader<T> reader) {
// TODO: implement
return null;
public <T> T get(long index, DBReader<T> reader) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.get(index, reader);
} else {
return fileIndices.get(index, reader);
}
}
@Override
public int getType(long index) {
// TODO: implement
return -1;
public int getType(long index) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.getType(index);
} else {
return fileIndices.getType(index);
}
}
@Override
public <T> void set(long index, DBDataOutput<T> writer) {
public <T> long add(DBDataOutput<T> writer) throws IOException {
return fileIndices.add(writer);
}
@Override
public void delete(long index) {
public <T> void set(long index, DBDataOutput<T> writer) throws IOException {
if (cacheIndices.has(index)) {
cacheIndices.set(index, writer);
} else {
fileIndices.set(index, writer);
}
}
@Override
public void delete(long index) throws IOException {
cacheIndices.delete(index);
fileIndices.delete(index);
}
@Override
public boolean has(long index) {
return false;
return cacheIndices.has(index) || fileIndices.has(index);
}
@Override

View File

@ -6,17 +6,24 @@ import java.util.HashMap;
public class TypesManager {
private final Map<Integer, DBTypeParser<?>> types;
public TypesManager() {
private final Map<Class<?>, DBTypeParser<?>> typesByClass;
public TypesManager(JCWDatabase db) {
types = new HashMap<>();
DBStandardTypes.registerStandardTypes(this);
typesByClass = new HashMap<>();
DBStandardTypes.registerStandardTypes(db, this);
}
public <T> void registerType(int type, DBTypeParser<T> parser) {
public <T> void registerType(Class<T> clazz, int type, DBTypeParser<T> parser) {
this.types.put(type, parser);
this.typesByClass.put(clazz, parser);
}
public <T> DBTypeParser<T> get(int type) {
return types.get(type).cast();
}
public <T> DBTypeParser<T> get(Class<T> type) {
return typesByClass.get(type).cast();
}
}