From ce9f5a20a8f68c3dcb086192b4eaf331b8b9f0c8 Mon Sep 17 00:00:00 2001 From: Cavallium Date: Tue, 20 Nov 2018 18:39:48 +0100 Subject: [PATCH] Initial implementation --- src/main/java/org/warp/App.java | 13 -- .../org/warp/jcwdb/CacheIndexManager.java | 22 ++- src/main/java/org/warp/jcwdb/Castable.java | 5 + .../java/org/warp/jcwdb/DBDataOutput.java | 28 ++++ .../java/org/warp/jcwdb/DBStandardTypes.java | 18 +++ .../java/org/warp/jcwdb/DBStringParser.java | 23 +++ .../java/org/warp/jcwdb/DBTypeParser.java | 4 +- .../java/org/warp/jcwdb/DBTypeParserImpl.java | 9 ++ .../java/org/warp/jcwdb/EntryReference.java | 14 +- .../java/org/warp/jcwdb/FileAllocator.java | 118 ++++++---------- .../java/org/warp/jcwdb/FileIndexManager.java | 133 +++++++++++++++++- .../java/org/warp/jcwdb/IndexDetails.java | 4 + .../java/org/warp/jcwdb/IndexManager.java | 6 +- src/main/java/org/warp/jcwdb/JCWDatabase.java | 24 +++- .../org/warp/jcwdb/MixedIndexDatabase.java | 26 +++- .../java/org/warp/jcwdb/TypesManager.java | 22 +++ 16 files changed, 356 insertions(+), 113 deletions(-) delete mode 100644 src/main/java/org/warp/App.java create mode 100644 src/main/java/org/warp/jcwdb/Castable.java create mode 100644 src/main/java/org/warp/jcwdb/DBDataOutput.java create mode 100644 src/main/java/org/warp/jcwdb/DBStandardTypes.java create mode 100644 src/main/java/org/warp/jcwdb/DBStringParser.java create mode 100644 src/main/java/org/warp/jcwdb/DBTypeParserImpl.java create mode 100644 src/main/java/org/warp/jcwdb/TypesManager.java diff --git a/src/main/java/org/warp/App.java b/src/main/java/org/warp/App.java deleted file mode 100644 index c3a2ec8..0000000 --- a/src/main/java/org/warp/App.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.warp; - -/** - * Hello world! - * - */ -public class App -{ - public static void main( String[] args ) - { - System.out.println( "Hello World!" ); - } -} diff --git a/src/main/java/org/warp/jcwdb/CacheIndexManager.java b/src/main/java/org/warp/jcwdb/CacheIndexManager.java index 2e6ecb6..9a3f082 100644 --- a/src/main/java/org/warp/jcwdb/CacheIndexManager.java +++ b/src/main/java/org/warp/jcwdb/CacheIndexManager.java @@ -3,17 +3,25 @@ package org.warp.jcwdb; import java.io.IOException; public class CacheIndexManager implements IndexManager { - public CacheIndexManager() { - + private final TypesManager typesManager; + + public CacheIndexManager(TypesManager typesManager) { + this.typesManager = typesManager; } @Override - public T get(long index, int type, DBReader reader) { + public T get(long index, DBReader reader) { return null; } @Override - public void set(long index, int type, DBWriter writer) { + public int getType(long index) throws IOException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public void set(long index, DBDataOutput writer) { } @@ -26,4 +34,10 @@ public class CacheIndexManager implements IndexManager { public boolean has(long index) { return false; } + + @Override + public void close() throws IOException { + // TODO Auto-generated method stub + + } } diff --git a/src/main/java/org/warp/jcwdb/Castable.java b/src/main/java/org/warp/jcwdb/Castable.java new file mode 100644 index 0000000..bc5c336 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/Castable.java @@ -0,0 +1,5 @@ +package org.warp.jcwdb; + +public interface Castable { + public T cast(); +} diff --git a/src/main/java/org/warp/jcwdb/DBDataOutput.java b/src/main/java/org/warp/jcwdb/DBDataOutput.java new file mode 100644 index 0000000..719ddcf --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBDataOutput.java @@ -0,0 +1,28 @@ +package org.warp.jcwdb; + +public interface DBDataOutput { + public int getSize(); + public int getType(); + public DBWriter getWriter(); + + public static DBDataOutput create(DBWriter writer, int type, int size) { + return new DBDataOutput() { + + @Override + public int getSize() { + return size; + } + + @Override + public int getType() { + return type; + } + + @Override + public DBWriter getWriter() { + return writer; + } + + }; + } +} diff --git a/src/main/java/org/warp/jcwdb/DBStandardTypes.java b/src/main/java/org/warp/jcwdb/DBStandardTypes.java new file mode 100644 index 0000000..8dcf5e2 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBStandardTypes.java @@ -0,0 +1,18 @@ +package org.warp.jcwdb; + +public class DBStandardTypes { + private static final int STD = 0xFFFFF000; + public static final int BOOLEAN = STD| 0; + public static final int BYTE = STD| 1; + public static final int SHORT = STD| 2; + public static final int CHAR = STD| 3; + public static final int INTEGER = STD| 4; + public static final int FLOAT = STD| 5; + public static final int DOUBLE = STD| 6; + public static final int STRING = STD| 7; + public static final int BYTE_ARRAY = STD| 8; + + public static void registerStandardTypes(TypesManager typesManager) { + typesManager.registerType(STRING, new DBStringParser()); + } +} diff --git a/src/main/java/org/warp/jcwdb/DBStringParser.java b/src/main/java/org/warp/jcwdb/DBStringParser.java new file mode 100644 index 0000000..ac359fa --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBStringParser.java @@ -0,0 +1,23 @@ +package org.warp.jcwdb; + +import java.nio.charset.StandardCharsets; + +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +public class DBStringParser extends DBTypeParserImpl { + private static final DBReader defaultReader = (i) -> { + return i.readString(); + }; + + public DBReader getReader() { + return defaultReader; + } + + public DBDataOutput getWriter(final String value) { + final byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + return DBDataOutput.create((o) -> { + o.write(bytes); + }, DBStandardTypes.STRING, bytes.length); + } +} diff --git a/src/main/java/org/warp/jcwdb/DBTypeParser.java b/src/main/java/org/warp/jcwdb/DBTypeParser.java index be64f52..3fa53f8 100644 --- a/src/main/java/org/warp/jcwdb/DBTypeParser.java +++ b/src/main/java/org/warp/jcwdb/DBTypeParser.java @@ -1,4 +1,6 @@ package org.warp.jcwdb; -public interface DBTypeParser extends DBReader, DBWriter { +public interface DBTypeParser extends Castable { + public DBReader getReader(); + public DBDataOutput getWriter(final T value); } diff --git a/src/main/java/org/warp/jcwdb/DBTypeParserImpl.java b/src/main/java/org/warp/jcwdb/DBTypeParserImpl.java new file mode 100644 index 0000000..daa7dcd --- /dev/null +++ b/src/main/java/org/warp/jcwdb/DBTypeParserImpl.java @@ -0,0 +1,9 @@ +package org.warp.jcwdb; + +public abstract class DBTypeParserImpl implements DBTypeParser { + @SuppressWarnings("unchecked") + @Override + public T cast() { + return (T) this; + } +} diff --git a/src/main/java/org/warp/jcwdb/EntryReference.java b/src/main/java/org/warp/jcwdb/EntryReference.java index 3c7b79f..d10d122 100644 --- a/src/main/java/org/warp/jcwdb/EntryReference.java +++ b/src/main/java/org/warp/jcwdb/EntryReference.java @@ -1,19 +1,21 @@ package org.warp.jcwdb; -public abstract class EntryReference { +import java.io.IOException; + +public class EntryReference { private final JCWDatabase db; private final long entryId; - private final DBTypeParser parser; + private final DBTypeParser parser; public T value; - public EntryReference(JCWDatabase db, long entryId, DBTypeParser parser) { + public EntryReference(JCWDatabase db, long entryId, DBTypeParser parser) throws IOException { this.db = db; this.entryId = entryId; this.parser = parser; - this.value = db.get(entryId, parser); + this.value = db.getIndexManager().get(entryId, parser.getReader()); } - public void save() { - db.set(entryId, parser); + public void save() throws IOException { + db.getIndexManager().set(entryId, parser.getWriter(value)); } } diff --git a/src/main/java/org/warp/jcwdb/FileAllocator.java b/src/main/java/org/warp/jcwdb/FileAllocator.java index bc2bccc..c12e56a 100644 --- a/src/main/java/org/warp/jcwdb/FileAllocator.java +++ b/src/main/java/org/warp/jcwdb/FileAllocator.java @@ -4,89 +4,63 @@ import com.esotericsoftware.kryo.io.Output; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.util.*; public class FileAllocator { + private final SeekableByteChannel dataFileChannel; + private volatile long allocableOffset; + private volatile boolean closed; + private final Object closeLock = new Object(); + + public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException { + this.dataFileChannel = dataFileChannel; + this.allocableOffset = this.dataFileChannel.size(); + } + /** - * Edit this using editIndex() - */ - private final Map indicesOffset; - /** - * Edit this using editIndex() - */ - private final Set dirtyIndices; - private final SeekableByteChannel fileChannel; - - public FileAllocator(SeekableByteChannel fileChannel) { - indicesOffset = new HashMap<>(); - dirtyIndices = new HashSet<>(); - this.fileChannel = fileChannel; - } - - public void write(final long index, final int type, final DBWriter w) throws IOException { - IndexDetails indexDetails = indicesOffset.getOrDefault(index, null); - if (indexDetails == null) { - allocateAndWrite(index, type, w); - } else { - write(index, indexDetails, w); - } - } - - private ByteArrayOutputStream getBytes(DBWriter w) { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - w.write(new Output(baos)); - return baos; - } - - private void write(final long index, final IndexDetails indexDetails, DBWriter w) throws IOException { - final int size = indexDetails.getSize(); - final long offset = indexDetails.getOffset(); - final int type = indexDetails.getType(); - - final ByteArrayOutputStream baos = getBytes(w); - long newOffset; - int newSize = baos.size(); - if (newSize > size) { - newOffset = allocate(newSize); - } else { - newOffset = offset; - } - if (size != newSize) { - editIndex(index, newOffset, newSize, type); - } - final Output o = new Output(Channels.newOutputStream(fileChannel.position(newOffset))); - o.write(baos.toByteArray()); - o.close(); - } - - private void allocateAndWrite(final long index, final int type, DBWriter w) throws IOException { - final ByteArrayOutputStream baos = getBytes(w); - final int size = baos.size(); - final long offset = allocate(size); - editIndex(index, offset, size, type); - - final Output o = new Output(Channels.newOutputStream(fileChannel.position(offset))); - o.write(baos.toByteArray()); - o.close(); - } - - /** - * Get an offset + * TODO: not implemented * @param size - * @return new offset + * @param type + * @return offset */ - private long allocate(int size) { - return 0; // TODO: fare + public long allocate(int size) { + checkClosed(); + long allocatedOffset = allocableOffset; + allocatedOffset += size; + return allocatedOffset; } - private void editIndex(long index, long offset, int size, int type) { - editIndex(index, new IndexDetails(offset, size, type)); + + public void close() throws IOException { + if (closed) { + return; + } + synchronized (closeLock) { + if (closed) { + return; + } + closed = true; + } + } + + /** + * Frees the unused bytes + * @param startPosition + * @param length + */ + public void markFree(long startPosition, int length) { + checkClosed(); + // TODO: advanced feature, not implemented. } - private void editIndex(long index, IndexDetails details) { - indicesOffset.put(index, details); - dirtyIndices.add(index); + + private void checkClosed() { + if (closed) { + throw new RuntimeException("Index Manager is closed."); + } } } diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java index 91a5042..6ac772e 100644 --- a/src/main/java/org/warp/jcwdb/FileIndexManager.java +++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java @@ -3,43 +3,164 @@ package org.warp.jcwdb; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; public class FileIndexManager implements IndexManager { + private final TypesManager typesManager; private final SeekableByteChannel dataFileChannel, metadataFileChannel; private final FileAllocator fileAllocator; + private volatile boolean closed; + private final Object closeLock = new Object(); - public FileIndexManager(Path dataFile, Path metadataFile) throws IOException { + /** + * Edit this using editIndex() + * Get using getIndexMetadata() + * This hashmap must contain all indices. + */ + private final Map loadedIndices; + /** + * Edit this using editIndex() + */ + private final Set dirtyLoadedIndices; + + public FileIndexManager(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { + this.typesManager = typesManager; + loadedIndices = new HashMap<>(); + dirtyLoadedIndices = new HashSet<>(); dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.CREATE); metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.CREATE); fileAllocator = new FileAllocator(dataFileChannel); } @Override - public T get(long l, int type, DBReader r) { - Input i = new Input(Channels.newInputStream(dataFileChannel)); - T result = r.read(i); + public T get(long index, DBReader reader) throws IOException { + checkClosed(); + IndexDetails details = getIndexMetadata(index); + Input i = new Input(Channels.newInputStream(dataFileChannel.position(details.getOffset()))); + T result = reader.read(i); i.close(); return result; } @Override - public void set(long index, int type, DBWriter w) throws IOException { - fileAllocator.write(index, type, w); + public int getType(long index) throws IOException { + return getIndexMetadata(index).getType(); + } + + @Override + public void set(long index, DBDataOutput data) throws IOException { + checkClosed(); + IndexDetails indexDetails = getIndexMetadataUnsafe(index); + if (indexDetails == null || indexDetails.getSize() < data.getSize()) { + allocateAndWrite(index, data); + } else { + if (indexDetails.getSize() > data.getSize()) { + editIndex(index, indexDetails.getOffset(), data.getSize(), indexDetails.getType()); + fileAllocator.markFree(indexDetails.getOffset()+data.getSize(), data.getSize()); + } + write(index, indexDetails, data); + } + } + + private void write(final long index, final IndexDetails indexDetails, DBDataOutput data) throws IOException { + final long offset = indexDetails.getOffset(); + + final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset))); + data.getWriter().write(o); + o.close(); + } + + private void allocateAndWrite(final long index, DBDataOutput w) throws IOException { + final int size = w.getSize(); + final int type = w.getType(); + final long offset = fileAllocator.allocate(size); + IndexDetails details = editIndex(index, offset, size, type); + write(index, details, w); } @Override public void delete(long index) { + checkClosed(); } @Override public boolean has(long index) { + checkClosed(); return false; } + + private IndexDetails editIndex(long index, long offset, int size, int type) { + IndexDetails indexDetails = new IndexDetails(offset, size, type); + editIndex(index, indexDetails); + return indexDetails; + } + + private IndexDetails getIndexMetadataUnsafe(long index) { + IndexDetails details = loadedIndices.getOrDefault(index, null); + if (details != null) return details; + + // TODO: implement index loading from file + return null; + } + + private IndexDetails getIndexMetadata(long index) throws IOException { + IndexDetails details = getIndexMetadataUnsafe(index); + if (details == null) + throw new IOException("Index " + index + " not found"); + else + return details; + } + + private void editIndex(long index, IndexDetails details) { + loadedIndices.put(index, details); + dirtyLoadedIndices.add(index); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + synchronized (closeLock) { + if (closed) { + return; + } + closed = true; + } + + SeekableByteChannel metadata = metadataFileChannel; + ByteBuffer metadataEntryBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES); + long lastIndex = -2; + for (Long index : dirtyLoadedIndices) { + IndexDetails indexDetails = loadedIndices.get(index); + if (index - lastIndex != 1) { + metadata = metadata.position(index * IndexDetails.TOTAL_BYTES); + } + metadataEntryBuffer.putLong(indexDetails.getOffset()); + metadataEntryBuffer.putInt(indexDetails.getSize()); + metadataEntryBuffer.putInt(indexDetails.getType()); + metadata.write(metadataEntryBuffer); + lastIndex = index; + } + fileAllocator.close(); + } + + private void checkClosed() { + if (closed) { + throw new RuntimeException("Index Manager is closed."); + } + } } diff --git a/src/main/java/org/warp/jcwdb/IndexDetails.java b/src/main/java/org/warp/jcwdb/IndexDetails.java index 0b1721d..5f43b0b 100644 --- a/src/main/java/org/warp/jcwdb/IndexDetails.java +++ b/src/main/java/org/warp/jcwdb/IndexDetails.java @@ -3,6 +3,10 @@ package org.warp.jcwdb; import java.util.Objects; public class IndexDetails { + public static final int OFFSET_BYTES = Long.SIZE; + public static final int DATA_SIZE_BYTES = Integer.SIZE; + public static final int TYPE_BYTES = Integer.SIZE; + public static final int TOTAL_BYTES = OFFSET_BYTES + DATA_SIZE_BYTES + TYPE_BYTES; private final long offset; private final int size; private final int type; diff --git a/src/main/java/org/warp/jcwdb/IndexManager.java b/src/main/java/org/warp/jcwdb/IndexManager.java index 82dad26..051b6e6 100644 --- a/src/main/java/org/warp/jcwdb/IndexManager.java +++ b/src/main/java/org/warp/jcwdb/IndexManager.java @@ -3,8 +3,10 @@ package org.warp.jcwdb; import java.io.IOException; public interface IndexManager { - public T get(long index, int type, DBReader reader) throws IOException; - public void set(long index, int type, DBWriter writer) throws IOException; + public T get(long index, DBReader reader) throws IOException; + int getType(long index) throws IOException; + public void set(long index, DBDataOutput writer) throws IOException; public void delete(long index) throws IOException; public boolean has(long index); + public void close() throws IOException; } diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java index d89545a..90d6beb 100644 --- a/src/main/java/org/warp/jcwdb/JCWDatabase.java +++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java @@ -1,15 +1,31 @@ package org.warp.jcwdb; -import com.esotericsoftware.kryo.Kryo; - import java.io.IOException; import java.nio.file.Path; public class JCWDatabase { - private static final Kryo kryo = new Kryo(); + private final TypesManager typesManager; private final MixedIndexDatabase indices; public JCWDatabase(Path dataFile, Path metadataFile) throws IOException { - this.indices = new MixedIndexDatabase(dataFile, metadataFile); + this.typesManager = new TypesManager(); + this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile); + } + + public EntryReference getRoot() { + // Get type of index 0 + int type = this.indices.getType(0); + // Get the parser + DBTypeParser parser = this.typesManager.get(type); + // Return the reference + try { + return new EntryReference(this, 0, parser); + } catch (IOException e) { + throw new RuntimeException("Can't load root!", e); + } + } + + public IndexManager getIndexManager() { + return indices; } } diff --git a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java index 42555c2..1054837 100644 --- a/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java +++ b/src/main/java/org/warp/jcwdb/MixedIndexDatabase.java @@ -7,23 +7,32 @@ import java.io.IOException; import java.nio.file.Path; public class MixedIndexDatabase implements IndexManager { + private final TypesManager typesManager; private final Long2LongMap mostAccessedIndices; private final FileIndexManager fileIndices; private final CacheIndexManager cacheIndices; - public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException { + public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException { + this.typesManager = typesManager; this.mostAccessedIndices = new Long2LongLinkedOpenHashMap(); - this.fileIndices = new FileIndexManager(dataFile, metadataFile); - this.cacheIndices = new CacheIndexManager(); + this.fileIndices = new FileIndexManager(typesManager, dataFile, metadataFile); + this.cacheIndices = new CacheIndexManager(typesManager); } @Override - public T get(long index, int type, DBReader reader) { + public T get(long index, DBReader reader) { + // TODO: implement return null; } @Override - public void set(long index, int type, DBWriter writer) { + public int getType(long index) { + // TODO: implement + return -1; + } + + @Override + public void set(long index, DBDataOutput writer) { } @@ -36,4 +45,11 @@ public class MixedIndexDatabase implements IndexManager { public boolean has(long index) { return false; } + + @Override + public void close() throws IOException { + // TODO: move all cached indices to filesIndices before closing. + this.cacheIndices.close(); + this.fileIndices.close(); + } } diff --git a/src/main/java/org/warp/jcwdb/TypesManager.java b/src/main/java/org/warp/jcwdb/TypesManager.java new file mode 100644 index 0000000..83688e5 --- /dev/null +++ b/src/main/java/org/warp/jcwdb/TypesManager.java @@ -0,0 +1,22 @@ +package org.warp.jcwdb; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.HashMap; + +public class TypesManager { + private final Map> types; + + public TypesManager() { + types = new HashMap<>(); + DBStandardTypes.registerStandardTypes(this); + } + + public void registerType(int type, DBTypeParser parser) { + this.types.put(type, parser); + } + + public DBTypeParser get(int type) { + return types.get(type).cast(); + } +}