andreacavalli/strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Initial implementation

Browse Source
This commit is contained in:
Cavallium 2018-11-20 18:39:48 +01:00
parent 8e37cfa4e0
commit ce9f5a20a8
16 changed files with 356 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/org/warp/App.java
View File

@ -1,13 +0,0 @@
package org.warp;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args )
{
System.out.println( "Hello World!" );
}
}

22
src/main/java/org/warp/jcwdb/CacheIndexManager.java
View File

@ -3,17 +3,25 @@ package org.warp.jcwdb;
import java.io.IOException;
public class CacheIndexManager implements IndexManager {
public CacheIndexManager() {
private final TypesManager typesManager;
public CacheIndexManager(TypesManager typesManager) {
this.typesManager = typesManager;
}
@Override
public <T> T get(long index, int type, DBReader<T> reader) {
public <T> T get(long index, DBReader<T> reader) {
return null;
}
@Override
public <T> void set(long index, int type, DBWriter<T> writer) {
public int getType(long index) throws IOException {
// TODO Auto-generated method stub
return 0;
}
@Override
public <T> void set(long index, DBDataOutput<T> writer) {
}
@ -26,4 +34,10 @@ public class CacheIndexManager implements IndexManager {
public boolean has(long index) {
return false;
}
@Override
public void close() throws IOException {
// TODO Auto-generated method stub
}
}

5
src/main/java/org/warp/jcwdb/Castable.java Normal file
View File

@ -0,0 +1,5 @@
package org.warp.jcwdb;
public interface Castable {
public <T> T cast();
}

28
src/main/java/org/warp/jcwdb/DBDataOutput.java Normal file
View File

@ -0,0 +1,28 @@
package org.warp.jcwdb;
public interface DBDataOutput<T> {
public int getSize();
public int getType();
public DBWriter<T> getWriter();
public static <T> DBDataOutput<T> create(DBWriter<T> writer, int type, int size) {
return new DBDataOutput<T>() {
@Override
public int getSize() {
return size;
}
@Override
public int getType() {
return type;
}
@Override
public DBWriter<T> getWriter() {
return writer;
}
};
}
}

18
src/main/java/org/warp/jcwdb/DBStandardTypes.java Normal file
View File

@ -0,0 +1,18 @@
package org.warp.jcwdb;
public class DBStandardTypes {
private static final int STD = 0xFFFFF000;
public static final int BOOLEAN = STD| 0;
public static final int BYTE = STD| 1;
public static final int SHORT = STD| 2;
public static final int CHAR = STD| 3;
public static final int INTEGER = STD| 4;
public static final int FLOAT = STD| 5;
public static final int DOUBLE = STD| 6;
public static final int STRING = STD| 7;
public static final int BYTE_ARRAY = STD| 8;
public static void registerStandardTypes(TypesManager typesManager) {
typesManager.registerType(STRING, new DBStringParser());
}
}

23
src/main/java/org/warp/jcwdb/DBStringParser.java Normal file
View File

@ -0,0 +1,23 @@
package org.warp.jcwdb;
import java.nio.charset.StandardCharsets;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
public class DBStringParser extends DBTypeParserImpl<String> {
private static final DBReader<String> defaultReader = (i) -> {
return i.readString();
};
public DBReader<String> getReader() {
return defaultReader;
}
public DBDataOutput<String> getWriter(final String value) {
final byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
return DBDataOutput.create((o) -> {
o.write(bytes);
}, DBStandardTypes.STRING, bytes.length);
}
}

4
src/main/java/org/warp/jcwdb/DBTypeParser.java
View File

@ -1,4 +1,6 @@
package org.warp.jcwdb;
public interface DBTypeParser<T> extends DBReader<T>, DBWriter<T> {
public interface DBTypeParser<T> extends Castable {
public DBReader<T> getReader();
public DBDataOutput<T> getWriter(final T value);
}

9
src/main/java/org/warp/jcwdb/DBTypeParserImpl.java Normal file
View File

@ -0,0 +1,9 @@
package org.warp.jcwdb;
public abstract class DBTypeParserImpl<T> implements DBTypeParser<T> {
@SuppressWarnings("unchecked")
@Override
public T cast() {
return (T) this;
}
}

14
src/main/java/org/warp/jcwdb/EntryReference.java
View File

@ -1,19 +1,21 @@
package org.warp.jcwdb;
public abstract class EntryReference<T> {
import java.io.IOException;
public class EntryReference<T> {
private final JCWDatabase db;
private final long entryId;
private final DBTypeParser parser;
private final DBTypeParser<T> parser;
public T value;
public EntryReference(JCWDatabase db, long entryId, DBTypeParser<T> parser) {
public EntryReference(JCWDatabase db, long entryId, DBTypeParser<T> parser) throws IOException {
this.db = db;
this.entryId = entryId;
this.parser = parser;
this.value = db.get(entryId, parser);
this.value = db.getIndexManager().get(entryId, parser.getReader());
}
public void save() {
db.set(entryId, parser);
public void save() throws IOException {
db.getIndexManager().set(entryId, parser.getWriter(value));
}
}

118
src/main/java/org/warp/jcwdb/FileAllocator.java
View File

@ -4,89 +4,63 @@ import com.esotericsoftware.kryo.io.Output;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.util.*;
public class FileAllocator {
private final SeekableByteChannel dataFileChannel;
private volatile long allocableOffset;
private volatile boolean closed;
private final Object closeLock = new Object();
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
this.dataFileChannel = dataFileChannel;
this.allocableOffset = this.dataFileChannel.size();
}
/**
* Edit this using editIndex()
*/
private final Map<Long, IndexDetails> indicesOffset;
/**
* Edit this using editIndex()
*/
private final Set<Long> dirtyIndices;
private final SeekableByteChannel fileChannel;
public FileAllocator(SeekableByteChannel fileChannel) {
indicesOffset = new HashMap<>();
dirtyIndices = new HashSet<>();
this.fileChannel = fileChannel;
}
public void write(final long index, final int type, final DBWriter w) throws IOException {
IndexDetails indexDetails = indicesOffset.getOrDefault(index, null);
if (indexDetails == null) {
allocateAndWrite(index, type, w);
} else {
write(index, indexDetails, w);
}
}
private ByteArrayOutputStream getBytes(DBWriter w) {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
w.write(new Output(baos));
return baos;
}
private void write(final long index, final IndexDetails indexDetails, DBWriter w) throws IOException {
final int size = indexDetails.getSize();
final long offset = indexDetails.getOffset();
final int type = indexDetails.getType();
final ByteArrayOutputStream baos = getBytes(w);
long newOffset;
int newSize = baos.size();
if (newSize > size) {
newOffset = allocate(newSize);
} else {
newOffset = offset;
}
if (size != newSize) {
editIndex(index, newOffset, newSize, type);
}
final Output o = new Output(Channels.newOutputStream(fileChannel.position(newOffset)));
o.write(baos.toByteArray());
o.close();
}
private void allocateAndWrite(final long index, final int type, DBWriter w) throws IOException {
final ByteArrayOutputStream baos = getBytes(w);
final int size = baos.size();
final long offset = allocate(size);
editIndex(index, offset, size, type);
final Output o = new Output(Channels.newOutputStream(fileChannel.position(offset)));
o.write(baos.toByteArray());
o.close();
}
/**
* Get an offset
* TODO: not implemented
* @param size
* @return new offset
* @param type
* @return offset
*/
private long allocate(int size) {
return 0; // TODO: fare
public long allocate(int size) {
checkClosed();
long allocatedOffset = allocableOffset;
allocatedOffset += size;
return allocatedOffset;
}
private void editIndex(long index, long offset, int size, int type) {
editIndex(index, new IndexDetails(offset, size, type));
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
/**
* Frees the unused bytes
* @param startPosition
* @param length
*/
public void markFree(long startPosition, int length) {
checkClosed();
// TODO: advanced feature, not implemented.
}
private void editIndex(long index, IndexDetails details) {
indicesOffset.put(index, details);
dirtyIndices.add(index);
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
}

133
src/main/java/org/warp/jcwdb/FileIndexManager.java
View File

@ -3,43 +3,164 @@ package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class FileIndexManager implements IndexManager {
private final TypesManager typesManager;
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private final FileAllocator fileAllocator;
private volatile boolean closed;
private final Object closeLock = new Object();
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
/**
* Edit this using editIndex()
* Get using getIndexMetadata()
* This hashmap must contain all indices.
*/
private final Map<Long, IndexDetails> loadedIndices;
/**
* Edit this using editIndex()
*/
private final Set<Long> dirtyLoadedIndices;
public FileIndexManager(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.typesManager = typesManager;
loadedIndices = new HashMap<>();
dirtyLoadedIndices = new HashSet<>();
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.CREATE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.CREATE);
fileAllocator = new FileAllocator(dataFileChannel);
}
@Override
public <T> T get(long l, int type, DBReader<T> r) {
Input i = new Input(Channels.newInputStream(dataFileChannel));
T result = r.read(i);
public <T> T get(long index, DBReader<T> reader) throws IOException {
checkClosed();
IndexDetails details = getIndexMetadata(index);
Input i = new Input(Channels.newInputStream(dataFileChannel.position(details.getOffset())));
T result = reader.read(i);
i.close();
return result;
}
@Override
public void set(long index, int type, DBWriter w) throws IOException {
fileAllocator.write(index, type, w);
public int getType(long index) throws IOException {
return getIndexMetadata(index).getType();
}
@Override
public <T> void set(long index, DBDataOutput<T> data) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < data.getSize()) {
allocateAndWrite(index, data);
} else {
if (indexDetails.getSize() > data.getSize()) {
editIndex(index, indexDetails.getOffset(), data.getSize(), indexDetails.getType());
fileAllocator.markFree(indexDetails.getOffset()+data.getSize(), data.getSize());
}
write(index, indexDetails, data);
}
}
private void write(final long index, final IndexDetails indexDetails, DBDataOutput<?> data) throws IOException {
final long offset = indexDetails.getOffset();
final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset)));
data.getWriter().write(o);
o.close();
}
private void allocateAndWrite(final long index, DBDataOutput<?> w) throws IOException {
final int size = w.getSize();
final int type = w.getType();
final long offset = fileAllocator.allocate(size);
IndexDetails details = editIndex(index, offset, size, type);
write(index, details, w);
}
@Override
public void delete(long index) {
checkClosed();
}
@Override
public boolean has(long index) {
checkClosed();
return false;
}
private IndexDetails editIndex(long index, long offset, int size, int type) {
IndexDetails indexDetails = new IndexDetails(offset, size, type);
editIndex(index, indexDetails);
return indexDetails;
}
private IndexDetails getIndexMetadataUnsafe(long index) {
IndexDetails details = loadedIndices.getOrDefault(index, null);
if (details != null) return details;
// TODO: implement index loading from file
return null;
}
private IndexDetails getIndexMetadata(long index) throws IOException {
IndexDetails details = getIndexMetadataUnsafe(index);
if (details == null)
throw new IOException("Index " + index + " not found");
else
return details;
}
private void editIndex(long index, IndexDetails details) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
SeekableByteChannel metadata = metadataFileChannel;
ByteBuffer metadataEntryBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
long lastIndex = -2;
for (Long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
}
metadataEntryBuffer.putLong(indexDetails.getOffset());
metadataEntryBuffer.putInt(indexDetails.getSize());
metadataEntryBuffer.putInt(indexDetails.getType());
metadata.write(metadataEntryBuffer);
lastIndex = index;
}
fileAllocator.close();
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
}

4
src/main/java/org/warp/jcwdb/IndexDetails.java
View File

@ -3,6 +3,10 @@ package org.warp.jcwdb;
import java.util.Objects;
public class IndexDetails {
public static final int OFFSET_BYTES = Long.SIZE;
public static final int DATA_SIZE_BYTES = Integer.SIZE;
public static final int TYPE_BYTES = Integer.SIZE;
public static final int TOTAL_BYTES = OFFSET_BYTES + DATA_SIZE_BYTES + TYPE_BYTES;
private final long offset;
private final int size;
private final int type;

6
src/main/java/org/warp/jcwdb/IndexManager.java
View File

@ -3,8 +3,10 @@ package org.warp.jcwdb;
import java.io.IOException;
public interface IndexManager {
public <T> T get(long index, int type, DBReader<T> reader) throws IOException;
public <T> void set(long index, int type, DBWriter<T> writer) throws IOException;
public <T> T get(long index, DBReader<T> reader) throws IOException;
int getType(long index) throws IOException;
public <T> void set(long index, DBDataOutput<T> writer) throws IOException;
public void delete(long index) throws IOException;
public boolean has(long index);
public void close() throws IOException;
}

24
src/main/java/org/warp/jcwdb/JCWDatabase.java
View File

@ -1,15 +1,31 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.Kryo;
import java.io.IOException;
import java.nio.file.Path;
public class JCWDatabase {
private static final Kryo kryo = new Kryo();
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.indices = new MixedIndexDatabase(dataFile, metadataFile);
this.typesManager = new TypesManager();
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
}
public <T> EntryReference<T> getRoot() {
// Get type of index 0
int type = this.indices.getType(0);
// Get the parser
DBTypeParser<T> parser = this.typesManager.get(type);
// Return the reference
try {
return new EntryReference<T>(this, 0, parser);
} catch (IOException e) {
throw new RuntimeException("Can't load root!", e);
}
}
public IndexManager getIndexManager() {
return indices;
}
}

26
src/main/java/org/warp/jcwdb/MixedIndexDatabase.java
View File

@ -7,23 +7,32 @@ import java.io.IOException;
import java.nio.file.Path;
public class MixedIndexDatabase implements IndexManager {
private final TypesManager typesManager;
private final Long2LongMap mostAccessedIndices;
private final FileIndexManager fileIndices;
private final CacheIndexManager cacheIndices;
public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.typesManager = typesManager;
this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager();
this.fileIndices = new FileIndexManager(typesManager, dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager(typesManager);
}
@Override
public <T> T get(long index, int type, DBReader<T> reader) {
public <T> T get(long index, DBReader<T> reader) {
// TODO: implement
return null;
}
@Override
public <T> void set(long index, int type, DBWriter<T> writer) {
public int getType(long index) {
// TODO: implement
return -1;
}
@Override
public <T> void set(long index, DBDataOutput<T> writer) {
}
@ -36,4 +45,11 @@ public class MixedIndexDatabase implements IndexManager {
public boolean has(long index) {
return false;
}
@Override
public void close() throws IOException {
// TODO: move all cached indices to filesIndices before closing.
this.cacheIndices.close();
this.fileIndices.close();
}
}

22
src/main/java/org/warp/jcwdb/TypesManager.java Normal file
View File

@ -0,0 +1,22 @@
package org.warp.jcwdb;
import java.util.Map;
import java.util.Map.Entry;
import java.util.HashMap;
public class TypesManager {
private final Map<Integer, DBTypeParser<?>> types;
public TypesManager() {
types = new HashMap<>();
DBStandardTypes.registerStandardTypes(this);
}
public <T> void registerType(int type, DBTypeParser<T> parser) {
this.types.put(type, parser);
}
public <T> DBTypeParser<T> get(int type) {
return types.get(type).cast();
}
}