andreacavalli/strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Better performance

Browse Source
This commit is contained in:
Cavallium 2018-11-27 10:26:01 +01:00
parent 0473fa4313
commit 0542c4c94a
5 changed files with 45 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/warp/jcwdb/CacheIndexManager.java
View File

@ -3,10 +3,8 @@ package org.warp.jcwdb;
import java.io.IOException;
public class CacheIndexManager implements IndexManager {
private final TypesManager typesManager;
public CacheIndexManager(TypesManager typesManager) {
this.typesManager = typesManager;
public CacheIndexManager() {
}
@Override

34
src/main/java/org/warp/jcwdb/FileIndexManager.java
View File

@ -3,6 +3,11 @@ package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.LongAVLTreeSet;
import it.unimi.dsi.fastutil.longs.LongSet;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
@ -10,14 +15,8 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class FileIndexManager implements IndexManager {
private final TypesManager typesManager;
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private final FileAllocator fileAllocator;
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
@ -25,25 +24,24 @@ public class FileIndexManager implements IndexManager {
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object metadataByteBufferLock = new Object();
private final Object maskByteBufferBufferLock = new Object();
private final Object maskByteBufferLock = new Object();
/**
* Edit this using editIndex()
* Get using getIndexMetadata()
* This hashmap must contain all indices.
*/
private final Map<Long, IndexDetails> loadedIndices;
private final Long2ObjectMap<IndexDetails> loadedIndices;
/**
* Edit this using editIndex()
*/
private final Set<Long> dirtyLoadedIndices, removedIndices;
private final LongSet dirtyLoadedIndices, removedIndices;
private long firstAllocableIndex;
public FileIndexManager(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.typesManager = typesManager;
loadedIndices = new HashMap<>();
dirtyLoadedIndices = new HashSet<>();
removedIndices = new HashSet<>();
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
loadedIndices = new Long2ObjectAVLTreeMap<>();
dirtyLoadedIndices = new LongAVLTreeSet();
removedIndices = new LongAVLTreeSet();
if (Files.notExists(dataFile)) {
Files.createFile(dataFile);
}
@ -162,7 +160,7 @@ public class FileIndexManager implements IndexManager {
}
private long createIndexMetadata(IndexDetails indexDetails) {
Long newIndex = firstAllocableIndex++;
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
removedIndices.remove(newIndex);
@ -223,7 +221,7 @@ public class FileIndexManager implements IndexManager {
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
for (Long index : dirtyLoadedIndices) {
for (long index : dirtyLoadedIndices) {
IndexDetails indexDetails = loadedIndices.get(index);
if (index - lastIndex != 1) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
@ -241,9 +239,9 @@ public class FileIndexManager implements IndexManager {
}
// Remove removed indices
for (Long index : removedIndices) {
for (long index : removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
synchronized (maskByteBufferBufferLock) {
synchronized (maskByteBufferLock) {
maskByteBuffer.rewind();
maskByteBuffer.putInt(IndexDetails.MASK_DELETED);
maskByteBuffer.rewind();

31
src/main/java/org/warp/jcwdb/JCWDatabase.java
View File

@ -1,17 +1,17 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.WeakHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
public class JCWDatabase implements AutoCloseable {
protected final TypesManager typesManager;
protected final MixedIndexDatabase indices;
private final WeakHashMap<Long, EntryReference<?>> references;
private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object indicesAccessLock = new Object();
@ -20,7 +20,7 @@ public class JCWDatabase implements AutoCloseable {
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
this.references = new WeakHashMap<>();
this.references = new Long2ObjectLinkedOpenHashMap<>();
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
JCWDatabase.this.close();
@ -43,15 +43,17 @@ public class JCWDatabase implements AutoCloseable {
public <T> EntryReference<T> get(long index) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
EntryReference<T> ref = (EntryReference<T>) this.references.getOrDefault(index, null);
if (ref == null) {
WeakReference<EntryReference<?>> refRef = this.references.getOrDefault(index, null);
EntryReference<T> ref;
if (refRef == null || (ref = (EntryReference<T>) refRef.get()) == null) {
int type;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
ref = new EntryReference<>(this, index, typeParser);
this.references.put(index, ref);
refRef = new WeakReference<EntryReference<?>>(ref);
this.references.put(index, refRef);
}
return ref;
}
@ -67,7 +69,7 @@ public class JCWDatabase implements AutoCloseable {
index = indices.add(typeParser.getWriter(value));
}
ref = new EntryReference<>(this, index, typeParser, value);
this.references.put(index, ref);
this.references.put(index, new WeakReference<EntryReference<?>>(ref));
return ref;
}
}
@ -95,7 +97,7 @@ public class JCWDatabase implements AutoCloseable {
indices.set(index, typeParser.getWriter(value));
}
ref = new EntryReference<>(this, index, typeParser);
this.references.put(index, ref);
this.references.put(index, new WeakReference<EntryReference<?>>(ref));
return ref;
}
}
@ -120,8 +122,11 @@ public class JCWDatabase implements AutoCloseable {
}
synchronized (referencesAccessLock) {
for (Map.Entry<Long, EntryReference<?>> reference : references.entrySet()) {
reference.getValue().save();
for (WeakReference<EntryReference<?>> referenceRef : references.values()) {
EntryReference<?> reference = referenceRef.get();
if (reference != null) {
reference.close();
}
}
}
synchronized (indicesAccessLock) {

4
src/main/java/org/warp/jcwdb/MixedIndexDatabase.java
View File

@ -13,8 +13,8 @@ public class MixedIndexDatabase implements IndexManager {
public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
this.fileIndices = new FileIndexManager(typesManager, dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager(typesManager);
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager();
}
@Override

9
src/main/java/org/warp/jcwdb/exampleimpl/App.java
View File

@ -26,12 +26,19 @@ public class App {
// for (int i = 0; i < root.size(); i++) {
// System.out.println(" - " + root.get(i));
// }
for (int i = 0; i < 500000000; i++) {
long prectime = System.currentTimeMillis();
for (int i = 0; i < 2000000; i++) {
root.add("Test " + i);
if (i > 0 && i % 200000 == 0) {
long precprectime = prectime;
prectime = System.currentTimeMillis();
System.out.println("Element "+i + " ("+(prectime-precprectime)+"ms)");
}
}
long time2 = System.currentTimeMillis();
System.out.println("Root size: "+root.size());
System.out.println("Time elapsed: " + (time2 - time1));
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Saving database...");
db.close();
long time3 = System.currentTimeMillis();