Use Micrometer registry, better optimistic commit failure logging

This commit is contained in:
Andrea Cavalli 2021-10-30 11:13:46 +02:00
parent 70f76b660d
commit 1ea004630e
19 changed files with 317 additions and 209 deletions

View File

@ -250,7 +250,6 @@
<dependency> <dependency>
<groupId>io.micrometer</groupId> <groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId> <artifactId>micrometer-core</artifactId>
<optional>true</optional>
</dependency> </dependency>
<dependency> <dependency>
<groupId>io.micrometer</groupId> <groupId>io.micrometer</groupId>

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.client; package it.cavallium.dbengine.client;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
@ -20,6 +21,8 @@ public interface CompositeDatabase {
BufferAllocator getAllocator(); BufferAllocator getAllocator();
MeterRegistry getMeterRegistry();
/** /**
* Find corrupted items * Find corrupted items
*/ */

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.client; package it.cavallium.dbengine.client;
import com.google.common.util.concurrent.Uninterruptibles;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
import it.cavallium.dbengine.client.IndexAction.Add; import it.cavallium.dbengine.client.IndexAction.Add;
import it.cavallium.dbengine.client.IndexAction.AddMulti; import it.cavallium.dbengine.client.IndexAction.AddMulti;
@ -24,11 +25,15 @@ import java.lang.ref.Cleaner;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.LockSupport;
import java.util.logging.Level;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
import org.warp.commonutils.type.ShortNamedThreadFactory;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.Sinks; import reactor.core.publisher.Sinks;
@ -42,108 +47,12 @@ import reactor.util.function.Tuple2;
public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> { public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
private static final Logger log = LoggerFactory.getLogger(LuceneIndex.class); private static final Logger log = LoggerFactory.getLogger(LuceneIndex.class);
private static final Cleaner cleaner = Cleaner.create();
private final LLLuceneIndex luceneIndex; private final LLLuceneIndex luceneIndex;
private final Indicizer<T,U> indicizer; private final Indicizer<T,U> indicizer;
private final Many<IndexAction> actions;
private final Empty<Void> actionsClosed;
public LuceneIndexImpl(LLLuceneIndex luceneIndex, Indicizer<T, U> indicizer) { public LuceneIndexImpl(LLLuceneIndex luceneIndex, Indicizer<T, U> indicizer) {
this.luceneIndex = luceneIndex; this.luceneIndex = luceneIndex;
this.indicizer = indicizer; this.indicizer = indicizer;
this.actions = Sinks
.many()
.unicast()
.onBackpressureBuffer(Queues.<IndexAction>get(1024).get());
this.actionsClosed = Sinks.empty();
subscribeToActions();
}
private void subscribeToActions() {
var d = actions
.asFlux()
.doAfterTerminate(actionsClosed::tryEmitEmpty)
.flatMap(this::onParallelAction)
.concatMap(this::onOrderedAction)
.then()
.subscribeOn(Schedulers.boundedElastic())
.subscribe();
cleaner.register(LuceneIndexImpl.this, d::dispose);
}
/**
* Actions that don't require any kind of order
*/
private Mono<IndexAction> onParallelAction(IndexAction action) {
return (switch (action.getType()) {
case TAKE_SNAPSHOT, RELEASE_SNAPSHOT, FLUSH, CLOSE -> Mono.empty();
case ADD -> luceneIndex.addDocument(((Add) action).key(), ((Add) action).doc())
.doOnError(e -> ((Add) action).addedFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((Add) action).addedFuture().success());
case ADD_MULTI -> luceneIndex.addDocuments(((AddMulti) action).docsFlux())
.doOnError(e -> ((AddMulti) action).addedMultiFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((AddMulti) action).addedMultiFuture().success());
case UPDATE -> luceneIndex
.updateDocument(((Update) action).key(),((Update) action).doc())
.doOnError(e -> ((Update) action).updatedFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((Update) action).updatedFuture().success());
case UPDATE_MULTI -> luceneIndex.updateDocuments(Mono.just(((UpdateMulti) action).docs()))
.doOnError(e -> ((UpdateMulti) action).updatedMultiFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((UpdateMulti) action).updatedMultiFuture().success());
case DELETE -> luceneIndex.deleteDocument(((Delete) action).key())
.doOnError(e -> ((Delete) action).deletedFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((Delete) action).deletedFuture().success());
case DELETE_ALL -> luceneIndex.deleteAll()
.doOnError(e -> ((DeleteAll) action).deletedAllFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((DeleteAll) action).deletedAllFuture().success());
case REFRESH -> luceneIndex.refresh(((Refresh) action).force())
.doOnError(e -> ((Refresh) action).refreshFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((Refresh) action).refreshFuture().success());
})
.doOnError(ex -> log.error("Uncaught error when handling parallel index action " + action.getType(), ex))
.onErrorResume(ex -> Mono.empty())
.thenReturn(action);
}
/**
* Actions that require absolute order
*/
private Mono<IndexAction> onOrderedAction(IndexAction action) {
return (switch (action.getType()) {
case ADD, REFRESH, DELETE_ALL, DELETE, UPDATE_MULTI, UPDATE, ADD_MULTI -> Mono.empty();
case TAKE_SNAPSHOT -> luceneIndex.takeSnapshot().single()
.doOnError(e -> ((TakeSnapshot) action).snapshotFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnNext(s -> ((TakeSnapshot) action).snapshotFuture().success(s));
case RELEASE_SNAPSHOT -> luceneIndex.releaseSnapshot(((ReleaseSnapshot) action).snapshot())
.doOnError(e -> ((ReleaseSnapshot) action).releasedFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((ReleaseSnapshot) action).releasedFuture().success());
case FLUSH -> luceneIndex.flush()
.doOnError(e -> ((Flush) action).flushFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((Flush) action).flushFuture().success());
case CLOSE -> luceneIndex.close()
.doOnError(e -> ((Close) action).closeFuture().error(e))
.onErrorResume(ex -> Mono.empty())
.doOnSuccess(s -> ((Close) action).closeFuture().success())
.doAfterTerminate(() -> emitActionOptimistically(null));
})
.doOnError(ex -> log.error("Uncaught error when handling ordered index action " + action.getType(), ex))
.onErrorResume(ex -> Mono.empty())
.onErrorResume(ex -> Mono.empty())
.thenReturn(action);
} }
private LLSnapshot resolveSnapshot(CompositeSnapshot snapshot) { private LLSnapshot resolveSnapshot(CompositeSnapshot snapshot) {
@ -158,45 +67,46 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
public Mono<Void> addDocument(T key, U value) { public Mono<Void> addDocument(T key, U value) {
return indicizer return indicizer
.toDocument(key, value) .toDocument(key, value)
.flatMap(doc -> Mono .flatMap(doc -> luceneIndex.addDocument(indicizer.toIndex(key), doc));
.create(sink -> emitActionOptimistically(new IndexAction.Add(indicizer.toIndex(key), doc, sink))));
} }
@Override @Override
public Mono<Void> addDocuments(Flux<Entry<T, U>> entries) { public Mono<Void> addDocuments(Flux<Entry<T, U>> entries) {
var convertedEntries = entries.flatMap(entry -> indicizer return luceneIndex
.toDocument(entry.getKey(), entry.getValue()) .addDocuments(entries
.map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc)) .flatMap(entry -> indicizer
); .toDocument(entry.getKey(), entry.getValue())
return Mono.create(sink -> emitActionOptimistically(new IndexAction.AddMulti(convertedEntries, sink))); .map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc)))
);
} }
@Override @Override
public Mono<Void> deleteDocument(T key) { public Mono<Void> deleteDocument(T key) {
LLTerm id = indicizer.toIndex(key); LLTerm id = indicizer.toIndex(key);
return Mono.create(sink -> emitActionOptimistically(new IndexAction.Delete(id, sink))); return luceneIndex.deleteDocument(id);
} }
@Override @Override
public Mono<Void> updateDocument(T key, @NotNull U value) { public Mono<Void> updateDocument(T key, @NotNull U value) {
return indicizer return indicizer
.toDocument(key, value) .toDocument(key, value)
.flatMap(doc -> Mono.create(sink -> emitActionOptimistically(new Update(indicizer.toIndex(key), doc, sink)))); .flatMap(doc -> luceneIndex.updateDocument(indicizer.toIndex(key), doc));
} }
@Override @Override
public Mono<Void> updateDocuments(Flux<Entry<T, U>> entries) { public Mono<Void> updateDocuments(Flux<Entry<T, U>> entries) {
return entries return luceneIndex
.flatMap(entry -> indicizer .updateDocuments(entries
.toDocument(entry.getKey(), entry.getValue()) .flatMap(entry -> indicizer
.map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc))) .toDocument(entry.getKey(), entry.getValue())
.collectMap(Entry::getKey, Entry::getValue) .map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc)))
.flatMap(docs -> Mono.create(sink -> emitActionOptimistically(new IndexAction.UpdateMulti(docs, sink)))); .collectMap(Entry::getKey, Entry::getValue)
);
} }
@Override @Override
public Mono<Void> deleteAll() { public Mono<Void> deleteAll() {
return Mono.create(sink -> emitActionOptimistically(new IndexAction.DeleteAll(sink))); return luceneIndex.deleteAll();
} }
@Override @Override
@ -255,19 +165,7 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
@Override @Override
public Mono<Void> close() { public Mono<Void> close() {
return Mono return luceneIndex.close();
.<Void>create(sink -> emitActionOptimistically(new Close(sink)))
.then(this.actionsClosed.asMono());
}
private void emitActionOptimistically(IndexAction action) {
EmitResult emitResult;
while ((emitResult = (action == null ? actions.tryEmitComplete() : actions.tryEmitNext(action)))
== EmitResult.FAIL_NON_SERIALIZED || emitResult == EmitResult.FAIL_OVERFLOW) {
// 10ms
LockSupport.parkNanos(10000000);
}
emitResult.orThrow();
} }
/** /**
@ -275,7 +173,7 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
*/ */
@Override @Override
public Mono<Void> flush() { public Mono<Void> flush() {
return Mono.create(sink -> emitActionOptimistically(new IndexAction.Flush(sink))); return luceneIndex.flush();
} }
/** /**
@ -283,16 +181,16 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
*/ */
@Override @Override
public Mono<Void> refresh(boolean force) { public Mono<Void> refresh(boolean force) {
return Mono.create(sink -> emitActionOptimistically(new IndexAction.Refresh(force, sink))); return luceneIndex.refresh(force);
} }
@Override @Override
public Mono<LLSnapshot> takeSnapshot() { public Mono<LLSnapshot> takeSnapshot() {
return Mono.create(sink -> emitActionOptimistically(new IndexAction.TakeSnapshot(sink))); return luceneIndex.takeSnapshot();
} }
@Override @Override
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) { public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
return Mono.create(sink -> emitActionOptimistically(new IndexAction.ReleaseSnapshot(snapshot, sink))); return luceneIndex.releaseSnapshot(snapshot);
} }
} }

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.database; package it.cavallium.dbengine.database;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerAnalyzers;
@ -15,6 +16,8 @@ public interface LLDatabaseConnection {
BufferAllocator getAllocator(); BufferAllocator getAllocator();
MeterRegistry getMeterRegistry();
Mono<? extends LLDatabaseConnection> connect(); Mono<? extends LLDatabaseConnection> connect();
Mono<? extends LLKeyValueDatabase> getDatabase(String name, Mono<? extends LLKeyValueDatabase> getDatabase(String name,

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs; import com.google.common.primitives.Longs;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.database.collections.DatabaseInt; import it.cavallium.dbengine.database.collections.DatabaseInt;
import it.cavallium.dbengine.database.collections.DatabaseLong; import it.cavallium.dbengine.database.collections.DatabaseLong;
@ -47,5 +48,7 @@ public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseS
BufferAllocator getAllocator(); BufferAllocator getAllocator();
MeterRegistry getMeterRegistry();
Mono<Void> close(); Mono<Void> close();
} }

View File

@ -1,27 +1,20 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
import static java.util.Objects.requireNonNullElse;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import io.net5.buffer.api.MemoryManager;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
import io.net5.util.internal.PlatformDependent; import io.net5.util.internal.PlatformDependent;
import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.database.LLDelta;
import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.LLUtils.DirectBuffer; import it.cavallium.dbengine.database.LLUtils.DirectBuffer;
import it.cavallium.dbengine.database.RepeatedElementList; import it.cavallium.dbengine.database.RepeatedElementList;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.lucene.ExponentialPageLimits;
import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List; import java.util.List;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.LockSupport;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyHandle;
@ -37,7 +30,6 @@ import org.rocksdb.WriteBatch;
import org.rocksdb.WriteOptions; import org.rocksdb.WriteOptions;
import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements RocksDBColumn public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements RocksDBColumn
@ -54,11 +46,24 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
private final BufferAllocator alloc; private final BufferAllocator alloc;
private final ColumnFamilyHandle cfh; private final ColumnFamilyHandle cfh;
public AbstractRocksDBColumn(T db, DatabaseOptions databaseOptions, BufferAllocator alloc, ColumnFamilyHandle cfh) { private final MeterRegistry meterRegistry;
private final AtomicInteger lastDataSizeMetric = new AtomicInteger(0);
public AbstractRocksDBColumn(T db,
DatabaseOptions databaseOptions,
BufferAllocator alloc,
ColumnFamilyHandle cfh,
MeterRegistry meterRegistry) {
this.db = db; this.db = db;
this.opts = databaseOptions; this.opts = databaseOptions;
this.alloc = alloc; this.alloc = alloc;
this.cfh = cfh; this.cfh = cfh;
this.meterRegistry = meterRegistry;
Gauge
.builder("it.cavallium.dbengine.database.disk.column.lastdatasize", lastDataSizeMetric::get)
.description("Last data size read using get()")
.register(meterRegistry);
} }
protected T getDb() { protected T getDb() {
@ -81,6 +86,15 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
if (Schedulers.isInNonBlockingThread()) { if (Schedulers.isInNonBlockingThread()) {
throw new UnsupportedOperationException("Called dbGet in a nonblocking thread"); throw new UnsupportedOperationException("Called dbGet in a nonblocking thread");
} }
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!readOptions.isOwningHandle()) {
throw new IllegalStateException("ReadOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
if (opts.allowNettyDirect()) { if (opts.allowNettyDirect()) {
//todo: implement keyMayExist if existsAlmostCertainly is false. //todo: implement keyMayExist if existsAlmostCertainly is false.
@ -91,7 +105,6 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
// Create a direct result buffer because RocksDB works only with direct buffers // Create a direct result buffer because RocksDB works only with direct buffers
try (Buffer resultBuf = alloc.allocate(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES)) { try (Buffer resultBuf = alloc.allocate(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES)) {
int valueSize; int valueSize;
int assertionReadData = -1;
ByteBuffer resultNioBuf; ByteBuffer resultNioBuf;
do { do {
// Create the result nio buffer to pass to RocksDB // Create the result nio buffer to pass to RocksDB
@ -115,6 +128,9 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
// of the result into the result buffer more than once. // of the result into the result buffer more than once.
assert resultNioBuf.limit() <= valueSize; assert resultNioBuf.limit() <= valueSize;
// Update data size metrics
this.lastDataSizeMetric.set(valueSize);
if (valueSize <= resultNioBuf.limit()) { if (valueSize <= resultNioBuf.limit()) {
// Return the result ready to be read // Return the result ready to be read
return resultBuf.readerOffset(0).writerOffset(valueSize).send(); return resultBuf.readerOffset(0).writerOffset(valueSize).send();
@ -175,6 +191,15 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
if (Schedulers.isInNonBlockingThread()) { if (Schedulers.isInNonBlockingThread()) {
throw new UnsupportedOperationException("Called dbPut in a nonblocking thread"); throw new UnsupportedOperationException("Called dbPut in a nonblocking thread");
} }
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!writeOptions.isOwningHandle()) {
throw new IllegalStateException("WriteOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
assert key.isAccessible(); assert key.isAccessible();
assert value.isAccessible(); assert value.isAccessible();
if (opts.allowNettyDirect()) { if (opts.allowNettyDirect()) {
@ -197,7 +222,7 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
} }
} }
} finally { } finally {
if (writeOptions != null && !(writeOptions instanceof UnreleasableWriteOptions)) { if (!(writeOptions instanceof UnreleasableWriteOptions)) {
writeOptions.close(); writeOptions.close();
} }
} }
@ -209,6 +234,15 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
if (Schedulers.isInNonBlockingThread()) { if (Schedulers.isInNonBlockingThread()) {
throw new UnsupportedOperationException("Called containsKey in a nonblocking thread"); throw new UnsupportedOperationException("Called containsKey in a nonblocking thread");
} }
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!readOptions.isOwningHandle()) {
throw new IllegalStateException("ReadOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
int size = RocksDB.NOT_FOUND; int size = RocksDB.NOT_FOUND;
byte[] keyBytes = LLUtils.toArray(key); byte[] keyBytes = LLUtils.toArray(key);
Holder<byte[]> data = new Holder<>(); Holder<byte[]> data = new Holder<>();
@ -221,7 +255,7 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
} }
} }
} finally { } finally {
if (readOptions != null && !(readOptions instanceof UnreleasableReadOptions)) { if (!(readOptions instanceof UnreleasableReadOptions)) {
readOptions.close(); readOptions.close();
} }
} }
@ -232,6 +266,15 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
@Override @Override
public void delete(WriteOptions writeOptions, Send<Buffer> keySend) throws RocksDBException { public void delete(WriteOptions writeOptions, Send<Buffer> keySend) throws RocksDBException {
try (var key = keySend.receive()) { try (var key = keySend.receive()) {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!writeOptions.isOwningHandle()) {
throw new IllegalStateException("WriteOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
if (opts.allowNettyDirect()) { if (opts.allowNettyDirect()) {
DirectBuffer keyNioBuffer = LLUtils.convertToReadableDirect(alloc, key.send()); DirectBuffer keyNioBuffer = LLUtils.convertToReadableDirect(alloc, key.send());
try { try {
@ -248,43 +291,106 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
@Override @Override
public void delete(WriteOptions writeOptions, byte[] key) throws RocksDBException { public void delete(WriteOptions writeOptions, byte[] key) throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!writeOptions.isOwningHandle()) {
throw new IllegalStateException("WriteOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
db.delete(cfh, writeOptions, key); db.delete(cfh, writeOptions, key);
} }
@Override @Override
public List<byte[]> multiGetAsList(ReadOptions readOptions, List<byte[]> keys) throws RocksDBException { public List<byte[]> multiGetAsList(ReadOptions readOptions, List<byte[]> keys) throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!readOptions.isOwningHandle()) {
throw new IllegalStateException("ReadOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
var columnFamilyHandles = new RepeatedElementList<>(cfh, keys.size()); var columnFamilyHandles = new RepeatedElementList<>(cfh, keys.size());
return db.multiGetAsList(readOptions, columnFamilyHandles, keys); return db.multiGetAsList(readOptions, columnFamilyHandles, keys);
} }
@Override @Override
public void suggestCompactRange() throws RocksDBException { public void suggestCompactRange() throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
db.suggestCompactRange(cfh); db.suggestCompactRange(cfh);
} }
@Override @Override
public void compactRange(byte[] begin, byte[] end, CompactRangeOptions options) public void compactRange(byte[] begin, byte[] end, CompactRangeOptions options)
throws RocksDBException { throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!options.isOwningHandle()) {
throw new IllegalStateException("CompactRangeOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
db.compactRange(cfh, begin, end, options); db.compactRange(cfh, begin, end, options);
} }
@Override @Override
public void flush(FlushOptions options) throws RocksDBException { public void flush(FlushOptions options) throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!options.isOwningHandle()) {
throw new IllegalStateException("FlushOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
db.flush(options, cfh); db.flush(options, cfh);
} }
@Override @Override
public void flushWal(boolean sync) throws RocksDBException { public void flushWal(boolean sync) throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
db.flushWal(sync); db.flushWal(sync);
} }
@Override @Override
public long getLongProperty(String property) throws RocksDBException { public long getLongProperty(String property) throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
return db.getLongProperty(cfh, property); return db.getLongProperty(cfh, property);
} }
@Override @Override
public void write(WriteOptions writeOptions, WriteBatch writeBatch) throws RocksDBException { public void write(WriteOptions writeOptions, WriteBatch writeBatch) throws RocksDBException {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!writeOptions.isOwningHandle()) {
throw new IllegalStateException("WriteOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
db.write(writeOptions, writeBatch); db.write(writeOptions, writeBatch);
} }
@ -298,6 +404,15 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
@Override @Override
@NotNull @NotNull
public RocksIterator newIterator(@NotNull ReadOptions readOptions) { public RocksIterator newIterator(@NotNull ReadOptions readOptions) {
if (!db.isOwningHandle()) {
throw new IllegalStateException("Database is closed");
}
if (!readOptions.isOwningHandle()) {
throw new IllegalStateException("ReadOptions is closed");
}
if (!cfh.isOwningHandle()) {
throw new IllegalStateException("Column family is closed");
}
return db.newIterator(cfh, readOptions); return db.newIterator(cfh, readOptions);
} }
@ -310,4 +425,8 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
public BufferAllocator getAllocator() { public BufferAllocator getAllocator() {
return alloc; return alloc;
} }
public MeterRegistry getMeterRegistry() {
return meterRegistry;
}
} }

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.client.IndicizerSimilarities;
@ -31,11 +32,13 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
private final AtomicBoolean connected = new AtomicBoolean(); private final AtomicBoolean connected = new AtomicBoolean();
private final BufferAllocator allocator; private final BufferAllocator allocator;
private final MeterRegistry meterRegistry;
private final Path basePath; private final Path basePath;
private final AtomicReference<LLTempLMDBEnv> env = new AtomicReference<>(); private final AtomicReference<LLTempLMDBEnv> env = new AtomicReference<>();
public LLLocalDatabaseConnection(BufferAllocator allocator, Path basePath) { public LLLocalDatabaseConnection(BufferAllocator allocator, MeterRegistry meterRegistry, Path basePath) {
this.allocator = allocator; this.allocator = allocator;
this.meterRegistry = meterRegistry;
this.basePath = basePath; this.basePath = basePath;
} }
@ -44,6 +47,10 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
return allocator; return allocator;
} }
public MeterRegistry getMeterRegistry() {
return meterRegistry;
}
@Override @Override
public Mono<LLDatabaseConnection> connect() { public Mono<LLDatabaseConnection> connect() {
return Mono return Mono
@ -70,6 +77,7 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
return Mono return Mono
.fromCallable(() -> new LLLocalKeyValueDatabase( .fromCallable(() -> new LLLocalKeyValueDatabase(
allocator, allocator,
meterRegistry,
name, name,
basePath.resolve("database_" + name), basePath.resolve("database_" + name),
columns, columns,
@ -93,6 +101,7 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
Objects.requireNonNull(env, "Environment not set"); Objects.requireNonNull(env, "Environment not set");
return new LLLocalMultiLuceneIndex(env, return new LLLocalMultiLuceneIndex(env,
luceneOptions.inMemory() ? null : basePath.resolve("lucene"), luceneOptions.inMemory() ? null : basePath.resolve("lucene"),
meterRegistry,
name, name,
instancesCount, instancesCount,
indicizerAnalyzers, indicizerAnalyzers,
@ -102,6 +111,7 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
); );
} else { } else {
return new LLLocalLuceneIndex(luceneOptions.inMemory() ? null : basePath.resolve("lucene"), return new LLLocalLuceneIndex(luceneOptions.inMemory() ? null : basePath.resolve("lucene"),
meterRegistry,
name, name,
indicizerAnalyzers, indicizerAnalyzers,
indicizerSimilarities, indicizerSimilarities,

View File

@ -808,7 +808,7 @@ public class LLLocalDictionary implements LLDictionary {
rangeSend -> Flux.using( rangeSend -> Flux.using(
() -> new LLLocalEntryReactiveRocksIterator(db, rangeSend, () -> new LLLocalEntryReactiveRocksIterator(db, rangeSend,
databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)), databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)),
llLocalEntryReactiveRocksIterator -> llLocalEntryReactiveRocksIterator.flux().subscribeOn(dbScheduler), iterator -> iterator.flux().subscribeOn(dbScheduler, false),
LLLocalReactiveRocksIterator::close LLLocalReactiveRocksIterator::close
).transform(LLUtils::handleDiscard), ).transform(LLUtils::handleDiscard),
rangeSend -> Mono.fromRunnable(rangeSend::close) rangeSend -> Mono.fromRunnable(rangeSend::close)
@ -821,7 +821,7 @@ public class LLLocalDictionary implements LLDictionary {
rangeSend -> Flux.using( rangeSend -> Flux.using(
() -> new LLLocalGroupedEntryReactiveRocksIterator(db, prefixLength, rangeSend, () -> new LLLocalGroupedEntryReactiveRocksIterator(db, prefixLength, rangeSend,
databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)), databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)),
reactiveRocksIterator -> reactiveRocksIterator.flux().subscribeOn(dbScheduler), iterator -> iterator.flux().subscribeOn(dbScheduler, false),
LLLocalGroupedReactiveRocksIterator::close LLLocalGroupedReactiveRocksIterator::close
).transform(LLUtils::handleDiscard), ).transform(LLUtils::handleDiscard),
rangeSend -> Mono.fromRunnable(rangeSend::close) rangeSend -> Mono.fromRunnable(rangeSend::close)
@ -852,7 +852,7 @@ public class LLLocalDictionary implements LLDictionary {
rangeSend -> Flux.using( rangeSend -> Flux.using(
() -> new LLLocalGroupedKeyReactiveRocksIterator(db, prefixLength, rangeSend, () -> new LLLocalGroupedKeyReactiveRocksIterator(db, prefixLength, rangeSend,
databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)), databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)),
reactiveRocksIterator -> reactiveRocksIterator.flux().subscribeOn(dbScheduler), iterator -> iterator.flux().subscribeOn(dbScheduler, false),
LLLocalGroupedReactiveRocksIterator::close LLLocalGroupedReactiveRocksIterator::close
).transform(LLUtils::handleDiscard), ).transform(LLUtils::handleDiscard),
rangeSend -> Mono.fromRunnable(rangeSend::close) rangeSend -> Mono.fromRunnable(rangeSend::close)
@ -951,7 +951,7 @@ public class LLLocalDictionary implements LLDictionary {
() -> new LLLocalKeyReactiveRocksIterator(db, rangeSend, () -> new LLLocalKeyReactiveRocksIterator(db, rangeSend,
databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot) databaseOptions.allowNettyDirect(), resolveSnapshot(snapshot)
), ),
llLocalKeyReactiveRocksIterator -> llLocalKeyReactiveRocksIterator.flux().subscribeOn(dbScheduler), iterator -> iterator.flux().subscribeOn(dbScheduler, false),
LLLocalReactiveRocksIterator::close LLLocalReactiveRocksIterator::close
).transform(LLUtils::handleDiscard), ).transform(LLUtils::handleDiscard),
rangeSend -> Mono.fromRunnable(rangeSend::close) rangeSend -> Mono.fromRunnable(rangeSend::close)

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import io.net5.util.internal.PlatformDependent; import io.net5.util.internal.PlatformDependent;
import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.Column;
@ -73,6 +74,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
RocksDB.DEFAULT_COLUMN_FAMILY); RocksDB.DEFAULT_COLUMN_FAMILY);
private final BufferAllocator allocator; private final BufferAllocator allocator;
private final MeterRegistry meterRegistry;
private final Scheduler dbScheduler; private final Scheduler dbScheduler;
// Configurations // Configurations
@ -89,6 +91,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
@SuppressWarnings("SwitchStatementWithTooFewBranches") @SuppressWarnings("SwitchStatementWithTooFewBranches")
public LLLocalKeyValueDatabase(BufferAllocator allocator, public LLLocalKeyValueDatabase(BufferAllocator allocator,
MeterRegistry meterRegistry,
String name, String name,
@Nullable Path path, @Nullable Path path,
List<Column> columns, List<Column> columns,
@ -96,6 +99,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
DatabaseOptions databaseOptions) throws IOException { DatabaseOptions databaseOptions) throws IOException {
this.name = name; this.name = name;
this.allocator = allocator; this.allocator = allocator;
this.meterRegistry = meterRegistry;
if (databaseOptions.allowNettyDirect()) { if (databaseOptions.allowNettyDirect()) {
if (!PlatformDependent.hasUnsafe()) { if (!PlatformDependent.hasUnsafe()) {
@ -496,11 +500,11 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
private RocksDBColumn getRocksDBColumn(RocksDB db, ColumnFamilyHandle cfh) { private RocksDBColumn getRocksDBColumn(RocksDB db, ColumnFamilyHandle cfh) {
if (db instanceof OptimisticTransactionDB optimisticTransactionDB) { if (db instanceof OptimisticTransactionDB optimisticTransactionDB) {
return new OptimisticRocksDBColumn(optimisticTransactionDB, databaseOptions, allocator, cfh); return new OptimisticRocksDBColumn(optimisticTransactionDB, databaseOptions, allocator, cfh, meterRegistry);
} else if (db instanceof TransactionDB) { } else if (db instanceof TransactionDB) {
return new PessimisticRocksDBColumn((TransactionDB) db, databaseOptions, allocator, cfh); return new PessimisticRocksDBColumn((TransactionDB) db, databaseOptions, allocator, cfh, meterRegistry);
} else { } else {
return new StandardRocksDBColumn(db, databaseOptions, allocator, cfh); return new StandardRocksDBColumn(db, databaseOptions, allocator, cfh, meterRegistry);
} }
} }
@ -547,6 +551,11 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
return allocator; return allocator;
} }
@Override
public MeterRegistry getMeterRegistry() {
return meterRegistry;
}
@Override @Override
public Mono<LLSnapshot> takeSnapshot() { public Mono<LLSnapshot> takeSnapshot() {
return Mono return Mono

View File

@ -3,6 +3,8 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_LUCENE; import static it.cavallium.dbengine.database.LLUtils.MARKER_LUCENE;
import static it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.NO_TRANSFORMATION; import static it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.NO_TRANSFORMATION;
import com.google.common.util.concurrent.Uninterruptibles;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
import it.cavallium.dbengine.client.DirectIOOptions; import it.cavallium.dbengine.client.DirectIOOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerAnalyzers;
@ -28,6 +30,9 @@ import java.nio.file.Path;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Phaser; import java.util.concurrent.Phaser;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
@ -49,8 +54,10 @@ import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Constants; import org.apache.lucene.util.Constants;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.functional.IORunnable;
import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
import org.warp.commonutils.type.ShortNamedThreadFactory;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Scheduler;
@ -67,7 +74,9 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
* concurrent commits or concurrent refreshes. * concurrent commits or concurrent refreshes.
*/ */
private static final Scheduler luceneHeavyTasksScheduler = Schedulers.single(Schedulers.boundedElastic()); private static final Scheduler luceneHeavyTasksScheduler = Schedulers.single(Schedulers.boundedElastic());
private static final ExecutorService SAFE_EXECUTOR = Executors.newCachedThreadPool(new ShortNamedThreadFactory("lucene-index-impl"));
private final MeterRegistry meterRegistry;
private final String luceneIndexName; private final String luceneIndexName;
private final IndexWriter indexWriter; private final IndexWriter indexWriter;
private final SnapshotsManager snapshotsManager; private final SnapshotsManager snapshotsManager;
@ -81,11 +90,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
private final AtomicBoolean closeRequested = new AtomicBoolean(); private final AtomicBoolean closeRequested = new AtomicBoolean();
public LLLocalLuceneIndex(@Nullable Path luceneBasePath, public LLLocalLuceneIndex(@Nullable Path luceneBasePath,
MeterRegistry meterRegistry,
String name, String name,
IndicizerAnalyzers indicizerAnalyzers, IndicizerAnalyzers indicizerAnalyzers,
IndicizerSimilarities indicizerSimilarities, IndicizerSimilarities indicizerSimilarities,
LuceneOptions luceneOptions, LuceneOptions luceneOptions,
@Nullable LuceneHacks luceneHacks) throws IOException { @Nullable LuceneHacks luceneHacks) throws IOException {
this.meterRegistry = meterRegistry;
Path directoryPath; Path directoryPath;
if (luceneOptions.inMemory() != (luceneBasePath == null)) { if (luceneOptions.inMemory() != (luceneBasePath == null)) {
throw new IllegalArgumentException(); throw new IllegalArgumentException();
@ -234,13 +245,45 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
} }
private <V> Mono<V> ensureOpen(Mono<V> mono) { private <V> Mono<V> ensureOpen(Mono<V> mono) {
return Mono.defer(() -> { return Mono.<Void>fromCallable(() -> {
if (closeRequested.get()) { if (closeRequested.get()) {
return Mono.error(new IllegalStateException("Lucene index is closed")); throw new IllegalStateException("Lucene index is closed");
} else { } else {
return mono; return null;
} }
}).doFirst(activeTasks::register).doFinally(s -> activeTasks.arriveAndDeregister()); }).then(mono).doFirst(activeTasks::register).doFinally(s -> activeTasks.arriveAndDeregister());
}
private <V> Mono<V> runSafe(Callable<V> callable) {
return Mono.<V>create(sink -> {
var future = SAFE_EXECUTOR.submit(() -> {
try {
var result = callable.call();
if (result != null) {
sink.success(result);
} else {
sink.success();
}
} catch (Throwable e) {
sink.error(e);
}
});
sink.onDispose(() -> future.cancel(false));
});
}
private <V> Mono<V> runSafe(IORunnable runnable) {
return Mono.create(sink -> {
var future = SAFE_EXECUTOR.submit(() -> {
try {
runnable.run();
sink.success();
} catch (Throwable e) {
sink.error(e);
}
});
sink.onDispose(() -> future.cancel(false));
});
} }
@Override @Override
@ -250,40 +293,29 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
@Override @Override
public Mono<Void> addDocument(LLTerm key, LLDocument doc) { public Mono<Void> addDocument(LLTerm key, LLDocument doc) {
return Mono.<Void>fromCallable(() -> { return this.<Void>runSafe(() -> indexWriter.addDocument(LLUtils.toDocument(doc))).transform(this::ensureOpen);
indexWriter.addDocument(LLUtils.toDocument(doc));
return null;
}).subscribeOn(Schedulers.boundedElastic()).transform(this::ensureOpen);
} }
@Override @Override
public Mono<Void> addDocuments(Flux<Entry<LLTerm, LLDocument>> documents) { public Mono<Void> addDocuments(Flux<Entry<LLTerm, LLDocument>> documents) {
return documents return documents
.collectList() .collectList()
.flatMap(documentsList -> Mono .flatMap(documentsList -> this.<Void>runSafe(() -> indexWriter.addDocuments(LLUtils
.<Void>fromCallable(() -> { .toDocumentsFromEntries(documentsList))))
indexWriter.addDocuments(LLUtils.toDocumentsFromEntries(documentsList));
return null;
}).subscribeOn(Schedulers.boundedElastic())
)
.transform(this::ensureOpen); .transform(this::ensureOpen);
} }
@Override @Override
public Mono<Void> deleteDocument(LLTerm id) { public Mono<Void> deleteDocument(LLTerm id) {
return Mono.<Void>fromCallable(() -> { return this.<Void>runSafe(() -> indexWriter.deleteDocuments(LLUtils.toTerm(id))).transform(this::ensureOpen);
indexWriter.deleteDocuments(LLUtils.toTerm(id));
return null;
}).subscribeOn(Schedulers.boundedElastic()).transform(this::ensureOpen);
} }
@Override @Override
public Mono<Void> updateDocument(LLTerm id, LLDocument document) { public Mono<Void> updateDocument(LLTerm id, LLDocument document) {
return Mono.<Void>fromCallable(() -> { return this
indexWriter.updateDocument(LLUtils.toTerm(id), LLUtils.toDocument(document)); .<Void>runSafe(() -> indexWriter.updateDocument(LLUtils.toTerm(id), LLUtils.toDocument(document)))
return null; .transform(this::ensureOpen);
}).subscribeOn(Schedulers.boundedElastic()).transform(this::ensureOpen);
} }
@Override @Override
@ -292,29 +324,21 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
} }
private Mono<Void> updateDocuments(Map<LLTerm, LLDocument> documentsMap) { private Mono<Void> updateDocuments(Map<LLTerm, LLDocument> documentsMap) {
return Mono return this.<Void>runSafe(() -> {
.<Void>fromCallable(() -> { for (Entry<LLTerm, LLDocument> entry : documentsMap.entrySet()) {
for (Entry<LLTerm, LLDocument> entry : documentsMap.entrySet()) { LLTerm key = entry.getKey();
LLTerm key = entry.getKey(); LLDocument value = entry.getValue();
LLDocument value = entry.getValue(); indexWriter.updateDocument(LLUtils.toTerm(key), LLUtils.toDocument(value));
indexWriter.updateDocument(LLUtils.toTerm(key), LLUtils.toDocument(value)); }
} }).transform(this::ensureOpen);
return null;
})
.subscribeOn(Schedulers.boundedElastic())
.transform(this::ensureOpen);
} }
@Override @Override
public Mono<Void> deleteAll() { public Mono<Void> deleteAll() {
return Mono.<Void>fromCallable(() -> { return this.<Void>runSafe(() -> {
//noinspection BlockingMethodInNonBlockingContext
indexWriter.deleteAll(); indexWriter.deleteAll();
//noinspection BlockingMethodInNonBlockingContext
indexWriter.forceMergeDeletes(true); indexWriter.forceMergeDeletes(true);
//noinspection BlockingMethodInNonBlockingContext
indexWriter.commit(); indexWriter.commit();
return null;
}).subscribeOn(luceneHeavyTasksScheduler).transform(this::ensureOpen); }).subscribeOn(luceneHeavyTasksScheduler).transform(this::ensureOpen);
} }

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.client.IndicizerSimilarities;
@ -39,6 +40,7 @@ import reactor.util.function.Tuple2;
public class LLLocalMultiLuceneIndex implements LLLuceneIndex { public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
private final MeterRegistry meterRegistry;
private final ConcurrentHashMap<Long, LLSnapshot[]> registeredSnapshots = new ConcurrentHashMap<>(); private final ConcurrentHashMap<Long, LLSnapshot[]> registeredSnapshots = new ConcurrentHashMap<>();
private final AtomicLong nextSnapshotNumber = new AtomicLong(1); private final AtomicLong nextSnapshotNumber = new AtomicLong(1);
private final LLLocalLuceneIndex[] luceneIndices; private final LLLocalLuceneIndex[] luceneIndices;
@ -49,6 +51,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
public LLLocalMultiLuceneIndex(LLTempLMDBEnv env, public LLLocalMultiLuceneIndex(LLTempLMDBEnv env,
Path lucene, Path lucene,
MeterRegistry meterRegistry,
String name, String name,
int instancesCount, int instancesCount,
IndicizerAnalyzers indicizerAnalyzers, IndicizerAnalyzers indicizerAnalyzers,
@ -60,6 +63,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
throw new IOException("Unsupported instances count: " + instancesCount); throw new IOException("Unsupported instances count: " + instancesCount);
} }
this.meterRegistry = meterRegistry;
LLLocalLuceneIndex[] luceneIndices = new LLLocalLuceneIndex[instancesCount]; LLLocalLuceneIndex[] luceneIndices = new LLLocalLuceneIndex[instancesCount];
for (int i = 0; i < instancesCount; i++) { for (int i = 0; i < instancesCount; i++) {
String instanceName; String instanceName;
@ -69,6 +73,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
instanceName = name + "_" + String.format("%03d", i); instanceName = name + "_" + String.format("%03d", i);
} }
luceneIndices[i] = new LLLocalLuceneIndex(lucene, luceneIndices[i] = new LLLocalLuceneIndex(lucene,
meterRegistry,
instanceName, instanceName,
indicizerAnalyzers, indicizerAnalyzers,
indicizerSimilarities, indicizerSimilarities,

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import io.net5.buffer.api.MemoryManager; import io.net5.buffer.api.MemoryManager;
@ -27,11 +28,14 @@ import reactor.core.scheduler.Schedulers;
public final class OptimisticRocksDBColumn extends AbstractRocksDBColumn<OptimisticTransactionDB> { public final class OptimisticRocksDBColumn extends AbstractRocksDBColumn<OptimisticTransactionDB> {
private static final boolean ALWAYS_PRINT_OPTIMISTIC_RETRIES = false;
public OptimisticRocksDBColumn(OptimisticTransactionDB db, public OptimisticRocksDBColumn(OptimisticTransactionDB db,
DatabaseOptions databaseOptions, DatabaseOptions databaseOptions,
BufferAllocator alloc, BufferAllocator alloc,
ColumnFamilyHandle cfh) { ColumnFamilyHandle cfh,
super(db, databaseOptions, alloc, cfh); MeterRegistry meterRegistry) {
super(db, databaseOptions, alloc, cfh, meterRegistry);
} }
@Override @Override
@ -156,20 +160,23 @@ public final class OptimisticRocksDBColumn extends AbstractRocksDBColumn<Optimis
sentCurData.close(); sentCurData.close();
} }
retries++; retries++;
if (retries >= 100 && retries % 100 == 0) {
logger.warn(MARKER_ROCKSDB, "Failed optimistic transaction {} (update):"
+ " waiting 5ms before retrying for the {} time", LLUtils.toStringSafe(key), retries);
} else if (logger.isDebugEnabled(MARKER_ROCKSDB)) {
logger.debug(MARKER_ROCKSDB, "Failed optimistic transaction {} (update):"
+ " waiting 5ms before retrying", LLUtils.toStringSafe(key));
}
if (retries == 1) { if (retries == 1) {
retryTime = new ExponentialPageLimits(0, 5, 2000); retryTime = new ExponentialPageLimits(0, 5, 2000);
} }
long retryMs = retryTime.getPageLimit(retries); long retryMs = retryTime.getPageLimit(retries);
// +- 20% // +- 20%
retryMs = retryMs + (long) (retryMs * 0.2d * ThreadLocalRandom.current().nextDouble(-1.0d, 1.0d)); retryMs = retryMs + (long) (retryMs * 0.2d * ThreadLocalRandom.current().nextDouble(-1.0d, 1.0d));
// Wait for 5ms
if (retries >= 5 && retries % 5 == 0 || ALWAYS_PRINT_OPTIMISTIC_RETRIES) {
logger.warn(MARKER_ROCKSDB, "Failed optimistic transaction {} (update):"
+ " waiting {} ms before retrying for the {} time", LLUtils.toStringSafe(key), retryMs, retries);
} else if (logger.isDebugEnabled(MARKER_ROCKSDB)) {
logger.debug(MARKER_ROCKSDB, "Failed optimistic transaction {} (update):"
+ " waiting {} ms before retrying for the {} time", LLUtils.toStringSafe(key), retryMs, retries);
}
// Wait for n milliseconds
try { try {
Thread.sleep(retryMs); Thread.sleep(retryMs);
} catch (InterruptedException e) { } catch (InterruptedException e) {

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import io.net5.buffer.api.MemoryManager; import io.net5.buffer.api.MemoryManager;
@ -29,8 +30,8 @@ public final class PessimisticRocksDBColumn extends AbstractRocksDBColumn<Transa
public PessimisticRocksDBColumn(TransactionDB db, public PessimisticRocksDBColumn(TransactionDB db,
DatabaseOptions databaseOptions, DatabaseOptions databaseOptions,
BufferAllocator alloc, BufferAllocator alloc,
ColumnFamilyHandle cfh) { ColumnFamilyHandle cfh, MeterRegistry meterRegistry) {
super(db, databaseOptions, alloc, cfh); super(db, databaseOptions, alloc, cfh, meterRegistry);
} }
@Override @Override

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
@ -62,5 +63,7 @@ public sealed interface RocksDBColumn permits AbstractRocksDBColumn {
BufferAllocator getAllocator(); BufferAllocator getAllocator();
MeterRegistry getMeterRegistry();
boolean supportsTransactions(); boolean supportsTransactions();
} }

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
@ -25,8 +26,8 @@ public final class StandardRocksDBColumn extends AbstractRocksDBColumn<RocksDB>
public StandardRocksDBColumn(RocksDB db, public StandardRocksDBColumn(RocksDB db,
DatabaseOptions databaseOptions, DatabaseOptions databaseOptions,
BufferAllocator alloc, BufferAllocator alloc,
ColumnFamilyHandle cfh) { ColumnFamilyHandle cfh, MeterRegistry meterRegistry) {
super(db, databaseOptions, alloc, cfh); super(db, databaseOptions, alloc, cfh, meterRegistry);
} }
@Override @Override

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.database.memory; package it.cavallium.dbengine.database.memory;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerAnalyzers;
@ -24,9 +25,11 @@ public class LLMemoryDatabaseConnection implements LLDatabaseConnection {
} }
private final BufferAllocator allocator; private final BufferAllocator allocator;
private final MeterRegistry meterRegistry;
public LLMemoryDatabaseConnection(BufferAllocator allocator) { public LLMemoryDatabaseConnection(BufferAllocator allocator, MeterRegistry meterRegistry) {
this.allocator = allocator; this.allocator = allocator;
this.meterRegistry = meterRegistry;
} }
@Override @Override
@ -34,6 +37,11 @@ public class LLMemoryDatabaseConnection implements LLDatabaseConnection {
return allocator; return allocator;
} }
@Override
public MeterRegistry getMeterRegistry() {
return meterRegistry;
}
@Override @Override
public Mono<LLDatabaseConnection> connect() { public Mono<LLDatabaseConnection> connect() {
return Mono.empty(); return Mono.empty();
@ -46,6 +54,7 @@ public class LLMemoryDatabaseConnection implements LLDatabaseConnection {
return Mono return Mono
.<LLKeyValueDatabase>fromCallable(() -> new LLMemoryKeyValueDatabase( .<LLKeyValueDatabase>fromCallable(() -> new LLMemoryKeyValueDatabase(
allocator, allocator,
meterRegistry,
name, name,
columns columns
)) ))
@ -61,6 +70,7 @@ public class LLMemoryDatabaseConnection implements LLDatabaseConnection {
@Nullable LuceneHacks luceneHacks) { @Nullable LuceneHacks luceneHacks) {
return Mono return Mono
.<LLLuceneIndex>fromCallable(() -> new LLLocalLuceneIndex(null, .<LLLuceneIndex>fromCallable(() -> new LLLocalLuceneIndex(null,
meterRegistry,
name, name,
indicizerAnalyzers, indicizerAnalyzers,
indicizerSimilarities, indicizerSimilarities,

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.database.memory; package it.cavallium.dbengine.database.memory;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator; import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLDictionary; import it.cavallium.dbengine.database.LLDictionary;
@ -18,6 +19,7 @@ import reactor.core.publisher.Mono;
public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase { public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
private final BufferAllocator allocator; private final BufferAllocator allocator;
private final MeterRegistry meterRegistry;
private final String name; private final String name;
private final AtomicLong nextSnapshotNumber = new AtomicLong(1); private final AtomicLong nextSnapshotNumber = new AtomicLong(1);
@ -25,8 +27,12 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
private final ConcurrentHashMap<String, ConcurrentSkipListMap<ByteList, ByteList>> mainDb; private final ConcurrentHashMap<String, ConcurrentSkipListMap<ByteList, ByteList>> mainDb;
private final ConcurrentHashMap<String, LLMemoryDictionary> singletons = new ConcurrentHashMap<>(); private final ConcurrentHashMap<String, LLMemoryDictionary> singletons = new ConcurrentHashMap<>();
public LLMemoryKeyValueDatabase(BufferAllocator allocator, String name, List<Column> columns) { public LLMemoryKeyValueDatabase(BufferAllocator allocator,
MeterRegistry meterRegistry,
String name,
List<Column> columns) {
this.allocator = allocator; this.allocator = allocator;
this.meterRegistry = meterRegistry;
this.name = name; this.name = name;
this.mainDb = new ConcurrentHashMap<>(); this.mainDb = new ConcurrentHashMap<>();
for (Column column : columns) { for (Column column : columns) {
@ -80,6 +86,11 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
return allocator; return allocator;
} }
@Override
public MeterRegistry getMeterRegistry() {
return meterRegistry;
}
@Override @Override
public Mono<Void> close() { public Mono<Void> close() {
return Mono return Mono

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.dbengine.DbTestUtils.TempDb; import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator; import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseOptions;
@ -56,7 +57,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
return null; return null;
}) })
.subscribeOn(Schedulers.boundedElastic()) .subscribeOn(Schedulers.boundedElastic())
.then(new LLLocalDatabaseConnection(allocator.allocator(), wrkspcPath).connect()) .then(new LLLocalDatabaseConnection(allocator.allocator(), new SimpleMeterRegistry(), wrkspcPath).connect())
.flatMap(conn -> { .flatMap(conn -> {
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher(); SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher); var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine; package it.cavallium.dbengine;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.dbengine.DbTestUtils.TempDb; import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator; import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseOptions;
@ -28,7 +29,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
public Mono<TempDb> openTempDb(TestAllocator allocator) { public Mono<TempDb> openTempDb(TestAllocator allocator) {
boolean canUseNettyDirect = DbTestUtils.computeCanUseNettyDirect(); boolean canUseNettyDirect = DbTestUtils.computeCanUseNettyDirect();
return Mono return Mono
.fromCallable(() -> new LLMemoryDatabaseConnection(allocator.allocator())) .fromCallable(() -> new LLMemoryDatabaseConnection(allocator.allocator(), new SimpleMeterRegistry()))
.flatMap(conn -> { .flatMap(conn -> {
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher(); SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher); var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);