Implement more micrometer metrics
This commit is contained in:
parent
5769bc7076
commit
68d8b5240c
1
pom.xml
1
pom.xml
@ -521,7 +521,6 @@
|
|||||||
<annotationProcessors>
|
<annotationProcessors>
|
||||||
<annotationProcessor>io.soabase.recordbuilder.processor.RecordBuilderProcessor</annotationProcessor>
|
<annotationProcessor>io.soabase.recordbuilder.processor.RecordBuilderProcessor</annotationProcessor>
|
||||||
</annotationProcessors>
|
</annotationProcessors>
|
||||||
<useIncrementalCompilation>false</useIncrementalCompilation>
|
|
||||||
<compilerArgs>--enable-preview</compilerArgs>
|
<compilerArgs>--enable-preview</compilerArgs>
|
||||||
<source>17</source>
|
<source>17</source>
|
||||||
<target>17</target>
|
<target>17</target>
|
||||||
|
@ -24,7 +24,8 @@ public interface LLDatabaseConnection {
|
|||||||
List<Column> columns,
|
List<Column> columns,
|
||||||
DatabaseOptions databaseOptions);
|
DatabaseOptions databaseOptions);
|
||||||
|
|
||||||
Mono<? extends LLLuceneIndex> getLuceneIndex(String name,
|
Mono<? extends LLLuceneIndex> getLuceneIndex(@Nullable String clusterName,
|
||||||
|
@Nullable String shardName,
|
||||||
int instancesCount,
|
int instancesCount,
|
||||||
IndicizerAnalyzers indicizerAnalyzers,
|
IndicizerAnalyzers indicizerAnalyzers,
|
||||||
IndicizerSimilarities indicizerSimilarities,
|
IndicizerSimilarities indicizerSimilarities,
|
||||||
|
@ -2,24 +2,22 @@ package it.cavallium.dbengine.database.disk;
|
|||||||
|
|
||||||
import io.micrometer.core.instrument.MeterRegistry;
|
import io.micrometer.core.instrument.MeterRegistry;
|
||||||
import io.net5.buffer.api.BufferAllocator;
|
import io.net5.buffer.api.BufferAllocator;
|
||||||
|
import it.cavallium.dbengine.client.DatabaseOptions;
|
||||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||||
import it.cavallium.dbengine.client.LuceneOptions;
|
import it.cavallium.dbengine.client.LuceneOptions;
|
||||||
import it.cavallium.dbengine.database.Column;
|
import it.cavallium.dbengine.database.Column;
|
||||||
import it.cavallium.dbengine.client.DatabaseOptions;
|
|
||||||
import it.cavallium.dbengine.database.LLDatabaseConnection;
|
import it.cavallium.dbengine.database.LLDatabaseConnection;
|
||||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||||
import it.cavallium.dbengine.netty.JMXNettyMonitoringManager;
|
import it.cavallium.dbengine.netty.JMXNettyMonitoringManager;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
import org.checkerframework.checker.units.qual.A;
|
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
import reactor.core.scheduler.Schedulers;
|
import reactor.core.scheduler.Schedulers;
|
||||||
@ -88,7 +86,8 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LLLuceneIndex> getLuceneIndex(String name,
|
public Mono<LLLuceneIndex> getLuceneIndex(@Nullable String clusterName,
|
||||||
|
@Nullable String shardName,
|
||||||
int instancesCount,
|
int instancesCount,
|
||||||
IndicizerAnalyzers indicizerAnalyzers,
|
IndicizerAnalyzers indicizerAnalyzers,
|
||||||
IndicizerSimilarities indicizerSimilarities,
|
IndicizerSimilarities indicizerSimilarities,
|
||||||
@ -97,12 +96,18 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
|
|||||||
return Mono
|
return Mono
|
||||||
.fromCallable(() -> {
|
.fromCallable(() -> {
|
||||||
var env = this.env.get();
|
var env = this.env.get();
|
||||||
|
if (clusterName == null && shardName == null) {
|
||||||
|
throw new IllegalArgumentException("Shard name and/or cluster name must be set");
|
||||||
|
}
|
||||||
if (instancesCount != 1) {
|
if (instancesCount != 1) {
|
||||||
|
if (shardName != null && !shardName.equals(clusterName)) {
|
||||||
|
throw new IllegalArgumentException("You shouldn't use a shard name for clustered instances");
|
||||||
|
}
|
||||||
Objects.requireNonNull(env, "Environment not set");
|
Objects.requireNonNull(env, "Environment not set");
|
||||||
return new LLLocalMultiLuceneIndex(env,
|
return new LLLocalMultiLuceneIndex(env,
|
||||||
luceneOptions.inMemory() ? null : basePath.resolve("lucene"),
|
luceneOptions.inMemory() ? null : basePath.resolve("lucene"),
|
||||||
meterRegistry,
|
meterRegistry,
|
||||||
name,
|
clusterName,
|
||||||
instancesCount,
|
instancesCount,
|
||||||
indicizerAnalyzers,
|
indicizerAnalyzers,
|
||||||
indicizerSimilarities,
|
indicizerSimilarities,
|
||||||
@ -112,7 +117,8 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
|
|||||||
} else {
|
} else {
|
||||||
return new LLLocalLuceneIndex(env, luceneOptions.inMemory() ? null : basePath.resolve("lucene"),
|
return new LLLocalLuceneIndex(env, luceneOptions.inMemory() ? null : basePath.resolve("lucene"),
|
||||||
meterRegistry,
|
meterRegistry,
|
||||||
name,
|
clusterName,
|
||||||
|
shardName,
|
||||||
indicizerAnalyzers,
|
indicizerAnalyzers,
|
||||||
indicizerSimilarities,
|
indicizerSimilarities,
|
||||||
luceneOptions,
|
luceneOptions,
|
||||||
|
@ -8,6 +8,8 @@ import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
|
|||||||
import static java.util.Objects.requireNonNull;
|
import static java.util.Objects.requireNonNull;
|
||||||
import static java.util.Objects.requireNonNullElse;
|
import static java.util.Objects.requireNonNullElse;
|
||||||
|
|
||||||
|
import io.micrometer.core.instrument.Counter;
|
||||||
|
import io.micrometer.core.instrument.Timer;
|
||||||
import io.net5.buffer.api.Buffer;
|
import io.net5.buffer.api.Buffer;
|
||||||
import io.net5.buffer.api.BufferAllocator;
|
import io.net5.buffer.api.BufferAllocator;
|
||||||
import io.net5.buffer.api.Resource;
|
import io.net5.buffer.api.Resource;
|
||||||
@ -40,6 +42,7 @@ import java.util.Optional;
|
|||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.ForkJoinPool;
|
import java.util.concurrent.ForkJoinPool;
|
||||||
import java.util.concurrent.ForkJoinTask;
|
import java.util.concurrent.ForkJoinTask;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.IntStream;
|
import java.util.stream.IntStream;
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
@ -131,6 +134,22 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
private final boolean nettyDirect;
|
private final boolean nettyDirect;
|
||||||
private final BufferAllocator alloc;
|
private final BufferAllocator alloc;
|
||||||
|
|
||||||
|
private final Counter startedUpdates;
|
||||||
|
private final Counter endedUpdates;
|
||||||
|
private final Timer updateTime;
|
||||||
|
private final Counter startedGet;
|
||||||
|
private final Counter endedGet;
|
||||||
|
private final Timer getTime;
|
||||||
|
private final Counter startedContains;
|
||||||
|
private final Counter endedContains;
|
||||||
|
private final Timer containsTime;
|
||||||
|
private final Counter startedPut;
|
||||||
|
private final Counter endedPut;
|
||||||
|
private final Timer putTime;
|
||||||
|
private final Counter startedRemove;
|
||||||
|
private final Counter endedRemove;
|
||||||
|
private final Timer removeTime;
|
||||||
|
|
||||||
public LLLocalDictionary(
|
public LLLocalDictionary(
|
||||||
BufferAllocator allocator,
|
BufferAllocator allocator,
|
||||||
@NotNull RocksDBColumn db,
|
@NotNull RocksDBColumn db,
|
||||||
@ -151,6 +170,48 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
this.databaseOptions = databaseOptions;
|
this.databaseOptions = databaseOptions;
|
||||||
alloc = allocator;
|
alloc = allocator;
|
||||||
this.nettyDirect = databaseOptions.allowNettyDirect() && alloc.getAllocationType() == OFF_HEAP;
|
this.nettyDirect = databaseOptions.allowNettyDirect() && alloc.getAllocationType() == OFF_HEAP;
|
||||||
|
var meterRegistry = db.getMeterRegistry();
|
||||||
|
|
||||||
|
this.startedGet = meterRegistry.counter("db.read.map.get.started.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.endedGet = meterRegistry.counter("db.read.map.get.ended.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.getTime = Timer
|
||||||
|
.builder("db.read.get.timer")
|
||||||
|
.publishPercentiles(0.2, 0.5, 0.95)
|
||||||
|
.publishPercentileHistogram()
|
||||||
|
.tags("db.name", databaseName, "db.column", columnName)
|
||||||
|
.register(meterRegistry);
|
||||||
|
this.startedContains = meterRegistry.counter("db.read.map.get.started.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.endedContains = meterRegistry.counter("db.read.map.get.ended.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.containsTime = Timer
|
||||||
|
.builder("db.read.get.timer")
|
||||||
|
.publishPercentiles(0.2, 0.5, 0.95)
|
||||||
|
.publishPercentileHistogram()
|
||||||
|
.tags("db.name", databaseName, "db.column", columnName)
|
||||||
|
.register(meterRegistry);
|
||||||
|
this.startedUpdates = meterRegistry.counter("db.write.map.update.started.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.endedUpdates = meterRegistry.counter("db.write.map.update.ended.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.updateTime = Timer
|
||||||
|
.builder("db.write.map.update.timer")
|
||||||
|
.publishPercentiles(0.2, 0.5, 0.95)
|
||||||
|
.publishPercentileHistogram()
|
||||||
|
.tags("db.name", databaseName, "db.column", columnName)
|
||||||
|
.register(meterRegistry);
|
||||||
|
this.startedPut = meterRegistry.counter("db.write.map.put.started.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.endedPut = meterRegistry.counter("db.write.map.put.ended.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.putTime = Timer
|
||||||
|
.builder("db.write.map.put.timer")
|
||||||
|
.publishPercentiles(0.2, 0.5, 0.95)
|
||||||
|
.publishPercentileHistogram()
|
||||||
|
.tags("db.name", databaseName, "db.column", columnName)
|
||||||
|
.register(meterRegistry);
|
||||||
|
this.startedRemove = meterRegistry.counter("db.write.map.remove.started.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.endedRemove = meterRegistry.counter("db.write.map.remove.ended.counter", "db.name", databaseName, "db.column", columnName);
|
||||||
|
this.removeTime = Timer
|
||||||
|
.builder("db.write.map.remove.timer")
|
||||||
|
.publishPercentiles(0.2, 0.5, 0.95)
|
||||||
|
.publishPercentileHistogram()
|
||||||
|
.tags("db.name", databaseName, "db.column", columnName)
|
||||||
|
.register(meterRegistry);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -205,7 +266,13 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
try (var key = keySend.receive()) {
|
try (var key = keySend.receive()) {
|
||||||
try {
|
try {
|
||||||
var readOptions = requireNonNullElse(resolveSnapshot(snapshot), EMPTY_READ_OPTIONS);
|
var readOptions = requireNonNullElse(resolveSnapshot(snapshot), EMPTY_READ_OPTIONS);
|
||||||
var result = db.get(readOptions, key, existsAlmostCertainly);
|
Buffer result;
|
||||||
|
startedGet.increment();
|
||||||
|
try {
|
||||||
|
result = getTime.recordCallable(() -> db.get(readOptions, key, existsAlmostCertainly));
|
||||||
|
} finally {
|
||||||
|
endedGet.increment();
|
||||||
|
}
|
||||||
logger.trace(MARKER_ROCKSDB, "Read {}: {}", () -> toStringSafe(key), () -> toStringSafe(result));
|
logger.trace(MARKER_ROCKSDB, "Read {}: {}", () -> toStringSafe(key), () -> toStringSafe(result));
|
||||||
if (result != null) {
|
if (result != null) {
|
||||||
sink.next(result.send());
|
sink.next(result.send());
|
||||||
@ -236,6 +303,9 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
|
|
||||||
public boolean containsRange(@Nullable LLSnapshot snapshot, LLRange range) throws RocksDBException {
|
public boolean containsRange(@Nullable LLSnapshot snapshot, LLRange range) throws RocksDBException {
|
||||||
assert !Schedulers.isInNonBlockingThread() : "Called containsRange in a nonblocking thread";
|
assert !Schedulers.isInNonBlockingThread() : "Called containsRange in a nonblocking thread";
|
||||||
|
startedContains.increment();
|
||||||
|
try {
|
||||||
|
var result = containsTime.recordCallable(() -> {
|
||||||
if (range.isSingle()) {
|
if (range.isSingle()) {
|
||||||
var unmodifiableReadOpts = resolveSnapshot(snapshot);
|
var unmodifiableReadOpts = resolveSnapshot(snapshot);
|
||||||
return db.exists(unmodifiableReadOpts, range.getSingleUnsafe());
|
return db.exists(unmodifiableReadOpts, range.getSingleUnsafe());
|
||||||
@ -283,6 +353,16 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
if (slice2 != null) slice2.close();
|
if (slice2 != null) slice2.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
assert result != null;
|
||||||
|
return result;
|
||||||
|
} catch (RocksDBException | RuntimeException e) {
|
||||||
|
throw e;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
} finally {
|
||||||
|
endedContains.increment();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Mono<Boolean> containsKey(@Nullable LLSnapshot snapshot, Mono<Send<Buffer>> keyMono) {
|
private Mono<Boolean> containsKey(@Nullable LLSnapshot snapshot, Mono<Send<Buffer>> keyMono) {
|
||||||
@ -298,8 +378,21 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private boolean containsKey(@Nullable LLSnapshot snapshot, Buffer key) throws RocksDBException {
|
private boolean containsKey(@Nullable LLSnapshot snapshot, Buffer key) throws RocksDBException {
|
||||||
|
startedContains.increment();
|
||||||
|
try {
|
||||||
|
var result = containsTime.recordCallable(() -> {
|
||||||
var unmodifiableReadOpts = resolveSnapshot(snapshot);
|
var unmodifiableReadOpts = resolveSnapshot(snapshot);
|
||||||
return db.exists(unmodifiableReadOpts, key);
|
return db.exists(unmodifiableReadOpts, key);
|
||||||
|
});
|
||||||
|
assert result != null;
|
||||||
|
return result;
|
||||||
|
} catch (RocksDBException | RuntimeException e) {
|
||||||
|
throw e;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
} finally {
|
||||||
|
endedContains.increment();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -340,7 +433,14 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
safeCloseable.close();
|
safeCloseable.close();
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.onErrorMap(cause -> new IOException("Failed to write", cause));
|
.onErrorMap(cause -> new IOException("Failed to write", cause))
|
||||||
|
.elapsed()
|
||||||
|
.map(tuple -> {
|
||||||
|
putTime.record(tuple.getT1(), TimeUnit.MILLISECONDS);
|
||||||
|
return tuple.getT2();
|
||||||
|
})
|
||||||
|
.doFirst(startedPut::increment)
|
||||||
|
.doFinally(s -> endedPut.increment());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -366,8 +466,15 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
case GET_NEW_VALUE -> UpdateAtomicResultMode.CURRENT;
|
case GET_NEW_VALUE -> UpdateAtomicResultMode.CURRENT;
|
||||||
case GET_OLD_VALUE -> UpdateAtomicResultMode.PREVIOUS;
|
case GET_OLD_VALUE -> UpdateAtomicResultMode.PREVIOUS;
|
||||||
};
|
};
|
||||||
UpdateAtomicResult result = db.updateAtomic(EMPTY_READ_OPTIONS, EMPTY_WRITE_OPTIONS, keySend, updater,
|
UpdateAtomicResult result;
|
||||||
existsAlmostCertainly, returnMode);
|
startedUpdates.increment();
|
||||||
|
try {
|
||||||
|
result = updateTime.recordCallable(() -> db.updateAtomic(EMPTY_READ_OPTIONS,
|
||||||
|
EMPTY_WRITE_OPTIONS, keySend, updater, existsAlmostCertainly, returnMode));
|
||||||
|
} finally {
|
||||||
|
endedUpdates.increment();
|
||||||
|
}
|
||||||
|
assert result != null;
|
||||||
return switch (updateReturnMode) {
|
return switch (updateReturnMode) {
|
||||||
case NOTHING -> null;
|
case NOTHING -> null;
|
||||||
case GET_NEW_VALUE -> ((UpdateAtomicResultCurrent) result).current();
|
case GET_NEW_VALUE -> ((UpdateAtomicResultCurrent) result).current();
|
||||||
@ -395,8 +502,15 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
throw new UnsupportedOperationException("update() is disallowed because the database doesn't support"
|
throw new UnsupportedOperationException("update() is disallowed because the database doesn't support"
|
||||||
+ "safe atomic operations");
|
+ "safe atomic operations");
|
||||||
}
|
}
|
||||||
UpdateAtomicResult result = db.updateAtomic(EMPTY_READ_OPTIONS, EMPTY_WRITE_OPTIONS, keySend, updater,
|
UpdateAtomicResult result;
|
||||||
existsAlmostCertainly, UpdateAtomicResultMode.DELTA);
|
startedUpdates.increment();
|
||||||
|
try {
|
||||||
|
result = updateTime.recordCallable(() -> db.updateAtomic(EMPTY_READ_OPTIONS,
|
||||||
|
EMPTY_WRITE_OPTIONS, keySend, updater, existsAlmostCertainly, UpdateAtomicResultMode.DELTA));
|
||||||
|
} finally {
|
||||||
|
endedUpdates.increment();
|
||||||
|
}
|
||||||
|
assert result != null;
|
||||||
return ((UpdateAtomicResultDelta) result).delta();
|
return ((UpdateAtomicResultDelta) result).delta();
|
||||||
}).onErrorMap(cause -> new IOException("Failed to read or write", cause)),
|
}).onErrorMap(cause -> new IOException("Failed to read or write", cause)),
|
||||||
keySend -> Mono.fromRunnable(keySend::close)).doOnDiscard(UpdateAtomicResult.class, uar -> {
|
keySend -> Mono.fromRunnable(keySend::close)).doOnDiscard(UpdateAtomicResult.class, uar -> {
|
||||||
@ -431,7 +545,10 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
)
|
)
|
||||||
.singleOrEmpty(),
|
.singleOrEmpty(),
|
||||||
keySend -> Mono.fromRunnable(keySend::close)
|
keySend -> Mono.fromRunnable(keySend::close)
|
||||||
);
|
).elapsed().map(tuple -> {
|
||||||
|
removeTime.record(tuple.getT1(), TimeUnit.MILLISECONDS);
|
||||||
|
return tuple.getT2();
|
||||||
|
}).doFirst(startedRemove::increment).doFinally(s -> endedRemove.increment());
|
||||||
}
|
}
|
||||||
|
|
||||||
private Mono<Send<Buffer>> getPreviousData(Mono<Send<Buffer>> keyMono, LLDictionaryResultType resultType,
|
private Mono<Send<Buffer>> getPreviousData(Mono<Send<Buffer>> keyMono, LLDictionaryResultType resultType,
|
||||||
|
@ -6,10 +6,11 @@ import static it.cavallium.dbengine.database.LLUtils.toDocument;
|
|||||||
import static it.cavallium.dbengine.database.LLUtils.toFields;
|
import static it.cavallium.dbengine.database.LLUtils.toFields;
|
||||||
import static it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.NO_TRANSFORMATION;
|
import static it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.NO_TRANSFORMATION;
|
||||||
|
|
||||||
|
import io.micrometer.core.instrument.Counter;
|
||||||
import io.micrometer.core.instrument.MeterRegistry;
|
import io.micrometer.core.instrument.MeterRegistry;
|
||||||
|
import io.micrometer.core.instrument.Timer;
|
||||||
import io.net5.buffer.api.Resource;
|
import io.net5.buffer.api.Resource;
|
||||||
import io.net5.buffer.api.Send;
|
import io.net5.buffer.api.Send;
|
||||||
import io.net5.buffer.api.internal.ResourceSupport;
|
|
||||||
import it.cavallium.dbengine.client.DirectIOOptions;
|
import it.cavallium.dbengine.client.DirectIOOptions;
|
||||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||||
@ -19,36 +20,33 @@ import it.cavallium.dbengine.client.query.QueryParser;
|
|||||||
import it.cavallium.dbengine.client.query.current.data.Query;
|
import it.cavallium.dbengine.client.query.current.data.Query;
|
||||||
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
||||||
import it.cavallium.dbengine.database.LLIndexRequest;
|
import it.cavallium.dbengine.database.LLIndexRequest;
|
||||||
import it.cavallium.dbengine.database.LLSoftUpdateDocument;
|
|
||||||
import it.cavallium.dbengine.database.LLUpdateDocument;
|
|
||||||
import it.cavallium.dbengine.database.LLItem;
|
|
||||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||||
import it.cavallium.dbengine.database.LLSearchResultShard;
|
import it.cavallium.dbengine.database.LLSearchResultShard;
|
||||||
import it.cavallium.dbengine.database.LLSnapshot;
|
import it.cavallium.dbengine.database.LLSnapshot;
|
||||||
|
import it.cavallium.dbengine.database.LLSoftUpdateDocument;
|
||||||
import it.cavallium.dbengine.database.LLTerm;
|
import it.cavallium.dbengine.database.LLTerm;
|
||||||
|
import it.cavallium.dbengine.database.LLUpdateDocument;
|
||||||
import it.cavallium.dbengine.database.LLUpdateFields;
|
import it.cavallium.dbengine.database.LLUpdateFields;
|
||||||
import it.cavallium.dbengine.database.LLUtils;
|
import it.cavallium.dbengine.database.LLUtils;
|
||||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
|
||||||
import it.cavallium.dbengine.lucene.AlwaysDirectIOFSDirectory;
|
import it.cavallium.dbengine.lucene.AlwaysDirectIOFSDirectory;
|
||||||
|
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||||
import it.cavallium.dbengine.lucene.collector.Buckets;
|
import it.cavallium.dbengine.lucene.collector.Buckets;
|
||||||
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
|
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
|
||||||
import it.cavallium.dbengine.lucene.searcher.BucketParams;
|
import it.cavallium.dbengine.lucene.searcher.BucketParams;
|
||||||
import it.cavallium.dbengine.lucene.searcher.DecimalBucketMultiSearcher;
|
import it.cavallium.dbengine.lucene.searcher.DecimalBucketMultiSearcher;
|
||||||
|
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
|
||||||
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
|
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
|
||||||
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
||||||
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
|
|
||||||
import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
import java.util.concurrent.Phaser;
|
import java.util.concurrent.Phaser;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
@ -60,11 +58,9 @@ import org.apache.lucene.index.ConcurrentMergeScheduler;
|
|||||||
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriter;
|
||||||
import org.apache.lucene.index.IndexWriterConfig;
|
import org.apache.lucene.index.IndexWriterConfig;
|
||||||
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
|
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
|
||||||
import org.apache.lucene.index.MergePolicy;
|
|
||||||
import org.apache.lucene.index.MergeScheduler;
|
import org.apache.lucene.index.MergeScheduler;
|
||||||
import org.apache.lucene.index.SerialMergeScheduler;
|
import org.apache.lucene.index.SerialMergeScheduler;
|
||||||
import org.apache.lucene.index.SnapshotDeletionPolicy;
|
import org.apache.lucene.index.SnapshotDeletionPolicy;
|
||||||
import org.apache.lucene.index.TieredMergePolicy;
|
|
||||||
import org.apache.lucene.misc.store.DirectIODirectory;
|
import org.apache.lucene.misc.store.DirectIODirectory;
|
||||||
import org.apache.lucene.search.similarities.Similarity;
|
import org.apache.lucene.search.similarities.Similarity;
|
||||||
import org.apache.lucene.store.ByteBuffersDirectory;
|
import org.apache.lucene.store.ByteBuffersDirectory;
|
||||||
@ -76,8 +72,6 @@ import org.apache.lucene.store.NRTCachingDirectory;
|
|||||||
import org.apache.lucene.util.Constants;
|
import org.apache.lucene.util.Constants;
|
||||||
import org.jetbrains.annotations.NotNull;
|
import org.jetbrains.annotations.NotNull;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.warp.commonutils.functional.IORunnable;
|
|
||||||
import org.warp.commonutils.type.ShortNamedThreadFactory;
|
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
import reactor.core.scheduler.Scheduler;
|
import reactor.core.scheduler.Scheduler;
|
||||||
@ -97,8 +91,15 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
private static final ReentrantLock shutdownLock = new ReentrantLock();
|
private static final ReentrantLock shutdownLock = new ReentrantLock();
|
||||||
private static final Scheduler luceneHeavyTasksScheduler = uninterruptibleScheduler(Schedulers.single(Schedulers.boundedElastic()));
|
private static final Scheduler luceneHeavyTasksScheduler = uninterruptibleScheduler(Schedulers.single(Schedulers.boundedElastic()));
|
||||||
|
|
||||||
|
private final Counter startedDocIndexings;
|
||||||
|
private final Counter endeddDocIndexings;
|
||||||
|
private final Timer docIndexingTime;
|
||||||
|
private final Timer snapshotTime;
|
||||||
|
private final Timer flushTime;
|
||||||
|
private final Timer commitTime;
|
||||||
|
private final Timer mergeTime;
|
||||||
|
private final Timer refreshTime;
|
||||||
|
|
||||||
private final MeterRegistry meterRegistry;
|
|
||||||
private final String luceneIndexName;
|
private final String luceneIndexName;
|
||||||
private final IndexWriter indexWriter;
|
private final IndexWriter indexWriter;
|
||||||
private final SnapshotsManager snapshotsManager;
|
private final SnapshotsManager snapshotsManager;
|
||||||
@ -114,21 +115,27 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
public LLLocalLuceneIndex(LLTempLMDBEnv env,
|
public LLLocalLuceneIndex(LLTempLMDBEnv env,
|
||||||
@Nullable Path luceneBasePath,
|
@Nullable Path luceneBasePath,
|
||||||
MeterRegistry meterRegistry,
|
MeterRegistry meterRegistry,
|
||||||
String name,
|
@Nullable String clusterName,
|
||||||
|
@Nullable String shardName,
|
||||||
IndicizerAnalyzers indicizerAnalyzers,
|
IndicizerAnalyzers indicizerAnalyzers,
|
||||||
IndicizerSimilarities indicizerSimilarities,
|
IndicizerSimilarities indicizerSimilarities,
|
||||||
LuceneOptions luceneOptions,
|
LuceneOptions luceneOptions,
|
||||||
@Nullable LuceneHacks luceneHacks) throws IOException {
|
@Nullable LuceneHacks luceneHacks) throws IOException {
|
||||||
this.meterRegistry = meterRegistry;
|
if (clusterName == null && shardName == null) {
|
||||||
|
throw new IllegalArgumentException("Clustern name and/or shard name must be set");
|
||||||
|
}
|
||||||
|
String logName = Objects.requireNonNullElse(clusterName, shardName);
|
||||||
|
String luceneIndexName = Objects.requireNonNullElse(shardName, clusterName);
|
||||||
|
|
||||||
Path directoryPath;
|
Path directoryPath;
|
||||||
if (luceneOptions.inMemory() != (luceneBasePath == null)) {
|
if (luceneOptions.inMemory() != (luceneBasePath == null)) {
|
||||||
throw new IllegalArgumentException();
|
throw new IllegalArgumentException();
|
||||||
} else if (luceneBasePath != null) {
|
} else if (luceneBasePath != null) {
|
||||||
directoryPath = luceneBasePath.resolve(name + ".lucene.db");
|
directoryPath = luceneBasePath.resolve(shardName + ".lucene.db");
|
||||||
} else {
|
} else {
|
||||||
directoryPath = null;
|
directoryPath = null;
|
||||||
}
|
}
|
||||||
if (name.length() == 0) {
|
if (luceneIndexName.length() == 0) {
|
||||||
throw new IOException("Empty lucene database name");
|
throw new IOException("Empty lucene database name");
|
||||||
}
|
}
|
||||||
if (!MMapDirectory.UNMAP_SUPPORTED) {
|
if (!MMapDirectory.UNMAP_SUPPORTED) {
|
||||||
@ -194,7 +201,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
this.directory = directory;
|
this.directory = directory;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.luceneIndexName = name;
|
this.luceneIndexName = luceneIndexName;
|
||||||
var snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
|
var snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
|
||||||
this.lowMemory = lowMemory;
|
this.lowMemory = lowMemory;
|
||||||
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
|
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
|
||||||
@ -251,6 +258,15 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
luceneOptions.queryRefreshDebounceTime()
|
luceneOptions.queryRefreshDebounceTime()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
this.startedDocIndexings = meterRegistry.counter("index.write.doc.started.counter", "index.name", logName);
|
||||||
|
this.endeddDocIndexings = meterRegistry.counter("index.write.doc.ended.counter", "index.name", logName);
|
||||||
|
this.docIndexingTime = Timer.builder("index.write.doc.timer").publishPercentiles(0.2, 0.5, 0.95).publishPercentileHistogram().tag("index.name", logName).register(meterRegistry);
|
||||||
|
this.snapshotTime = Timer.builder("index.write.snapshot.timer").publishPercentiles(0.2, 0.5, 0.95).publishPercentileHistogram().tag("index.name", logName).register(meterRegistry);
|
||||||
|
this.flushTime = Timer.builder("index.write.flush.timer").publishPercentiles(0.2, 0.5, 0.95).publishPercentileHistogram().tag("index.name", logName).register(meterRegistry);
|
||||||
|
this.commitTime = Timer.builder("index.write.commit.timer").publishPercentiles(0.2, 0.5, 0.95).publishPercentileHistogram().tag("index.name", logName).register(meterRegistry);
|
||||||
|
this.mergeTime = Timer.builder("index.write.merge.timer").publishPercentiles(0.2, 0.5, 0.95).publishPercentileHistogram().tag("index.name", logName).register(meterRegistry);
|
||||||
|
this.refreshTime = Timer.builder("index.search.refresh.timer").publishPercentiles(0.2, 0.5, 0.95).publishPercentileHistogram().tag("index.name", logName).register(meterRegistry);
|
||||||
|
|
||||||
// Start scheduled tasks
|
// Start scheduled tasks
|
||||||
var commitMillis = luceneOptions.commitDebounceTime().toMillis();
|
var commitMillis = luceneOptions.commitDebounceTime().toMillis();
|
||||||
luceneHeavyTasksScheduler.schedulePeriodically(this::scheduledCommit, commitMillis, commitMillis,
|
luceneHeavyTasksScheduler.schedulePeriodically(this::scheduledCommit, commitMillis, commitMillis,
|
||||||
@ -271,7 +287,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LLSnapshot> takeSnapshot() {
|
public Mono<LLSnapshot> takeSnapshot() {
|
||||||
return snapshotsManager.takeSnapshot().transform(this::ensureOpen);
|
return snapshotsManager.takeSnapshot().elapsed().map(elapsed -> {
|
||||||
|
snapshotTime.record(elapsed.getT1(), TimeUnit.MILLISECONDS);
|
||||||
|
return elapsed.getT2();
|
||||||
|
}).transform(this::ensureOpen);
|
||||||
}
|
}
|
||||||
|
|
||||||
private <V> Mono<V> ensureOpen(Mono<V> mono) {
|
private <V> Mono<V> ensureOpen(Mono<V> mono) {
|
||||||
@ -293,49 +312,75 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
|
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
|
||||||
return snapshotsManager.releaseSnapshot(snapshot);
|
return snapshotsManager
|
||||||
|
.releaseSnapshot(snapshot)
|
||||||
|
.elapsed()
|
||||||
|
.doOnNext(elapsed -> snapshotTime.record(elapsed.getT1(), TimeUnit.MILLISECONDS))
|
||||||
|
.then();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> addDocument(LLTerm key, LLUpdateDocument doc) {
|
public Mono<Void> addDocument(LLTerm key, LLUpdateDocument doc) {
|
||||||
return this.<Void>runSafe(() -> {
|
return this.<Void>runSafe(() -> docIndexingTime.recordCallable(() -> {
|
||||||
|
startedDocIndexings.increment();
|
||||||
|
try {
|
||||||
indexWriter.addDocument(toDocument(doc));
|
indexWriter.addDocument(toDocument(doc));
|
||||||
|
} finally {
|
||||||
|
endeddDocIndexings.increment();
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}).transform(this::ensureOpen);
|
})).transform(this::ensureOpen);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> addDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
|
public Mono<Void> addDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
|
||||||
return documents.collectList().flatMap(documentsList -> this.<Void>runSafe(() -> {
|
return documents.collectList().flatMap(documentsList -> this.<Void>runSafe(() -> docIndexingTime.recordCallable(() -> {
|
||||||
|
double count = documentsList.size();
|
||||||
|
startedDocIndexings.increment(count);
|
||||||
|
try {
|
||||||
indexWriter.addDocuments(LLUtils.toDocumentsFromEntries(documentsList));
|
indexWriter.addDocuments(LLUtils.toDocumentsFromEntries(documentsList));
|
||||||
|
} finally {
|
||||||
|
endeddDocIndexings.increment(count);
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
})).transform(this::ensureOpen);
|
}))).transform(this::ensureOpen);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> deleteDocument(LLTerm id) {
|
public Mono<Void> deleteDocument(LLTerm id) {
|
||||||
return this.<Void>runSafe(() -> {
|
return this.<Void>runSafe(() -> docIndexingTime.recordCallable(() -> {
|
||||||
|
startedDocIndexings.increment();
|
||||||
|
try {
|
||||||
indexWriter.deleteDocuments(LLUtils.toTerm(id));
|
indexWriter.deleteDocuments(LLUtils.toTerm(id));
|
||||||
|
} finally {
|
||||||
|
endeddDocIndexings.increment();
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}).transform(this::ensureOpen);
|
})).transform(this::ensureOpen);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> update(LLTerm id, LLIndexRequest request) {
|
public Mono<Void> update(LLTerm id, LLIndexRequest request) {
|
||||||
return this
|
return this
|
||||||
.<Void>runSafe(() -> {
|
.<Void>runSafe(() -> docIndexingTime.recordCallable(() -> {
|
||||||
|
startedDocIndexings.increment();
|
||||||
|
try {
|
||||||
switch (request) {
|
switch (request) {
|
||||||
case LLUpdateDocument updateDocument ->
|
case LLUpdateDocument updateDocument ->
|
||||||
indexWriter.updateDocument(LLUtils.toTerm(id), toDocument(updateDocument));
|
indexWriter.updateDocument(LLUtils.toTerm(id), toDocument(updateDocument));
|
||||||
case LLSoftUpdateDocument softUpdateDocument -> indexWriter.softUpdateDocument(LLUtils.toTerm(id),
|
case LLSoftUpdateDocument softUpdateDocument ->
|
||||||
toDocument(softUpdateDocument.items()), toFields(softUpdateDocument.softDeleteItems()));
|
indexWriter.softUpdateDocument(LLUtils.toTerm(id), toDocument(softUpdateDocument.items()),
|
||||||
case LLUpdateFields updateFields ->
|
toFields(softUpdateDocument.softDeleteItems()));
|
||||||
indexWriter.updateDocValues(LLUtils.toTerm(id), toFields(updateFields.items()));
|
case LLUpdateFields updateFields -> indexWriter.updateDocValues(LLUtils.toTerm(id),
|
||||||
|
toFields(updateFields.items()));
|
||||||
case null, default -> throw new UnsupportedOperationException("Unexpected request type: " + request);
|
case null, default -> throw new UnsupportedOperationException("Unexpected request type: " + request);
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
endeddDocIndexings.increment();
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
})
|
}))
|
||||||
.transform(this::ensureOpen);
|
.transform(this::ensureOpen);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -349,7 +394,15 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
for (Entry<LLTerm, LLUpdateDocument> entry : documentsMap.entrySet()) {
|
for (Entry<LLTerm, LLUpdateDocument> entry : documentsMap.entrySet()) {
|
||||||
LLTerm key = entry.getKey();
|
LLTerm key = entry.getKey();
|
||||||
LLUpdateDocument value = entry.getValue();
|
LLUpdateDocument value = entry.getValue();
|
||||||
|
startedDocIndexings.increment();
|
||||||
|
try {
|
||||||
|
docIndexingTime.recordCallable(() -> {
|
||||||
indexWriter.updateDocument(LLUtils.toTerm(key), toDocument(value));
|
indexWriter.updateDocument(LLUtils.toTerm(key), toDocument(value));
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
endeddDocIndexings.increment();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}).transform(this::ensureOpen);
|
}).transform(this::ensureOpen);
|
||||||
@ -469,7 +522,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
if (activeTasks.isTerminated()) return null;
|
if (activeTasks.isTerminated()) return null;
|
||||||
shutdownLock.lock();
|
shutdownLock.lock();
|
||||||
try {
|
try {
|
||||||
|
flushTime.recordCallable(() -> {
|
||||||
indexWriter.flush();
|
indexWriter.flush();
|
||||||
|
return null;
|
||||||
|
});
|
||||||
} finally {
|
} finally {
|
||||||
shutdownLock.unlock();
|
shutdownLock.unlock();
|
||||||
}
|
}
|
||||||
@ -488,11 +544,14 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
if (activeTasks.isTerminated()) return null;
|
if (activeTasks.isTerminated()) return null;
|
||||||
shutdownLock.lock();
|
shutdownLock.lock();
|
||||||
try {
|
try {
|
||||||
|
refreshTime.recordCallable(() -> {
|
||||||
if (force) {
|
if (force) {
|
||||||
searcherManager.maybeRefreshBlocking();
|
searcherManager.maybeRefreshBlocking();
|
||||||
} else {
|
} else {
|
||||||
searcherManager.maybeRefresh();
|
searcherManager.maybeRefresh();
|
||||||
}
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
} finally {
|
} finally {
|
||||||
shutdownLock.unlock();
|
shutdownLock.unlock();
|
||||||
}
|
}
|
||||||
@ -507,8 +566,11 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
private void scheduledCommit() {
|
private void scheduledCommit() {
|
||||||
shutdownLock.lock();
|
shutdownLock.lock();
|
||||||
try {
|
try {
|
||||||
|
commitTime.recordCallable(() -> {
|
||||||
indexWriter.commit();
|
indexWriter.commit();
|
||||||
} catch (IOException ex) {
|
return null;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
logger.error(MARKER_LUCENE, "Failed to execute a scheduled commit", ex);
|
logger.error(MARKER_LUCENE, "Failed to execute a scheduled commit", ex);
|
||||||
} finally {
|
} finally {
|
||||||
shutdownLock.unlock();
|
shutdownLock.unlock();
|
||||||
@ -518,8 +580,11 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
private void scheduledMerge() {
|
private void scheduledMerge() {
|
||||||
shutdownLock.lock();
|
shutdownLock.lock();
|
||||||
try {
|
try {
|
||||||
|
mergeTime.recordCallable(() -> {
|
||||||
indexWriter.maybeMerge();
|
indexWriter.maybeMerge();
|
||||||
} catch (IOException ex) {
|
return null;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
logger.error(MARKER_LUCENE, "Failed to execute a scheduled merge", ex);
|
logger.error(MARKER_LUCENE, "Failed to execute a scheduled merge", ex);
|
||||||
} finally {
|
} finally {
|
||||||
shutdownLock.unlock();
|
shutdownLock.unlock();
|
||||||
|
@ -65,7 +65,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
|
|||||||
public LLLocalMultiLuceneIndex(LLTempLMDBEnv env,
|
public LLLocalMultiLuceneIndex(LLTempLMDBEnv env,
|
||||||
Path lucene,
|
Path lucene,
|
||||||
MeterRegistry meterRegistry,
|
MeterRegistry meterRegistry,
|
||||||
String name,
|
String clusterName,
|
||||||
int instancesCount,
|
int instancesCount,
|
||||||
IndicizerAnalyzers indicizerAnalyzers,
|
IndicizerAnalyzers indicizerAnalyzers,
|
||||||
IndicizerSimilarities indicizerSimilarities,
|
IndicizerSimilarities indicizerSimilarities,
|
||||||
@ -79,16 +79,17 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
|
|||||||
this.meterRegistry = meterRegistry;
|
this.meterRegistry = meterRegistry;
|
||||||
LLLocalLuceneIndex[] luceneIndices = new LLLocalLuceneIndex[instancesCount];
|
LLLocalLuceneIndex[] luceneIndices = new LLLocalLuceneIndex[instancesCount];
|
||||||
for (int i = 0; i < instancesCount; i++) {
|
for (int i = 0; i < instancesCount; i++) {
|
||||||
String instanceName;
|
String shardName;
|
||||||
if (i == 0) {
|
if (i == 0) {
|
||||||
instanceName = name;
|
shardName = clusterName;
|
||||||
} else {
|
} else {
|
||||||
instanceName = name + "_" + String.format("%03d", i);
|
shardName = clusterName + "_" + String.format("%03d", i);
|
||||||
}
|
}
|
||||||
luceneIndices[i] = new LLLocalLuceneIndex(env,
|
luceneIndices[i] = new LLLocalLuceneIndex(env,
|
||||||
lucene,
|
lucene,
|
||||||
meterRegistry,
|
meterRegistry,
|
||||||
instanceName,
|
clusterName,
|
||||||
|
shardName,
|
||||||
indicizerAnalyzers,
|
indicizerAnalyzers,
|
||||||
indicizerSimilarities,
|
indicizerSimilarities,
|
||||||
luceneOptions,
|
luceneOptions,
|
||||||
|
@ -79,7 +79,8 @@ public class LLMemoryDatabaseConnection implements LLDatabaseConnection {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LLLuceneIndex> getLuceneIndex(String name,
|
public Mono<LLLuceneIndex> getLuceneIndex(@Nullable String clusterName,
|
||||||
|
@Nullable String shardName,
|
||||||
int instancesCount,
|
int instancesCount,
|
||||||
IndicizerAnalyzers indicizerAnalyzers,
|
IndicizerAnalyzers indicizerAnalyzers,
|
||||||
IndicizerSimilarities indicizerSimilarities,
|
IndicizerSimilarities indicizerSimilarities,
|
||||||
@ -91,7 +92,8 @@ public class LLMemoryDatabaseConnection implements LLDatabaseConnection {
|
|||||||
return new LLLocalLuceneIndex(env,
|
return new LLLocalLuceneIndex(env,
|
||||||
null,
|
null,
|
||||||
meterRegistry,
|
meterRegistry,
|
||||||
name,
|
clusterName,
|
||||||
|
shardName,
|
||||||
indicizerAnalyzers,
|
indicizerAnalyzers,
|
||||||
indicizerSimilarities,
|
indicizerSimilarities,
|
||||||
luceneOptions,
|
luceneOptions,
|
||||||
|
@ -80,7 +80,8 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
|||||||
new DatabaseOptions(List.of(), Map.of(), true, false, true, false,
|
new DatabaseOptions(List.of(), Map.of(), true, false, true, false,
|
||||||
true, canUseNettyDirect, false, -1, null)
|
true, canUseNettyDirect, false, -1, null)
|
||||||
),
|
),
|
||||||
conn.getLuceneIndex("testluceneindex1",
|
conn.getLuceneIndex(null,
|
||||||
|
"testluceneindex1",
|
||||||
1,
|
1,
|
||||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
||||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||||
@ -88,6 +89,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
|||||||
luceneHacks
|
luceneHacks
|
||||||
),
|
),
|
||||||
conn.getLuceneIndex("testluceneindex16",
|
conn.getLuceneIndex("testluceneindex16",
|
||||||
|
null,
|
||||||
3,
|
3,
|
||||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
||||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||||
|
@ -11,8 +11,8 @@ import it.cavallium.dbengine.client.IndicizerSimilarities;
|
|||||||
import it.cavallium.dbengine.client.LuceneOptions;
|
import it.cavallium.dbengine.client.LuceneOptions;
|
||||||
import it.cavallium.dbengine.client.NRTCachingOptions;
|
import it.cavallium.dbengine.client.NRTCachingOptions;
|
||||||
import it.cavallium.dbengine.database.Column;
|
import it.cavallium.dbengine.database.Column;
|
||||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
|
||||||
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
|
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
|
||||||
|
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
@ -51,7 +51,8 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
|||||||
List.of(Column.dictionary("testmap"), Column.special("ints"), Column.special("longs")),
|
List.of(Column.dictionary("testmap"), Column.special("ints"), Column.special("longs")),
|
||||||
new DatabaseOptions(List.of(), Map.of(), true, false, true, false, true, canUseNettyDirect, true, -1, null)
|
new DatabaseOptions(List.of(), Map.of(), true, false, true, false, true, canUseNettyDirect, true, -1, null)
|
||||||
),
|
),
|
||||||
conn.getLuceneIndex("testluceneindex1",
|
conn.getLuceneIndex(null,
|
||||||
|
"testluceneindex1",
|
||||||
1,
|
1,
|
||||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
||||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||||
@ -59,6 +60,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
|||||||
luceneHacks
|
luceneHacks
|
||||||
),
|
),
|
||||||
conn.getLuceneIndex("testluceneindex16",
|
conn.getLuceneIndex("testluceneindex16",
|
||||||
|
null,
|
||||||
3,
|
3,
|
||||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple),
|
||||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||||
|
Loading…
Reference in New Issue
Block a user