andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update dependencies

Browse Source
This commit is contained in:
Andrea Cavalli 2022-03-19 00:08:23 +01:00
parent aad7195acb
commit bbc77df56b
12 changed files with 400 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

544
pom.xml
View File

@ -71,274 +71,6 @@
<developerConnection>scm:git:https://git.ignuranza.net/andreacavalli/CavalliumDBEngine.git</developerConnection> <developerConnection>scm:git:https://git.ignuranza.net/andreacavalli/CavalliumDBEngine.git</developerConnection>
<tag>HEAD</tag> <tag>HEAD</tag>
</scm> </scm>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.warp</groupId>
<artifactId>common-utils</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty5-buffer</artifactId>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.22.0</version>
<scope>test</scope>
</dependency>
<!-- This will get hamcrest-core automatically -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.17.1</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.17.1</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>3.4.4</version>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-join</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-icu</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-codecs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queries</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-misc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-facet</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-tools</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
</dependency>
<dependency>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-codec-native-quic</artifactId>
<version>0.0.25.Final</version>
<classifier>linux-x86_64</classifier>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.projectreactor.netty.incubator</groupId>
<artifactId>reactor-netty-incubator-quic</artifactId>
<version>0.0.4</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.69</version>
</dependency>
<dependency>
<groupId>org.novasearch</groupId>
<artifactId>lucene-relevance</artifactId>
</dependency>
<dependency>
<groupId>io.soabase.record-builder</groupId>
<artifactId>record-builder-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>it.cavallium</groupId>
<artifactId>data-generator-runtime</artifactId>
<exclusions>
<exclusion>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-jmx</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.lmdbjava</groupId>
<artifactId>lmdbjava</artifactId>
<version>0.8.2</version>
</dependency>
</dependencies>
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
<dependency> <dependency>
@ -511,6 +243,282 @@
</dependency> </dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.warp</groupId>
<artifactId>common-utils</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty5-buffer</artifactId>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.22.0</version>
<scope>test</scope>
</dependency>
<!-- This will get hamcrest-core automatically -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.17.1</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.17.1</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>3.4.4</version>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-join</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-icu</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-codecs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queries</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-misc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-facet</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor.addons</groupId>
<artifactId>reactor-extra</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-tools</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
</dependency>
<dependency>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-codec-native-quic</artifactId>
<version>0.0.26.Final</version>
<classifier>linux-x86_64</classifier>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.projectreactor.netty.incubator</groupId>
<artifactId>reactor-netty-incubator-quic</artifactId>
<version>0.0.5</version>
<exclusions>
<exclusion>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-codec-native-quic</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>reactor-netty-core</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.69</version>
</dependency>
<dependency>
<groupId>org.novasearch</groupId>
<artifactId>lucene-relevance</artifactId>
</dependency>
<dependency>
<groupId>io.soabase.record-builder</groupId>
<artifactId>record-builder-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>it.cavallium</groupId>
<artifactId>data-generator-runtime</artifactId>
<exclusions>
<exclusion>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-jmx</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.lmdbjava</groupId>
<artifactId>lmdbjava</artifactId>
<version>0.8.2</version>
</dependency>
</dependencies>
<build> <build>
<testSourceDirectory>src/test/java</testSourceDirectory> <testSourceDirectory>src/test/java</testSourceDirectory>
<resources> <resources>

4
src/main/java/it/cavallium/dbengine/client/LuceneIndex.java
View File

@ -22,13 +22,13 @@ public interface LuceneIndex<T, U> extends LLSnapshottable {
Mono<Void> addDocument(T key, U value); Mono<Void> addDocument(T key, U value);
Mono<Void> addDocuments(boolean atomic, Flux<Entry<T, U>> entries); Mono<Long> addDocuments(boolean atomic, Flux<Entry<T, U>> entries);
Mono<Void> deleteDocument(T key); Mono<Void> deleteDocument(T key);
Mono<Void> updateDocument(T key, @NotNull U value); Mono<Void> updateDocument(T key, @NotNull U value);
Mono<Void> updateDocuments(Flux<Entry<T, U>> entries); Mono<Long> updateDocuments(Flux<Entry<T, U>> entries);
default Mono<Void> updateOrDeleteDocument(T key, @Nullable U value) { default Mono<Void> updateOrDeleteDocument(T key, @Nullable U value) {
if (value == null) { if (value == null) {

22
src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java
View File

@ -9,6 +9,7 @@ import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLSearchResultShard; import it.cavallium.dbengine.database.LLSearchResultShard;
import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm; import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.LLUpdateDocument;
import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.collector.Buckets; import it.cavallium.dbengine.lucene.collector.Buckets;
import it.cavallium.dbengine.lucene.searcher.BucketParams; import it.cavallium.dbengine.lucene.searcher.BucketParams;
@ -19,10 +20,12 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Objects; import java.util.Objects;
import java.util.logging.Level;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> { public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
@ -50,7 +53,7 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
} }
@Override @Override
public Mono<Void> addDocuments(boolean atomic, Flux<Entry<T, U>> entries) { public Mono<Long> addDocuments(boolean atomic, Flux<Entry<T, U>> entries) {
return luceneIndex.addDocuments(atomic, entries.flatMap(entry -> indicizer return luceneIndex.addDocuments(atomic, entries.flatMap(entry -> indicizer
.toDocument(entry.getKey(), entry.getValue()) .toDocument(entry.getKey(), entry.getValue())
.map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc)))); .map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc))));
@ -70,12 +73,17 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
} }
@Override @Override
public Mono<Void> updateDocuments(Flux<Entry<T, U>> entries) { public Mono<Long> updateDocuments(Flux<Entry<T, U>> entries) {
return luceneIndex.updateDocuments(entries.flatMap(entry -> Mono.zip( Flux<Entry<LLTerm, LLUpdateDocument>> mappedEntries = entries
Mono.just(indicizer.toIndex(entry.getKey())), .flatMap(entry -> Mono
indicizer.toDocument(entry.getKey(), entry.getValue()).single(), .zip(Mono.just(indicizer.toIndex(entry.getKey())),
Map::entry indicizer.toDocument(entry.getKey(), entry.getValue()).single(),
))); Map::entry
)
.single()
)
.log("impl-update-documents", Level.FINEST, false, SignalType.ON_NEXT, SignalType.ON_COMPLETE);
return luceneIndex.updateDocuments(mappedEntries);
} }
@Override @Override

4
src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java
View File

@ -21,13 +21,13 @@ public interface LLLuceneIndex extends LLSnapshottable {
Mono<Void> addDocument(LLTerm id, LLUpdateDocument doc); Mono<Void> addDocument(LLTerm id, LLUpdateDocument doc);
Mono<Void> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents); Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents);
Mono<Void> deleteDocument(LLTerm id); Mono<Void> deleteDocument(LLTerm id);
Mono<Void> update(LLTerm id, LLIndexRequest request); Mono<Void> update(LLTerm id, LLIndexRequest request);
Mono<Void> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents); Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents);
Mono<Void> deleteAll(); Mono<Void> deleteAll();

18
src/main/java/it/cavallium/dbengine/database/LLMultiLuceneIndex.java
View File

@ -22,10 +22,12 @@ import java.util.Map.Entry;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
public class LLMultiLuceneIndex implements LLLuceneIndex { public class LLMultiLuceneIndex implements LLLuceneIndex {
@ -84,14 +86,14 @@ public class LLMultiLuceneIndex implements LLLuceneIndex {
} }
@Override @Override
public Mono<Void> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return documents return documents
.groupBy(term -> LuceneUtils.getLuceneIndexId(term.getKey(), totalShards)) .groupBy(term -> LuceneUtils.getLuceneIndexId(term.getKey(), totalShards))
.flatMap(group -> { .flatMap(group -> {
var index = luceneIndicesById[group.key()]; var index = luceneIndicesById[group.key()];
return index.addDocuments(atomic, group); return index.addDocuments(atomic, group);
}) })
.then(); .reduce(0L, Long::sum);
} }
@Override @Override
@ -105,14 +107,12 @@ public class LLMultiLuceneIndex implements LLLuceneIndex {
} }
@Override @Override
public Mono<Void> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return documents return documents
.groupBy(term -> LuceneUtils.getLuceneIndexId(term.getKey(), totalShards)) .log("multi-update-documents", Level.FINEST, false, SignalType.ON_NEXT, SignalType.ON_COMPLETE)
.flatMap(group -> { .groupBy(term -> getLuceneIndex(term.getKey()))
var index = luceneIndicesById[group.key()]; .flatMap(groupFlux -> groupFlux.key().updateDocuments(groupFlux))
return index.updateDocuments(group); .reduce(0L, Long::sum);
})
.then();
} }
@Override @Override

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -1725,6 +1725,6 @@ public class LLLocalDictionary implements LLDictionary {
seekTo = () -> {}; seekTo = () -> {};
rocksIterator.seekToFirst(); rocksIterator.seekToFirst();
} }
return new RocksIteratorTuple(rocksIterator, sliceMin, sliceMax, seekTo); return new RocksIteratorTuple(List.of(readOptions), rocksIterator, sliceMin, sliceMax, seekTo);
} }
} }

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -44,6 +44,7 @@ import org.jetbrains.annotations.Nullable;
import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter; import org.rocksdb.BloomFilter;
import org.rocksdb.Cache; import org.rocksdb.Cache;
import org.rocksdb.ChecksumType;
import org.rocksdb.ClockCache; import org.rocksdb.ClockCache;
import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyHandle;
@ -194,6 +195,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
tableOptions.setOptimizeFiltersForMemory(true); tableOptions.setOptimizeFiltersForMemory(true);
} }
tableOptions tableOptions
.setChecksumType(ChecksumType.kxxHash64)
.setBlockCacheCompressed(optionsWithCache.compressedCache()) .setBlockCacheCompressed(optionsWithCache.compressedCache())
.setBlockCache(optionsWithCache.standardCache()) .setBlockCache(optionsWithCache.standardCache())
.setBlockSize(16 * 1024); // 16KiB .setBlockSize(16 * 1024); // 16KiB

103
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -44,11 +44,13 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.Phaser; import java.util.concurrent.Phaser;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Level;
import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.lang3.time.StopWatch;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
@ -70,6 +72,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
@ -135,12 +138,12 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
logger.debug("Lucene MMap is supported"); logger.debug("Lucene MMap is supported");
} }
this.lowMemory = luceneOptions.lowMemory(); this.lowMemory = luceneOptions.lowMemory();
this.shardName = LuceneUtils.getStandardName(clusterName, shardIndex);
this.directory = LuceneUtils.createLuceneDirectory(luceneOptions.directoryOptions(), this.directory = LuceneUtils.createLuceneDirectory(luceneOptions.directoryOptions(),
LuceneUtils.getStandardName(clusterName, shardIndex), shardName,
rocksDBManager); rocksDBManager);
boolean isFilesystemCompressed = LuceneUtils.getIsFilesystemCompressed(luceneOptions.directoryOptions()); boolean isFilesystemCompressed = LuceneUtils.getIsFilesystemCompressed(luceneOptions.directoryOptions());
this.shardName = clusterName;
var snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); var snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers); this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
this.luceneSimilarity = LuceneUtils.toPerFieldSimilarityWrapper(indicizerSimilarities); this.luceneSimilarity = LuceneUtils.toPerFieldSimilarityWrapper(indicizerSimilarities);
@ -264,19 +267,23 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
@Override @Override
public Mono<Void> addDocument(LLTerm key, LLUpdateDocument doc) { public Mono<Void> addDocument(LLTerm key, LLUpdateDocument doc) {
return this.<Void>runSafe(() -> docIndexingTime.recordCallable(() -> { return this.<Void>runSafe(() -> {
startedDocIndexings.increment(); docIndexingTime.recordCallable(() -> {
try { startedDocIndexings.increment();
indexWriter.addDocument(toDocument(doc)); try {
} finally { indexWriter.addDocument(toDocument(doc));
endeddDocIndexings.increment(); } finally {
} endeddDocIndexings.increment();
}
return null;
});
logger.trace(MARKER_LUCENE, "Added document {}: {}", key, doc);
return null; return null;
})).transform(this::ensureOpen); }).transform(this::ensureOpen);
} }
@Override @Override
public Mono<Void> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
if (!atomic) { if (!atomic) {
return documents return documents
.publishOn(bulkScheduler) .publishOn(bulkScheduler)
@ -294,15 +301,16 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
} finally { } finally {
endeddDocIndexings.increment(); endeddDocIndexings.increment();
} }
sink.complete(); logger.trace(MARKER_LUCENE, "Added document: {}", document);
sink.next(true);
}) })
.then() .count()
.transform(this::ensureOpen); .transform(this::ensureOpen);
} else { } else {
return documents return documents
.collectList() .collectList()
.publishOn(bulkScheduler) .publishOn(bulkScheduler)
.handle((documentsList, sink) -> { .<Long>handle((documentsList, sink) -> {
var count = documentsList.size(); var count = documentsList.size();
StopWatch stopWatch = StopWatch.createStarted(); StopWatch stopWatch = StopWatch.createStarted();
try { try {
@ -320,9 +328,8 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
TimeUnit.MILLISECONDS TimeUnit.MILLISECONDS
); );
} }
sink.complete(); sink.next((long) documentsList.size());
}) })
.then()
.transform(this::ensureOpen); .transform(this::ensureOpen);
} }
} }
@ -343,30 +350,36 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
@Override @Override
public Mono<Void> update(LLTerm id, LLIndexRequest request) { public Mono<Void> update(LLTerm id, LLIndexRequest request) {
return this.<Void>runSafe(() -> docIndexingTime.recordCallable(() -> { return this.<Void>runSafe(() -> {
startedDocIndexings.increment(); docIndexingTime.recordCallable(() -> {
try { startedDocIndexings.increment();
if (request instanceof LLUpdateDocument updateDocument) { try {
indexWriter.updateDocument(LLUtils.toTerm(id), toDocument(updateDocument)); if (request instanceof LLUpdateDocument updateDocument) {
} else if (request instanceof LLSoftUpdateDocument softUpdateDocument) { indexWriter.updateDocument(LLUtils.toTerm(id), toDocument(updateDocument));
indexWriter.softUpdateDocument(LLUtils.toTerm(id), } else if (request instanceof LLSoftUpdateDocument softUpdateDocument) {
toDocument(softUpdateDocument.items()), indexWriter.softUpdateDocument(LLUtils.toTerm(id),
toFields(softUpdateDocument.softDeleteItems())); toDocument(softUpdateDocument.items()),
} else if (request instanceof LLUpdateFields updateFields) { toFields(softUpdateDocument.softDeleteItems())
indexWriter.updateDocValues(LLUtils.toTerm(id), toFields(updateFields.items())); );
} else { } else if (request instanceof LLUpdateFields updateFields) {
throw new UnsupportedOperationException("Unexpected request type: " + request); indexWriter.updateDocValues(LLUtils.toTerm(id), toFields(updateFields.items()));
} else {
throw new UnsupportedOperationException("Unexpected request type: " + request);
}
} finally {
endeddDocIndexings.increment();
} }
} finally { return null;
endeddDocIndexings.increment(); });
} logger.trace(MARKER_LUCENE, "Updated document {}: {}", id, request);
return null; return null;
})).transform(this::ensureOpen); }).transform(this::ensureOpen);
} }
@Override @Override
public Mono<Void> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return documents return documents
.log("local-update-documents", Level.FINEST, false, SignalType.ON_NEXT, SignalType.ON_COMPLETE)
.publishOn(bulkScheduler) .publishOn(bulkScheduler)
.handle((document, sink) -> { .handle((document, sink) -> {
LLTerm key = document.getKey(); LLTerm key = document.getKey();
@ -384,9 +397,9 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
} finally { } finally {
endeddDocIndexings.increment(); endeddDocIndexings.increment();
} }
sink.complete(); sink.next(true);
}) })
.then() .count()
.transform(this::ensureOpen); .transform(this::ensureOpen);
} }
@ -589,4 +602,22 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
return lowMemory; return lowMemory;
} }
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LLLocalLuceneIndex that = (LLLocalLuceneIndex) o;
return Objects.equals(shardName, that.shardName);
}
@Override
public int hashCode() {
return shardName.hashCode();
}
} }

26
src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java
View File

@ -42,6 +42,7 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
@ -51,7 +52,9 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
import reactor.math.MathFlux;
public class LLLocalMultiLuceneIndex implements LLLuceneIndex { public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
@ -163,13 +166,13 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
} }
@Override @Override
public Mono<Void> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
if (BYPASS_GROUPBY_BUG) { if (BYPASS_GROUPBY_BUG) {
return documents return documents
.buffer(8192) .buffer(8192)
.flatMap(inputEntries -> { .flatMap(inputEntries -> {
List<Entry<LLTerm, LLUpdateDocument>>[] sortedEntries = new List[totalShards]; List<Entry<LLTerm, LLUpdateDocument>>[] sortedEntries = new List[totalShards];
Mono<Void>[] results = new Mono[totalShards]; Mono<Long>[] results = new Mono[totalShards];
// Sort entries // Sort entries
for(var inputEntry : inputEntries) { for(var inputEntry : inputEntries) {
@ -192,14 +195,14 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
luceneIndexId++; luceneIndexId++;
} }
return Mono.when(results); return Flux.merge(results).reduce(0L, Long::sum);
}) })
.then(); .reduce(0L, Long::sum);
} else { } else {
return documents return documents
.groupBy(term -> getLuceneIndex(term.getKey())) .groupBy(term -> getLuceneIndex(term.getKey()))
.flatMap(group -> group.key().addDocuments(atomic, group)) .flatMap(group -> group.key().addDocuments(atomic, group))
.then(); .reduce(0L, Long::sum);
} }
} }
@ -214,7 +217,9 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
} }
@Override @Override
public Mono<Void> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
documents = documents
.log("local-multi-update-documents", Level.FINEST, false, SignalType.ON_NEXT, SignalType.ON_COMPLETE);
if (BYPASS_GROUPBY_BUG) { if (BYPASS_GROUPBY_BUG) {
int bufferSize = 8192; int bufferSize = 8192;
return documents return documents
@ -222,20 +227,19 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
.flatMap(bufferFlux -> bufferFlux .flatMap(bufferFlux -> bufferFlux
.collect(Collectors.groupingBy(inputEntry -> LuceneUtils.getLuceneIndexId(inputEntry.getKey(), totalShards), .collect(Collectors.groupingBy(inputEntry -> LuceneUtils.getLuceneIndexId(inputEntry.getKey(), totalShards),
Collectors.collectingAndThen(Collectors.toList(), docs -> { Collectors.collectingAndThen(Collectors.toList(), docs -> {
int luceneIndexId = LuceneUtils.getLuceneIndexId(docs.get(0).getKey(), totalShards); var luceneIndex = getLuceneIndex(docs.get(0).getKey());
LLLocalLuceneIndex luceneIndex = Objects.requireNonNull(luceneIndicesById[luceneIndexId]);
return luceneIndex.updateDocuments(Flux.fromIterable(docs)); return luceneIndex.updateDocuments(Flux.fromIterable(docs));
})) }))
) )
.map(Map::values) .map(Map::values)
.flatMap(Mono::whenDelayError) .flatMap(parts -> Flux.merge(parts).reduce(0L, Long::sum))
) )
.then(); .reduce(0L, Long::sum);
} else { } else {
return documents return documents
.groupBy(term -> getLuceneIndex(term.getKey())) .groupBy(term -> getLuceneIndex(term.getKey()))
.flatMap(group -> group.key().updateDocuments(group)) .flatMap(group -> group.key().updateDocuments(group))
.then(); .reduce(0L, Long::sum);
} }
} }

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalReactiveRocksIterator.java
View File

@ -155,7 +155,7 @@ public abstract class LLLocalReactiveRocksIterator<T> extends
@Override @Override
protected Owned<LLLocalReactiveRocksIterator<T>> prepareSend() { protected Owned<LLLocalReactiveRocksIterator<T>> prepareSend() {
var range = this.rangeShared.send(); var range = this.rangeShared.send();
var readOptions = new ReadOptions(this.readOptions); var readOptions = this.readOptions;
return drop -> new LLLocalReactiveRocksIterator<>(db, range, allowNettyDirect, readOptions, readValues) { return drop -> new LLLocalReactiveRocksIterator<>(db, range, allowNettyDirect, readOptions, readValues) {
@Override @Override
public T getEntry(@Nullable Send<Buffer> key, @Nullable Send<Buffer> value) { public T getEntry(@Nullable Send<Buffer> key, @Nullable Send<Buffer> value) {

10
src/main/java/it/cavallium/dbengine/database/disk/RocksIteratorTuple.java
View File

@ -1,15 +1,21 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import it.cavallium.dbengine.database.SafeCloseable; import it.cavallium.dbengine.database.SafeCloseable;
import java.util.List;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.rocksdb.AbstractImmutableNativeReference;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksIterator; import org.rocksdb.RocksIterator;
import org.rocksdb.RocksObject;
public record RocksIteratorTuple(@NotNull RocksIterator iterator, @NotNull ReleasableSlice sliceMin, public record RocksIteratorTuple(List<RocksObject> refs, @NotNull RocksIterator iterator,
@NotNull ReleasableSlice sliceMax, @NotNull SafeCloseable seekTo) implements @NotNull ReleasableSlice sliceMin, @NotNull ReleasableSlice sliceMax,
@NotNull SafeCloseable seekTo) implements
SafeCloseable { SafeCloseable {
@Override @Override
public void close() { public void close() {
refs.forEach(AbstractImmutableNativeReference::close);
iterator.close(); iterator.close();
sliceMin.close(); sliceMin.close();
sliceMax.close(); sliceMax.close();

4
src/main/java/it/cavallium/dbengine/database/remote/LLQuicConnection.java
View File

@ -375,7 +375,7 @@ public class LLQuicConnection implements LLDatabaseConnection {
} }
@Override @Override
public Mono<Void> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> addDocuments(boolean atomic, Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return null; return null;
} }
@ -390,7 +390,7 @@ public class LLQuicConnection implements LLDatabaseConnection {
} }
@Override @Override
public Mono<Void> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) { public Mono<Long> updateDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents) {
return null; return null;
} }