andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use LuceneUtils to create simple shards

Browse Source
This commit is contained in:
Andrea Cavalli 2022-03-07 01:42:33 +01:00
parent 48f3a54e72
commit f0533a17c9
4 changed files with 26 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
src/main/java/it/cavallium/dbengine/client/LuceneDirectoryOptions.java
View File

@ -1,175 +0,0 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.lucene.directory.RocksdbDirectory;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.misc.store.DirectIODirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Constants;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDB;
public sealed interface LuceneDirectoryOptions {
Directory createLuceneDirectory(String directoryName) throws IOException;
Optional<Path> getManagedPath();
boolean isStorageCompressed();
record ByteBuffersDirectory() implements LuceneDirectoryOptions {
@Override
public Directory createLuceneDirectory(String directoryName) {
return new org.apache.lucene.store.ByteBuffersDirectory();
}
@Override
public Optional<Path> getManagedPath() {
return Optional.empty();
}
@Override
public boolean isStorageCompressed() {
return false;
}
}
record MemoryMappedFSDirectory(Path managedPath) implements StandardFSDirectoryOptions {
@Override
public FSDirectory createLuceneDirectory(String directoryName) throws IOException {
return FSDirectory.open(managedPath.resolve(directoryName + ".lucene.db"));
}
@Override
public boolean isStorageCompressed() {
return false;
}
}
record NIOFSDirectory(Path managedPath) implements StandardFSDirectoryOptions {
@Override
public FSDirectory createLuceneDirectory(String directoryName) throws IOException {
return org.apache.lucene.store.NIOFSDirectory.open(managedPath.resolve(directoryName + ".lucene.db"));
}
@Override
public boolean isStorageCompressed() {
return false;
}
}
record DirectIOFSDirectory(StandardFSDirectoryOptions delegate, Optional<Integer> mergeBufferSize,
Optional<Long> minBytesDirect) implements LuceneDirectoryOptions {
private static final Logger logger = LogManager.getLogger(DirectIOFSDirectory.class);
@Override
public Directory createLuceneDirectory(String directoryName) throws IOException {
var delegateDirectory = delegate.createLuceneDirectory(directoryName);
if (Constants.LINUX || Constants.MAC_OS_X) {
try {
int mergeBufferSize = mergeBufferSize().orElse(DirectIODirectory.DEFAULT_MERGE_BUFFER_SIZE);
long minBytesDirect = minBytesDirect().orElse(DirectIODirectory.DEFAULT_MIN_BYTES_DIRECT);
return new DirectIODirectory(delegateDirectory, mergeBufferSize, minBytesDirect);
} catch (UnsupportedOperationException ex) {
logger.warn("Failed to open FSDirectory with DIRECT flag", ex);
return delegateDirectory;
}
} else {
logger.warn("Failed to open FSDirectory with DIRECT flag because the operating system is Windows");
return delegateDirectory;
}
}
@Override
public Optional<Path> getManagedPath() {
return delegate.getManagedPath();
}
@Override
public boolean isStorageCompressed() {
return delegate.isStorageCompressed();
}
}
record RocksDBStandaloneDirectory(Path managedPath, int blockSize) implements PathDirectoryOptions {
@Override
public Directory createLuceneDirectory(String directoryName) throws IOException {
return new RocksdbDirectory(managedPath.resolve(directoryName + ".lucene.db"), blockSize);
}
@Override
public boolean isStorageCompressed() {
return true;
}
}
record RocksDBSharedDirectory(RocksDB db, Map<String, ColumnFamilyHandle> handles, int blockSize) implements
LuceneDirectoryOptions {
@Override
public Directory createLuceneDirectory(String directoryName) throws IOException {
return new RocksdbDirectory(db, handles, directoryName, blockSize);
}
@Override
public Optional<Path> getManagedPath() {
return Optional.empty();
}
@Override
public boolean isStorageCompressed() {
return true;
}
}
record NRTCachingDirectory(LuceneDirectoryOptions delegate, long maxMergeSizeBytes, long maxCachedBytes) implements
LuceneDirectoryOptions {
@Override
public Directory createLuceneDirectory(String directoryName) throws IOException {
var delegateDirectory = delegate.createLuceneDirectory(directoryName);
return new org.apache.lucene.store.NRTCachingDirectory(delegateDirectory,
maxMergeSizeBytes / 1024D / 1024D,
maxCachedBytes / 1024D / 1024D
);
}
@Override
public Optional<Path> getManagedPath() {
return delegate.getManagedPath();
}
@Override
public boolean isStorageCompressed() {
return delegate.isStorageCompressed();
}
}
sealed interface StandardFSDirectoryOptions extends PathDirectoryOptions {
@Override
FSDirectory createLuceneDirectory(String directoryName) throws IOException;
}
sealed interface PathDirectoryOptions extends LuceneDirectoryOptions {
Path managedPath();
@Override
default Optional<Path> getManagedPath() {
return Optional.of(managedPath());
}
}
}

21
src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java
View File

@ -31,14 +31,16 @@ import it.cavallium.dbengine.rpc.current.data.DirectIOFSDirectory;
import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import it.cavallium.dbengine.rpc.current.data.LuceneDirectoryOptions;
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
import it.cavallium.dbengine.rpc.current.data.MemoryMappedFSDirectory;
import it.cavallium.dbengine.rpc.current.data.NIOFSDirectory;
import it.cavallium.dbengine.rpc.current.data.NRTCachingDirectory;
import it.cavallium.dbengine.rpc.current.data.RocksDBSharedDirectory;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.io.EOFException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
@ -136,6 +138,7 @@ public class LuceneUtils {
// TODO: remove this default page limits and make the limits configurable into QueryParams
private static final PageLimits DEFAULT_PAGE_LIMITS = new ExponentialPageLimits();
private static final CharArraySet ENGLISH_AND_ITALIAN_STOP_WORDS;
private static final LuceneIndexStructure SINGLE_STRUCTURE = new LuceneIndexStructure(1, IntList.of(0));
static {
var cas = new CharArraySet(
@ -634,4 +637,20 @@ public class LuceneUtils {
throw new UnsupportedOperationException("Unsupported directory: " + directoryOptions);
}
}
public static IntList intListTo(int to) {
var il = new IntArrayList(to);
for (int i = 0; i < to; i++) {
il.add(i);
}
return il;
}
public static LuceneIndexStructure singleStructure() {
return SINGLE_STRUCTURE;
}
public static LuceneIndexStructure shardsStructure(int count) {
return new LuceneIndexStructure(count, intListTo(count));
}
}

7
src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java
View File

@ -15,13 +15,12 @@ import it.cavallium.dbengine.database.ColumnUtils;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import it.unimi.dsi.fastutil.ints.IntList;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -97,14 +96,14 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
)
),
conn.getLuceneIndex("testluceneindex1",
new LuceneIndexStructure(1, IntList.of(0)),
LuceneUtils.singleStructure(),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
conn.getLuceneIndex("testluceneindex16",
new LuceneIndexStructure(3, IntList.of(0, 1, 2)),
LuceneUtils.shardsStructure(3),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,

5
src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java
View File

@ -13,6 +13,7 @@ import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.database.ColumnUtils;
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
@ -66,14 +67,14 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
)
),
conn.getLuceneIndex("testluceneindex1",
new LuceneIndexStructure(1, IntList.of(0)),
LuceneUtils.singleStructure(),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
conn.getLuceneIndex("testluceneindex16",
new LuceneIndexStructure(3, IntList.of(0, 1, 2)),
LuceneUtils.shardsStructure(3),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,