From 04623b754c858383b6e731a75792ffc0e27c3a13 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Sat, 30 Apr 2022 21:56:42 +0200 Subject: [PATCH] Update luene, update rocksdb --- pom.xml | 25 ++++++++++--------- .../disk/LLLocalKeyValueDatabase.java | 1 + .../database/disk/LLLocalLuceneIndex.java | 4 +-- .../cavallium/dbengine/lucene/FullDocs.java | 2 +- .../dbengine/lucene/HugePqComparator.java | 22 ++++++++-------- .../dbengine/lucene/LLSlotDocCodec.java | 2 +- .../lucene/RandomFieldComparatorSource.java | 2 +- .../lucene/comparators/DoubleComparator.java | 4 +-- .../lucene/comparators/FloatComparator.java | 4 +-- .../lucene/comparators/IntComparator.java | 4 +-- .../lucene/comparators/LongComparator.java | 4 +-- ... Lucene91CodecWithNoFieldCompression.java} | 7 +++--- .../comparators/HugePqDocComparator.java | 6 ++--- .../services/org.apache.lucene.codecs.Codec | 2 +- 14 files changed, 45 insertions(+), 44 deletions(-) rename src/main/java/it/cavallium/dbengine/lucene/directory/{Lucene90CodecWithNoFieldCompression.java => Lucene91CodecWithNoFieldCompression.java} (65%) diff --git a/pom.xml b/pom.xml index ae28a5d..9f206f3 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,7 @@ 0-SNAPSHOT false 1.7.4 + 9.1.0 @@ -133,57 +134,57 @@ org.apache.lucene lucene-core - 9.0.0 + ${lucene.version} org.apache.lucene lucene-join - 9.0.0 + ${lucene.version} org.apache.lucene lucene-analysis-common - 9.0.0 + ${lucene.version} org.apache.lucene lucene-analysis-icu - 9.0.0 + ${lucene.version} org.apache.lucene lucene-codecs - 9.0.0 + ${lucene.version} org.apache.lucene lucene-backward-codecs - 9.0.0 + ${lucene.version} org.apache.lucene lucene-queries - 9.0.0 + ${lucene.version} org.apache.lucene lucene-queryparser - 9.0.0 + ${lucene.version} org.apache.lucene lucene-misc - 9.0.0 + ${lucene.version} org.apache.lucene lucene-facet - 9.0.0 + ${lucene.version} org.apache.lucene lucene-test-framework - 9.0.0 + ${lucene.version} test @@ -322,7 +323,7 @@ org.rocksdb rocksdbjni - 7.1.1 + 7.1.2 org.apache.lucene diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index e9078bf..c0abf9f 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java index 45fc9bc..3e5d446 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java @@ -29,7 +29,7 @@ import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneRocksDBManager; import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.collector.Buckets; -import it.cavallium.dbengine.lucene.directory.Lucene90CodecWithNoFieldCompression; +import it.cavallium.dbengine.lucene.directory.Lucene91CodecWithNoFieldCompression; import it.cavallium.dbengine.lucene.mlt.MoreLikeThisTransformer; import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher; import it.cavallium.dbengine.lucene.searcher.BucketParams; @@ -194,7 +194,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { } if (isFilesystemCompressed) { indexWriterConfig.setUseCompoundFile(false); - indexWriterConfig.setCodec(new Lucene90CodecWithNoFieldCompression()); + indexWriterConfig.setCodec(new Lucene91CodecWithNoFieldCompression()); } logger.trace("WriterSchedulerMaxThreadCount: {}", writerSchedulerMaxThreadCount); indexWriterConfig.setMergeScheduler(mergeScheduler); diff --git a/src/main/java/it/cavallium/dbengine/lucene/FullDocs.java b/src/main/java/it/cavallium/dbengine/lucene/FullDocs.java index 5c3c25a..b99f3bc 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/FullDocs.java +++ b/src/main/java/it/cavallium/dbengine/lucene/FullDocs.java @@ -109,7 +109,7 @@ public interface FullDocs extends ResourceIterable { for (int compIDX = 0; compIDX < sortFields.length; ++compIDX) { SortField sortField = sortFields[compIDX]; - comparators[compIDX] = sortField.getComparator(1, compIDX); + comparators[compIDX] = sortField.getComparator(1, compIDX == 0); reverseMul[compIDX] = sortField.getReverse() ? -1 : 1; } diff --git a/src/main/java/it/cavallium/dbengine/lucene/HugePqComparator.java b/src/main/java/it/cavallium/dbengine/lucene/HugePqComparator.java index c21e7bf..7d6b952 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/HugePqComparator.java +++ b/src/main/java/it/cavallium/dbengine/lucene/HugePqComparator.java @@ -23,7 +23,7 @@ import it.cavallium.dbengine.lucene.hugepq.search.comparators.HugePqDocComparato public class HugePqComparator { public static FieldComparator getComparator(LLTempHugePqEnv env, SortField sortField, - int numHits, int sortPos) { + int numHits, boolean enableSkipping) { var sortFieldClass = sortField.getClass(); if (sortFieldClass == org.apache.lucene.search.SortedNumericSortField.class) { var nf = (org.apache.lucene.search.SortedNumericSortField) sortField; @@ -32,7 +32,7 @@ public class HugePqComparator { var reverse = nf.getReverse(); var selector = nf.getSelector(); final FieldComparator fieldComparator = switch (type) { - case INT -> new IntComparator(env, numHits, nf.getField(), (Integer) missingValue, reverse, sortPos) { + case INT -> new IntComparator(env, numHits, nf.getField(), (Integer) missingValue, reverse, enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new IntLeafComparator(context) { @@ -44,7 +44,7 @@ public class HugePqComparator { }; } }; - case FLOAT -> new FloatComparator(env, numHits, nf.getField(), (Float) missingValue, reverse, sortPos) { + case FLOAT -> new FloatComparator(env, numHits, nf.getField(), (Float) missingValue, reverse, enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new FloatLeafComparator(context) { @@ -56,7 +56,7 @@ public class HugePqComparator { }; } }; - case LONG -> new LongComparator(env, numHits, nf.getField(), (Long) missingValue, reverse, sortPos) { + case LONG -> new LongComparator(env, numHits, nf.getField(), (Long) missingValue, reverse, enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { @@ -68,7 +68,7 @@ public class HugePqComparator { }; } }; - case DOUBLE -> new DoubleComparator(env, numHits, nf.getField(), (Double) missingValue, reverse, sortPos) { + case DOUBLE -> new DoubleComparator(env, numHits, nf.getField(), (Double) missingValue, reverse, enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { @@ -93,18 +93,18 @@ public class HugePqComparator { var comparatorSource = sortField.getComparatorSource(); return switch (sortField.getType()) { case SCORE -> new RelevanceComparator(env, numHits); - case DOC -> new HugePqDocComparator(env, numHits, reverse, sortPos); + case DOC -> new HugePqDocComparator(env, numHits, reverse, enableSkipping); case INT -> new IntComparator(env, numHits, field, (Integer) missingValue, - reverse, sortPos); + reverse, enableSkipping); case FLOAT -> new FloatComparator(env, numHits, field, (Float) missingValue, - reverse, sortPos); + reverse, enableSkipping); case LONG -> new LongComparator(env, numHits, field, (Long) missingValue, - reverse, sortPos); + reverse, enableSkipping); case DOUBLE -> new DoubleComparator(env, numHits, field, (Double) missingValue, - reverse, sortPos); + reverse, enableSkipping); case CUSTOM -> { assert comparatorSource != null; - yield comparatorSource.newComparator(field, numHits, sortPos, reverse); + yield comparatorSource.newComparator(field, numHits, enableSkipping, reverse); } case STRING -> new TermOrdValComparator(env, numHits, field, missingValue == STRING_LAST); case STRING_VAL -> throw new NotImplementedException("String val sort field not implemented"); diff --git a/src/main/java/it/cavallium/dbengine/lucene/LLSlotDocCodec.java b/src/main/java/it/cavallium/dbengine/lucene/LLSlotDocCodec.java index 3686cd5..569c95c 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/LLSlotDocCodec.java +++ b/src/main/java/it/cavallium/dbengine/lucene/LLSlotDocCodec.java @@ -40,7 +40,7 @@ public class LLSlotDocCodec implements HugePqCodec, FieldValueHitQueu for (int i = 0; i < numComparators; ++i) { SortField field = fields[i]; reverseMul[i] = field.getReverse() ? -1 : 1; - comparators[i] = HugePqComparator.getComparator(env, field, numHits, i); + comparators[i] = HugePqComparator.getComparator(env, field, numHits, i == 0); } comparator = new AbstractComparator(new ComparatorOptions().setMaxReusedBufferSize(0)) { @Override diff --git a/src/main/java/it/cavallium/dbengine/lucene/RandomFieldComparatorSource.java b/src/main/java/it/cavallium/dbengine/lucene/RandomFieldComparatorSource.java index 988d372..22437a3 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/RandomFieldComparatorSource.java +++ b/src/main/java/it/cavallium/dbengine/lucene/RandomFieldComparatorSource.java @@ -14,7 +14,7 @@ public class RandomFieldComparatorSource extends FieldComparatorSource { } @Override - public FieldComparator newComparator(String fieldName, int numHits, int sortPos, boolean reversed) { + public FieldComparator newComparator(String fieldName, int numHits, boolean enableSkipping, boolean reversed) { return new RandomFieldComparator(rand.iterator(), numHits); } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/comparators/DoubleComparator.java b/src/main/java/it/cavallium/dbengine/lucene/comparators/DoubleComparator.java index 06b7065..0b96587 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/comparators/DoubleComparator.java +++ b/src/main/java/it/cavallium/dbengine/lucene/comparators/DoubleComparator.java @@ -39,8 +39,8 @@ public class DoubleComparator extends NumericComparator implements SafeC protected double bottom; public DoubleComparator(LLTempHugePqEnv env, - int numHits, String field, Double missingValue, boolean reverse, int sortPos) { - super(field, missingValue != null ? missingValue : 0.0, reverse, sortPos, Double.BYTES); + int numHits, String field, Double missingValue, boolean reverse, boolean enableSkipping) { + super(field, missingValue != null ? missingValue : 0.0, reverse, enableSkipping, Double.BYTES); values = new HugePqArray<>(env, new DoubleCodec(), numHits, 0d); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/comparators/FloatComparator.java b/src/main/java/it/cavallium/dbengine/lucene/comparators/FloatComparator.java index f95b62a..c3ca981 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/comparators/FloatComparator.java +++ b/src/main/java/it/cavallium/dbengine/lucene/comparators/FloatComparator.java @@ -39,8 +39,8 @@ public class FloatComparator extends NumericComparator implements SafeClo protected float bottom; public FloatComparator(LLTempHugePqEnv env, - int numHits, String field, Float missingValue, boolean reverse, int sortPos) { - super(field, missingValue != null ? missingValue : 0.0f, reverse, sortPos, Float.BYTES); + int numHits, String field, Float missingValue, boolean reverse, boolean enableSkipping) { + super(field, missingValue != null ? missingValue : 0.0f, reverse, enableSkipping, Float.BYTES); values = new HugePqArray<>(env, new FloatCodec(), numHits, 0f); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/comparators/IntComparator.java b/src/main/java/it/cavallium/dbengine/lucene/comparators/IntComparator.java index 351389d..eae8fbc 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/comparators/IntComparator.java +++ b/src/main/java/it/cavallium/dbengine/lucene/comparators/IntComparator.java @@ -39,8 +39,8 @@ public class IntComparator extends NumericComparator implements SafeClo protected int bottom; public IntComparator(LLTempHugePqEnv env, - int numHits, String field, Integer missingValue, boolean reverse, int sortPos) { - super(field, missingValue != null ? missingValue : 0, reverse, sortPos, Integer.BYTES); + int numHits, String field, Integer missingValue, boolean reverse, boolean enableSkipping) { + super(field, missingValue != null ? missingValue : 0, reverse, enableSkipping, Integer.BYTES); values = new HugePqArray<>(env, new IntCodec(), numHits, 0); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/comparators/LongComparator.java b/src/main/java/it/cavallium/dbengine/lucene/comparators/LongComparator.java index b151be1..2d9fb84 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/comparators/LongComparator.java +++ b/src/main/java/it/cavallium/dbengine/lucene/comparators/LongComparator.java @@ -42,8 +42,8 @@ public class LongComparator extends NumericComparator implements SafeClose protected long bottom; public LongComparator(LLTempHugePqEnv env, - int numHits, String field, Long missingValue, boolean reverse, int sortPos) { - super(field, missingValue != null ? missingValue : 0L, reverse, sortPos, Long.BYTES); + int numHits, String field, Long missingValue, boolean reverse, boolean enableSkipping) { + super(field, missingValue != null ? missingValue : 0L, reverse, enableSkipping, Long.BYTES); values = new HugePqArray<>(env, new LongCodec(), numHits, 0L); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/directory/Lucene90CodecWithNoFieldCompression.java b/src/main/java/it/cavallium/dbengine/lucene/directory/Lucene91CodecWithNoFieldCompression.java similarity index 65% rename from src/main/java/it/cavallium/dbengine/lucene/directory/Lucene90CodecWithNoFieldCompression.java rename to src/main/java/it/cavallium/dbengine/lucene/directory/Lucene91CodecWithNoFieldCompression.java index 4163468..31ffb00 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/directory/Lucene90CodecWithNoFieldCompression.java +++ b/src/main/java/it/cavallium/dbengine/lucene/directory/Lucene91CodecWithNoFieldCompression.java @@ -1,15 +1,14 @@ package it.cavallium.dbengine.lucene.directory; +import org.apache.lucene.backward_codecs.lucene90.Lucene90Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene90.Lucene90Codec; -import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -public final class Lucene90CodecWithNoFieldCompression extends FilterCodec { +public final class Lucene91CodecWithNoFieldCompression extends FilterCodec { private final StoredFieldsFormat storedFieldsFormat; - public Lucene90CodecWithNoFieldCompression() { + public Lucene91CodecWithNoFieldCompression() { super("Lucene410CodecWithNoFieldCompression", new Lucene90Codec()); storedFieldsFormat = new Lucene90NoCompressionStoredFieldsFormat(); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/hugepq/search/comparators/HugePqDocComparator.java b/src/main/java/it/cavallium/dbengine/lucene/hugepq/search/comparators/HugePqDocComparator.java index 4fd04ef..386e0b7 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/hugepq/search/comparators/HugePqDocComparator.java +++ b/src/main/java/it/cavallium/dbengine/lucene/hugepq/search/comparators/HugePqDocComparator.java @@ -43,11 +43,11 @@ public class HugePqDocComparator extends org.apache.lucene.search.comparators.Do private boolean hitsThresholdReached; /** Creates a new comparator based on document ids for {@code numHits} */ - public HugePqDocComparator(LLTempHugePqEnv env, int numHits, boolean reverse, int sortPost) { - super(0, reverse, sortPost); + public HugePqDocComparator(LLTempHugePqEnv env, int numHits, boolean reverse, boolean enableSkipping) { + super(0, reverse, enableSkipping); this.docIDs = new HugePqArray<>(env, new IntCodec(), numHits, 0); // skipping functionality is enabled if we are sorting by _doc in asc order as a primary sort - this.enableSkipping = (!reverse && sortPost == 0); + this.enableSkipping = (!reverse && enableSkipping); } @Override diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 6cd6697..1439d5c 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1 +1 @@ -it.cavallium.dbengine.lucene.directory.Lucene90CodecWithNoFieldCompression +it.cavallium.dbengine.lucene.directory.Lucene91CodecWithNoFieldCompression