Fix tests
This commit is contained in:
parent
59f9f01268
commit
3f88ff8f83
@ -13,7 +13,7 @@ import java.util.stream.Stream;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public interface LuceneIndex<T, U> extends LLSnapshottable {
|
||||
public interface LuceneIndex<T, U> extends LLSnapshottable, AutoCloseable {
|
||||
|
||||
void addDocument(T key, U value);
|
||||
|
||||
|
@ -69,11 +69,11 @@ public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeClosea
|
||||
false,
|
||||
timeout == null ? Long.MAX_VALUE : timeout.toMillis()
|
||||
);
|
||||
return this
|
||||
.search(snapshot, params, null)
|
||||
.parallel()
|
||||
.map(LLSearchResultShard::totalHitsCount)
|
||||
.reduce(TotalHitsCount.of(0, true), (a, b) -> TotalHitsCount.of(a.value() + b.value(), a.exact() && b.exact()));
|
||||
try (var stream = this.search(snapshot, params, null)) {
|
||||
return stream.parallel().map(LLSearchResultShard::totalHitsCount).reduce(TotalHitsCount.of(0, true),
|
||||
(a, b) -> TotalHitsCount.of(a.value() + b.value(), a.exact() && b.exact())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
boolean isLowMemoryMode();
|
||||
|
@ -1,5 +1,7 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.collectClose;
|
||||
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.buffers.BufDataInput;
|
||||
import it.cavallium.dbengine.buffers.BufDataOutput;
|
||||
@ -164,7 +166,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
|
||||
|
||||
@Override
|
||||
public Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
|
||||
var map = dictionary
|
||||
var map = collectClose(dictionary
|
||||
.getRange(resolveSnapshot(snapshot), range, false, true)
|
||||
.map(entry -> {
|
||||
Entry<T, U> deserializedEntry;
|
||||
@ -180,8 +182,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
|
||||
U value = valueSerializer.deserialize(serializedValue);
|
||||
deserializedEntry = Map.entry(key, value);
|
||||
return deserializedEntry;
|
||||
})
|
||||
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
|
||||
}), Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
|
||||
return map.isEmpty() ? null : map;
|
||||
}
|
||||
|
||||
|
@ -499,15 +499,17 @@ public class LLLocalLuceneIndex extends SimpleResource implements IBackuppable,
|
||||
public Stream<LLSearchResultShard> search(@Nullable LLSnapshot snapshot, QueryParams queryParams,
|
||||
@Nullable String keyFieldName) {
|
||||
var result = searchInternal(snapshot, queryParams, keyFieldName);
|
||||
return Stream.of(LLSearchResultShard.withResource(result.results(), result.totalHitsCount(), result));
|
||||
var shard = LLSearchResultShard.withResource(result.results(), result.totalHitsCount(), result);
|
||||
return Stream.of(shard).onClose(shard::close);
|
||||
}
|
||||
|
||||
public LuceneSearchResult searchInternal(@Nullable LLSnapshot snapshot, QueryParams queryParams,
|
||||
@Nullable String keyFieldName) {
|
||||
LocalQueryParams localQueryParams = LuceneUtils.toLocalQueryParams(queryParams, luceneAnalyzer);
|
||||
var searcher = searcherManager.retrieveSearcher(snapshot);
|
||||
try (var searcher = searcherManager.retrieveSearcher(snapshot)) {
|
||||
|
||||
return localSearcher.collect(searcher, localQueryParams, keyFieldName, NO_REWRITE);
|
||||
return localSearcher.collect(searcher, localQueryParams, keyFieldName, NO_REWRITE);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -227,7 +227,8 @@ public class LLLocalMultiLuceneIndex extends SimpleResource implements LLLuceneI
|
||||
@Nullable String keyFieldName) {
|
||||
LuceneSearchResult result = searchInternal(snapshot, queryParams, keyFieldName);
|
||||
// Transform the result type
|
||||
return Stream.of(new LLSearchResultShard(result.results(), result.totalHitsCount()));
|
||||
var shard = new LLSearchResultShard(result.results(), result.totalHitsCount());
|
||||
return Stream.of(shard).onClose(shard::close);
|
||||
}
|
||||
|
||||
private LuceneSearchResult searchInternal(@Nullable LLSnapshot snapshot,
|
||||
|
@ -147,17 +147,23 @@ public class SimpleIndexSearcherManager extends SimpleResource implements IndexS
|
||||
}
|
||||
refreshSubscription.cancel(true);
|
||||
LOG.debug("Closed IndexSearcherManager");
|
||||
LOG.debug("Closing refreshes...");
|
||||
LOG.debug("Closing refresh tasks...");
|
||||
initTime = System.nanoTime();
|
||||
while (activeRefreshes.get() > 0 && (System.nanoTime() - initTime) <= 15000000000L) {
|
||||
LockSupport.parkNanos(50000000);
|
||||
}
|
||||
LOG.debug("Closed refreshes...");
|
||||
if (activeRefreshes.get() > 0) {
|
||||
LOG.warn("Some refresh tasks remained active after shutdown: {}", activeRefreshes.get());
|
||||
}
|
||||
LOG.debug("Closed refresh tasks");
|
||||
LOG.debug("Closing active searchers...");
|
||||
initTime = System.nanoTime();
|
||||
while (activeSearchers.get() > 0 && (System.nanoTime() - initTime) <= 15000000000L) {
|
||||
LockSupport.parkNanos(50000000);
|
||||
}
|
||||
if (activeSearchers.get() > 0) {
|
||||
LOG.warn("Some searchers remained active after shutdown: {}", activeSearchers.get());
|
||||
}
|
||||
LOG.debug("Closed active searchers");
|
||||
LOG.debug("Stopping searcher executor...");
|
||||
SEARCH_EXECUTOR.shutdown();
|
||||
|
@ -3,12 +3,14 @@ package it.cavallium.dbengine.utils;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.common.collect.Streams;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Spliterator;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collector;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
@ -74,6 +76,18 @@ public class StreamUtils {
|
||||
return Streams.stream(it);
|
||||
}
|
||||
|
||||
public static <X> List<X> toListClose(Stream<X> stream) {
|
||||
try (stream) {
|
||||
return stream.toList();
|
||||
}
|
||||
}
|
||||
|
||||
public static <T, R, A> R collectClose(Stream<T> stream, Collector<? super T, A, R> collector) {
|
||||
try (stream) {
|
||||
return stream.collect(collector);
|
||||
}
|
||||
}
|
||||
|
||||
private record BatchSpliterator<E>(Spliterator<E> base, int batchSize) implements Spliterator<List<E>> {
|
||||
|
||||
@Override
|
||||
|
@ -99,15 +99,20 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||
public void closeTempDb(TempDb tempDb) throws IOException {
|
||||
tempDb.db().close();
|
||||
tempDb.connection().disconnect();
|
||||
tempDb.swappableLuceneSearcher().close();
|
||||
tempDb.luceneMulti().close();
|
||||
tempDb.luceneSingle().close();
|
||||
ensureNoLeaks(false, false);
|
||||
if (Files.exists(tempDb.path())) {
|
||||
Files.walk(tempDb.path()).sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
try (var walk = Files.walk(tempDb.path())) {
|
||||
walk.sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
|
||||
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -69,7 +70,11 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeTempDb(TempDb db) {
|
||||
db.db().close();
|
||||
public void closeTempDb(TempDb tempDb) throws IOException {
|
||||
tempDb.db().close();
|
||||
tempDb.connection().disconnect();
|
||||
tempDb.swappableLuceneSearcher().close();
|
||||
tempDb.luceneMulti().close();
|
||||
tempDb.luceneSingle().close();
|
||||
}
|
||||
}
|
||||
|
@ -29,9 +29,9 @@ public abstract class TestDictionaryMap {
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {}
|
||||
|
||||
private static Stream<Arguments> provideArgumentsPut() {
|
||||
var goodKeys = List.of("12345");
|
||||
|
@ -43,9 +43,9 @@ public abstract class TestDictionaryMapDeep {
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {}
|
||||
|
||||
private static Stream<Arguments> provideArgumentsSet() {
|
||||
var goodKeys = Set.of("12345");
|
||||
|
@ -23,9 +23,9 @@ public abstract class TestDictionaryMapDeepHashMap {
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {}
|
||||
|
||||
private static Stream<Arguments> provideArgumentsPut() {
|
||||
var goodKeys1 = isCIMode() ? List.of("12345") : List.of("12345", "zebra");
|
||||
|
@ -2,6 +2,7 @@ package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.toListClose;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
@ -12,6 +13,7 @@ import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import it.cavallium.dbengine.utils.StreamUtils;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
@ -163,8 +165,8 @@ public abstract class TestLLDictionary {
|
||||
var afterSize = dict.sizeRange(null, LLRange.all(), false);
|
||||
Assertions.assertEquals(1, afterSize - beforeSize);
|
||||
|
||||
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).toList().contains("test-nonexistent"));
|
||||
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).toList().contains("test-nonexistent"));
|
||||
Assertions.assertTrue(toListClose(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString)).contains("test-nonexistent"));
|
||||
Assertions.assertTrue(toListClose(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString)).contains("test-nonexistent"));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ -211,15 +213,13 @@ public abstract class TestLLDictionary {
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
|
||||
if (updateMode != UpdateMode.DISALLOW) {
|
||||
Assertions.assertTrue(dict
|
||||
Assertions.assertTrue(toListClose(dict
|
||||
.getRangeKeys(null, RANGE_ALL, false, false)
|
||||
.map(this::toString)
|
||||
.toList()
|
||||
.map(this::toString))
|
||||
.contains("test-nonexistent"));
|
||||
Assertions.assertTrue(dict
|
||||
Assertions.assertTrue(toListClose(dict
|
||||
.getRangeKeys(null, RANGE_ALL, true, false)
|
||||
.map(this::toString)
|
||||
.toList()
|
||||
.map(this::toString))
|
||||
.contains("test-nonexistent"));
|
||||
}
|
||||
}
|
||||
|
@ -82,19 +82,19 @@ public class TestLuceneIndex {
|
||||
return new Object[] {getT1, getT2};
|
||||
}
|
||||
}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2, getT3};
|
||||
}
|
||||
}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2, getT3, getT4};
|
||||
}
|
||||
}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2, getT3, getT4, getT5};
|
||||
@ -192,43 +192,48 @@ public class TestLuceneIndex {
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetLuceneIndex(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
Assertions.assertNotNull(luceneIndex);
|
||||
try (var luceneIndex = getLuceneIndex(shards, null)) {
|
||||
Assertions.assertNotNull(luceneIndex);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testDeleteAll(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
luceneIndex.deleteAll();
|
||||
assertCount(luceneIndex, 0);
|
||||
try (var luceneIndex = getLuceneIndex(shards, null)) {
|
||||
luceneIndex.deleteAll();
|
||||
assertCount(luceneIndex, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testDelete(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
var prevCount = getCount(luceneIndex);
|
||||
luceneIndex.deleteDocument("test-key-1");
|
||||
assertCount(luceneIndex, prevCount - 1);
|
||||
try (var luceneIndex = getLuceneIndex(shards, null)) {
|
||||
var prevCount = getCount(luceneIndex);
|
||||
luceneIndex.deleteDocument("test-key-1");
|
||||
assertCount(luceneIndex, prevCount - 1);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testUpdateSameDoc(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
var prevCount = getCount(luceneIndex);
|
||||
luceneIndex.updateDocument("test-key-1", "new-value");
|
||||
assertCount(luceneIndex, prevCount );
|
||||
try (var luceneIndex = getLuceneIndex(shards, null)) {
|
||||
var prevCount = getCount(luceneIndex);
|
||||
luceneIndex.updateDocument("test-key-1", "new-value");
|
||||
assertCount(luceneIndex, prevCount);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testUpdateNewDoc(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
var prevCount = getCount(luceneIndex);
|
||||
luceneIndex.updateDocument("test-key-new", "new-value");
|
||||
assertCount(luceneIndex, prevCount + 1);
|
||||
try (var luceneIndex = getLuceneIndex(shards, null)) {
|
||||
var prevCount = getCount(luceneIndex);
|
||||
luceneIndex.updateDocument("test-key-new", "new-value");
|
||||
assertCount(luceneIndex, prevCount + 1);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user