andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix tests

This commit is contained in:
Andrea Cavalli 2023-02-22 23:31:05 +01:00
parent 59f9f01268
commit 3f88ff8f83
14 changed files with 100 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/it/cavallium/dbengine/client/LuceneIndex.java
View File

@ -13,7 +13,7 @@ import java.util.stream.Stream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public interface LuceneIndex<T, U> extends LLSnapshottable {
public interface LuceneIndex<T, U> extends LLSnapshottable, AutoCloseable {
void addDocument(T key, U value);

10
src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java
View File

@ -69,11 +69,11 @@ public interface LLLuceneIndex extends LLSnapshottable, IBackuppable, SafeClosea
false,
timeout == null ? Long.MAX_VALUE : timeout.toMillis()
);
return this
.search(snapshot, params, null)
.parallel()
.map(LLSearchResultShard::totalHitsCount)
.reduce(TotalHitsCount.of(0, true), (a, b) -> TotalHitsCount.of(a.value() + b.value(), a.exact() && b.exact()));
try (var stream = this.search(snapshot, params, null)) {
return stream.parallel().map(LLSearchResultShard::totalHitsCount).reduce(TotalHitsCount.of(0, true),
(a, b) -> TotalHitsCount.of(a.value() + b.value(), a.exact() && b.exact())
);
}
}
boolean isLowMemoryMode();

7
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionary.java
View File

@ -1,5 +1,7 @@
package it.cavallium.dbengine.database.collections;
import static it.cavallium.dbengine.utils.StreamUtils.collectClose;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
@ -164,7 +166,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
@Override
public Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
var map = dictionary
var map = collectClose(dictionary
.getRange(resolveSnapshot(snapshot), range, false, true)
.map(entry -> {
Entry<T, U> deserializedEntry;
@ -180,8 +182,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
U value = valueSerializer.deserialize(serializedValue);
deserializedEntry = Map.entry(key, value);
return deserializedEntry;
})
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
}), Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
return map.isEmpty() ? null : map;
}

8
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -499,15 +499,17 @@ public class LLLocalLuceneIndex extends SimpleResource implements IBackuppable,
public Stream<LLSearchResultShard> search(@Nullable LLSnapshot snapshot, QueryParams queryParams,
@Nullable String keyFieldName) {
var result = searchInternal(snapshot, queryParams, keyFieldName);
return Stream.of(LLSearchResultShard.withResource(result.results(), result.totalHitsCount(), result));
var shard = LLSearchResultShard.withResource(result.results(), result.totalHitsCount(), result);
return Stream.of(shard).onClose(shard::close);
}
public LuceneSearchResult searchInternal(@Nullable LLSnapshot snapshot, QueryParams queryParams,
@Nullable String keyFieldName) {
LocalQueryParams localQueryParams = LuceneUtils.toLocalQueryParams(queryParams, luceneAnalyzer);
var searcher = searcherManager.retrieveSearcher(snapshot);
try (var searcher = searcherManager.retrieveSearcher(snapshot)) {
return localSearcher.collect(searcher, localQueryParams, keyFieldName, NO_REWRITE);
return localSearcher.collect(searcher, localQueryParams, keyFieldName, NO_REWRITE);
}
}
@Override

3
src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java
View File

@ -227,7 +227,8 @@ public class LLLocalMultiLuceneIndex extends SimpleResource implements LLLuceneI
@Nullable String keyFieldName) {
LuceneSearchResult result = searchInternal(snapshot, queryParams, keyFieldName);
// Transform the result type
return Stream.of(new LLSearchResultShard(result.results(), result.totalHitsCount()));
var shard = new LLSearchResultShard(result.results(), result.totalHitsCount());
return Stream.of(shard).onClose(shard::close);
}
private LuceneSearchResult searchInternal(@Nullable LLSnapshot snapshot,

10
src/main/java/it/cavallium/dbengine/database/disk/SimpleIndexSearcherManager.java
View File

@ -147,17 +147,23 @@ public class SimpleIndexSearcherManager extends SimpleResource implements IndexS
}
refreshSubscription.cancel(true);
LOG.debug("Closed IndexSearcherManager");
LOG.debug("Closing refreshes...");
LOG.debug("Closing refresh tasks...");
initTime = System.nanoTime();
while (activeRefreshes.get() > 0 && (System.nanoTime() - initTime) <= 15000000000L) {
LockSupport.parkNanos(50000000);
}
LOG.debug("Closed refreshes...");
if (activeRefreshes.get() > 0) {
LOG.warn("Some refresh tasks remained active after shutdown: {}", activeRefreshes.get());
}
LOG.debug("Closed refresh tasks");
LOG.debug("Closing active searchers...");
initTime = System.nanoTime();
while (activeSearchers.get() > 0 && (System.nanoTime() - initTime) <= 15000000000L) {
LockSupport.parkNanos(50000000);
}
if (activeSearchers.get() > 0) {
LOG.warn("Some searchers remained active after shutdown: {}", activeSearchers.get());
}
LOG.debug("Closed active searchers");
LOG.debug("Stopping searcher executor...");
SEARCH_EXECUTOR.shutdown();

14
src/main/java/it/cavallium/dbengine/utils/StreamUtils.java
View File

@ -3,12 +3,14 @@ package it.cavallium.dbengine.utils;
import com.google.common.collect.Iterators;
import com.google.common.collect.Streams;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Spliterator;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@ -74,6 +76,18 @@ public class StreamUtils {
return Streams.stream(it);
}
public static <X> List<X> toListClose(Stream<X> stream) {
try (stream) {
return stream.toList();
}
}
public static <T, R, A> R collectClose(Stream<T> stream, Collector<? super T, A, R> collector) {
try (stream) {
return stream.collect(collector);
}
}
private record BatchSpliterator<E>(Spliterator<E> base, int batchSize) implements Spliterator<List<E>> {
@Override

19
src/test/java/it/cavallium/dbengine/tests/LocalTemporaryDbGenerator.java
View File

@ -99,15 +99,20 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
public void closeTempDb(TempDb tempDb) throws IOException {
tempDb.db().close();
tempDb.connection().disconnect();
tempDb.swappableLuceneSearcher().close();
tempDb.luceneMulti().close();
tempDb.luceneSingle().close();
ensureNoLeaks(false, false);
if (Files.exists(tempDb.path())) {
Files.walk(tempDb.path()).sorted(Comparator.reverseOrder()).forEach(file -> {
try {
Files.delete(file);
} catch (IOException ex) {
throw new CompletionException(ex);
}
});
try (var walk = Files.walk(tempDb.path())) {
walk.sorted(Comparator.reverseOrder()).forEach(file -> {
try {
Files.delete(file);
} catch (IOException ex) {
throw new CompletionException(ex);
}
});
}
}
}
}

9
src/test/java/it/cavallium/dbengine/tests/MemoryTemporaryDbGenerator.java
View File

@ -18,6 +18,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import java.io.IOException;
import java.time.Duration;
import java.util.List;
import java.util.Map;
@ -69,7 +70,11 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
}
@Override
public void closeTempDb(TempDb db) {
db.db().close();
public void closeTempDb(TempDb tempDb) throws IOException {
tempDb.db().close();
tempDb.connection().disconnect();
tempDb.swappableLuceneSearcher().close();
tempDb.luceneMulti().close();
tempDb.luceneSingle().close();
}
}

6
src/test/java/it/cavallium/dbengine/tests/TestDictionaryMap.java
View File

@ -29,9 +29,9 @@ public abstract class TestDictionaryMap {
protected abstract TemporaryDbGenerator getTempDbGenerator();
record Tuple2<X, Y>(X getT1, Y getT2) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {}
private static Stream<Arguments> provideArgumentsPut() {
var goodKeys = List.of("12345");

6
src/test/java/it/cavallium/dbengine/tests/TestDictionaryMapDeep.java
View File

@ -43,9 +43,9 @@ public abstract class TestDictionaryMapDeep {
protected abstract TemporaryDbGenerator getTempDbGenerator();
record Tuple2<X, Y>(X getT1, Y getT2) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {}
private static Stream<Arguments> provideArgumentsSet() {
var goodKeys = Set.of("12345");

6
src/test/java/it/cavallium/dbengine/tests/TestDictionaryMapDeepHashMap.java
View File

@ -23,9 +23,9 @@ public abstract class TestDictionaryMapDeepHashMap {
protected abstract TemporaryDbGenerator getTempDbGenerator();
record Tuple2<X, Y>(X getT1, Y getT2) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {}
private static Stream<Arguments> provideArgumentsPut() {
var goodKeys1 = isCIMode() ? List.of("12345") : List.of("12345", "zebra");

16
src/test/java/it/cavallium/dbengine/tests/TestLLDictionary.java
View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
import static it.cavallium.dbengine.utils.StreamUtils.toListClose;
import static org.junit.jupiter.api.Assertions.assertEquals;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
@ -12,6 +13,7 @@ import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.utils.StreamUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@ -163,8 +165,8 @@ public abstract class TestLLDictionary {
var afterSize = dict.sizeRange(null, LLRange.all(), false);
Assertions.assertEquals(1, afterSize - beforeSize);
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).toList().contains("test-nonexistent"));
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).toList().contains("test-nonexistent"));
Assertions.assertTrue(toListClose(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString)).contains("test-nonexistent"));
Assertions.assertTrue(toListClose(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString)).contains("test-nonexistent"));
}
@ParameterizedTest
@ -211,15 +213,13 @@ public abstract class TestLLDictionary {
assertEquals(expected, afterSize - beforeSize);
if (updateMode != UpdateMode.DISALLOW) {
Assertions.assertTrue(dict
Assertions.assertTrue(toListClose(dict
.getRangeKeys(null, RANGE_ALL, false, false)
.map(this::toString)
.toList()
.map(this::toString))
.contains("test-nonexistent"));
Assertions.assertTrue(dict
Assertions.assertTrue(toListClose(dict
.getRangeKeys(null, RANGE_ALL, true, false)
.map(this::toString)
.toList()
.map(this::toString))
.contains("test-nonexistent"));
}
}

45
src/test/java/it/cavallium/dbengine/tests/TestLuceneIndex.java
View File

@ -82,19 +82,19 @@ public class TestLuceneIndex {
return new Object[] {getT1, getT2};
}
}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {
record Tuple3<X, Y, Z>(X getT1, Y getT2, Z getT3) {
public Object[] toArray() {
return new Object[] {getT1, getT2, getT3};
}
}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Z getT3, W getT4) {
public Object[] toArray() {
return new Object[] {getT1, getT2, getT3, getT4};
}
}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Z getT3, W getT4, X1 getT5) {
public Object[] toArray() {
return new Object[] {getT1, getT2, getT3, getT4, getT5};
@ -192,43 +192,48 @@ public class TestLuceneIndex {
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetLuceneIndex(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
Assertions.assertNotNull(luceneIndex);
try (var luceneIndex = getLuceneIndex(shards, null)) {
Assertions.assertNotNull(luceneIndex);
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testDeleteAll(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
luceneIndex.deleteAll();
assertCount(luceneIndex, 0);
try (var luceneIndex = getLuceneIndex(shards, null)) {
luceneIndex.deleteAll();
assertCount(luceneIndex, 0);
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testDelete(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
var prevCount = getCount(luceneIndex);
luceneIndex.deleteDocument("test-key-1");
assertCount(luceneIndex, prevCount - 1);
try (var luceneIndex = getLuceneIndex(shards, null)) {
var prevCount = getCount(luceneIndex);
luceneIndex.deleteDocument("test-key-1");
assertCount(luceneIndex, prevCount - 1);
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testUpdateSameDoc(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
var prevCount = getCount(luceneIndex);
luceneIndex.updateDocument("test-key-1", "new-value");
assertCount(luceneIndex, prevCount );
try (var luceneIndex = getLuceneIndex(shards, null)) {
var prevCount = getCount(luceneIndex);
luceneIndex.updateDocument("test-key-1", "new-value");
assertCount(luceneIndex, prevCount);
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testUpdateNewDoc(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
var prevCount = getCount(luceneIndex);
luceneIndex.updateDocument("test-key-new", "new-value");
assertCount(luceneIndex, prevCount + 1);
try (var luceneIndex = getLuceneIndex(shards, null)) {
var prevCount = getCount(luceneIndex);
luceneIndex.updateDocument("test-key-new", "new-value");
assertCount(luceneIndex, prevCount + 1);
}
}
}