CavalliumDBEngine/src/test/java/it/cavallium/dbengine/DbTestUtils.java

234 lines
8.2 KiB
Java
Raw Normal View History

2021-05-02 19:18:15 +02:00
package it.cavallium.dbengine;
2021-08-28 22:42:51 +02:00
import static org.junit.jupiter.api.Assertions.assertEquals;
2022-03-16 13:47:56 +01:00
import io.netty5.buffer.api.Buffer;
import io.netty5.buffer.api.MemoryManager;
import io.netty5.buffer.api.Send;
import io.netty5.buffer.api.pool.MetricUtils;
import io.netty5.buffer.api.pool.PoolArenaMetric;
import io.netty5.buffer.api.pool.PooledBufferAllocator;
import io.netty5.util.internal.PlatformDependent;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.LuceneIndexImpl;
2021-08-28 22:42:51 +02:00
import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionary;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryHashed;
2021-05-03 18:07:18 +02:00
import it.cavallium.dbengine.database.collections.DatabaseStageEntry;
import it.cavallium.dbengine.database.collections.DatabaseStageMap;
import it.cavallium.dbengine.database.collections.SubStageGetterHashMap;
import it.cavallium.dbengine.database.collections.SubStageGetterMap;
2021-09-02 21:14:26 +02:00
import it.cavallium.dbengine.database.disk.MemorySegmentUtils;
2021-10-19 00:22:05 +02:00
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
2021-12-18 18:16:56 +01:00
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.nio.file.Path;
import java.util.Map;
2021-09-22 18:33:28 +02:00
import java.util.Objects;
import java.util.function.Function;
2021-05-03 18:07:18 +02:00
import org.jetbrains.annotations.NotNull;
2021-09-22 18:33:28 +02:00
import org.jetbrains.annotations.Nullable;
import org.reactivestreams.Publisher;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class DbTestUtils {
2021-09-06 01:15:13 +02:00
public static final String BIG_STRING = generateBigString();
2021-12-12 23:40:30 +01:00
public static final int MAX_IN_MEMORY_RESULT_ENTRIES = 8192;
2021-09-06 01:15:13 +02:00
private static String generateBigString() {
2021-09-08 00:22:39 +02:00
return "0123456789".repeat(1024);
2021-09-06 01:15:13 +02:00
}
2021-08-31 15:50:11 +02:00
public static record TestAllocator(PooledBufferAllocator allocator) {}
2021-08-28 22:42:51 +02:00
2021-08-29 01:15:51 +02:00
public static TestAllocator newAllocator() {
return new TestAllocator(new PooledBufferAllocator(MemoryManager.instance(), true, 1, 8192, 9, 0, 0, true));
2021-08-28 22:42:51 +02:00
}
2021-08-29 01:15:51 +02:00
public static void destroyAllocator(TestAllocator testAllocator) {
2021-08-31 15:50:11 +02:00
testAllocator.allocator().close();
2021-08-28 22:42:51 +02:00
}
@SuppressWarnings("SameParameterValue")
private static long getActiveAllocations(PooledBufferAllocator allocator, boolean printStats) {
2021-08-31 15:50:11 +02:00
allocator.trimCurrentThreadCache();
var metrics = MetricUtils.getPoolArenaMetrics(allocator);
int allocations = 0;
int deallocations = 0;
int activeAllocations = 0;
for (PoolArenaMetric metric : metrics) {
allocations += metric.numAllocations();
deallocations += metric.numDeallocations();
activeAllocations += metric.numActiveAllocations();
}
2021-08-29 01:15:51 +02:00
if (printStats) {
System.out.println("allocations=" + allocations + ", deallocations=" + deallocations
+ ", activeAllocations=" + activeAllocations);
2021-08-29 01:15:51 +02:00
}
return activeAllocations;
2021-08-28 22:42:51 +02:00
}
2021-09-10 13:34:26 +02:00
public static boolean isCIMode() {
return System.getProperty("dbengine.ci", "false").equalsIgnoreCase("true");
}
2021-09-06 01:15:13 +02:00
public static <U> Flux<U> tempDb(TemporaryDbGenerator temporaryDbGenerator,
TestAllocator alloc,
Function<LLKeyValueDatabase, Publisher<U>> action) {
return Flux.usingWhen(
temporaryDbGenerator.openTempDb(alloc),
2021-09-08 00:22:39 +02:00
tempDb -> Flux.from(action.apply(tempDb.db())).doOnDiscard(Object.class, o -> {
System.out.println("Discarded: " + o.getClass().getName() + ", " + o);
}),
2021-09-06 01:15:13 +02:00
temporaryDbGenerator::closeTempDb
);
}
2021-05-02 19:18:15 +02:00
2021-08-29 01:15:51 +02:00
public static record TempDb(TestAllocator allocator, LLDatabaseConnection connection, LLKeyValueDatabase db,
LLLuceneIndex luceneSingle,
LLLuceneIndex luceneMulti,
SwappableLuceneSearcher swappableLuceneSearcher,
2021-08-28 22:42:51 +02:00
Path path) {}
2021-09-06 01:15:13 +02:00
static boolean computeCanUseNettyDirect() {
2021-09-02 21:14:26 +02:00
boolean canUse = true;
if (!PlatformDependent.hasUnsafe()) {
System.err.println("Warning! Unsafe is not available!"
+ " Netty direct buffers will not be used in tests!");
canUse = false;
}
if (!MemorySegmentUtils.isSupported()) {
System.err.println("Warning! Foreign Memory Access API is not available!"
+ " Netty direct buffers will not be used in tests!"
2021-09-22 11:03:39 +02:00
+ " Please set \"" + MemorySegmentUtils.getSuggestedArgs() + "\"");
2021-09-02 21:14:26 +02:00
if (MemorySegmentUtils.getUnsupportedCause() != null) {
System.err.println("\tCause: " + MemorySegmentUtils.getUnsupportedCause().getClass().getName()
+ ":" + MemorySegmentUtils.getUnsupportedCause().getLocalizedMessage());
}
canUse = false;
}
return canUse;
}
public static void ensureNoLeaks(PooledBufferAllocator allocator, boolean printStats, boolean useClassicException) {
2021-08-28 22:42:51 +02:00
if (allocator != null) {
var allocs = getActiveAllocations(allocator, printStats);
if (useClassicException) {
if (allocs != 0) {
throw new IllegalStateException("Active allocations: " + allocs);
}
} else {
assertEquals(0L, allocs);
}
2021-08-28 22:42:51 +02:00
}
}
public static Mono<? extends LLDictionary> tempDictionary(LLKeyValueDatabase database, UpdateMode updateMode) {
return tempDictionary(database, "testmap", updateMode);
}
public static Mono<? extends LLDictionary> tempDictionary(LLKeyValueDatabase database,
String name,
UpdateMode updateMode) {
return database.getDictionary(name, updateMode);
}
public static Mono<? extends LuceneIndex<String, String>> tempLuceneIndex(LLLuceneIndex index) {
return Mono.fromCallable(() -> new LuceneIndexImpl<>(index, new StringIndicizer()));
}
2021-05-03 18:07:18 +02:00
2021-09-06 01:15:13 +02:00
public enum MapType {
2021-05-03 18:07:18 +02:00
MAP,
HASH_MAP
}
public static DatabaseStageMap<String, String, DatabaseStageEntry<String>> tempDatabaseMapDictionaryMap(
LLDictionary dictionary,
2021-09-06 01:15:13 +02:00
MapType mapType,
int keyBytes) {
2021-09-06 01:15:13 +02:00
if (mapType == MapType.MAP) {
2021-05-03 21:41:51 +02:00
return DatabaseMapDictionary.simple(dictionary,
2021-10-19 00:22:05 +02:00
SerializerFixedBinaryLength.utf8(keyBytes),
Serializer.UTF8_SERIALIZER,
2021-10-01 19:17:33 +02:00
null
2021-05-03 21:41:51 +02:00
);
2021-05-03 18:07:18 +02:00
} else {
return DatabaseMapDictionaryHashed.simple(dictionary,
2021-10-19 00:22:05 +02:00
Serializer.UTF8_SERIALIZER,
Serializer.UTF8_SERIALIZER,
2021-05-08 03:09:00 +02:00
s -> (short) s.hashCode(),
2021-05-03 18:07:18 +02:00
new SerializerFixedBinaryLength<>() {
@Override
public int getSerializedBinaryLength() {
2021-05-08 03:09:00 +02:00
return Short.BYTES;
2021-05-03 18:07:18 +02:00
}
@Override
2021-10-19 00:22:05 +02:00
public @NotNull Short deserialize(@NotNull Buffer serialized) throws SerializationException {
Objects.requireNonNull(serialized);
var val = serialized.readShort();
return val;
2021-05-03 18:07:18 +02:00
}
@Override
2021-10-19 00:22:05 +02:00
public void serialize(@NotNull Short deserialized, Buffer output) throws SerializationException {
output.writeShort(deserialized);
2021-05-03 18:07:18 +02:00
}
2021-09-23 20:57:28 +02:00
},
2021-10-01 19:17:33 +02:00
null
2021-05-03 18:07:18 +02:00
);
}
}
2021-12-18 18:16:56 +01:00
public static DatabaseMapDictionaryDeep<String, Object2ObjectSortedMap<String, String>,
2021-05-08 03:09:00 +02:00
DatabaseMapDictionary<String, String>> tempDatabaseMapDictionaryDeepMap(
LLDictionary dictionary,
int key1Bytes,
int key2Bytes) {
return DatabaseMapDictionaryDeep.deepTail(dictionary,
2021-10-19 00:22:05 +02:00
SerializerFixedBinaryLength.utf8(key1Bytes),
key2Bytes,
2021-10-19 00:22:05 +02:00
new SubStageGetterMap<>(SerializerFixedBinaryLength.utf8(key2Bytes),
Serializer.UTF8_SERIALIZER
2021-09-23 20:57:28 +02:00
),
2021-10-01 19:17:33 +02:00
null
2021-05-08 03:09:00 +02:00
);
}
2021-12-18 18:16:56 +01:00
public static DatabaseMapDictionaryDeep<String, Object2ObjectSortedMap<String, String>,
2021-05-08 03:09:00 +02:00
DatabaseMapDictionaryHashed<String, String, Integer>> tempDatabaseMapDictionaryDeepMapHashMap(
LLDictionary dictionary,
int key1Bytes) {
return DatabaseMapDictionaryDeep.deepTail(dictionary,
2021-10-19 00:22:05 +02:00
SerializerFixedBinaryLength.utf8(key1Bytes),
2021-05-08 03:09:00 +02:00
Integer.BYTES,
2021-10-19 00:22:05 +02:00
new SubStageGetterHashMap<>(Serializer.UTF8_SERIALIZER,
Serializer.UTF8_SERIALIZER,
2021-05-08 03:09:00 +02:00
String::hashCode,
2021-08-28 22:42:51 +02:00
SerializerFixedBinaryLength.intSerializer(dictionary.getAllocator())
2021-09-23 20:57:28 +02:00
),
2021-10-01 19:17:33 +02:00
null
);
}
public static <T, U> DatabaseMapDictionaryHashed<String, String, Integer> tempDatabaseMapDictionaryHashMap(
LLDictionary dictionary) {
return DatabaseMapDictionaryHashed.simple(dictionary,
2021-10-19 00:22:05 +02:00
Serializer.UTF8_SERIALIZER,
Serializer.UTF8_SERIALIZER,
String::hashCode,
2021-09-23 20:57:28 +02:00
SerializerFixedBinaryLength.intSerializer(dictionary.getAllocator()),
2021-10-01 19:17:33 +02:00
null
);
}
}