From 8bd0e7cf12dce90adc46d3604e3b47f7a17c0448 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Thu, 19 Sep 2024 00:45:35 +0200 Subject: [PATCH] Update rocksdb, set optimize filters for memory, add cache tests --- pom.xml | 2 +- .../disk/LLLocalKeyValueDatabase.java | 4 +- .../tests/TestBlockCacheCapacity.java | 108 ++++++++++++++++++ 3 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 src/test/java/it/cavallium/dbengine/tests/TestBlockCacheCapacity.java diff --git a/pom.xml b/pom.xml index dbbfd45..5faab6a 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ 0-SNAPSHOT false 1.10.4 - 9.5.2 + 9.6.1 5.9.0 1.0.26 diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index d38fccd..053c41e 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -320,10 +320,9 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa columnFamilyOptions.setCompressionPerLevel(compressionTypes); } - final TableFormatConfig tableOptions = inMemory ? new PlainTableConfig() : new BlockBasedTableConfig(); + final TableFormatConfig tableOptions = inMemory && Boolean.parseBoolean(System.getProperty("rocksdb.debugging.useplaintablewheninmemory", "true")) ? new PlainTableConfig() : new BlockBasedTableConfig(); if (!FOLLOW_ROCKSDB_OPTIMIZATIONS) { if (!databaseOptions.lowMemory()) { - // tableOptions.setOptimizeFiltersForMemory(true); columnFamilyOptions.setWriteBufferSize(256 * SizeUnit.MB); } } @@ -332,6 +331,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa } columnFamilyOptions.setMaxWriteBufferNumberToMaintain(1); if (tableOptions instanceof BlockBasedTableConfig blockBasedTableConfig) { + blockBasedTableConfig.setOptimizeFiltersForMemory(true); blockBasedTableConfig.setVerifyCompression(VERIFY_COMPRESSION); } if (columnOptions.filter().isPresent()) { diff --git a/src/test/java/it/cavallium/dbengine/tests/TestBlockCacheCapacity.java b/src/test/java/it/cavallium/dbengine/tests/TestBlockCacheCapacity.java new file mode 100644 index 0000000..7d1f40c --- /dev/null +++ b/src/test/java/it/cavallium/dbengine/tests/TestBlockCacheCapacity.java @@ -0,0 +1,108 @@ +package it.cavallium.dbengine.tests; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import it.cavallium.datagen.nativedata.Nullablelong; +import it.cavallium.dbengine.database.RocksDBLongProperty; +import it.cavallium.dbengine.database.disk.LLLocalKeyValueDatabase; +import it.cavallium.dbengine.rpc.current.data.Column; +import it.cavallium.dbengine.rpc.current.data.DatabaseOptions; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.rocksdb.ColumnFamilyHandle; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.CompletionException; + +import static it.cavallium.dbengine.client.DefaultDatabaseOptions.DEFAULT_DATABASE_OPTIONS; +import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks; + +public class TestBlockCacheCapacity { + + private long cacheSmallSize = 64 * 1024 * 1024; // 64MiB + private @NotNull DatabaseOptions opts; + private Path tmpDir; + private LLLocalKeyValueDatabase db; + private ArrayList handles; + + @BeforeEach + public void setUp() throws IOException { + System.setProperty("it.cavallium.dbengine.log.levelcode", "2"); + System.setProperty("rocksdb.debugging.useplaintablewheninmemory", "false"); + tmpDir = Files.createTempDirectory("rocksdb"); + opts = DEFAULT_DATABASE_OPTIONS.setBlockCache(Nullablelong.of(cacheSmallSize)); + } + + @AfterEach + public void tearDown() throws IOException { + try { + db.close(); + } catch (Throwable ex) { + ex.printStackTrace(); + } + deleteDirectory(tmpDir); + } + + private void startDb() { + var cols = new ArrayList(); + cols.add(new Column("main")); + handles = new ArrayList(); + db = new LLLocalKeyValueDatabase(new SimpleMeterRegistry(), "test", true, tmpDir, cols, handles, opts); + } + + @Test + public void testBlockCacheCapeacity() throws IOException { + startDb(); + testConditions(1.0f, 1.5f); + } + + @Test + public void testSpinningBlockCacheCapeacity() throws IOException { + opts.setSpinning(true); + startDb(); + testConditions(1.0f, 1.5f); + } + + @Test + public void testLRUCacheCapacity() throws IOException { + System.setProperty("it.cavallium.dbengine.clockcache.enable", "false"); + startDb(); + testConditions(1.0f, 1.5f); + } + + @Test + public void testHyperClockCacheCapacity() throws IOException { + System.setProperty("it.cavallium.dbengine.clockcache.enable", "true"); + startDb(); + testConditions(1.0f, 1.5f); + } + + private void testConditions(float min, float max) { + var property = db.getAggregatedLongProperty(RocksDBLongProperty.BLOCK_CACHE_CAPACITY); + System.out.println("expected block cache capacity: " + cacheSmallSize / 1024 / 1024 + "MiB"); + System.out.println("block cache capacity: " + property / 1024 / 1024 + "MiB"); + var err = "Cache is not correct: %.1f != %.1f".formatted(property / 1024 / 1024d, cacheSmallSize / 1024 / 1024d); + Assertions.assertTrue(property <= cacheSmallSize * max, err); + Assertions.assertTrue(property >= cacheSmallSize * min, err); + } + + private static void deleteDirectory(Path path) throws IOException { + if (Files.exists(path)) { + try (var walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).forEach(file -> { + try { + Files.delete(file); + } catch (IOException ex) { + throw new CompletionException(ex); + } + }); + } + } + } +}