From 4f19d2fff31ea8ef87527648ddd2972f32fbc0af Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Sat, 15 Jan 2022 20:00:10 +0100 Subject: [PATCH] Add memory stats method --- .../dbengine/client/MemoryStats.java | 5 ++ .../dbengine/database/LLKeyValueDatabase.java | 3 + .../disk/LLLocalKeyValueDatabase.java | 55 +++++++++++++++---- .../memory/LLMemoryKeyValueDatabase.java | 6 ++ 4 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 src/main/java/it/cavallium/dbengine/client/MemoryStats.java diff --git a/src/main/java/it/cavallium/dbengine/client/MemoryStats.java b/src/main/java/it/cavallium/dbengine/client/MemoryStats.java new file mode 100644 index 0000000..c0f48a3 --- /dev/null +++ b/src/main/java/it/cavallium/dbengine/client/MemoryStats.java @@ -0,0 +1,5 @@ +package it.cavallium.dbengine.client; + +public record MemoryStats(long estimateTableReadersMem, long sizeAllMemTables, + long curSizeAllMemTables, long estimateNumKeys, long blockCacheUsage, + long blockCachePinnedUsage) {} diff --git a/src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java index cad9d34..7d76858 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java @@ -4,6 +4,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import io.micrometer.core.instrument.MeterRegistry; import io.net5.buffer.api.BufferAllocator; +import it.cavallium.dbengine.client.MemoryStats; import it.cavallium.dbengine.database.collections.DatabaseInt; import it.cavallium.dbengine.database.collections.DatabaseLong; import java.nio.charset.StandardCharsets; @@ -44,6 +45,8 @@ public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseS Mono getProperty(String propertyName); + Mono getMemoryStats(); + Mono verifyChecksum(); BufferAllocator getAllocator(); diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index 2692369..4f8766a 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -5,11 +5,13 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static java.util.Objects.requireNonNullElse; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Timer; import io.net5.buffer.api.BufferAllocator; import io.net5.util.internal.PlatformDependent; import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseVolume; +import it.cavallium.dbengine.client.MemoryStats; import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.LLKeyValueDatabase; import it.cavallium.dbengine.database.LLSnapshot; @@ -75,8 +77,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { } protected static final Logger logger = LogManager.getLogger(LLLocalKeyValueDatabase.class); - private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor( - RocksDB.DEFAULT_COLUMN_FAMILY); + private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY + = new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY); private final BufferAllocator allocator; private final MeterRegistry meterRegistry; @@ -96,6 +98,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { private final Map handles; private final ConcurrentHashMap snapshotsHandles = new ConcurrentHashMap<>(); private final AtomicLong nextSnapshotNumbers = new AtomicLong(1); + private volatile boolean closed = false; @SuppressWarnings("SwitchStatementWithTooFewBranches") public LLLocalKeyValueDatabase(BufferAllocator allocator, @@ -220,6 +223,30 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { } catch (RocksDBException ex) { throw new IOException(ex); } + + registerGauge(meterRegistry, name, "rocksdb.estimate-table-readers-mem"); + registerGauge(meterRegistry, name, "rocksdb.size-all-mem-tables"); + registerGauge(meterRegistry, name, "rocksdb.cur-size-all-mem-tables"); + registerGauge(meterRegistry, name, "rocksdb.estimate-num-keys"); + registerGauge(meterRegistry, name, "rocksdb.block-cache-usage"); + registerGauge(meterRegistry, name, "rocksdb.block-cache-pinned-usage"); + } + + private void registerGauge(MeterRegistry meterRegistry, String name, String propertyName) { + meterRegistry.gauge("rocksdb.property.value", + List.of(Tag.of("db.name", name), Tag.of("db.property.name", propertyName)), + db, + database -> { + if (closed) { + return 0d; + } + try { + return database.getAggregatedLongProperty(propertyName); + } catch (RocksDBException e) { + throw new RuntimeException(e); + } + } + ); } @Override @@ -562,19 +589,24 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { @Override public Mono getProperty(String propertyName) { return Mono.fromCallable(() -> db.getAggregatedLongProperty(propertyName)) - .map(val -> { - // Sometimes this two properties return a negative value, I don't know why. - if (Objects.equals(propertyName, "rocksdb.cur-size-all-mem-tables") - || Objects.equals(propertyName, "rocksdb.size-all-mem-tables")) { - return Math.abs(val); - } else { - return val; - } - }) .onErrorMap(cause -> new IOException("Failed to read " + propertyName, cause)) .subscribeOn(dbScheduler); } + @Override + public Mono getMemoryStats() { + return Mono + .fromCallable(() -> new MemoryStats(db.getAggregatedLongProperty("rocksdb.estimate-table-readers-mem"), + db.getAggregatedLongProperty("rocksdb.size-all-mem-tables"), + db.getAggregatedLongProperty("rocksdb.cur-size-all-mem-tables"), + db.getAggregatedLongProperty("rocksdb.estimate-num-keys"), + db.getAggregatedLongProperty("rocksdb.block-cache-usage"), + db.getAggregatedLongProperty("rocksdb.block-cache-pinned-usage") + )) + .onErrorMap(cause -> new IOException("Failed to read memory stats", cause)) + .subscribeOn(dbScheduler); + } + @Override public Mono verifyChecksum() { return Mono @@ -628,6 +660,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { return Mono .fromCallable(() -> { try { + closed = true; flushAndCloseDb(db, new ArrayList<>(handles.values())); deleteUnusedOldLogFiles(); } catch (RocksDBException e) { diff --git a/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryKeyValueDatabase.java index cdc3657..ccaad62 100644 --- a/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryKeyValueDatabase.java @@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.memory; import io.micrometer.core.instrument.MeterRegistry; import io.net5.buffer.api.BufferAllocator; +import it.cavallium.dbengine.client.MemoryStats; import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.LLDictionary; import it.cavallium.dbengine.database.LLKeyValueDatabase; @@ -76,6 +77,11 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase { return Mono.empty(); } + @Override + public Mono getMemoryStats() { + return Mono.just(new MemoryStats(0, 0, 0, 0, 0, 0)); + } + @Override public Mono verifyChecksum() { return Mono.empty();