andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add memory stats method

Browse Source
This commit is contained in:
Andrea Cavalli 2022-01-15 20:00:10 +01:00
parent d2abe044cc
commit 4f19d2fff3
4 changed files with 58 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/it/cavallium/dbengine/client/MemoryStats.java Normal file
View File

@ -0,0 +1,5 @@
package it.cavallium.dbengine.client;
public record MemoryStats(long estimateTableReadersMem, long sizeAllMemTables,
long curSizeAllMemTables, long estimateNumKeys, long blockCacheUsage,
long blockCachePinnedUsage) {}

3
src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java
View File

@ -4,6 +4,7 @@ import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.client.MemoryStats;
import it.cavallium.dbengine.database.collections.DatabaseInt;
import it.cavallium.dbengine.database.collections.DatabaseLong;
import java.nio.charset.StandardCharsets;
@ -44,6 +45,8 @@ public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseS
Mono<Long> getProperty(String propertyName);
Mono<MemoryStats> getMemoryStats();
Mono<Void> verifyChecksum();
BufferAllocator getAllocator();

55
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -5,11 +5,13 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static java.util.Objects.requireNonNullElse;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Timer;
import io.net5.buffer.api.BufferAllocator;
import io.net5.util.internal.PlatformDependent;
import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.client.DatabaseVolume;
import it.cavallium.dbengine.client.MemoryStats;
import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLSnapshot;
@ -75,8 +77,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
}
protected static final Logger logger = LogManager.getLogger(LLLocalKeyValueDatabase.class);
private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor(
RocksDB.DEFAULT_COLUMN_FAMILY);
private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY
= new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY);
private final BufferAllocator allocator;
private final MeterRegistry meterRegistry;
@ -96,6 +98,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
private final Map<Column, ColumnFamilyHandle> handles;
private final ConcurrentHashMap<Long, Snapshot> snapshotsHandles = new ConcurrentHashMap<>();
private final AtomicLong nextSnapshotNumbers = new AtomicLong(1);
private volatile boolean closed = false;
@SuppressWarnings("SwitchStatementWithTooFewBranches")
public LLLocalKeyValueDatabase(BufferAllocator allocator,
@ -220,6 +223,30 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
} catch (RocksDBException ex) {
throw new IOException(ex);
}
registerGauge(meterRegistry, name, "rocksdb.estimate-table-readers-mem");
registerGauge(meterRegistry, name, "rocksdb.size-all-mem-tables");
registerGauge(meterRegistry, name, "rocksdb.cur-size-all-mem-tables");
registerGauge(meterRegistry, name, "rocksdb.estimate-num-keys");
registerGauge(meterRegistry, name, "rocksdb.block-cache-usage");
registerGauge(meterRegistry, name, "rocksdb.block-cache-pinned-usage");
}
private void registerGauge(MeterRegistry meterRegistry, String name, String propertyName) {
meterRegistry.gauge("rocksdb.property.value",
List.of(Tag.of("db.name", name), Tag.of("db.property.name", propertyName)),
db,
database -> {
if (closed) {
return 0d;
}
try {
return database.getAggregatedLongProperty(propertyName);
} catch (RocksDBException e) {
throw new RuntimeException(e);
}
}
);
}
@Override
@ -562,19 +589,24 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
@Override
public Mono<Long> getProperty(String propertyName) {
return Mono.fromCallable(() -> db.getAggregatedLongProperty(propertyName))
.map(val -> {
// Sometimes this two properties return a negative value, I don't know why.
if (Objects.equals(propertyName, "rocksdb.cur-size-all-mem-tables")
|| Objects.equals(propertyName, "rocksdb.size-all-mem-tables")) {
return Math.abs(val);
} else {
return val;
}
})
.onErrorMap(cause -> new IOException("Failed to read " + propertyName, cause))
.subscribeOn(dbScheduler);
}
@Override
public Mono<MemoryStats> getMemoryStats() {
return Mono
.fromCallable(() -> new MemoryStats(db.getAggregatedLongProperty("rocksdb.estimate-table-readers-mem"),
db.getAggregatedLongProperty("rocksdb.size-all-mem-tables"),
db.getAggregatedLongProperty("rocksdb.cur-size-all-mem-tables"),
db.getAggregatedLongProperty("rocksdb.estimate-num-keys"),
db.getAggregatedLongProperty("rocksdb.block-cache-usage"),
db.getAggregatedLongProperty("rocksdb.block-cache-pinned-usage")
))
.onErrorMap(cause -> new IOException("Failed to read memory stats", cause))
.subscribeOn(dbScheduler);
}
@Override
public Mono<Void> verifyChecksum() {
return Mono
@ -628,6 +660,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
return Mono
.<Void>fromCallable(() -> {
try {
closed = true;
flushAndCloseDb(db, new ArrayList<>(handles.values()));
deleteUnusedOldLogFiles();
} catch (RocksDBException e) {

6
src/main/java/it/cavallium/dbengine/database/memory/LLMemoryKeyValueDatabase.java
View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.memory;
import io.micrometer.core.instrument.MeterRegistry;
import io.net5.buffer.api.BufferAllocator;
import it.cavallium.dbengine.client.MemoryStats;
import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
@ -76,6 +77,11 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
return Mono.empty();
}
@Override
public Mono<MemoryStats> getMemoryStats() {
return Mono.just(new MemoryStats(0, 0, 0, 0, 0, 0));
}
@Override
public Mono<Void> verifyChecksum() {
return Mono.empty();