From 3df0dcf36a03e7530ae81308b30fdc86c7461c30 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Tue, 26 Sep 2023 01:18:24 +0200 Subject: [PATCH] Rename badBlocks, check failures in rocksdb iterator --- .../java/it/cavallium/dbengine/database/LLDictionary.java | 2 +- .../database/collections/DatabaseMapDictionaryDeep.java | 4 ++-- .../database/collections/DatabaseMapDictionaryHashed.java | 4 ++-- .../dbengine/database/collections/DatabaseMapSingle.java | 4 ++-- .../database/collections/DatabaseSingleBucket.java | 4 ++-- .../database/collections/DatabaseSingleMapped.java | 4 ++-- .../dbengine/database/collections/DatabaseSingleton.java | 2 +- .../dbengine/database/collections/DatabaseStage.java | 2 +- .../dbengine/database/disk/LLLocalDictionary.java | 2 +- .../dbengine/database/disk/LLLocalKeyValueDatabase.java | 3 +-- .../disk/LLLocalMigrationReactiveRocksIterator.java | 6 +++++- .../dbengine/database/disk/rocksdb/RocksIteratorObj.java | 8 ++++---- .../dbengine/database/memory/LLMemoryDictionary.java | 2 +- src/repair/java/it/cavallium/dbengine/repair/Repair.java | 2 +- 14 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/main/java/it/cavallium/dbengine/database/LLDictionary.java b/src/main/java/it/cavallium/dbengine/database/LLDictionary.java index ad2c6d1..71a4f76 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLDictionary.java +++ b/src/main/java/it/cavallium/dbengine/database/LLDictionary.java @@ -64,7 +64,7 @@ public interface LLDictionary extends LLKeyValueDatabaseStructure { int prefixLength, boolean smallRange); - Stream badBlocks(LLRange range); + Stream verifyChecksum(LLRange range); void setRange(LLRange range, Stream entries, boolean smallRange); diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java index e5a2148..37b1df4 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java @@ -269,8 +269,8 @@ public class DatabaseMapDictionaryDeep> implem } @Override - public Stream badBlocks() { - return dictionary.badBlocks(range); + public Stream verifyChecksum() { + return dictionary.verifyChecksum(range); } @Override diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryHashed.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryHashed.java index a4777b7..abdd110 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryHashed.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryHashed.java @@ -148,8 +148,8 @@ public class DatabaseMapDictionaryHashed implements DatabaseStageMap badBlocks() { - return this.subDictionary.badBlocks(); + public Stream verifyChecksum() { + return this.subDictionary.verifyChecksum(); } @Override diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapSingle.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapSingle.java index 6ba1c83..c7cf60c 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapSingle.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapSingle.java @@ -121,8 +121,8 @@ public final class DatabaseMapSingle implements DatabaseStageEntry { } @Override - public Stream badBlocks() { - return dictionary.badBlocks(LLRange.single(key)); + public Stream verifyChecksum() { + return dictionary.verifyChecksum(LLRange.single(key)); } } \ No newline at end of file diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleBucket.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleBucket.java index 5254882..7d05136 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleBucket.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleBucket.java @@ -124,8 +124,8 @@ public class DatabaseSingleBucket implements DatabaseStageEntry { } @Override - public Stream badBlocks() { - return bucketStage.badBlocks(); + public Stream verifyChecksum() { + return bucketStage.verifyChecksum(); } @Nullable diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleMapped.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleMapped.java index 973c895..e2cb7e3 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleMapped.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleMapped.java @@ -107,8 +107,8 @@ public class DatabaseSingleMapped implements DatabaseStageEntry { } @Override - public Stream badBlocks() { - return this.serializedSingle.badBlocks(); + public Stream verifyChecksum() { + return this.serializedSingle.verifyChecksum(); } private A unMap(B bytes) throws SerializationException { diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleton.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleton.java index 2440633..001d27b 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleton.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleton.java @@ -119,7 +119,7 @@ public class DatabaseSingleton implements DatabaseStageEntry { } @Override - public Stream badBlocks() { + public Stream verifyChecksum() { return Stream.empty(); } } \ No newline at end of file diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStage.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStage.java index bc77c49..c45337e 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStage.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStage.java @@ -64,5 +64,5 @@ public interface DatabaseStage extends DatabaseStageWithEntry { return leavesCount(snapshot, false) <= 0; } - Stream badBlocks(); + Stream verifyChecksum(); } diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java index e2d7a30..1d40d5b 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java @@ -637,7 +637,7 @@ public class LLLocalDictionary implements LLDictionary { } @Override - public Stream badBlocks(LLRange rangeFull) { + public Stream verifyChecksum(LLRange rangeFull) { Set brokenFiles = new ConcurrentHashMap().keySet(true); LongAdder totalScanned = new LongAdder(); AtomicLong totalEstimate = new AtomicLong(); diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index 87321df..cc5937b 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -438,8 +438,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa } this.db = RocksDB.openAsSecondary(rocksdbOptions, dbPathString, - secondaryPath - .toString(), + secondaryPath.toString(), descriptors, handles ); diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMigrationReactiveRocksIterator.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMigrationReactiveRocksIterator.java index cc6cfaa..781dd49 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMigrationReactiveRocksIterator.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMigrationReactiveRocksIterator.java @@ -47,7 +47,11 @@ public final class LLLocalMigrationReactiveRocksIterator { if (iterator.isValid()) { var key = iterator.keyBuf().copy(); var value = iterator.valueBuf().copy(); - iterator.next(false); + try { + iterator.next(false); + } catch (RocksDBException e) { + throw new CompletionException(e); + } return LLEntry.of(key, value); } else { return null; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/rocksdb/RocksIteratorObj.java b/src/main/java/it/cavallium/dbengine/database/disk/rocksdb/RocksIteratorObj.java index 07cc785..590f6bf 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/rocksdb/RocksIteratorObj.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/rocksdb/RocksIteratorObj.java @@ -50,7 +50,6 @@ public class RocksIteratorObj extends SimpleResource { public synchronized void seek(byte[] seekArray) throws RocksDBException { ensureOpen(); - rocksIterator.status(); startedIterSeek.increment(); try { iterSeekTime.record(() -> rocksIterator.seek(seekArray)); @@ -102,7 +101,6 @@ public class RocksIteratorObj extends SimpleResource { */ public synchronized void seekTo(Buf key) throws RocksDBException { ensureOpen(); - rocksIterator.status(); var keyArray = LLUtils.asArray(key); startedIterSeek.increment(); try { @@ -166,7 +164,7 @@ public class RocksIteratorObj extends SimpleResource { next(true); } - public synchronized void next(boolean traceStats) { + public synchronized void next(boolean traceStats) throws RocksDBException { ensureOpen(); if (traceStats) { startedIterNext.increment(); @@ -175,13 +173,14 @@ public class RocksIteratorObj extends SimpleResource { } else { rocksIterator.next(); } + rocksIterator.status(); } public void prev() throws RocksDBException { prev(true); } - public synchronized void prev(boolean traceStats) { + public synchronized void prev(boolean traceStats) throws RocksDBException { ensureOpen(); if (traceStats) { startedIterNext.increment(); @@ -190,6 +189,7 @@ public class RocksIteratorObj extends SimpleResource { } else { rocksIterator.prev(); } + rocksIterator.status(); } @Override diff --git a/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDictionary.java b/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDictionary.java index 30a31df..09f2335 100644 --- a/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDictionary.java +++ b/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDictionary.java @@ -358,7 +358,7 @@ public class LLMemoryDictionary implements LLDictionary { } @Override - public Stream badBlocks(LLRange range) { + public Stream verifyChecksum(LLRange range) { return Stream.empty(); } diff --git a/src/repair/java/it/cavallium/dbengine/repair/Repair.java b/src/repair/java/it/cavallium/dbengine/repair/Repair.java index fa91015..f38ed0f 100644 --- a/src/repair/java/it/cavallium/dbengine/repair/Repair.java +++ b/src/repair/java/it/cavallium/dbengine/repair/Repair.java @@ -76,7 +76,7 @@ public class Repair { db.getAllColumnFamilyHandles().forEach((column, cfh) -> { System.err.printf("Scanning column: %s%n", column.name()); LLLocalDictionary dict = db.getDictionary(column.name().getBytes(StandardCharsets.UTF_8), UpdateMode.DISALLOW); - StreamUtils.collectOn(StreamUtils.ROCKSDB_POOL, dict.badBlocks(LLRange.all()), StreamUtils.executing(block -> { + StreamUtils.collectOn(StreamUtils.ROCKSDB_POOL, dict.verifyChecksum(LLRange.all()), StreamUtils.executing(block -> { synchronized (Repair.class) { switch (block) { case null -> {}