Rename badBlocks, check failures in rocksdb iterator

This commit is contained in:
Andrea Cavalli 2023-09-26 01:18:24 +02:00
parent 0cb1ad55a8
commit 3df0dcf36a
14 changed files with 26 additions and 23 deletions

View File

@ -64,7 +64,7 @@ public interface LLDictionary extends LLKeyValueDatabaseStructure {
int prefixLength, int prefixLength,
boolean smallRange); boolean smallRange);
Stream<VerificationProgress> badBlocks(LLRange range); Stream<VerificationProgress> verifyChecksum(LLRange range);
void setRange(LLRange range, Stream<LLEntry> entries, boolean smallRange); void setRange(LLRange range, Stream<LLEntry> entries, boolean smallRange);

View File

@ -269,8 +269,8 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
} }
@Override @Override
public Stream<VerificationProgress> badBlocks() { public Stream<VerificationProgress> verifyChecksum() {
return dictionary.badBlocks(range); return dictionary.verifyChecksum(range);
} }
@Override @Override

View File

@ -148,8 +148,8 @@ public class DatabaseMapDictionaryHashed<T, U, TH> implements DatabaseStageMap<T
} }
@Override @Override
public Stream<VerificationProgress> badBlocks() { public Stream<VerificationProgress> verifyChecksum() {
return this.subDictionary.badBlocks(); return this.subDictionary.verifyChecksum();
} }
@Override @Override

View File

@ -121,8 +121,8 @@ public final class DatabaseMapSingle<U> implements DatabaseStageEntry<U> {
} }
@Override @Override
public Stream<VerificationProgress> badBlocks() { public Stream<VerificationProgress> verifyChecksum() {
return dictionary.badBlocks(LLRange.single(key)); return dictionary.verifyChecksum(LLRange.single(key));
} }
} }

View File

@ -124,8 +124,8 @@ public class DatabaseSingleBucket<K, V, TH> implements DatabaseStageEntry<V> {
} }
@Override @Override
public Stream<VerificationProgress> badBlocks() { public Stream<VerificationProgress> verifyChecksum() {
return bucketStage.badBlocks(); return bucketStage.verifyChecksum();
} }
@Nullable @Nullable

View File

@ -107,8 +107,8 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
} }
@Override @Override
public Stream<VerificationProgress> badBlocks() { public Stream<VerificationProgress> verifyChecksum() {
return this.serializedSingle.badBlocks(); return this.serializedSingle.verifyChecksum();
} }
private A unMap(B bytes) throws SerializationException { private A unMap(B bytes) throws SerializationException {

View File

@ -119,7 +119,7 @@ public class DatabaseSingleton<U> implements DatabaseStageEntry<U> {
} }
@Override @Override
public Stream<VerificationProgress> badBlocks() { public Stream<VerificationProgress> verifyChecksum() {
return Stream.empty(); return Stream.empty();
} }
} }

View File

@ -64,5 +64,5 @@ public interface DatabaseStage<T> extends DatabaseStageWithEntry<T> {
return leavesCount(snapshot, false) <= 0; return leavesCount(snapshot, false) <= 0;
} }
Stream<VerificationProgress> badBlocks(); Stream<VerificationProgress> verifyChecksum();
} }

View File

@ -637,7 +637,7 @@ public class LLLocalDictionary implements LLDictionary {
} }
@Override @Override
public Stream<VerificationProgress> badBlocks(LLRange rangeFull) { public Stream<VerificationProgress> verifyChecksum(LLRange rangeFull) {
Set<String> brokenFiles = new ConcurrentHashMap<String, Boolean>().keySet(true); Set<String> brokenFiles = new ConcurrentHashMap<String, Boolean>().keySet(true);
LongAdder totalScanned = new LongAdder(); LongAdder totalScanned = new LongAdder();
AtomicLong totalEstimate = new AtomicLong(); AtomicLong totalEstimate = new AtomicLong();

View File

@ -438,8 +438,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
} }
this.db = RocksDB.openAsSecondary(rocksdbOptions, this.db = RocksDB.openAsSecondary(rocksdbOptions,
dbPathString, dbPathString,
secondaryPath secondaryPath.toString(),
.toString(),
descriptors, descriptors,
handles handles
); );

View File

@ -47,7 +47,11 @@ public final class LLLocalMigrationReactiveRocksIterator {
if (iterator.isValid()) { if (iterator.isValid()) {
var key = iterator.keyBuf().copy(); var key = iterator.keyBuf().copy();
var value = iterator.valueBuf().copy(); var value = iterator.valueBuf().copy();
iterator.next(false); try {
iterator.next(false);
} catch (RocksDBException e) {
throw new CompletionException(e);
}
return LLEntry.of(key, value); return LLEntry.of(key, value);
} else { } else {
return null; return null;

View File

@ -50,7 +50,6 @@ public class RocksIteratorObj extends SimpleResource {
public synchronized void seek(byte[] seekArray) throws RocksDBException { public synchronized void seek(byte[] seekArray) throws RocksDBException {
ensureOpen(); ensureOpen();
rocksIterator.status();
startedIterSeek.increment(); startedIterSeek.increment();
try { try {
iterSeekTime.record(() -> rocksIterator.seek(seekArray)); iterSeekTime.record(() -> rocksIterator.seek(seekArray));
@ -102,7 +101,6 @@ public class RocksIteratorObj extends SimpleResource {
*/ */
public synchronized void seekTo(Buf key) throws RocksDBException { public synchronized void seekTo(Buf key) throws RocksDBException {
ensureOpen(); ensureOpen();
rocksIterator.status();
var keyArray = LLUtils.asArray(key); var keyArray = LLUtils.asArray(key);
startedIterSeek.increment(); startedIterSeek.increment();
try { try {
@ -166,7 +164,7 @@ public class RocksIteratorObj extends SimpleResource {
next(true); next(true);
} }
public synchronized void next(boolean traceStats) { public synchronized void next(boolean traceStats) throws RocksDBException {
ensureOpen(); ensureOpen();
if (traceStats) { if (traceStats) {
startedIterNext.increment(); startedIterNext.increment();
@ -175,13 +173,14 @@ public class RocksIteratorObj extends SimpleResource {
} else { } else {
rocksIterator.next(); rocksIterator.next();
} }
rocksIterator.status();
} }
public void prev() throws RocksDBException { public void prev() throws RocksDBException {
prev(true); prev(true);
} }
public synchronized void prev(boolean traceStats) { public synchronized void prev(boolean traceStats) throws RocksDBException {
ensureOpen(); ensureOpen();
if (traceStats) { if (traceStats) {
startedIterNext.increment(); startedIterNext.increment();
@ -190,6 +189,7 @@ public class RocksIteratorObj extends SimpleResource {
} else { } else {
rocksIterator.prev(); rocksIterator.prev();
} }
rocksIterator.status();
} }
@Override @Override

View File

@ -358,7 +358,7 @@ public class LLMemoryDictionary implements LLDictionary {
} }
@Override @Override
public Stream<VerificationProgress> badBlocks(LLRange range) { public Stream<VerificationProgress> verifyChecksum(LLRange range) {
return Stream.empty(); return Stream.empty();
} }

View File

@ -76,7 +76,7 @@ public class Repair {
db.getAllColumnFamilyHandles().forEach((column, cfh) -> { db.getAllColumnFamilyHandles().forEach((column, cfh) -> {
System.err.printf("Scanning column: %s%n", column.name()); System.err.printf("Scanning column: %s%n", column.name());
LLLocalDictionary dict = db.getDictionary(column.name().getBytes(StandardCharsets.UTF_8), UpdateMode.DISALLOW); LLLocalDictionary dict = db.getDictionary(column.name().getBytes(StandardCharsets.UTF_8), UpdateMode.DISALLOW);
StreamUtils.collectOn(StreamUtils.ROCKSDB_POOL, dict.badBlocks(LLRange.all()), StreamUtils.executing(block -> { StreamUtils.collectOn(StreamUtils.ROCKSDB_POOL, dict.verifyChecksum(LLRange.all()), StreamUtils.executing(block -> {
synchronized (Repair.class) { synchronized (Repair.class) {
switch (block) { switch (block) {
case null -> {} case null -> {}