diff --git a/pom.xml b/pom.xml
index 711e5c5..7e41226 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,7 +16,7 @@
9.7.0
8.5.3
5.9.0
- 1.0.12
+ 1.0.13
diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
index 1d40d5b..afe0ece 100644
--- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
+++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
@@ -4,6 +4,7 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
import static it.cavallium.dbengine.database.LLUtils.mapList;
import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
+import static it.cavallium.dbengine.database.disk.LLLocalKeyValueDatabase.PRINT_ALL_CHECKSUM_VERIFICATION_STEPS;
import static it.cavallium.dbengine.database.disk.UpdateAtomicResultMode.DELTA;
import static it.cavallium.dbengine.utils.StreamUtils.ROCKSDB_POOL;
import static it.cavallium.dbengine.utils.StreamUtils.collectOn;
@@ -678,6 +679,9 @@ public class LLLocalDictionary implements LLDictionary {
}
ro.setVerifyChecksums(true);
return resourceStream(() -> db.newRocksIterator(ro, rangePartial, false), rocksIterator -> {
+ if (PRINT_ALL_CHECKSUM_VERIFICATION_STEPS) {
+ logger.info("Seeking to {}->{}->first on file {}", databaseName, column.name(), path);
+ }
rocksIterator.seekToFirst();
return StreamUtils.>streamWhileNonNull(() -> {
if (!rocksIterator.isValid()) {
@@ -694,6 +698,9 @@ public class LLLocalDictionary implements LLDictionary {
try {
rawKey = rocksIterator.keyBuf().copy();
shouldSendStatus = fileScanned.incrementAndGet() % 1_000_000 == 0;
+ if (PRINT_ALL_CHECKSUM_VERIFICATION_STEPS) {
+ logger.info("Checking {}->{}->{} on file {}", databaseName, column.name(), rawKey, path);
+ }
rocksIterator.next();
} catch (RocksDBException ex) {
return Optional.of(new BlockBad(databaseName, column, rawKey, path, ex));
diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
index cc5937b..06428f7 100644
--- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
+++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
@@ -106,6 +106,16 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.clockcache.enable", "false"));
private static final boolean PARANOID_CHECKS
= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.paranoid", "true"));
+ private static final boolean VERIFY_COMPRESSION
+ = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.compression", "false"));
+ private static final boolean VERIFY_FILE_SIZE
+ = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.filesize", "false"));
+ private static final boolean PARANOID_FILE_CHECKS
+ = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.paranoidfilechecks", "false"));
+ private static final boolean FORCE_COLUMN_FAMILY_CONSISTENCY_CHECKS
+ = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.forcecolumnfamilyconsistencychecks", "true"));
+ static final boolean PRINT_ALL_CHECKSUM_VERIFICATION_STEPS
+ = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.verification.print", "false"));
private static final CacheFactory CACHE_FACTORY = USE_CLOCK_CACHE ? new ClockCacheFactory() : new LRUCacheFactory();
private static final boolean ALLOW_SNAPSHOTS = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.snapshots.allow", "true"));
@@ -193,6 +203,10 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
var columnFamilyOptions = new ColumnFamilyOptions();
refs.track(columnFamilyOptions);
+ columnFamilyOptions
+ .setForceConsistencyChecks(FORCE_COLUMN_FAMILY_CONSISTENCY_CHECKS)
+ .setParanoidFileChecks(PARANOID_FILE_CHECKS);
+
var columnOptions = databaseOptions
.columnOptions()
.stream()
@@ -312,7 +326,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
}
columnFamilyOptions.setMaxWriteBufferNumberToMaintain(1);
if (tableOptions instanceof BlockBasedTableConfig blockBasedTableConfig) {
- blockBasedTableConfig.setVerifyCompression(false);
+ blockBasedTableConfig.setVerifyCompression(VERIFY_COMPRESSION);
}
if (columnOptions.filter().isPresent()) {
var filterOptions = columnOptions.filter().get();
@@ -363,6 +377,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
.setIndexType(columnOptions.partitionFilters().orElse(false) ? IndexType.kTwoLevelIndexSearch : IndexType.kBinarySearch)
.setChecksumType(ChecksumType.kXXH3)
+ .setVerifyCompression(VERIFY_COMPRESSION)
// Spinning disks: 64KiB to 256KiB (also 512KiB). SSDs: 16KiB
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
// https://nightlies.apache.org/flink/flink-docs-release-1.3/api/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.html
@@ -908,7 +923,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
var options = new DBOptions();
refs.track(options);
options.setParanoidChecks(PARANOID_CHECKS);
- options.setSkipCheckingSstFileSizesOnDbOpen(true);
+ options.setSkipCheckingSstFileSizesOnDbOpen(!VERIFY_FILE_SIZE);
options.setEnablePipelinedWrite(true);
var maxSubCompactions = Integer.parseInt(System.getProperty("it.cavallium.dbengine.compactions.max.sub", "-1"));
if (maxSubCompactions > 0) {
diff --git a/src/repair/java/it/cavallium/dbengine/repair/Repair.java b/src/repair/java/it/cavallium/dbengine/repair/Repair.java
index f38ed0f..d190b86 100644
--- a/src/repair/java/it/cavallium/dbengine/repair/Repair.java
+++ b/src/repair/java/it/cavallium/dbengine/repair/Repair.java
@@ -1,8 +1,11 @@
package it.cavallium.dbengine.repair;
import io.micrometer.core.instrument.Clock;
+import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.logging.LoggingMeterRegistry;
+import io.micrometer.core.instrument.noop.NoopMeter;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.datagen.nativedata.NullableString;
import it.cavallium.datagen.nativedata.Nullableboolean;
import it.cavallium.datagen.nativedata.Nullableint;
@@ -40,13 +43,14 @@ import org.rocksdb.RocksDBException;
public class Repair {
- public static final MeterRegistry METER = LoggingMeterRegistry
- .builder(key -> null)
- .clock(Clock.SYSTEM)
- .loggingSink(System.err::println)
- .build();
+ public static final MeterRegistry METER = new SimpleMeterRegistry(); // LoggingMeterRegistry.builder(key -> null).clock(Clock.SYSTEM).loggingSink(System.err::println).build();
public static void main(String[] argsArray) throws RocksDBException {
+ System.setProperty("it.cavallium.dbengine.checks.compression", "true");
+ System.setProperty("it.cavallium.dbengine.checks.paranoid", "true");
+ System.setProperty("it.cavallium.dbengine.checks.filesize", "true");
+ System.setProperty("it.cavallium.dbengine.checks.paranoidfilechecks", "true");
+ System.setProperty("it.cavallium.dbengine.checks.forcecolumnfamilyconsistencychecks", "true");
ObjectList initialArgs = ObjectArrayList.wrap(argsArray), args = initialArgs;
if (args.isEmpty() || args.contains("--help")) {
printHelp(initialArgs);
diff --git a/src/repair/resources/log4j2.xml b/src/repair/resources/log4j2.xml
index b33854b..eee070c 100644
--- a/src/repair/resources/log4j2.xml
+++ b/src/repair/resources/log4j2.xml
@@ -7,17 +7,13 @@
pattern="%highlight{${LOG_LEVEL_PATTERN:-%p}}{FATAL=red blink, ERROR=red, WARN=yellow bold, INFO=green, DEBUG=green bold, TRACE=blue}: %m%n"/>
-
-
-
-
-
+
-
-
+
+
\ No newline at end of file