Customizable checks

pom.xml

@@ -16,7 +16,7 @@
 		<lucene.version>9.7.0</lucene.version>
     <rocksdb.version>8.5.3</rocksdb.version>
 		<junit.jupiter.version>5.9.0</junit.jupiter.version>
-    <data.generator.version>1.0.12</data.generator.version>
+    <data.generator.version>1.0.13</data.generator.version>
 	</properties>
 	<repositories>
 		<repository>

src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java

@@ -4,6 +4,7 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
 import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
 import static it.cavallium.dbengine.database.LLUtils.mapList;
 import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
+import static it.cavallium.dbengine.database.disk.LLLocalKeyValueDatabase.PRINT_ALL_CHECKSUM_VERIFICATION_STEPS;
 import static it.cavallium.dbengine.database.disk.UpdateAtomicResultMode.DELTA;
 import static it.cavallium.dbengine.utils.StreamUtils.ROCKSDB_POOL;
 import static it.cavallium.dbengine.utils.StreamUtils.collectOn;
@@ -678,6 +679,9 @@ public class LLLocalDictionary implements LLDictionary {
 										}
 										ro.setVerifyChecksums(true);
 										return resourceStream(() -> db.newRocksIterator(ro, rangePartial, false), rocksIterator -> {
+											if (PRINT_ALL_CHECKSUM_VERIFICATION_STEPS) {
+												logger.info("Seeking to {}->{}->first on file {}", databaseName, column.name(), path);
+											}
 											rocksIterator.seekToFirst();
 											return StreamUtils.<Optional<VerificationProgress>>streamWhileNonNull(() -> {
 												if (!rocksIterator.isValid()) {
@@ -694,6 +698,9 @@ public class LLLocalDictionary implements LLDictionary {
 												try {
 													rawKey = rocksIterator.keyBuf().copy();
 													shouldSendStatus = fileScanned.incrementAndGet() % 1_000_000 == 0;
+													if (PRINT_ALL_CHECKSUM_VERIFICATION_STEPS) {
+														logger.info("Checking {}->{}->{} on file {}", databaseName, column.name(), rawKey, path);
+													}
 													rocksIterator.next();
 												} catch (RocksDBException ex) {
 													return Optional.of(new BlockBad(databaseName, column, rawKey, path, ex));

src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java

@@ -106,6 +106,16 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
 			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.clockcache.enable", "false"));
 	private static final boolean PARANOID_CHECKS
 			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.paranoid", "true"));
+	private static final boolean VERIFY_COMPRESSION
+			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.compression", "false"));
+	private static final boolean VERIFY_FILE_SIZE
+			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.filesize", "false"));
+	private static final boolean PARANOID_FILE_CHECKS
+			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.paranoidfilechecks", "false"));
+	private static final boolean FORCE_COLUMN_FAMILY_CONSISTENCY_CHECKS
+			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.forcecolumnfamilyconsistencychecks", "true"));
+	static final boolean PRINT_ALL_CHECKSUM_VERIFICATION_STEPS
+			= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.checks.verification.print", "false"));
 
 	private static final CacheFactory CACHE_FACTORY = USE_CLOCK_CACHE ? new ClockCacheFactory() : new LRUCacheFactory();
 	private static final boolean ALLOW_SNAPSHOTS = Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.snapshots.allow", "true"));
@@ -193,6 +203,10 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
 				var columnFamilyOptions = new ColumnFamilyOptions();
 				refs.track(columnFamilyOptions);
 
+				columnFamilyOptions
+						.setForceConsistencyChecks(FORCE_COLUMN_FAMILY_CONSISTENCY_CHECKS)
+						.setParanoidFileChecks(PARANOID_FILE_CHECKS);
+
 				var columnOptions = databaseOptions
 						.columnOptions()
 						.stream()
@@ -312,7 +326,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
 				}
 				columnFamilyOptions.setMaxWriteBufferNumberToMaintain(1);
 				if (tableOptions instanceof BlockBasedTableConfig blockBasedTableConfig) {
-					blockBasedTableConfig.setVerifyCompression(false);
+					blockBasedTableConfig.setVerifyCompression(VERIFY_COMPRESSION);
 				}
 				if (columnOptions.filter().isPresent()) {
 					var filterOptions = columnOptions.filter().get();
@@ -363,6 +377,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
 							// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
 							.setIndexType(columnOptions.partitionFilters().orElse(false) ? IndexType.kTwoLevelIndexSearch : IndexType.kBinarySearch)
 							.setChecksumType(ChecksumType.kXXH3)
+							.setVerifyCompression(VERIFY_COMPRESSION)
 							// Spinning disks: 64KiB to 256KiB (also 512KiB). SSDs: 16KiB
 							// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
 							// https://nightlies.apache.org/flink/flink-docs-release-1.3/api/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.html
@@ -908,7 +923,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
 			var options = new DBOptions();
 			refs.track(options);
 			options.setParanoidChecks(PARANOID_CHECKS);
-			options.setSkipCheckingSstFileSizesOnDbOpen(true);
+			options.setSkipCheckingSstFileSizesOnDbOpen(!VERIFY_FILE_SIZE);
 			options.setEnablePipelinedWrite(true);
 			var maxSubCompactions = Integer.parseInt(System.getProperty("it.cavallium.dbengine.compactions.max.sub", "-1"));
 			if (maxSubCompactions > 0) {

src/repair/java/it/cavallium/dbengine/repair/Repair.java

@@ -1,8 +1,11 @@
 package it.cavallium.dbengine.repair;
 
 import io.micrometer.core.instrument.Clock;
+import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.logging.LoggingMeterRegistry;
+import io.micrometer.core.instrument.noop.NoopMeter;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import it.cavallium.datagen.nativedata.NullableString;
 import it.cavallium.datagen.nativedata.Nullableboolean;
 import it.cavallium.datagen.nativedata.Nullableint;
@@ -40,13 +43,14 @@ import org.rocksdb.RocksDBException;
 
 public class Repair {
 
-	public static final MeterRegistry METER = LoggingMeterRegistry
+	public static final MeterRegistry METER = new SimpleMeterRegistry(); // LoggingMeterRegistry.builder(key -> null).clock(Clock.SYSTEM).loggingSink(System.err::println).build();
-			.builder(key -> null)
-			.clock(Clock.SYSTEM)
-			.loggingSink(System.err::println)
-			.build();
 
 	public static void main(String[] argsArray) throws RocksDBException {
+		System.setProperty("it.cavallium.dbengine.checks.compression", "true");
+		System.setProperty("it.cavallium.dbengine.checks.paranoid", "true");
+		System.setProperty("it.cavallium.dbengine.checks.filesize", "true");
+		System.setProperty("it.cavallium.dbengine.checks.paranoidfilechecks", "true");
+		System.setProperty("it.cavallium.dbengine.checks.forcecolumnfamilyconsistencychecks", "true");
 		ObjectList<String> initialArgs = ObjectArrayList.wrap(argsArray), args = initialArgs;
 		if (args.isEmpty() || args.contains("--help")) {
 			printHelp(initialArgs);

src/repair/resources/log4j2.xml

@@ -7,17 +7,13 @@
         pattern="%highlight{${LOG_LEVEL_PATTERN:-%p}}{FATAL=red blink, ERROR=red, WARN=yellow bold, INFO=green, DEBUG=green bold, TRACE=blue}: %m%n"/>
     </Console>
 
-    <Async name="Async">
-      <AppenderRef ref="Console"/>
-    </Async>
-
   </Appenders>
   <Loggers>
-    <AsyncRoot level="INFO">
+    <Root level="INFO">
       <filters>
         <MarkerFilter marker="NETWORK_PACKETS" onMatch="DENY" onMismatch="NEUTRAL"/>
       </filters>
-      <AppenderRef ref="Async"/>
+      <AppenderRef ref="Console"/>
-    </AsyncRoot>
+    </Root>
   </Loggers>
 </Configuration>