2020-12-07 22:15:18 +01:00
|
|
|
package it.cavallium.dbengine.database.disk;
|
|
|
|
|
2022-05-12 19:14:27 +02:00
|
|
|
import static it.cavallium.dbengine.database.LLUtils.ALLOW_STATIC_OPTIONS;
|
2021-09-10 12:13:52 +02:00
|
|
|
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
|
2022-04-15 16:49:01 +02:00
|
|
|
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
|
2021-12-17 01:48:49 +01:00
|
|
|
import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
|
2022-04-01 01:30:56 +02:00
|
|
|
import static it.cavallium.dbengine.database.disk.UpdateAtomicResultMode.DELTA;
|
2021-08-29 23:18:03 +02:00
|
|
|
import static java.util.Objects.requireNonNull;
|
2023-02-09 23:34:25 +01:00
|
|
|
import static it.cavallium.dbengine.utils.StreamUtils.batches;
|
2021-08-29 23:18:03 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
import com.google.common.collect.Lists;
|
|
|
|
import com.google.common.collect.Streams;
|
2021-12-30 17:28:06 +01:00
|
|
|
import io.micrometer.core.instrument.Counter;
|
|
|
|
import io.micrometer.core.instrument.Timer;
|
2023-02-09 23:34:25 +01:00
|
|
|
import it.cavallium.dbengine.buffers.Buf;
|
2021-06-27 11:58:12 +02:00
|
|
|
import it.cavallium.dbengine.client.BadBlock;
|
2022-03-02 12:34:30 +01:00
|
|
|
import it.cavallium.dbengine.database.ColumnUtils;
|
2021-08-29 23:18:03 +02:00
|
|
|
import it.cavallium.dbengine.database.LLDelta;
|
2021-01-17 18:31:25 +01:00
|
|
|
import it.cavallium.dbengine.database.LLDictionary;
|
|
|
|
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
2021-08-28 22:42:51 +02:00
|
|
|
import it.cavallium.dbengine.database.LLEntry;
|
2021-01-30 00:24:55 +01:00
|
|
|
import it.cavallium.dbengine.database.LLRange;
|
2021-01-17 18:31:25 +01:00
|
|
|
import it.cavallium.dbengine.database.LLSnapshot;
|
|
|
|
import it.cavallium.dbengine.database.LLUtils;
|
2022-05-21 15:28:52 +02:00
|
|
|
import it.cavallium.dbengine.database.OptionalBuf;
|
2021-02-13 01:31:24 +01:00
|
|
|
import it.cavallium.dbengine.database.UpdateMode;
|
2021-05-08 03:09:00 +02:00
|
|
|
import it.cavallium.dbengine.database.UpdateReturnMode;
|
2021-11-08 10:49:59 +01:00
|
|
|
import it.cavallium.dbengine.database.serialization.KVSerializationFunction;
|
2022-03-02 12:34:30 +01:00
|
|
|
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
2023-02-09 23:34:25 +01:00
|
|
|
import it.cavallium.dbengine.utils.DBException;
|
2020-12-07 22:15:18 +01:00
|
|
|
import java.io.IOException;
|
2021-04-30 19:15:04 +02:00
|
|
|
import java.nio.ByteBuffer;
|
2022-05-21 23:49:06 +02:00
|
|
|
import java.time.Duration;
|
2021-02-01 02:21:53 +01:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
2023-02-09 23:34:25 +01:00
|
|
|
import java.util.Objects;
|
2021-03-18 19:53:32 +01:00
|
|
|
import java.util.concurrent.Callable;
|
2023-02-09 23:34:25 +01:00
|
|
|
import java.util.concurrent.CompletionException;
|
2021-03-18 19:53:32 +01:00
|
|
|
import java.util.concurrent.ForkJoinPool;
|
|
|
|
import java.util.concurrent.ForkJoinTask;
|
2020-12-07 22:15:18 +01:00
|
|
|
import java.util.function.Function;
|
2021-03-18 19:53:32 +01:00
|
|
|
import java.util.stream.IntStream;
|
2023-02-09 23:34:25 +01:00
|
|
|
import java.util.stream.Stream;
|
2021-03-18 19:53:32 +01:00
|
|
|
import org.apache.commons.lang3.tuple.Pair;
|
2021-12-17 01:48:49 +01:00
|
|
|
import org.apache.logging.log4j.LogManager;
|
|
|
|
import org.apache.logging.log4j.Logger;
|
2021-12-27 17:34:44 +01:00
|
|
|
import org.apache.logging.log4j.util.Supplier;
|
2020-12-07 22:15:18 +01:00
|
|
|
import org.jetbrains.annotations.NotNull;
|
|
|
|
import org.jetbrains.annotations.Nullable;
|
|
|
|
import org.rocksdb.ColumnFamilyHandle;
|
2021-03-20 12:41:11 +01:00
|
|
|
import org.rocksdb.CompactRangeOptions;
|
2020-12-07 22:15:18 +01:00
|
|
|
import org.rocksdb.FlushOptions;
|
|
|
|
import org.rocksdb.ReadOptions;
|
|
|
|
import org.rocksdb.RocksDBException;
|
2021-03-13 19:01:36 +01:00
|
|
|
import org.rocksdb.Slice;
|
2020-12-07 22:15:18 +01:00
|
|
|
import org.rocksdb.Snapshot;
|
2021-05-02 19:18:15 +02:00
|
|
|
import org.rocksdb.WriteBatch;
|
2020-12-07 22:15:18 +01:00
|
|
|
import org.rocksdb.WriteOptions;
|
|
|
|
|
|
|
|
public class LLLocalDictionary implements LLDictionary {
|
|
|
|
|
2021-12-17 01:48:49 +01:00
|
|
|
protected static final Logger logger = LogManager.getLogger(LLLocalDictionary.class);
|
2021-03-18 19:53:32 +01:00
|
|
|
private static final boolean USE_CURRENT_FASTSIZE_FOR_OLD_SNAPSHOTS = false;
|
2020-12-07 22:15:18 +01:00
|
|
|
static final int RESERVED_WRITE_BATCH_SIZE = 2 * 1024 * 1024; // 2MiB
|
|
|
|
static final long MAX_WRITE_BATCH_SIZE = 1024L * 1024L * 1024L; // 1GiB
|
|
|
|
static final int CAPPED_WRITE_BATCH_CAP = 50000; // 50K operations
|
2021-09-06 15:54:44 +02:00
|
|
|
static final int MULTI_GET_WINDOW = 16;
|
2022-05-20 10:20:00 +02:00
|
|
|
private static final ReadOptions EMPTY_READ_OPTIONS = LLUtils.ALLOW_STATIC_OPTIONS ? new ReadOptions() : null;
|
2022-03-24 21:14:17 +01:00
|
|
|
static final boolean PREFER_AUTO_SEEK_BOUND = false;
|
2021-06-27 11:58:12 +02:00
|
|
|
/**
|
2021-08-16 10:36:54 +02:00
|
|
|
* It used to be false,
|
|
|
|
* now it's true to avoid crashes during iterations on completely corrupted files
|
2021-06-27 11:58:12 +02:00
|
|
|
*/
|
2022-04-15 16:49:01 +02:00
|
|
|
static final boolean VERIFY_CHECKSUMS_WHEN_NOT_NEEDED = !LLUtils.FORCE_DISABLE_CHECKSUM_VERIFICATION;
|
2021-05-03 02:45:29 +02:00
|
|
|
/**
|
|
|
|
* Default: true. Use false to debug problems with windowing.
|
|
|
|
*/
|
|
|
|
static final boolean USE_WINDOW_IN_SET_RANGE = true;
|
2021-05-02 19:18:15 +02:00
|
|
|
/**
|
|
|
|
* Default: true. Use false to debug problems with write batches.
|
|
|
|
*/
|
2021-05-03 12:29:15 +02:00
|
|
|
static final boolean USE_WRITE_BATCHES_IN_PUT_MULTI = true;
|
|
|
|
/**
|
|
|
|
* Default: true. Use false to debug problems with write batches.
|
|
|
|
*/
|
|
|
|
static final boolean USE_WRITE_BATCHES_IN_SET_RANGE = true;
|
2021-05-02 19:18:15 +02:00
|
|
|
/**
|
|
|
|
* Default: true. Use false to debug problems with capped write batches.
|
|
|
|
*/
|
2021-05-03 02:45:29 +02:00
|
|
|
static final boolean USE_CAPPED_WRITE_BATCH_IN_SET_RANGE = true;
|
2021-05-03 12:29:15 +02:00
|
|
|
/**
|
|
|
|
* Default: true. Use false to debug problems with write batches deletes.
|
|
|
|
*/
|
|
|
|
static final boolean USE_WRITE_BATCH_IN_SET_RANGE_DELETE = false;
|
2021-03-18 19:53:32 +01:00
|
|
|
static final boolean PARALLEL_EXACT_SIZE = true;
|
2023-01-05 02:21:09 +01:00
|
|
|
private static final boolean USE_NUM_ENTRIES_PRECISE_COUNTER = true;
|
2020-12-07 22:15:18 +01:00
|
|
|
|
2021-01-30 00:24:55 +01:00
|
|
|
private static final byte[] FIRST_KEY = new byte[]{};
|
2021-06-27 11:58:12 +02:00
|
|
|
/**
|
|
|
|
* 1KiB dummy buffer, write only, used for debugging purposes
|
|
|
|
*/
|
|
|
|
private static final ByteBuffer DUMMY_WRITE_ONLY_BYTE_BUFFER = ByteBuffer.allocateDirect(1024);
|
2021-04-30 19:15:04 +02:00
|
|
|
|
2021-10-20 01:51:34 +02:00
|
|
|
private final RocksDBColumn db;
|
2022-05-20 10:20:00 +02:00
|
|
|
private final ColumnFamilyHandle cfh;
|
2020-12-07 22:15:18 +01:00
|
|
|
private final String databaseName;
|
2021-06-26 02:35:33 +02:00
|
|
|
private final String columnName;
|
2022-05-20 10:20:00 +02:00
|
|
|
private final Function<LLSnapshot, Snapshot> snapshotResolver;
|
2021-02-13 01:31:24 +01:00
|
|
|
private final UpdateMode updateMode;
|
2020-12-07 22:15:18 +01:00
|
|
|
|
2021-12-30 17:28:06 +01:00
|
|
|
private final Counter startedUpdates;
|
|
|
|
private final Counter endedUpdates;
|
|
|
|
private final Timer updateTime;
|
|
|
|
private final Counter startedGet;
|
|
|
|
private final Counter endedGet;
|
|
|
|
private final Timer getTime;
|
|
|
|
private final Counter startedContains;
|
|
|
|
private final Counter endedContains;
|
|
|
|
private final Timer containsTime;
|
|
|
|
private final Counter startedPut;
|
|
|
|
private final Counter endedPut;
|
|
|
|
private final Timer putTime;
|
|
|
|
private final Counter startedRemove;
|
|
|
|
private final Counter endedRemove;
|
|
|
|
private final Timer removeTime;
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
public LLLocalDictionary(
|
2021-10-20 01:51:34 +02:00
|
|
|
@NotNull RocksDBColumn db,
|
2020-12-07 22:15:18 +01:00
|
|
|
String databaseName,
|
2021-06-26 02:35:33 +02:00
|
|
|
String columnName,
|
2022-05-20 10:20:00 +02:00
|
|
|
Function<LLSnapshot, Snapshot> snapshotResolver,
|
2021-06-29 23:31:02 +02:00
|
|
|
UpdateMode updateMode,
|
|
|
|
DatabaseOptions databaseOptions) {
|
2021-08-29 23:18:03 +02:00
|
|
|
requireNonNull(db);
|
2020-12-07 22:15:18 +01:00
|
|
|
this.db = db;
|
2021-10-20 01:51:34 +02:00
|
|
|
this.cfh = db.getColumnFamilyHandle();
|
2020-12-07 22:15:18 +01:00
|
|
|
this.databaseName = databaseName;
|
2021-06-26 02:35:33 +02:00
|
|
|
this.columnName = columnName;
|
2020-12-07 22:15:18 +01:00
|
|
|
this.snapshotResolver = snapshotResolver;
|
2021-02-13 01:31:24 +01:00
|
|
|
this.updateMode = updateMode;
|
2021-12-30 17:28:06 +01:00
|
|
|
var meterRegistry = db.getMeterRegistry();
|
|
|
|
|
|
|
|
this.startedGet = meterRegistry.counter("db.read.map.get.started.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.endedGet = meterRegistry.counter("db.read.map.get.ended.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.getTime = Timer
|
2021-12-30 18:20:56 +01:00
|
|
|
.builder("db.read.map.get.timer")
|
2021-12-30 17:28:06 +01:00
|
|
|
.publishPercentiles(0.2, 0.5, 0.95)
|
|
|
|
.publishPercentileHistogram()
|
|
|
|
.tags("db.name", databaseName, "db.column", columnName)
|
|
|
|
.register(meterRegistry);
|
2021-12-30 18:20:56 +01:00
|
|
|
this.startedContains = meterRegistry.counter("db.read.map.contains.started.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.endedContains = meterRegistry.counter("db.read.map.contains.ended.counter", "db.name", databaseName, "db.column", columnName);
|
2021-12-30 17:28:06 +01:00
|
|
|
this.containsTime = Timer
|
2021-12-30 18:20:56 +01:00
|
|
|
.builder("db.read.map.contains.timer")
|
2021-12-30 17:28:06 +01:00
|
|
|
.publishPercentiles(0.2, 0.5, 0.95)
|
|
|
|
.publishPercentileHistogram()
|
|
|
|
.tags("db.name", databaseName, "db.column", columnName)
|
|
|
|
.register(meterRegistry);
|
|
|
|
this.startedUpdates = meterRegistry.counter("db.write.map.update.started.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.endedUpdates = meterRegistry.counter("db.write.map.update.ended.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.updateTime = Timer
|
|
|
|
.builder("db.write.map.update.timer")
|
|
|
|
.publishPercentiles(0.2, 0.5, 0.95)
|
|
|
|
.publishPercentileHistogram()
|
|
|
|
.tags("db.name", databaseName, "db.column", columnName)
|
|
|
|
.register(meterRegistry);
|
|
|
|
this.startedPut = meterRegistry.counter("db.write.map.put.started.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.endedPut = meterRegistry.counter("db.write.map.put.ended.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.putTime = Timer
|
|
|
|
.builder("db.write.map.put.timer")
|
|
|
|
.publishPercentiles(0.2, 0.5, 0.95)
|
|
|
|
.publishPercentileHistogram()
|
|
|
|
.tags("db.name", databaseName, "db.column", columnName)
|
|
|
|
.register(meterRegistry);
|
|
|
|
this.startedRemove = meterRegistry.counter("db.write.map.remove.started.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.endedRemove = meterRegistry.counter("db.write.map.remove.ended.counter", "db.name", databaseName, "db.column", columnName);
|
|
|
|
this.removeTime = Timer
|
|
|
|
.builder("db.write.map.remove.timer")
|
|
|
|
.publishPercentiles(0.2, 0.5, 0.95)
|
|
|
|
.publishPercentileHistogram()
|
|
|
|
.tags("db.name", databaseName, "db.column", columnName)
|
|
|
|
.register(meterRegistry);
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getDatabaseName() {
|
|
|
|
return databaseName;
|
|
|
|
}
|
|
|
|
|
2021-06-26 02:35:33 +02:00
|
|
|
public String getColumnName() {
|
|
|
|
return columnName;
|
|
|
|
}
|
|
|
|
|
2022-05-10 16:57:41 +02:00
|
|
|
@NotNull
|
2022-05-20 10:20:00 +02:00
|
|
|
private ReadOptions generateReadOptionsOrStatic(LLSnapshot snapshot) {
|
2022-05-12 19:14:27 +02:00
|
|
|
var resolved = generateReadOptions(snapshot != null ? snapshotResolver.apply(snapshot) : null, true);
|
|
|
|
if (resolved != null) {
|
|
|
|
return resolved;
|
|
|
|
} else {
|
2022-05-20 10:20:00 +02:00
|
|
|
return new ReadOptions();
|
2022-05-12 19:14:27 +02:00
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
|
2022-05-10 16:57:41 +02:00
|
|
|
@Nullable
|
2022-05-20 10:20:00 +02:00
|
|
|
private ReadOptions generateReadOptionsOrNull(LLSnapshot snapshot) {
|
2022-05-10 16:57:41 +02:00
|
|
|
return generateReadOptions(snapshot != null ? snapshotResolver.apply(snapshot) : null, false);
|
|
|
|
}
|
|
|
|
|
2022-05-11 00:29:42 +02:00
|
|
|
@NotNull
|
2022-05-20 10:20:00 +02:00
|
|
|
private ReadOptions generateReadOptionsOrNew(LLSnapshot snapshot) {
|
2022-05-11 00:29:42 +02:00
|
|
|
var result = generateReadOptions(snapshot != null ? snapshotResolver.apply(snapshot) : null, false);
|
|
|
|
if (result != null) {
|
|
|
|
return result;
|
|
|
|
} else {
|
2022-05-20 10:20:00 +02:00
|
|
|
return new ReadOptions();
|
2022-05-11 00:29:42 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-20 10:20:00 +02:00
|
|
|
private ReadOptions generateReadOptions(Snapshot snapshot, boolean orStaticOpts) {
|
2020-12-07 22:15:18 +01:00
|
|
|
if (snapshot != null) {
|
2022-05-20 10:20:00 +02:00
|
|
|
return new ReadOptions().setSnapshot(snapshot);
|
2022-05-12 19:14:27 +02:00
|
|
|
} else if (ALLOW_STATIC_OPTIONS && orStaticOpts) {
|
2020-12-07 22:15:18 +01:00
|
|
|
return EMPTY_READ_OPTIONS;
|
2022-05-10 16:57:41 +02:00
|
|
|
} else {
|
|
|
|
return null;
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Buf get(@Nullable LLSnapshot snapshot, Buf key) {
|
|
|
|
return this.getSync(snapshot, key);
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Buf getSync(LLSnapshot snapshot, Buf key) {
|
2022-05-21 15:28:52 +02:00
|
|
|
logger.trace(MARKER_ROCKSDB, "Reading {}", () -> toStringSafe(key));
|
|
|
|
try {
|
|
|
|
var readOptions = generateReadOptionsOrStatic(snapshot);
|
2023-02-09 23:34:25 +01:00
|
|
|
Buf result;
|
2022-05-21 15:28:52 +02:00
|
|
|
startedGet.increment();
|
2022-05-20 18:31:05 +02:00
|
|
|
try {
|
2022-05-21 23:49:06 +02:00
|
|
|
var initTime = System.nanoTime();
|
|
|
|
result = db.get(readOptions, key);
|
|
|
|
getTime.record(Duration.ofNanos(System.nanoTime() - initTime));
|
2022-05-21 15:28:52 +02:00
|
|
|
} finally {
|
|
|
|
endedGet.increment();
|
|
|
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
|
|
|
readOptions.close();
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
|
|
|
}
|
2022-05-21 15:28:52 +02:00
|
|
|
logger.trace(MARKER_ROCKSDB, "Read {}: {}", () -> toStringSafe(key), () -> toStringSafe(result));
|
|
|
|
return result;
|
|
|
|
} catch (RocksDBException ex) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new DBException("Failed to read " + toStringSafe(key) + ": " + ex.getMessage());
|
2022-05-21 15:28:52 +02:00
|
|
|
}
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
|
|
|
|
2020-12-07 22:15:18 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public boolean isRangeEmpty(@Nullable LLSnapshot snapshot, LLRange range, boolean fillCache) {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called isRangeEmpty in a nonblocking thread";
|
|
|
|
startedContains.increment();
|
|
|
|
try {
|
|
|
|
Boolean isRangeEmpty = containsTime.recordCallable(() -> {
|
|
|
|
if (range.isSingle()) {
|
|
|
|
return !containsKey(snapshot, range.getSingleUnsafe());
|
|
|
|
} else {
|
|
|
|
// Temporary resources to release after finished
|
|
|
|
|
|
|
|
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot),
|
|
|
|
true,
|
|
|
|
isBoundedRange(range),
|
|
|
|
true
|
|
|
|
)) {
|
|
|
|
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
|
|
|
readOpts.setFillCache(fillCache);
|
|
|
|
var rocksIterator = db.newIterator(readOpts, range.getMin(), range.getMax());
|
|
|
|
try {
|
|
|
|
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
|
|
|
var seekArray = LLUtils.asArray(range.getMin());
|
|
|
|
rocksIterator.seek(seekArray);
|
|
|
|
} else {
|
|
|
|
rocksIterator.seekToFirst();
|
2021-12-30 17:28:06 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
return !rocksIterator.isValid();
|
|
|
|
} finally {
|
|
|
|
rocksIterator.close();
|
2021-12-30 17:28:06 +01:00
|
|
|
}
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
}
|
|
|
|
});
|
|
|
|
assert isRangeEmpty != null;
|
|
|
|
return isRangeEmpty;
|
|
|
|
} catch (Exception ex) {
|
|
|
|
throw new DBException("Failed to read range " + LLUtils.toStringSafe(range), ex);
|
|
|
|
} finally {
|
|
|
|
endedContains.increment();
|
|
|
|
}
|
2021-05-12 01:25:59 +02:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private boolean containsKey(@Nullable LLSnapshot snapshot, Buf key) throws RocksDBException {
|
2021-12-30 17:28:06 +01:00
|
|
|
startedContains.increment();
|
|
|
|
try {
|
|
|
|
var result = containsTime.recordCallable(() -> {
|
2022-05-10 16:57:41 +02:00
|
|
|
var readOptions = generateReadOptionsOrStatic(snapshot);
|
|
|
|
try {
|
|
|
|
return db.exists(readOptions, key);
|
|
|
|
} finally {
|
|
|
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
|
|
|
readOptions.close();
|
|
|
|
}
|
|
|
|
}
|
2021-12-30 17:28:06 +01:00
|
|
|
});
|
|
|
|
assert result != null;
|
|
|
|
return result;
|
|
|
|
} catch (RocksDBException | RuntimeException e) {
|
|
|
|
throw e;
|
|
|
|
} catch (Exception ex) {
|
|
|
|
throw new RuntimeException(ex);
|
|
|
|
} finally {
|
|
|
|
endedContains.increment();
|
|
|
|
}
|
2021-01-30 00:24:55 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Buf put(Buf key, Buf value, LLDictionaryResultType resultType) {
|
2021-12-17 01:48:49 +01:00
|
|
|
// Obtain the previous value from the database
|
2023-02-09 23:34:25 +01:00
|
|
|
var previousData = this.getPreviousData(key, resultType);
|
|
|
|
putInternal(key, value);
|
|
|
|
return previousData;
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private void putInternal(Buf key, Buf value) {
|
2022-01-26 21:18:43 +01:00
|
|
|
if (logger.isTraceEnabled(MARKER_ROCKSDB)) {
|
|
|
|
var varargs = new Supplier<?>[]{() -> toStringSafe(key), () -> toStringSafe(value)};
|
|
|
|
logger.trace(MARKER_ROCKSDB, "Writing {}: {}", varargs);
|
|
|
|
}
|
|
|
|
startedPut.increment();
|
2022-05-20 10:20:00 +02:00
|
|
|
try (var writeOptions = new WriteOptions()) {
|
2022-01-26 21:18:43 +01:00
|
|
|
putTime.recordCallable(() -> {
|
2022-05-12 19:14:27 +02:00
|
|
|
db.put(writeOptions, key, value);
|
2022-01-26 21:18:43 +01:00
|
|
|
return null;
|
|
|
|
});
|
|
|
|
} catch (RocksDBException ex) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new DBException("Failed to write: " + ex.getMessage());
|
2022-01-26 21:18:43 +01:00
|
|
|
} catch (RuntimeException ex) {
|
|
|
|
throw ex;
|
|
|
|
} catch (Exception ex) {
|
|
|
|
throw new RuntimeException("Failed to write", ex);
|
|
|
|
} finally {
|
|
|
|
endedPut.increment();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-02 19:18:15 +02:00
|
|
|
@Override
|
2022-09-12 20:14:56 +02:00
|
|
|
public UpdateMode getUpdateMode() {
|
|
|
|
return updateMode;
|
2021-05-02 19:18:15 +02:00
|
|
|
}
|
|
|
|
|
2021-05-08 03:09:00 +02:00
|
|
|
@SuppressWarnings("DuplicatedCode")
|
2021-02-06 19:21:31 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Buf update(Buf key,
|
2022-04-01 01:30:56 +02:00
|
|
|
BinarySerializationFunction updater,
|
2022-03-02 18:33:58 +01:00
|
|
|
UpdateReturnMode updateReturnMode) {
|
2023-02-09 23:34:25 +01:00
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called update in a nonblocking thread";
|
|
|
|
if (updateMode == UpdateMode.DISALLOW) {
|
|
|
|
throw new UnsupportedOperationException("update() is disallowed");
|
|
|
|
}
|
|
|
|
UpdateAtomicResultMode returnMode = switch (updateReturnMode) {
|
|
|
|
case NOTHING -> UpdateAtomicResultMode.NOTHING;
|
|
|
|
case GET_NEW_VALUE -> UpdateAtomicResultMode.CURRENT;
|
|
|
|
case GET_OLD_VALUE -> UpdateAtomicResultMode.PREVIOUS;
|
|
|
|
};
|
|
|
|
UpdateAtomicResult result = null;
|
|
|
|
try {
|
|
|
|
var readOptions = generateReadOptionsOrStatic(null);
|
|
|
|
startedUpdates.increment();
|
|
|
|
try (var writeOptions = new WriteOptions()) {
|
|
|
|
result = updateTime.recordCallable(() -> db.updateAtomic(readOptions, writeOptions, key, updater, returnMode));
|
|
|
|
} finally {
|
|
|
|
endedUpdates.increment();
|
|
|
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
|
|
|
readOptions.close();
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2022-05-20 23:59:56 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
assert result != null;
|
|
|
|
return switch (updateReturnMode) {
|
|
|
|
case NOTHING -> {
|
|
|
|
yield null;
|
|
|
|
}
|
|
|
|
case GET_NEW_VALUE -> ((UpdateAtomicResultCurrent) result).current();
|
|
|
|
case GET_OLD_VALUE -> ((UpdateAtomicResultPrevious) result).previous();
|
|
|
|
};
|
|
|
|
} catch (Exception ex) {
|
|
|
|
throw new DBException("Failed to update key-value pair", ex);
|
|
|
|
}
|
2021-10-17 19:52:43 +02:00
|
|
|
}
|
|
|
|
|
2021-10-20 01:51:34 +02:00
|
|
|
@SuppressWarnings("DuplicatedCode")
|
2021-05-08 03:09:00 +02:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public LLDelta updateAndGetDelta(Buf key, BinarySerializationFunction updater) {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called update in a nonblocking thread";
|
|
|
|
if (updateMode == UpdateMode.DISALLOW) {
|
|
|
|
throw new UnsupportedOperationException("update() is disallowed");
|
|
|
|
}
|
|
|
|
if (updateMode == UpdateMode.ALLOW && !db.supportsTransactions()) {
|
|
|
|
throw new UnsupportedOperationException("update() is disallowed because the database doesn't support"
|
|
|
|
+ "safe atomic operations");
|
|
|
|
}
|
2022-01-26 19:03:51 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
UpdateAtomicResultDelta result = null;
|
|
|
|
try {
|
|
|
|
var readOptions = generateReadOptionsOrStatic(null);
|
|
|
|
startedUpdates.increment();
|
|
|
|
try (var writeOptions = new WriteOptions()) {
|
|
|
|
result = updateTime.recordCallable(() ->
|
|
|
|
(UpdateAtomicResultDelta) db.updateAtomic(readOptions, writeOptions, key, updater, DELTA));
|
|
|
|
} finally {
|
|
|
|
endedUpdates.increment();
|
|
|
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
|
|
|
readOptions.close();
|
2022-05-20 23:59:56 +02:00
|
|
|
}
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
assert result != null;
|
|
|
|
return result.delta();
|
|
|
|
} catch (Exception ex) {
|
|
|
|
throw new DBException("Failed to update key-value pair and/or return the delta", ex);
|
|
|
|
}
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
|
|
|
|
2020-12-07 22:15:18 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Buf remove(Buf key, LLDictionaryResultType resultType) {
|
2022-01-26 19:03:51 +01:00
|
|
|
// Obtain the previous value from the database
|
2023-02-09 23:34:25 +01:00
|
|
|
Buf previousData = this.getPreviousData(key, resultType);
|
2022-01-26 19:03:51 +01:00
|
|
|
// Delete the value from the database
|
2023-02-09 23:34:25 +01:00
|
|
|
try {
|
|
|
|
logger.trace(MARKER_ROCKSDB, "Deleting {}", () -> toStringSafe(key));
|
|
|
|
startedRemove.increment();
|
|
|
|
try (var writeOptions = new WriteOptions()) {
|
|
|
|
removeTime.recordCallable(() -> {
|
|
|
|
db.delete(writeOptions, key);
|
|
|
|
return null;
|
|
|
|
});
|
|
|
|
} finally {
|
|
|
|
endedRemove.increment();
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
return previousData;
|
|
|
|
} catch (Exception ex) {
|
|
|
|
throw new DBException("Failed to delete", ex);
|
|
|
|
}
|
2021-01-30 00:24:55 +01:00
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Buf getPreviousData(Buf key, LLDictionaryResultType resultType) {
|
|
|
|
try {
|
|
|
|
return switch (resultType) {
|
|
|
|
case PREVIOUS_VALUE_EXISTENCE -> {
|
|
|
|
var contained = containsKey(null, key);
|
|
|
|
yield LLUtils.booleanToResponseByteBuffer(contained);
|
|
|
|
}
|
|
|
|
case PREVIOUS_VALUE -> {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called getPreviousData in a nonblocking thread";
|
|
|
|
Buf result;
|
|
|
|
var readOptions = generateReadOptionsOrStatic(null);
|
|
|
|
try {
|
|
|
|
result = db.get(readOptions, key);
|
|
|
|
} finally {
|
|
|
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
|
|
|
readOptions.close();
|
|
|
|
}
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
logger.trace(MARKER_ROCKSDB, "Read {}: {}", () -> toStringSafe(key), () -> toStringSafe(result));
|
|
|
|
yield result;
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
case VOID -> null;
|
|
|
|
};
|
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new DBException("Failed to read previous data");
|
|
|
|
}
|
2021-08-16 10:27:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<OptionalBuf> getMulti(@Nullable LLSnapshot snapshot, Stream<Buf> keys) {
|
2023-02-22 16:59:35 +01:00
|
|
|
return keys.map(key -> OptionalBuf.ofNullable(getSync(snapshot, key)));
|
2021-08-29 23:18:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public void putMulti(Stream<LLEntry> entries) {
|
|
|
|
batches(entries, Math.min(MULTI_GET_WINDOW, CAPPED_WRITE_BATCH_CAP)).parallel().forEach(entriesWindow -> {
|
|
|
|
try (var writeOptions = new WriteOptions()) {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called putMulti in a nonblocking thread";
|
|
|
|
if (USE_WRITE_BATCHES_IN_PUT_MULTI) {
|
|
|
|
try (var batch = new CappedWriteBatch(db,
|
|
|
|
CAPPED_WRITE_BATCH_CAP,
|
|
|
|
RESERVED_WRITE_BATCH_SIZE,
|
|
|
|
MAX_WRITE_BATCH_SIZE,
|
|
|
|
writeOptions
|
|
|
|
)) {
|
|
|
|
for (LLEntry entry : entriesWindow) {
|
|
|
|
batch.put(cfh, entry.getKey(), entry.getValue());
|
2022-01-26 19:03:51 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
batch.flush();
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
for (LLEntry entry : entriesWindow) {
|
|
|
|
db.put(writeOptions, entry.getKey(), entry.getValue());
|
2022-01-26 19:03:51 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
}
|
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new CompletionException(new DBException("Failed to write: " + ex.getMessage()));
|
|
|
|
}
|
|
|
|
});
|
2021-02-01 02:21:53 +01:00
|
|
|
}
|
|
|
|
|
2021-07-17 11:52:08 +02:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public <K> Stream<Boolean> updateMulti(Stream<K> keys, Stream<Buf> serializedKeys,
|
|
|
|
KVSerializationFunction<K, @Nullable Buf, @Nullable Buf> updateFunction) {
|
|
|
|
record Key<K>(K key, Buf serializedKey) {}
|
|
|
|
record MappedInput<K>(K key, Buf serializedKey, OptionalBuf mapped) {}
|
|
|
|
return batches(Streams.zip(keys, serializedKeys, Key<K>::new), Math.min(MULTI_GET_WINDOW, CAPPED_WRITE_BATCH_CAP))
|
|
|
|
.parallel()
|
|
|
|
.flatMap(entriesWindow -> {
|
2022-05-20 10:20:00 +02:00
|
|
|
try (var writeOptions = new WriteOptions()) {
|
2023-02-09 23:34:25 +01:00
|
|
|
if (LLUtils.isInNonBlockingThread()) {
|
2021-09-05 14:23:46 +02:00
|
|
|
throw new UnsupportedOperationException("Called updateMulti in a nonblocking thread");
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
List<Buf> keyBufsWindow = new ArrayList<>(entriesWindow.size());
|
|
|
|
for (Key<K> objects : entriesWindow) {
|
|
|
|
keyBufsWindow.add(objects.serializedKey());
|
2021-08-29 23:18:03 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
ArrayList<MappedInput<K>> mappedInputs;
|
2021-10-17 19:52:43 +02:00
|
|
|
{
|
2022-05-10 16:57:41 +02:00
|
|
|
var readOptions = generateReadOptionsOrStatic(null);
|
|
|
|
try {
|
2023-02-09 23:34:25 +01:00
|
|
|
var inputs = db.multiGetAsList(readOptions, Lists.transform(keyBufsWindow, Buf::asArray));
|
2022-05-10 16:57:41 +02:00
|
|
|
mappedInputs = new ArrayList<>(inputs.size());
|
|
|
|
for (int i = 0; i < inputs.size(); i++) {
|
|
|
|
var val = inputs.get(i);
|
|
|
|
if (val != null) {
|
|
|
|
inputs.set(i, null);
|
2023-02-09 23:34:25 +01:00
|
|
|
mappedInputs.add(new MappedInput<>(
|
|
|
|
entriesWindow.get(i).key(),
|
2022-05-20 10:20:00 +02:00
|
|
|
keyBufsWindow.get(i),
|
2023-02-09 23:34:25 +01:00
|
|
|
OptionalBuf.of(Buf.wrap(val))
|
2022-05-10 16:57:41 +02:00
|
|
|
));
|
|
|
|
} else {
|
2023-02-09 23:34:25 +01:00
|
|
|
mappedInputs.add(new MappedInput<>(
|
|
|
|
entriesWindow.get(i).key(),
|
2022-05-20 10:20:00 +02:00
|
|
|
keyBufsWindow.get(i),
|
2022-05-21 15:28:52 +02:00
|
|
|
OptionalBuf.empty()
|
2022-05-10 16:57:41 +02:00
|
|
|
));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
|
|
|
readOptions.close();
|
2021-10-17 19:52:43 +02:00
|
|
|
}
|
2021-08-29 23:18:03 +02:00
|
|
|
}
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
var updatedValuesToWrite = new ArrayList<Buf>(mappedInputs.size());
|
2021-11-08 10:49:59 +01:00
|
|
|
var valueChangedResult = new ArrayList<Boolean>(mappedInputs.size());
|
2023-02-09 23:34:25 +01:00
|
|
|
for (var mappedInput : mappedInputs) {
|
|
|
|
var updatedValue = updateFunction.apply(mappedInput.key(), mappedInput.serializedKey());
|
|
|
|
var t3 = mappedInput.mapped().orElse(null);
|
|
|
|
valueChangedResult.add(!LLUtils.equals(t3, updatedValue));
|
|
|
|
updatedValuesToWrite.add(updatedValue);
|
2021-10-17 19:52:43 +02:00
|
|
|
}
|
2021-07-17 11:52:08 +02:00
|
|
|
|
2021-10-17 19:52:43 +02:00
|
|
|
if (USE_WRITE_BATCHES_IN_PUT_MULTI) {
|
2023-02-09 23:34:25 +01:00
|
|
|
try (var batch = new CappedWriteBatch(db,
|
|
|
|
CAPPED_WRITE_BATCH_CAP,
|
|
|
|
RESERVED_WRITE_BATCH_SIZE,
|
|
|
|
MAX_WRITE_BATCH_SIZE,
|
|
|
|
writeOptions
|
|
|
|
)) {
|
2022-05-12 19:14:27 +02:00
|
|
|
int i = 0;
|
2023-02-09 23:34:25 +01:00
|
|
|
for (Key<K> entry : entriesWindow) {
|
|
|
|
var valueToWrite = updatedValuesToWrite.get(i);
|
|
|
|
if (valueToWrite == null) {
|
|
|
|
batch.delete(cfh, entry.serializedKey());
|
|
|
|
} else {
|
|
|
|
batch.put(cfh, entry.serializedKey(), valueToWrite);
|
2021-11-08 16:33:41 +01:00
|
|
|
}
|
2022-05-12 19:14:27 +02:00
|
|
|
i++;
|
2021-07-17 11:52:08 +02:00
|
|
|
}
|
2022-05-20 10:20:00 +02:00
|
|
|
batch.flush();
|
2021-10-17 19:52:43 +02:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
int i = 0;
|
2023-02-09 23:34:25 +01:00
|
|
|
for (Key<K> entry : entriesWindow) {
|
|
|
|
db.put(writeOptions, entry.serializedKey(), updatedValuesToWrite.get(i));
|
2021-10-17 19:52:43 +02:00
|
|
|
i++;
|
2021-08-29 23:18:03 +02:00
|
|
|
}
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
return valueChangedResult.stream();
|
|
|
|
} catch (RocksDBException e) {
|
|
|
|
throw new CompletionException(new DBException("Failed to update multiple key-value pairs", e));
|
2021-08-29 23:18:03 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
});
|
2021-07-17 11:52:08 +02:00
|
|
|
}
|
|
|
|
|
2020-12-07 22:15:18 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<LLEntry> getRange(@Nullable LLSnapshot snapshot,
|
|
|
|
LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
boolean reverse,
|
|
|
|
boolean smallRange) {
|
2023-02-09 23:34:25 +01:00
|
|
|
if (range.isSingle()) {
|
|
|
|
var rangeSingle = range.getSingle();
|
|
|
|
return getRangeSingle(snapshot, rangeSingle);
|
|
|
|
} else {
|
|
|
|
return getRangeMulti(snapshot, range, reverse, smallRange);
|
|
|
|
}
|
2021-02-02 00:09:46 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<List<LLEntry>> getRangeGrouped(@Nullable LLSnapshot snapshot,
|
|
|
|
LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
int prefixLength,
|
|
|
|
boolean smallRange) {
|
2023-02-09 23:34:25 +01:00
|
|
|
if (range.isSingle()) {
|
|
|
|
var rangeSingle = range.getSingle();
|
2022-07-19 23:45:39 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
return Stream.of(getRangeSingle(snapshot, rangeSingle).toList());
|
|
|
|
} else {
|
|
|
|
return getRangeMultiGrouped(snapshot, range, prefixLength, smallRange);
|
|
|
|
}
|
2021-01-30 00:24:55 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Stream<LLEntry> getRangeSingle(LLSnapshot snapshot, Buf key) {
|
|
|
|
var val = this.get(snapshot, key);
|
|
|
|
if (val == null) return Stream.of();
|
|
|
|
return Stream.of(LLEntry.of(key, val));
|
2021-01-30 20:16:14 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Stream<LLEntry> getRangeMulti(LLSnapshot snapshot,
|
|
|
|
LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
boolean reverse,
|
|
|
|
boolean smallRange) {
|
2022-07-19 23:45:39 +02:00
|
|
|
return new LLLocalEntryReactiveRocksIterator(db,
|
2023-02-09 23:34:25 +01:00
|
|
|
range,
|
2022-07-19 23:45:39 +02:00
|
|
|
() -> generateReadOptionsOrNull(snapshot),
|
|
|
|
reverse,
|
|
|
|
smallRange
|
2023-02-09 23:34:25 +01:00
|
|
|
).stream();
|
2021-02-02 00:09:46 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Stream<List<LLEntry>> getRangeMultiGrouped(LLSnapshot snapshot, LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
int prefixLength, boolean smallRange) {
|
2022-07-19 23:45:39 +02:00
|
|
|
return new LLLocalGroupedEntryReactiveRocksIterator(db,
|
|
|
|
prefixLength,
|
2023-02-09 23:34:25 +01:00
|
|
|
range,
|
2022-07-19 23:45:39 +02:00
|
|
|
() -> generateReadOptionsOrNull(snapshot),
|
|
|
|
smallRange
|
2023-02-09 23:34:25 +01:00
|
|
|
).stream();
|
2021-01-30 00:24:55 +01:00
|
|
|
}
|
|
|
|
|
2021-01-30 20:16:14 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<Buf> getRangeKeys(@Nullable LLSnapshot snapshot,
|
|
|
|
LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
boolean reverse,
|
|
|
|
boolean smallRange) {
|
2023-02-09 23:34:25 +01:00
|
|
|
if (range.isSingle()) {
|
|
|
|
return this.getRangeKeysSingle(snapshot, range.getSingle());
|
|
|
|
} else {
|
|
|
|
return this.getRangeKeysMulti(snapshot, range, reverse, smallRange);
|
|
|
|
}
|
2021-02-02 00:09:46 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<List<Buf>> getRangeKeysGrouped(@Nullable LLSnapshot snapshot,
|
|
|
|
LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
int prefixLength,
|
|
|
|
boolean smallRange) {
|
2022-07-19 23:45:39 +02:00
|
|
|
return new LLLocalGroupedKeyReactiveRocksIterator(db,
|
|
|
|
prefixLength,
|
2023-02-09 23:34:25 +01:00
|
|
|
range,
|
2022-07-19 23:45:39 +02:00
|
|
|
() -> generateReadOptionsOrNull(snapshot),
|
|
|
|
smallRange
|
2023-02-09 23:34:25 +01:00
|
|
|
).stream();
|
2021-03-14 13:24:46 +01:00
|
|
|
}
|
|
|
|
|
2021-06-27 11:58:12 +02:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<BadBlock> badBlocks(LLRange range) {
|
|
|
|
try {
|
|
|
|
var ro = LLUtils.generateCustomReadOptions(null,
|
|
|
|
false,
|
|
|
|
isBoundedRange(range),
|
|
|
|
false
|
|
|
|
);
|
|
|
|
ro.setFillCache(false);
|
|
|
|
if (!range.isSingle()) {
|
|
|
|
if (LLUtils.MANUAL_READAHEAD) {
|
|
|
|
ro.setReadaheadSize(32 * 1024);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ro.setVerifyChecksums(true);
|
|
|
|
var rocksIterator = db.newRocksIterator(ro, range, false);
|
|
|
|
try {
|
|
|
|
rocksIterator.seekToFirst();
|
|
|
|
} catch (Exception ex) {
|
|
|
|
rocksIterator.close();
|
|
|
|
ro.close();
|
|
|
|
throw new DBException("Failed to open rocksdb iterator", ex);
|
|
|
|
}
|
|
|
|
return Stream.generate(() -> {
|
|
|
|
if (!rocksIterator.isValid()) return null;
|
|
|
|
Buf rawKey = null;
|
|
|
|
try {
|
|
|
|
rawKey = rocksIterator.keyBuf().copy();
|
|
|
|
rocksIterator.next();
|
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
return new BadBlock(databaseName, ColumnUtils.special(columnName), rawKey, ex);
|
|
|
|
}
|
|
|
|
return null;
|
|
|
|
}).takeWhile(x -> rocksIterator.isValid()).filter(Objects::nonNull).onClose(() -> {
|
|
|
|
rocksIterator.close();
|
|
|
|
ro.close();
|
|
|
|
});
|
|
|
|
} catch (RocksDBException e) {
|
|
|
|
throw new DBException("Failed to get bad blocks", e);
|
|
|
|
}
|
2021-06-27 11:58:12 +02:00
|
|
|
}
|
|
|
|
|
2021-03-14 13:24:46 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<Buf> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot, LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
int prefixLength, boolean smallRange) {
|
2022-07-19 23:45:39 +02:00
|
|
|
return new LLLocalKeyPrefixReactiveRocksIterator(db,
|
|
|
|
prefixLength,
|
2023-02-09 23:34:25 +01:00
|
|
|
range,
|
2022-07-19 23:45:39 +02:00
|
|
|
() -> generateReadOptionsOrNull(snapshot),
|
|
|
|
true,
|
|
|
|
smallRange
|
2023-02-09 23:34:25 +01:00
|
|
|
).stream();
|
2021-01-30 20:16:14 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Stream<Buf> getRangeKeysSingle(LLSnapshot snapshot, Buf key) {
|
|
|
|
try {
|
2022-05-20 18:31:05 +02:00
|
|
|
if (containsKey(snapshot, key)) {
|
2023-02-09 23:34:25 +01:00
|
|
|
return Stream.of(key);
|
2022-05-20 18:31:05 +02:00
|
|
|
} else {
|
2023-02-09 23:34:25 +01:00
|
|
|
return Stream.empty();
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
} catch (RocksDBException e) {
|
|
|
|
throw new DBException("Failed to get range keys", e);
|
2022-05-12 19:14:27 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private Stream<Buf> getRangeKeysMulti(LLSnapshot snapshot,
|
|
|
|
LLRange range,
|
2022-03-24 23:56:23 +01:00
|
|
|
boolean reverse,
|
|
|
|
boolean smallRange) {
|
2022-07-19 23:45:39 +02:00
|
|
|
return new LLLocalKeyReactiveRocksIterator(db,
|
2023-02-09 23:34:25 +01:00
|
|
|
range,
|
2022-07-19 23:45:39 +02:00
|
|
|
() -> generateReadOptionsOrNull(snapshot),
|
|
|
|
reverse,
|
|
|
|
smallRange
|
2023-02-09 23:34:25 +01:00
|
|
|
).stream();
|
2021-01-30 20:16:14 +01:00
|
|
|
}
|
|
|
|
|
2020-12-07 22:15:18 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public void setRange(LLRange range, Stream<LLEntry> entries, boolean smallRange) {
|
2022-01-26 21:18:43 +01:00
|
|
|
if (USE_WINDOW_IN_SET_RANGE) {
|
2023-02-09 23:34:25 +01:00
|
|
|
try (var writeOptions = new WriteOptions()) {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called setRange in a nonblocking thread";
|
|
|
|
if (!USE_WRITE_BATCH_IN_SET_RANGE_DELETE || !USE_WRITE_BATCHES_IN_SET_RANGE) {
|
|
|
|
try (var opts = LLUtils.generateCustomReadOptions(null, true, isBoundedRange(range), smallRange)) {
|
|
|
|
try (var it = db.newIterator(opts, range.getMin(), range.getMax())) {
|
|
|
|
if (!PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
|
|
|
it.seekTo(range.getMin());
|
|
|
|
} else {
|
|
|
|
it.seekToFirst();
|
|
|
|
}
|
|
|
|
while (it.isValid()) {
|
|
|
|
db.delete(writeOptions, it.key());
|
|
|
|
it.next();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (USE_CAPPED_WRITE_BATCH_IN_SET_RANGE) {
|
|
|
|
try (var batch = new CappedWriteBatch(db,
|
|
|
|
CAPPED_WRITE_BATCH_CAP,
|
|
|
|
RESERVED_WRITE_BATCH_SIZE,
|
|
|
|
MAX_WRITE_BATCH_SIZE,
|
|
|
|
writeOptions
|
|
|
|
)) {
|
|
|
|
if (range.isSingle()) {
|
|
|
|
batch.delete(cfh, range.getSingle());
|
|
|
|
} else {
|
|
|
|
deleteSmallRangeWriteBatch(batch, range.copy());
|
|
|
|
}
|
|
|
|
batch.flush();
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
try (var batch = new WriteBatch(RESERVED_WRITE_BATCH_SIZE)) {
|
|
|
|
if (range.isSingle()) {
|
|
|
|
batch.delete(cfh, LLUtils.asArray(range.getSingleUnsafe()));
|
|
|
|
} else {
|
|
|
|
deleteSmallRangeWriteBatch(batch, range.copy());
|
|
|
|
}
|
|
|
|
db.write(writeOptions, batch);
|
|
|
|
batch.clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new DBException("Failed to set a range: " + ex.getMessage());
|
|
|
|
}
|
|
|
|
|
|
|
|
batches(entries, MULTI_GET_WINDOW)
|
|
|
|
.forEach(entriesList -> {
|
2022-05-20 23:59:56 +02:00
|
|
|
try (var writeOptions = new WriteOptions()) {
|
2023-02-09 23:34:25 +01:00
|
|
|
if (!USE_WRITE_BATCHES_IN_SET_RANGE) {
|
|
|
|
for (LLEntry entry : entriesList) {
|
|
|
|
db.put(writeOptions, entry.getKey(), entry.getValue());
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
|
|
|
} else if (USE_CAPPED_WRITE_BATCH_IN_SET_RANGE) {
|
2023-02-09 23:34:25 +01:00
|
|
|
|
|
|
|
try (var batch = new CappedWriteBatch(db,
|
|
|
|
CAPPED_WRITE_BATCH_CAP,
|
|
|
|
RESERVED_WRITE_BATCH_SIZE,
|
|
|
|
MAX_WRITE_BATCH_SIZE,
|
|
|
|
writeOptions
|
|
|
|
)) {
|
|
|
|
for (LLEntry entry : entriesList) {
|
|
|
|
batch.put(cfh, entry.getKey(), entry.getValue());
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2022-05-20 10:20:00 +02:00
|
|
|
batch.flush();
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
try (var batch = new WriteBatch(RESERVED_WRITE_BATCH_SIZE)) {
|
2023-02-09 23:34:25 +01:00
|
|
|
for (LLEntry entry : entriesList) {
|
|
|
|
batch.put(cfh, LLUtils.asArray(entry.getKey()), LLUtils.asArray(entry.getValue()));
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2022-05-12 19:14:27 +02:00
|
|
|
db.write(writeOptions, batch);
|
2022-01-26 21:18:43 +01:00
|
|
|
batch.clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (RocksDBException ex) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new CompletionException(new DBException("Failed to write range", ex));
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
});
|
2022-01-26 21:18:43 +01:00
|
|
|
} else {
|
|
|
|
if (USE_WRITE_BATCHES_IN_SET_RANGE) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new UnsupportedOperationException("Can't use write batches in setRange without window. Please fix the parameters");
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
this.getRange(null, range, false, smallRange).forEach(oldValue -> {
|
|
|
|
try (var writeOptions = new WriteOptions()) {
|
|
|
|
db.delete(writeOptions, oldValue.getKey());
|
2022-05-20 18:31:05 +02:00
|
|
|
} catch (RocksDBException ex) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new CompletionException(new DBException("Failed to write range", ex));
|
2022-05-20 18:31:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
});
|
2022-01-26 21:18:43 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
entries.forEach(entry -> {
|
2023-02-22 16:59:35 +01:00
|
|
|
if (entry.getKey() != null && entry.getValue() != null) {
|
|
|
|
this.putInternal(entry.getKey(), entry.getValue());
|
2023-02-09 23:34:25 +01:00
|
|
|
}
|
|
|
|
});
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private void deleteSmallRangeWriteBatch(WriteBatch writeBatch, LLRange range)
|
2021-03-20 12:41:11 +01:00
|
|
|
throws RocksDBException {
|
2023-02-09 23:34:25 +01:00
|
|
|
try (var readOpts = LLUtils.generateCustomReadOptions(null, false, isBoundedRange(range), true)) {
|
|
|
|
try (var rocksIterator = db.newIterator(readOpts, range.getMin(), range.getMax())) {
|
2022-05-12 19:14:27 +02:00
|
|
|
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
2023-02-09 23:34:25 +01:00
|
|
|
rocksIterator.seekTo(range.getMin());
|
2021-05-03 02:57:08 +02:00
|
|
|
} else {
|
2022-05-12 19:14:27 +02:00
|
|
|
rocksIterator.seekToFirst();
|
2021-05-03 02:57:08 +02:00
|
|
|
}
|
2022-05-12 19:14:27 +02:00
|
|
|
while (rocksIterator.isValid()) {
|
2023-02-09 23:34:25 +01:00
|
|
|
writeBatch.delete(cfh, rocksIterator.key());
|
2022-05-12 19:14:27 +02:00
|
|
|
rocksIterator.next();
|
2021-05-03 02:57:08 +02:00
|
|
|
}
|
2021-03-20 12:41:11 +01:00
|
|
|
}
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
public void clear() {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called clear in a nonblocking thread";
|
|
|
|
boolean shouldCompactLater = false;
|
|
|
|
try (var writeOptions = new WriteOptions();
|
|
|
|
var readOpts = LLUtils.generateCustomReadOptions(null, false, false, false)) {
|
|
|
|
if (VERIFY_CHECKSUMS_WHEN_NOT_NEEDED) {
|
|
|
|
readOpts.setVerifyChecksums(true);
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
// readOpts.setIgnoreRangeDeletions(true);
|
|
|
|
if (LLUtils.MANUAL_READAHEAD) {
|
|
|
|
readOpts.setReadaheadSize(32 * 1024); // 32KiB
|
|
|
|
}
|
|
|
|
try (CappedWriteBatch writeBatch = new CappedWriteBatch(db,
|
|
|
|
CAPPED_WRITE_BATCH_CAP,
|
|
|
|
RESERVED_WRITE_BATCH_SIZE,
|
|
|
|
MAX_WRITE_BATCH_SIZE,
|
|
|
|
writeOptions
|
|
|
|
)) {
|
|
|
|
|
|
|
|
byte[] firstDeletedKey = null;
|
|
|
|
byte[] lastDeletedKey = null;
|
|
|
|
try (var rocksIterator = db.newIterator(readOpts, null, null)) {
|
|
|
|
// If the database supports transactions, delete each key one by one
|
|
|
|
if (db.supportsTransactions()) {
|
|
|
|
rocksIterator.seekToFirst();
|
|
|
|
while (rocksIterator.isValid()) {
|
|
|
|
writeBatch.delete(cfh, rocksIterator.key());
|
|
|
|
rocksIterator.next();
|
2022-03-20 14:33:27 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
} else {
|
|
|
|
rocksIterator.seekToLast();
|
2020-12-07 22:15:18 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
if (rocksIterator.isValid()) {
|
|
|
|
firstDeletedKey = FIRST_KEY;
|
|
|
|
lastDeletedKey = rocksIterator.key().clone();
|
|
|
|
writeBatch.deleteRange(cfh, FIRST_KEY, lastDeletedKey);
|
|
|
|
writeBatch.delete(cfh, lastDeletedKey);
|
|
|
|
shouldCompactLater = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
writeBatch.flush();
|
|
|
|
|
|
|
|
if (shouldCompactLater) {
|
|
|
|
// Compact range
|
|
|
|
db.suggestCompactRange();
|
|
|
|
if (lastDeletedKey != null) {
|
|
|
|
try (var cro = new CompactRangeOptions()
|
|
|
|
.setAllowWriteStall(false)
|
|
|
|
.setExclusiveManualCompaction(false)
|
|
|
|
.setChangeLevel(false)) {
|
|
|
|
db.compactRange(firstDeletedKey, lastDeletedKey, cro);
|
2021-06-27 16:52:45 +02:00
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
try (var fo = new FlushOptions().setWaitForFlush(true).setAllowWriteStall(true)) {
|
|
|
|
db.flush(fo);
|
|
|
|
}
|
|
|
|
db.flushWal(true);
|
|
|
|
}
|
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new DBException("Failed to clear", ex);
|
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public long sizeRange(@Nullable LLSnapshot snapshot, LLRange range, boolean fast) {
|
|
|
|
try {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called sizeRange in a nonblocking thread";
|
|
|
|
if (range.isAll()) {
|
|
|
|
return fast ? fastSizeAll(snapshot) : exactSizeAll(snapshot);
|
|
|
|
} else {
|
|
|
|
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot),
|
|
|
|
false,
|
|
|
|
isBoundedRange(range),
|
|
|
|
false
|
|
|
|
)) {
|
|
|
|
readOpts.setFillCache(false);
|
|
|
|
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
|
|
|
if (fast) {
|
|
|
|
readOpts.setIgnoreRangeDeletions(true);
|
2022-05-12 19:14:27 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
}
|
|
|
|
try (var rocksIterator = db.newIterator(readOpts, range.getMin(), range.getMax())) {
|
|
|
|
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
|
|
|
rocksIterator.seekTo(range.getMin());
|
|
|
|
} else {
|
|
|
|
rocksIterator.seekToFirst();
|
2021-10-19 00:22:05 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
long i = 0;
|
|
|
|
while (rocksIterator.isValid()) {
|
|
|
|
rocksIterator.next();
|
|
|
|
i++;
|
2021-08-29 23:18:03 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
return i;
|
2021-10-19 00:22:05 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new DBException("Failed to get size of range", ex);
|
|
|
|
}
|
2021-05-12 01:25:59 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public LLEntry getOne(@Nullable LLSnapshot snapshot, LLRange range) {
|
|
|
|
try {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called getOne in a nonblocking thread";
|
|
|
|
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot), true, true, true)) {
|
|
|
|
try (var rocksIterator = db.newIterator(readOpts, range.getMin(), range.getMax())) {
|
|
|
|
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
|
|
|
rocksIterator.seekTo(range.getMin());
|
|
|
|
} else {
|
|
|
|
rocksIterator.seekToFirst();
|
|
|
|
}
|
|
|
|
if (rocksIterator.isValid()) {
|
|
|
|
var keyView = rocksIterator.keyBuf();
|
|
|
|
var valueView = rocksIterator.valueBuf();
|
|
|
|
return LLEntry.copyOf(keyView, valueView);
|
|
|
|
} else {
|
|
|
|
return null;
|
2021-08-16 10:27:47 +02:00
|
|
|
}
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new DBException("Failed to get one entry", ex);
|
|
|
|
}
|
2021-02-02 15:36:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Buf getOneKey(@Nullable LLSnapshot snapshot, LLRange range) {
|
|
|
|
try {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called getOneKey in a nonblocking thread";
|
|
|
|
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot), true, true, true)) {
|
|
|
|
try (var rocksIterator = db.newIterator(readOpts, range.getMin(), range.getMax())) {
|
|
|
|
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
|
|
|
rocksIterator.seekTo(range.getMin());
|
|
|
|
} else {
|
|
|
|
rocksIterator.seekToFirst();
|
|
|
|
}
|
|
|
|
if (rocksIterator.isValid()) {
|
|
|
|
return rocksIterator.keyBuf();
|
|
|
|
} else {
|
|
|
|
return null;
|
2021-08-16 10:27:47 +02:00
|
|
|
}
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new DBException("Failed to get one key", ex);
|
|
|
|
}
|
2021-02-02 15:36:11 +01:00
|
|
|
}
|
|
|
|
|
2021-06-27 11:58:12 +02:00
|
|
|
private long fastSizeAll(@Nullable LLSnapshot snapshot) throws RocksDBException {
|
2022-05-11 00:29:42 +02:00
|
|
|
try (var rocksdbSnapshot = generateReadOptionsOrNew(snapshot)) {
|
2022-05-20 10:20:00 +02:00
|
|
|
if (USE_CURRENT_FASTSIZE_FOR_OLD_SNAPSHOTS || rocksdbSnapshot.snapshot() == null) {
|
2021-06-19 21:55:20 +02:00
|
|
|
try {
|
2023-01-05 02:21:09 +01:00
|
|
|
if (USE_NUM_ENTRIES_PRECISE_COUNTER) {
|
|
|
|
return exactSizeAll(null);
|
|
|
|
}
|
2021-10-20 01:51:34 +02:00
|
|
|
return db.getLongProperty("rocksdb.estimate-num-keys");
|
2021-06-19 21:55:20 +02:00
|
|
|
} catch (RocksDBException e) {
|
2021-09-10 12:13:52 +02:00
|
|
|
logger.error(MARKER_ROCKSDB, "Failed to get RocksDB estimated keys count property", e);
|
2021-06-19 21:55:20 +02:00
|
|
|
return 0;
|
|
|
|
}
|
2023-01-05 02:21:09 +01:00
|
|
|
} else if (USE_NUM_ENTRIES_PRECISE_COUNTER || PARALLEL_EXACT_SIZE) {
|
2021-06-19 21:55:20 +02:00
|
|
|
return exactSizeAll(snapshot);
|
|
|
|
} else {
|
2022-05-20 10:20:00 +02:00
|
|
|
rocksdbSnapshot.setFillCache(false);
|
|
|
|
rocksdbSnapshot.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
|
|
|
rocksdbSnapshot.setIgnoreRangeDeletions(true);
|
2021-06-19 21:55:20 +02:00
|
|
|
long count = 0;
|
2022-05-12 19:14:27 +02:00
|
|
|
try (var rocksIterator = db.newIterator(rocksdbSnapshot, null, null)) {
|
2021-06-27 11:58:12 +02:00
|
|
|
rocksIterator.seekToFirst();
|
2021-06-19 21:55:20 +02:00
|
|
|
// If it's a fast size of a snapshot, count only up to 100'000 elements
|
2021-06-27 11:58:12 +02:00
|
|
|
while (rocksIterator.isValid() && count < 100_000) {
|
2021-06-19 21:55:20 +02:00
|
|
|
count++;
|
2021-06-27 11:58:12 +02:00
|
|
|
rocksIterator.next();
|
2021-06-19 21:55:20 +02:00
|
|
|
}
|
|
|
|
return count;
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-30 00:24:55 +01:00
|
|
|
private long exactSizeAll(@Nullable LLSnapshot snapshot) {
|
2023-02-09 23:34:25 +01:00
|
|
|
if (LLUtils.isInNonBlockingThread()) {
|
2021-09-05 14:23:46 +02:00
|
|
|
throw new UnsupportedOperationException("Called exactSizeAll in a nonblocking thread");
|
|
|
|
}
|
2023-01-05 02:21:09 +01:00
|
|
|
if (snapshot == null && USE_NUM_ENTRIES_PRECISE_COUNTER) {
|
|
|
|
try {
|
|
|
|
return db.getNumEntries();
|
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new IllegalStateException("Failed to read exact size", ex);
|
|
|
|
}
|
|
|
|
}
|
2022-05-10 16:57:41 +02:00
|
|
|
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot), false, false, false)) {
|
2022-03-20 14:33:27 +01:00
|
|
|
if (LLUtils.MANUAL_READAHEAD) {
|
2022-05-20 10:20:00 +02:00
|
|
|
readOpts.setReadaheadSize(128 * 1024); // 128KiB
|
2022-03-20 14:33:27 +01:00
|
|
|
}
|
2022-05-20 10:20:00 +02:00
|
|
|
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
2021-06-19 21:55:20 +02:00
|
|
|
|
|
|
|
if (PARALLEL_EXACT_SIZE) {
|
|
|
|
var commonPool = ForkJoinPool.commonPool();
|
|
|
|
var futures = IntStream
|
|
|
|
.range(-1, LLUtils.LEXICONOGRAPHIC_ITERATION_SEEKS.length)
|
|
|
|
.mapToObj(idx -> Pair.of(idx == -1 ? new byte[0] : LLUtils.LEXICONOGRAPHIC_ITERATION_SEEKS[idx],
|
|
|
|
idx + 1 >= LLUtils.LEXICONOGRAPHIC_ITERATION_SEEKS.length ? null
|
|
|
|
: LLUtils.LEXICONOGRAPHIC_ITERATION_SEEKS[idx + 1]
|
|
|
|
))
|
|
|
|
.map(range -> (Callable<Long>) () -> {
|
|
|
|
long partialCount = 0;
|
2022-05-20 10:20:00 +02:00
|
|
|
try (var rangeReadOpts = new ReadOptions(readOpts)) {
|
2021-06-19 21:55:20 +02:00
|
|
|
Slice sliceBegin;
|
|
|
|
if (range.getKey() != null) {
|
|
|
|
sliceBegin = new Slice(range.getKey());
|
|
|
|
} else {
|
|
|
|
sliceBegin = null;
|
|
|
|
}
|
|
|
|
Slice sliceEnd;
|
|
|
|
if (range.getValue() != null) {
|
|
|
|
sliceEnd = new Slice(range.getValue());
|
|
|
|
} else {
|
|
|
|
sliceEnd = null;
|
|
|
|
}
|
|
|
|
try {
|
|
|
|
if (sliceBegin != null) {
|
2022-05-20 10:20:00 +02:00
|
|
|
rangeReadOpts.setIterateLowerBound(sliceBegin);
|
2021-06-19 21:55:20 +02:00
|
|
|
}
|
2022-05-11 20:32:56 +02:00
|
|
|
if (sliceEnd != null) {
|
2022-05-20 10:20:00 +02:00
|
|
|
rangeReadOpts.setIterateUpperBound(sliceEnd);
|
2021-06-19 21:55:20 +02:00
|
|
|
}
|
2022-05-12 19:14:27 +02:00
|
|
|
try (var rocksIterator = db.newIterator(rangeReadOpts, null, null)) {
|
2021-06-27 11:58:12 +02:00
|
|
|
rocksIterator.seekToFirst();
|
|
|
|
while (rocksIterator.isValid()) {
|
2021-06-19 21:55:20 +02:00
|
|
|
partialCount++;
|
2021-06-27 11:58:12 +02:00
|
|
|
rocksIterator.next();
|
2021-06-19 21:55:20 +02:00
|
|
|
}
|
|
|
|
return partialCount;
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
if (sliceBegin != null) {
|
|
|
|
sliceBegin.close();
|
|
|
|
}
|
|
|
|
if (sliceEnd != null) {
|
|
|
|
sliceEnd.close();
|
|
|
|
}
|
2021-03-18 19:53:32 +01:00
|
|
|
}
|
|
|
|
}
|
2021-06-19 21:55:20 +02:00
|
|
|
})
|
2022-07-03 01:32:13 +02:00
|
|
|
.map(task -> commonPool.submit(task))
|
2021-12-17 01:48:49 +01:00
|
|
|
.toList();
|
2021-06-19 21:55:20 +02:00
|
|
|
long count = 0;
|
|
|
|
for (ForkJoinTask<Long> future : futures) {
|
|
|
|
count += future.join();
|
2021-03-18 19:53:32 +01:00
|
|
|
}
|
|
|
|
return count;
|
2021-06-19 21:55:20 +02:00
|
|
|
} else {
|
|
|
|
long count = 0;
|
2022-05-12 19:14:27 +02:00
|
|
|
try (var rocksIterator = db.newIterator(readOpts, null, null)) {
|
2022-03-30 15:15:53 +02:00
|
|
|
rocksIterator.seekToFirst();
|
|
|
|
while (rocksIterator.isValid()) {
|
2021-06-19 21:55:20 +02:00
|
|
|
count++;
|
2022-03-30 15:15:53 +02:00
|
|
|
rocksIterator.next();
|
2021-06-19 21:55:20 +02:00
|
|
|
}
|
|
|
|
return count;
|
2022-03-30 15:15:53 +02:00
|
|
|
} catch (RocksDBException ex) {
|
|
|
|
throw new IllegalStateException("Failed to read exact size", ex);
|
2021-06-19 21:55:20 +02:00
|
|
|
}
|
2021-03-18 19:53:32 +01:00
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public LLEntry removeOne(LLRange range) {
|
|
|
|
assert !LLUtils.isInNonBlockingThread() : "Called removeOne in a nonblocking thread";
|
|
|
|
try (var readOpts = new ReadOptions();
|
|
|
|
var writeOpts = new WriteOptions()) {
|
|
|
|
try (var rocksIterator = db.newIterator(readOpts, range.getMin(), range.getMax())) {
|
|
|
|
if (!LLLocalDictionary.PREFER_AUTO_SEEK_BOUND && range.hasMin()) {
|
|
|
|
rocksIterator.seekTo(range.getMin());
|
|
|
|
} else {
|
|
|
|
rocksIterator.seekToFirst();
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
if (!rocksIterator.isValid()) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
Buf key = rocksIterator.keyBuf().copy();
|
|
|
|
Buf value = rocksIterator.valueBuf().copy();
|
|
|
|
db.delete(writeOpts, key);
|
|
|
|
return LLEntry.of(key, value);
|
|
|
|
} catch (RocksDBException e) {
|
|
|
|
throw new DBException("Failed to remove key", e);
|
2022-01-26 21:18:43 +01:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
}
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|
2021-04-03 19:09:06 +02:00
|
|
|
|
2020-12-07 22:15:18 +01:00
|
|
|
}
|