2021-03-14 03:13:19 +01:00
|
|
|
package it.cavallium.dbengine.database.disk;
|
|
|
|
|
2021-09-23 11:30:44 +02:00
|
|
|
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
|
|
|
|
|
2021-09-17 16:56:28 +02:00
|
|
|
import io.net5.buffer.api.Buffer;
|
|
|
|
import io.net5.buffer.api.BufferAllocator;
|
|
|
|
import io.net5.buffer.api.Send;
|
2021-03-14 03:13:19 +01:00
|
|
|
import it.cavallium.dbengine.database.LLRange;
|
2021-04-30 19:15:04 +02:00
|
|
|
import it.cavallium.dbengine.database.LLUtils;
|
2021-03-14 03:13:19 +01:00
|
|
|
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
|
|
|
import java.util.List;
|
2021-09-22 18:33:28 +02:00
|
|
|
import org.jetbrains.annotations.Nullable;
|
2021-03-14 03:13:19 +01:00
|
|
|
import org.rocksdb.ColumnFamilyHandle;
|
|
|
|
import org.rocksdb.ReadOptions;
|
|
|
|
import org.rocksdb.RocksDB;
|
2021-06-27 11:58:12 +02:00
|
|
|
import org.rocksdb.RocksDBException;
|
2021-09-23 11:30:44 +02:00
|
|
|
import org.warp.commonutils.log.Logger;
|
|
|
|
import org.warp.commonutils.log.LoggerFactory;
|
2021-03-14 03:13:19 +01:00
|
|
|
import reactor.core.publisher.Flux;
|
|
|
|
|
2021-03-14 23:06:46 +01:00
|
|
|
public abstract class LLLocalGroupedReactiveRocksIterator<T> {
|
2021-03-14 03:13:19 +01:00
|
|
|
|
2021-09-23 11:30:44 +02:00
|
|
|
protected static final Logger logger = LoggerFactory.getLogger(LLLocalGroupedReactiveRocksIterator.class);
|
2021-03-14 03:13:19 +01:00
|
|
|
private final RocksDB db;
|
2021-08-29 23:18:03 +02:00
|
|
|
private final BufferAllocator alloc;
|
2021-03-14 03:13:19 +01:00
|
|
|
private final ColumnFamilyHandle cfh;
|
|
|
|
private final int prefixLength;
|
|
|
|
private final LLRange range;
|
2021-06-29 23:31:02 +02:00
|
|
|
private final boolean allowNettyDirect;
|
2021-03-14 03:13:19 +01:00
|
|
|
private final ReadOptions readOptions;
|
2021-03-22 20:02:19 +01:00
|
|
|
private final boolean canFillCache;
|
2021-03-14 03:13:19 +01:00
|
|
|
private final boolean readValues;
|
|
|
|
|
2021-08-29 23:18:03 +02:00
|
|
|
public LLLocalGroupedReactiveRocksIterator(RocksDB db, BufferAllocator alloc, ColumnFamilyHandle cfh,
|
2021-03-14 03:13:19 +01:00
|
|
|
int prefixLength,
|
2021-08-29 23:18:03 +02:00
|
|
|
Send<LLRange> range,
|
2021-06-29 23:31:02 +02:00
|
|
|
boolean allowNettyDirect,
|
2021-03-14 03:13:19 +01:00
|
|
|
ReadOptions readOptions,
|
2021-03-22 20:02:19 +01:00
|
|
|
boolean canFillCache,
|
2021-04-03 19:09:06 +02:00
|
|
|
boolean readValues) {
|
2021-08-31 09:14:46 +02:00
|
|
|
try (range) {
|
|
|
|
this.db = db;
|
|
|
|
this.alloc = alloc;
|
|
|
|
this.cfh = cfh;
|
|
|
|
this.prefixLength = prefixLength;
|
|
|
|
this.range = range.receive();
|
|
|
|
this.allowNettyDirect = allowNettyDirect;
|
|
|
|
this.readOptions = readOptions;
|
|
|
|
this.canFillCache = canFillCache;
|
|
|
|
this.readValues = readValues;
|
|
|
|
}
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|
|
|
|
|
2021-03-14 13:08:03 +01:00
|
|
|
|
|
|
|
public Flux<List<T>> flux() {
|
|
|
|
return Flux
|
2021-03-14 03:13:19 +01:00
|
|
|
.generate(() -> {
|
2021-03-14 19:38:20 +01:00
|
|
|
var readOptions = new ReadOptions(this.readOptions);
|
2021-03-22 20:02:19 +01:00
|
|
|
readOptions.setFillCache(canFillCache && range.hasMin() && range.hasMax());
|
2021-09-23 11:30:44 +02:00
|
|
|
if (logger.isTraceEnabled()) {
|
|
|
|
logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range));
|
|
|
|
}
|
2021-09-01 00:01:56 +02:00
|
|
|
return LLLocalDictionary.getRocksIterator(alloc, allowNettyDirect, readOptions, range.copy().send(), db, cfh);
|
2021-03-14 19:38:20 +01:00
|
|
|
}, (tuple, sink) -> {
|
2021-04-30 19:15:04 +02:00
|
|
|
try {
|
2021-05-02 19:18:15 +02:00
|
|
|
var rocksIterator = tuple.getT1();
|
|
|
|
ObjectArrayList<T> values = new ObjectArrayList<>();
|
2021-08-29 23:18:03 +02:00
|
|
|
Buffer firstGroupKey = null;
|
2021-05-02 19:18:15 +02:00
|
|
|
try {
|
2021-06-27 11:58:12 +02:00
|
|
|
rocksIterator.status();
|
2021-05-02 19:18:15 +02:00
|
|
|
while (rocksIterator.isValid()) {
|
2021-09-22 18:33:28 +02:00
|
|
|
try (Buffer key = LLUtils.readDirectNioBuffer(alloc, rocksIterator::key)) {
|
2021-05-02 19:18:15 +02:00
|
|
|
if (firstGroupKey == null) {
|
2021-08-31 09:14:46 +02:00
|
|
|
firstGroupKey = key.copy();
|
|
|
|
} else if (!LLUtils.equals(firstGroupKey, firstGroupKey.readerOffset(),
|
|
|
|
key, key.readerOffset(), prefixLength)) {
|
2021-05-02 19:18:15 +02:00
|
|
|
break;
|
|
|
|
}
|
2021-09-22 18:33:28 +02:00
|
|
|
@Nullable Buffer value;
|
2021-05-02 19:18:15 +02:00
|
|
|
if (readValues) {
|
2021-09-22 18:33:28 +02:00
|
|
|
value = LLUtils.readDirectNioBuffer(alloc, rocksIterator::value);
|
2021-05-02 19:18:15 +02:00
|
|
|
} else {
|
2021-09-22 18:33:28 +02:00
|
|
|
value = null;
|
2021-05-02 19:18:15 +02:00
|
|
|
}
|
2021-09-23 11:30:44 +02:00
|
|
|
|
|
|
|
if (logger.isTraceEnabled()) {
|
|
|
|
logger.trace(MARKER_ROCKSDB,
|
|
|
|
"Range {} is reading {}: {}",
|
|
|
|
LLUtils.toStringSafe(range),
|
|
|
|
LLUtils.toStringSafe(key),
|
|
|
|
LLUtils.toStringSafe(value)
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2021-05-02 19:18:15 +02:00
|
|
|
try {
|
|
|
|
rocksIterator.next();
|
2021-06-27 11:58:12 +02:00
|
|
|
rocksIterator.status();
|
2021-09-22 18:33:28 +02:00
|
|
|
T entry = getEntry(key.send(), value == null ? null : value.send());
|
2021-05-02 19:18:15 +02:00
|
|
|
values.add(entry);
|
|
|
|
} finally {
|
2021-09-22 18:33:28 +02:00
|
|
|
if (value != null) {
|
|
|
|
value.close();
|
|
|
|
}
|
2021-05-02 19:18:15 +02:00
|
|
|
}
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
2021-05-02 19:18:15 +02:00
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
if (firstGroupKey != null) {
|
2021-08-31 09:14:46 +02:00
|
|
|
firstGroupKey.close();
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
|
|
|
}
|
2021-05-02 19:18:15 +02:00
|
|
|
if (!values.isEmpty()) {
|
|
|
|
sink.next(values);
|
|
|
|
} else {
|
2021-09-23 11:30:44 +02:00
|
|
|
if (logger.isTraceEnabled()) {
|
|
|
|
logger.trace(MARKER_ROCKSDB, "Range {} ended", LLUtils.toStringSafe(range));
|
|
|
|
}
|
2021-05-02 19:18:15 +02:00
|
|
|
sink.complete();
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|
2021-06-27 11:58:12 +02:00
|
|
|
} catch (RocksDBException ex) {
|
2021-09-23 11:30:44 +02:00
|
|
|
if (logger.isTraceEnabled()) {
|
|
|
|
logger.trace(MARKER_ROCKSDB, "Range {} failed", LLUtils.toStringSafe(range));
|
|
|
|
}
|
2021-06-27 11:58:12 +02:00
|
|
|
sink.error(ex);
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|
2021-06-27 11:58:12 +02:00
|
|
|
return tuple;
|
2021-03-14 19:38:20 +01:00
|
|
|
}, tuple -> {
|
|
|
|
var rocksIterator = tuple.getT1();
|
|
|
|
rocksIterator.close();
|
2021-08-31 09:14:46 +02:00
|
|
|
tuple.getT2().close();
|
|
|
|
tuple.getT3().close();
|
2021-09-01 00:01:56 +02:00
|
|
|
tuple.getT4().close();
|
2021-03-14 14:16:07 +01:00
|
|
|
});
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|
|
|
|
|
2021-09-22 18:33:28 +02:00
|
|
|
public abstract T getEntry(@Nullable Send<Buffer> key, @Nullable Send<Buffer> value);
|
2021-05-02 19:18:15 +02:00
|
|
|
|
|
|
|
public void release() {
|
2021-08-31 09:14:46 +02:00
|
|
|
range.close();
|
2021-05-02 19:18:15 +02:00
|
|
|
}
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|