Fixed more leaks
This commit is contained in:
parent
99e101914d
commit
fab06db239
@ -723,7 +723,7 @@ public class LLUtils {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate a ReadOptions, with some parameters modified to help with bulk iterations
|
* Generate a ReadOptions, with some parameters modified to help with bulk iterations
|
||||||
* @param readOptions the read options to start with
|
* @param readOptions the read options to start with, it will be modified
|
||||||
* @param canFillCache true to fill the cache. If closedRange is false, this field will be ignored
|
* @param canFillCache true to fill the cache. If closedRange is false, this field will be ignored
|
||||||
* @param boundedRange true if the range is bounded from both sides
|
* @param boundedRange true if the range is bounded from both sides
|
||||||
* @param smallRange true if the range is small
|
* @param smallRange true if the range is small
|
||||||
|
@ -222,6 +222,16 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
return generateReadOptions(snapshot != null ? snapshotResolver.apply(snapshot) : null, false);
|
return generateReadOptions(snapshot != null ? snapshotResolver.apply(snapshot) : null, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@NotNull
|
||||||
|
private ReadOptions generateReadOptionsOrNew(LLSnapshot snapshot) {
|
||||||
|
var result = generateReadOptions(snapshot != null ? snapshotResolver.apply(snapshot) : null, false);
|
||||||
|
if (result != null) {
|
||||||
|
return result;
|
||||||
|
} else {
|
||||||
|
return new ReadOptions();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private ReadOptions generateReadOptions(Snapshot snapshot, boolean orStaticOpts) {
|
private ReadOptions generateReadOptions(Snapshot snapshot, boolean orStaticOpts) {
|
||||||
if (snapshot != null) {
|
if (snapshot != null) {
|
||||||
return new ReadOptions().setSnapshot(snapshot);
|
return new ReadOptions().setSnapshot(snapshot);
|
||||||
@ -589,7 +599,7 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
var readOptions = generateReadOptionsOrStatic(snapshot);
|
var readOptions = generateReadOptionsOrStatic(snapshot);
|
||||||
try {
|
try {
|
||||||
List<byte[]> results = db.multiGetAsList(readOptions, LLUtils.toArray(keyBufsWindow));
|
List<byte[]> results = db.multiGetAsList(readOptions, LLUtils.toArray(keyBufsWindow));
|
||||||
mappedResults = new ArrayList<Optional<Buffer>>(results.size());
|
mappedResults = new ArrayList<>(results.size());
|
||||||
for (int i = 0; i < results.size(); i++) {
|
for (int i = 0; i < results.size(); i++) {
|
||||||
byte[] val = results.get(i);
|
byte[] val = results.get(i);
|
||||||
Optional<Buffer> valueOpt;
|
Optional<Buffer> valueOpt;
|
||||||
@ -832,8 +842,8 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
boolean reverse,
|
boolean reverse,
|
||||||
boolean smallRange) {
|
boolean smallRange) {
|
||||||
Mono<LLLocalEntryReactiveRocksIterator> iteratorMono = rangeMono.map(rangeSend -> {
|
Mono<LLLocalEntryReactiveRocksIterator> iteratorMono = rangeMono.map(rangeSend -> {
|
||||||
ReadOptions resolvedSnapshot = generateReadOptionsOrNull(snapshot);
|
ReadOptions readOptions = generateReadOptionsOrNull(snapshot);
|
||||||
return new LLLocalEntryReactiveRocksIterator(db, rangeSend, nettyDirect, resolvedSnapshot, reverse, smallRange);
|
return new LLLocalEntryReactiveRocksIterator(db, rangeSend, nettyDirect, readOptions, reverse, smallRange);
|
||||||
});
|
});
|
||||||
return Flux.usingWhen(iteratorMono,
|
return Flux.usingWhen(iteratorMono,
|
||||||
iterator -> iterator.flux().subscribeOn(dbRScheduler, false),
|
iterator -> iterator.flux().subscribeOn(dbRScheduler, false),
|
||||||
@ -844,12 +854,12 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
private Flux<List<Send<LLEntry>>> getRangeMultiGrouped(LLSnapshot snapshot, Mono<Send<LLRange>> rangeMono,
|
private Flux<List<Send<LLEntry>>> getRangeMultiGrouped(LLSnapshot snapshot, Mono<Send<LLRange>> rangeMono,
|
||||||
int prefixLength, boolean smallRange) {
|
int prefixLength, boolean smallRange) {
|
||||||
Mono<LLLocalGroupedEntryReactiveRocksIterator> iteratorMono = rangeMono.map(rangeSend -> {
|
Mono<LLLocalGroupedEntryReactiveRocksIterator> iteratorMono = rangeMono.map(rangeSend -> {
|
||||||
ReadOptions resolvedSnapshot = generateReadOptionsOrNull(snapshot);
|
ReadOptions readOptions = generateReadOptionsOrNull(snapshot);
|
||||||
return new LLLocalGroupedEntryReactiveRocksIterator(db,
|
return new LLLocalGroupedEntryReactiveRocksIterator(db,
|
||||||
prefixLength,
|
prefixLength,
|
||||||
rangeSend,
|
rangeSend,
|
||||||
nettyDirect,
|
nettyDirect,
|
||||||
resolvedSnapshot,
|
readOptions,
|
||||||
smallRange
|
smallRange
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@ -882,12 +892,12 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
int prefixLength,
|
int prefixLength,
|
||||||
boolean smallRange) {
|
boolean smallRange) {
|
||||||
Mono<LLLocalGroupedKeyReactiveRocksIterator> iteratorMono = rangeMono.map(rangeSend -> {
|
Mono<LLLocalGroupedKeyReactiveRocksIterator> iteratorMono = rangeMono.map(rangeSend -> {
|
||||||
ReadOptions resolvedSnapshot = generateReadOptionsOrNull(snapshot);
|
ReadOptions readOptions = generateReadOptionsOrNull(snapshot);
|
||||||
return new LLLocalGroupedKeyReactiveRocksIterator(db,
|
return new LLLocalGroupedKeyReactiveRocksIterator(db,
|
||||||
prefixLength,
|
prefixLength,
|
||||||
rangeSend,
|
rangeSend,
|
||||||
nettyDirect,
|
nettyDirect,
|
||||||
resolvedSnapshot,
|
readOptions,
|
||||||
smallRange
|
smallRange
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@ -943,8 +953,13 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
public Flux<Send<Buffer>> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> rangeMono,
|
public Flux<Send<Buffer>> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> rangeMono,
|
||||||
int prefixLength, boolean smallRange) {
|
int prefixLength, boolean smallRange) {
|
||||||
Mono<LLLocalKeyPrefixReactiveRocksIterator> iteratorMono = rangeMono.map(range -> {
|
Mono<LLLocalKeyPrefixReactiveRocksIterator> iteratorMono = rangeMono.map(range -> {
|
||||||
ReadOptions resolvedSnapshot = generateReadOptionsOrNull(snapshot);
|
ReadOptions readOptions = generateReadOptionsOrNull(snapshot);
|
||||||
return new LLLocalKeyPrefixReactiveRocksIterator(db, prefixLength, range, nettyDirect, resolvedSnapshot, true,
|
return new LLLocalKeyPrefixReactiveRocksIterator(db,
|
||||||
|
prefixLength,
|
||||||
|
range,
|
||||||
|
nettyDirect,
|
||||||
|
readOptions,
|
||||||
|
true,
|
||||||
smallRange
|
smallRange
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@ -976,8 +991,8 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
boolean reverse,
|
boolean reverse,
|
||||||
boolean smallRange) {
|
boolean smallRange) {
|
||||||
Mono<LLLocalKeyReactiveRocksIterator> iteratorMono = rangeMono.map(range -> {
|
Mono<LLLocalKeyReactiveRocksIterator> iteratorMono = rangeMono.map(range -> {
|
||||||
ReadOptions resolvedSnapshot = generateReadOptionsOrNull(snapshot);
|
ReadOptions readOptions = generateReadOptionsOrNull(snapshot);
|
||||||
return new LLLocalKeyReactiveRocksIterator(db, range, nettyDirect, resolvedSnapshot, reverse, smallRange);
|
return new LLLocalKeyReactiveRocksIterator(db, range, nettyDirect, readOptions, reverse, smallRange);
|
||||||
});
|
});
|
||||||
return Flux.usingWhen(iteratorMono,
|
return Flux.usingWhen(iteratorMono,
|
||||||
iterator -> iterator.flux().subscribeOn(dbRScheduler, false),
|
iterator -> iterator.flux().subscribeOn(dbRScheduler, false),
|
||||||
@ -1169,8 +1184,7 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
private void deleteSmallRangeWriteBatch(CappedWriteBatch writeBatch, Send<LLRange> rangeToReceive)
|
private void deleteSmallRangeWriteBatch(CappedWriteBatch writeBatch, Send<LLRange> rangeToReceive)
|
||||||
throws RocksDBException {
|
throws RocksDBException {
|
||||||
var range = rangeToReceive.receive();
|
var range = rangeToReceive.receive();
|
||||||
var readOpts = generateReadOptionsOrStatic(null);
|
try (var readOpts = generateReadOptionsOrNew(null)) {
|
||||||
try {
|
|
||||||
readOpts.setFillCache(false);
|
readOpts.setFillCache(false);
|
||||||
ReleasableSlice minBound;
|
ReleasableSlice minBound;
|
||||||
if (range.hasMin()) {
|
if (range.hasMin()) {
|
||||||
@ -1212,17 +1226,13 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
range.close();
|
range.close();
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
|
||||||
if (readOpts != EMPTY_READ_OPTIONS) {
|
|
||||||
readOpts.close();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteSmallRangeWriteBatch(WriteBatch writeBatch, Send<LLRange> rangeToReceive)
|
private void deleteSmallRangeWriteBatch(WriteBatch writeBatch, Send<LLRange> rangeToReceive)
|
||||||
throws RocksDBException {
|
throws RocksDBException {
|
||||||
try (var range = rangeToReceive.receive()) {
|
try (var range = rangeToReceive.receive()) {
|
||||||
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(null), true, isBoundedRange(range), true)) {
|
try (var readOpts = LLUtils.generateCustomReadOptions(null, true, isBoundedRange(range), true)) {
|
||||||
readOpts.setFillCache(false);
|
readOpts.setFillCache(false);
|
||||||
ReleasableSlice minBound;
|
ReleasableSlice minBound;
|
||||||
if (range.hasMin()) {
|
if (range.hasMin()) {
|
||||||
@ -1286,7 +1296,7 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
.<Void>fromCallable(() -> {
|
.<Void>fromCallable(() -> {
|
||||||
assert !Schedulers.isInNonBlockingThread() : "Called clear in a nonblocking thread";
|
assert !Schedulers.isInNonBlockingThread() : "Called clear in a nonblocking thread";
|
||||||
boolean shouldCompactLater = false;
|
boolean shouldCompactLater = false;
|
||||||
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(null), false, false, false)) {
|
try (var readOpts = LLUtils.generateCustomReadOptions(null, false, false, false)) {
|
||||||
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
||||||
|
|
||||||
// readOpts.setIgnoreRangeDeletions(true);
|
// readOpts.setIgnoreRangeDeletions(true);
|
||||||
@ -1331,15 +1341,18 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
// Compact range
|
// Compact range
|
||||||
db.suggestCompactRange();
|
db.suggestCompactRange();
|
||||||
if (lastDeletedKey != null) {
|
if (lastDeletedKey != null) {
|
||||||
db.compactRange(firstDeletedKey, lastDeletedKey, new CompactRangeOptions()
|
try (var cro = new CompactRangeOptions()
|
||||||
.setAllowWriteStall(false)
|
.setAllowWriteStall(false)
|
||||||
.setExclusiveManualCompaction(false)
|
.setExclusiveManualCompaction(false)
|
||||||
.setChangeLevel(false)
|
.setChangeLevel(false)) {
|
||||||
);
|
db.compactRange(firstDeletedKey, lastDeletedKey, cro);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
db.flush(new FlushOptions().setWaitForFlush(true).setAllowWriteStall(true));
|
try (var fo = new FlushOptions().setWaitForFlush(true).setAllowWriteStall(true)) {
|
||||||
|
db.flush(fo);
|
||||||
|
}
|
||||||
db.flushWal(true);
|
db.flushWal(true);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@ -1358,7 +1371,11 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
if (range.isAll()) {
|
if (range.isAll()) {
|
||||||
sink.next(fast ? fastSizeAll(snapshot) : exactSizeAll(snapshot));
|
sink.next(fast ? fastSizeAll(snapshot) : exactSizeAll(snapshot));
|
||||||
} else {
|
} else {
|
||||||
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot), false, isBoundedRange(range), false)) {
|
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot),
|
||||||
|
false,
|
||||||
|
isBoundedRange(range),
|
||||||
|
false
|
||||||
|
)) {
|
||||||
readOpts.setFillCache(false);
|
readOpts.setFillCache(false);
|
||||||
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
|
||||||
ReleasableSlice minBound;
|
ReleasableSlice minBound;
|
||||||
@ -1521,11 +1538,7 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private long fastSizeAll(@Nullable LLSnapshot snapshot) throws RocksDBException {
|
private long fastSizeAll(@Nullable LLSnapshot snapshot) throws RocksDBException {
|
||||||
var rocksdbSnapshot = generateReadOptionsOrNull(snapshot);
|
try (var rocksdbSnapshot = generateReadOptionsOrNew(snapshot)) {
|
||||||
if (rocksdbSnapshot == null) {
|
|
||||||
rocksdbSnapshot = new ReadOptions();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (USE_CURRENT_FASTSIZE_FOR_OLD_SNAPSHOTS || rocksdbSnapshot.snapshot() == null) {
|
if (USE_CURRENT_FASTSIZE_FOR_OLD_SNAPSHOTS || rocksdbSnapshot.snapshot() == null) {
|
||||||
try {
|
try {
|
||||||
return db.getLongProperty("rocksdb.estimate-num-keys");
|
return db.getLongProperty("rocksdb.estimate-num-keys");
|
||||||
@ -1550,8 +1563,6 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
rocksdbSnapshot.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1641,8 +1652,7 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
return rangeMono.publishOn(dbWScheduler).handle((rangeSend, sink) -> {
|
return rangeMono.publishOn(dbWScheduler).handle((rangeSend, sink) -> {
|
||||||
try (var range = rangeSend.receive()) {
|
try (var range = rangeSend.receive()) {
|
||||||
assert !Schedulers.isInNonBlockingThread() : "Called removeOne in a nonblocking thread";
|
assert !Schedulers.isInNonBlockingThread() : "Called removeOne in a nonblocking thread";
|
||||||
var readOpts = generateReadOptionsOrStatic(null);
|
try (var readOpts = new ReadOptions()) {
|
||||||
try {
|
|
||||||
ReleasableSlice minBound;
|
ReleasableSlice minBound;
|
||||||
if (range.hasMin()) {
|
if (range.hasMin()) {
|
||||||
minBound = AbstractRocksDBColumn.setIterateBound(nettyDirect, readOpts, IterateBound.LOWER, range.getMinUnsafe());
|
minBound = AbstractRocksDBColumn.setIterateBound(nettyDirect, readOpts, IterateBound.LOWER, range.getMinUnsafe());
|
||||||
@ -1684,10 +1694,6 @@ public class LLLocalDictionary implements LLDictionary {
|
|||||||
} finally {
|
} finally {
|
||||||
minBound.close();
|
minBound.close();
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
if (readOpts != EMPTY_READ_OPTIONS) {
|
|
||||||
readOpts.close();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (RocksDBException ex) {
|
} catch (RocksDBException ex) {
|
||||||
sink.error(ex);
|
sink.error(ex);
|
||||||
|
@ -79,7 +79,7 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> extends
|
|||||||
this.prefixLength = prefixLength;
|
this.prefixLength = prefixLength;
|
||||||
this.range = range.receive();
|
this.range = range.receive();
|
||||||
this.allowNettyDirect = allowNettyDirect;
|
this.allowNettyDirect = allowNettyDirect;
|
||||||
this.readOptions = readOptions;
|
this.readOptions = readOptions != null ? readOptions : new ReadOptions();
|
||||||
this.canFillCache = canFillCache;
|
this.canFillCache = canFillCache;
|
||||||
this.readValues = readValues;
|
this.readValues = readValues;
|
||||||
this.smallRange = smallRange;
|
this.smallRange = smallRange;
|
||||||
@ -162,7 +162,6 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> extends
|
|||||||
}, t -> {
|
}, t -> {
|
||||||
t.getT2().close();
|
t.getT2().close();
|
||||||
t.getT1().close();
|
t.getT1().close();
|
||||||
this.close();
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73,7 +73,7 @@ public class LLLocalKeyPrefixReactiveRocksIterator extends
|
|||||||
this.prefixLength = prefixLength;
|
this.prefixLength = prefixLength;
|
||||||
this.rangeShared = range.receive();
|
this.rangeShared = range.receive();
|
||||||
this.allowNettyDirect = allowNettyDirect;
|
this.allowNettyDirect = allowNettyDirect;
|
||||||
this.readOptions = readOptions;
|
this.readOptions = readOptions != null ? readOptions : new ReadOptions();
|
||||||
this.canFillCache = canFillCache;
|
this.canFillCache = canFillCache;
|
||||||
this.smallRange = smallRange;
|
this.smallRange = smallRange;
|
||||||
}
|
}
|
||||||
@ -153,7 +153,6 @@ public class LLLocalKeyPrefixReactiveRocksIterator extends
|
|||||||
}, t -> {
|
}, t -> {
|
||||||
t.getT2().close();
|
t.getT2().close();
|
||||||
t.getT1().close();
|
t.getT1().close();
|
||||||
this.close();
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -783,10 +783,12 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
|||||||
try {
|
try {
|
||||||
// Range rangeToCompact = db.suggestCompactRange(cfh);
|
// Range rangeToCompact = db.suggestCompactRange(cfh);
|
||||||
logger.info("Compacting range {}", r);
|
logger.info("Compacting range {}", r);
|
||||||
db.compactRange(cfh, null, null, new CompactRangeOptions()
|
try (var cro = new CompactRangeOptions()
|
||||||
.setAllowWriteStall(true)
|
.setAllowWriteStall(true)
|
||||||
.setExclusiveManualCompaction(true)
|
.setExclusiveManualCompaction(true)
|
||||||
.setChangeLevel(false));
|
.setChangeLevel(false)) {
|
||||||
|
db.compactRange(cfh, null, null, cro);
|
||||||
|
}
|
||||||
} catch (RocksDBException e) {
|
} catch (RocksDBException e) {
|
||||||
if ("Database shutdown".equalsIgnoreCase(e.getMessage())) {
|
if ("Database shutdown".equalsIgnoreCase(e.getMessage())) {
|
||||||
logger.warn("Compaction cancelled: database shutdown");
|
logger.warn("Compaction cancelled: database shutdown");
|
||||||
|
@ -148,7 +148,6 @@ public abstract class LLLocalReactiveRocksIterator<T> extends
|
|||||||
}, t -> {
|
}, t -> {
|
||||||
t.getT2().close();
|
t.getT2().close();
|
||||||
t.getT1().close();
|
t.getT1().close();
|
||||||
this.close();
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -90,7 +90,15 @@ public class LLLocalSingleton implements LLSingleton {
|
|||||||
public Mono<Send<Buffer>> get(@Nullable LLSnapshot snapshot) {
|
public Mono<Send<Buffer>> get(@Nullable LLSnapshot snapshot) {
|
||||||
return nameMono.publishOn(dbRScheduler).handle((nameSend, sink) -> {
|
return nameMono.publishOn(dbRScheduler).handle((nameSend, sink) -> {
|
||||||
try (Buffer name = nameSend.receive()) {
|
try (Buffer name = nameSend.receive()) {
|
||||||
Buffer result = db.get(generateReadOptions(snapshot, true), name);
|
Buffer result;
|
||||||
|
var readOptions = generateReadOptions(snapshot, true);
|
||||||
|
try {
|
||||||
|
result = db.get(readOptions, name);
|
||||||
|
} finally {
|
||||||
|
if (readOptions != EMPTY_READ_OPTIONS) {
|
||||||
|
readOptions.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
if (result != null) {
|
if (result != null) {
|
||||||
sink.next(result.send());
|
sink.next(result.send());
|
||||||
} else {
|
} else {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user