andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Schedule commits and merges

Browse Source
This commit is contained in:
Andrea Cavalli 2021-12-15 16:47:59 +01:00
parent 8ad622db0a
commit b7ca57a215
2 changed files with 40 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -250,6 +250,9 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
var commitMillis = luceneOptions.commitDebounceTime().toMillis();
luceneHeavyTasksScheduler.schedulePeriodically(this::scheduledCommit, commitMillis, commitMillis,
TimeUnit.MILLISECONDS);
// Maybe merge every 5 commits
luceneHeavyTasksScheduler.schedulePeriodically(this::scheduledMerge, commitMillis * 5, commitMillis * 5,
TimeUnit.MILLISECONDS);
}
private Similarity getLuceneSimilarity() {
@ -263,7 +266,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
@Override
public Mono<LLSnapshot> takeSnapshot() {
return snapshotsManager.takeSnapshot().subscribeOn(luceneHeavyTasksScheduler).transform(this::ensureOpen);
return snapshotsManager.takeSnapshot().transform(this::ensureOpen);
}
private <V> Mono<V> ensureOpen(Mono<V> mono) {
@ -503,6 +506,14 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
}
}
private void scheduledMerge() {
try {
indexWriter.maybeMerge();
} catch (IOException ex) {
logger.error(MARKER_LUCENE, "Failed to execute a scheduled merge", ex);
}
}
@Override
public boolean isLowMemoryMode() {
return lowMemory;

49
src/main/java/it/cavallium/dbengine/database/disk/SnapshotsManager.java
View File

@ -57,29 +57,34 @@ public class SnapshotsManager {
*/
private Mono<IndexCommit> takeLuceneSnapshot() {
return Mono
.fromCallable(snapshotter::snapshot)
.subscribeOn(Schedulers.boundedElastic())
.onErrorResume(ex -> Mono
.defer(() -> {
if (ex instanceof IllegalStateException && "No index commit to snapshot".equals(ex.getMessage())) {
return Mono.fromCallable(() -> {
activeTasks.register();
try {
indexWriter.commit();
return snapshotter.snapshot();
} finally {
activeTasks.arriveAndDeregister();
}
});
} else {
return Mono.error(ex);
.<IndexCommit>create(sink -> Schedulers.boundedElastic().schedule(() -> {
try {
sink.success(snapshotter.snapshot());
} catch (Throwable ex) {
sink.error(ex);
}
}))
.onErrorResume(ex -> {
if (ex instanceof IllegalStateException && "No index commit to snapshot".equals(ex.getMessage())) {
return Mono.create(sink -> Schedulers.boundedElastic().schedule(() -> {
activeTasks.register();
try {
indexWriter.commit();
sink.success(snapshotter.snapshot());
} catch (Throwable e) {
sink.error(e);
} finally {
activeTasks.arriveAndDeregister();
}
})
);
}));
} else {
return Mono.error(ex);
}
});
}
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
return Mono.<Void>fromCallable(() -> {
return Mono.create(sink -> Schedulers.boundedElastic().schedule(() -> {
activeTasks.register();
try {
var indexSnapshot = this.snapshots.remove(snapshot.getSequenceNumber());
@ -93,11 +98,13 @@ public class SnapshotsManager {
snapshotter.release(luceneIndexSnapshot);
// Delete unused files after releasing the snapshot
indexWriter.deleteUnusedFiles();
return null;
sink.success();
} catch (Throwable ex) {
sink.error(ex);
} finally {
activeTasks.arriveAndDeregister();
}
}).subscribeOn(Schedulers.boundedElastic());
}));
}
public void close() {