Use single scheduler for lucene scheduled tasks
This commit is contained in:
parent
68dc7306c6
commit
f855949a48
2
pom.xml
2
pom.xml
@ -38,7 +38,7 @@
|
||||
<dependency>
|
||||
<groupId>org.warp</groupId>
|
||||
<artifactId>common-utils</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<version>1.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.xml.bind</groupId>
|
||||
|
@ -34,6 +34,7 @@ import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.concurrency.executor.ScheduledTaskLifecycle;
|
||||
import org.warp.commonutils.functional.IOFunction;
|
||||
import org.warp.commonutils.type.ShortNamedThreadFactory;
|
||||
import it.cavallium.dbengine.database.LLDocument;
|
||||
@ -59,14 +60,19 @@ import reactor.util.function.Tuples;
|
||||
public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||
|
||||
private static final LuceneStreamSearcher streamSearcher = new AdaptiveStreamSearcher();
|
||||
private final Duration queryRefreshDebounceTime;
|
||||
private final Duration commitDebounceTime;
|
||||
/**
|
||||
* Global lucene index scheduler.
|
||||
* There is only a single thread globally to not overwhelm the disk with
|
||||
* parallel commits or parallel refreshes.
|
||||
*/
|
||||
private static final ScheduledExecutorService scheduler
|
||||
= Executors.newSingleThreadScheduledExecutor(new ShortNamedThreadFactory("Lucene"));
|
||||
|
||||
private final String luceneIndexName;
|
||||
private final SnapshotDeletionPolicy snapshotter;
|
||||
private final IndexWriter indexWriter;
|
||||
private final SearcherManager searcherManager;
|
||||
private final Directory directory;
|
||||
private final AtomicLong lastSearcherRefresh = new AtomicLong(0);
|
||||
/**
|
||||
* Last snapshot sequence number. 0 is not used
|
||||
*/
|
||||
@ -75,9 +81,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||
* Snapshot seq no to index commit point
|
||||
*/
|
||||
private final ConcurrentHashMap<Long, LuceneIndexSnapshot> snapshots = new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService scheduler;
|
||||
private final boolean lowMemory;
|
||||
|
||||
private final ScheduledTaskLifecycle scheduledTasksLifecycle;
|
||||
|
||||
public LLLocalLuceneIndex(Path luceneBasePath,
|
||||
String name,
|
||||
TextFieldsAnalyzer analyzer,
|
||||
@ -105,20 +112,21 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||
}
|
||||
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
|
||||
this.searcherManager = new SearcherManager(indexWriter, false, false, null);
|
||||
this.queryRefreshDebounceTime = queryRefreshDebounceTime;
|
||||
this.commitDebounceTime = commitDebounceTime;
|
||||
this.lastSearcherRefresh.set(System.currentTimeMillis());
|
||||
this.scheduler = Executors.newSingleThreadScheduledExecutor(new ShortNamedThreadFactory("Lucene"));
|
||||
scheduler.scheduleAtFixedRate(this::scheduledCommit,
|
||||
commitDebounceTime.toMillis(),
|
||||
commitDebounceTime.toMillis(),
|
||||
|
||||
// Create scheduled tasks lifecycle manager
|
||||
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();
|
||||
|
||||
// Start scheduled tasks
|
||||
registerScheduledFixedTask(this::scheduledCommit, commitDebounceTime);
|
||||
registerScheduledFixedTask(this::scheduledQueryRefresh, queryRefreshDebounceTime);
|
||||
}
|
||||
|
||||
private void registerScheduledFixedTask(Runnable task, Duration duration) {
|
||||
scheduledTasksLifecycle.registerScheduledTask(scheduler.scheduleAtFixedRate(task,
|
||||
duration.toMillis(),
|
||||
duration.toMillis(),
|
||||
TimeUnit.MILLISECONDS
|
||||
);
|
||||
scheduler.scheduleAtFixedRate(this::scheduledQueryRefresh,
|
||||
queryRefreshDebounceTime.toMillis(),
|
||||
queryRefreshDebounceTime.toMillis(),
|
||||
TimeUnit.MILLISECONDS
|
||||
);
|
||||
));
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -368,17 +376,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
scheduler.shutdown();
|
||||
try {
|
||||
scheduler.awaitTermination(10, TimeUnit.MINUTES);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
if (!scheduler.isTerminated()) {
|
||||
System.err.println("Terminating lucene scheduler");
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
indexWriter.commit();
|
||||
scheduledTasksLifecycle.cancelAndWait();
|
||||
indexWriter.close();
|
||||
directory.close();
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user