Use single scheduler for lucene scheduled tasks
This commit is contained in:
parent
68dc7306c6
commit
f855949a48
2
pom.xml
2
pom.xml
|
@ -38,7 +38,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.warp</groupId>
|
<groupId>org.warp</groupId>
|
||||||
<artifactId>common-utils</artifactId>
|
<artifactId>common-utils</artifactId>
|
||||||
<version>1.1.1</version>
|
<version>1.1.2</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>javax.xml.bind</groupId>
|
<groupId>javax.xml.bind</groupId>
|
||||||
|
|
|
@ -34,6 +34,7 @@ import org.apache.lucene.search.TopDocs;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.FSDirectory;
|
import org.apache.lucene.store.FSDirectory;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
|
import org.warp.commonutils.concurrency.executor.ScheduledTaskLifecycle;
|
||||||
import org.warp.commonutils.functional.IOFunction;
|
import org.warp.commonutils.functional.IOFunction;
|
||||||
import org.warp.commonutils.type.ShortNamedThreadFactory;
|
import org.warp.commonutils.type.ShortNamedThreadFactory;
|
||||||
import it.cavallium.dbengine.database.LLDocument;
|
import it.cavallium.dbengine.database.LLDocument;
|
||||||
|
@ -59,14 +60,19 @@ import reactor.util.function.Tuples;
|
||||||
public class LLLocalLuceneIndex implements LLLuceneIndex {
|
public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||||
|
|
||||||
private static final LuceneStreamSearcher streamSearcher = new AdaptiveStreamSearcher();
|
private static final LuceneStreamSearcher streamSearcher = new AdaptiveStreamSearcher();
|
||||||
private final Duration queryRefreshDebounceTime;
|
/**
|
||||||
private final Duration commitDebounceTime;
|
* Global lucene index scheduler.
|
||||||
|
* There is only a single thread globally to not overwhelm the disk with
|
||||||
|
* parallel commits or parallel refreshes.
|
||||||
|
*/
|
||||||
|
private static final ScheduledExecutorService scheduler
|
||||||
|
= Executors.newSingleThreadScheduledExecutor(new ShortNamedThreadFactory("Lucene"));
|
||||||
|
|
||||||
private final String luceneIndexName;
|
private final String luceneIndexName;
|
||||||
private final SnapshotDeletionPolicy snapshotter;
|
private final SnapshotDeletionPolicy snapshotter;
|
||||||
private final IndexWriter indexWriter;
|
private final IndexWriter indexWriter;
|
||||||
private final SearcherManager searcherManager;
|
private final SearcherManager searcherManager;
|
||||||
private final Directory directory;
|
private final Directory directory;
|
||||||
private final AtomicLong lastSearcherRefresh = new AtomicLong(0);
|
|
||||||
/**
|
/**
|
||||||
* Last snapshot sequence number. 0 is not used
|
* Last snapshot sequence number. 0 is not used
|
||||||
*/
|
*/
|
||||||
|
@ -75,9 +81,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||||
* Snapshot seq no to index commit point
|
* Snapshot seq no to index commit point
|
||||||
*/
|
*/
|
||||||
private final ConcurrentHashMap<Long, LuceneIndexSnapshot> snapshots = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LuceneIndexSnapshot> snapshots = new ConcurrentHashMap<>();
|
||||||
private final ScheduledExecutorService scheduler;
|
|
||||||
private final boolean lowMemory;
|
private final boolean lowMemory;
|
||||||
|
|
||||||
|
private final ScheduledTaskLifecycle scheduledTasksLifecycle;
|
||||||
|
|
||||||
public LLLocalLuceneIndex(Path luceneBasePath,
|
public LLLocalLuceneIndex(Path luceneBasePath,
|
||||||
String name,
|
String name,
|
||||||
TextFieldsAnalyzer analyzer,
|
TextFieldsAnalyzer analyzer,
|
||||||
|
@ -105,20 +112,21 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||||
}
|
}
|
||||||
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
|
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
|
||||||
this.searcherManager = new SearcherManager(indexWriter, false, false, null);
|
this.searcherManager = new SearcherManager(indexWriter, false, false, null);
|
||||||
this.queryRefreshDebounceTime = queryRefreshDebounceTime;
|
|
||||||
this.commitDebounceTime = commitDebounceTime;
|
// Create scheduled tasks lifecycle manager
|
||||||
this.lastSearcherRefresh.set(System.currentTimeMillis());
|
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();
|
||||||
this.scheduler = Executors.newSingleThreadScheduledExecutor(new ShortNamedThreadFactory("Lucene"));
|
|
||||||
scheduler.scheduleAtFixedRate(this::scheduledCommit,
|
// Start scheduled tasks
|
||||||
commitDebounceTime.toMillis(),
|
registerScheduledFixedTask(this::scheduledCommit, commitDebounceTime);
|
||||||
commitDebounceTime.toMillis(),
|
registerScheduledFixedTask(this::scheduledQueryRefresh, queryRefreshDebounceTime);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void registerScheduledFixedTask(Runnable task, Duration duration) {
|
||||||
|
scheduledTasksLifecycle.registerScheduledTask(scheduler.scheduleAtFixedRate(task,
|
||||||
|
duration.toMillis(),
|
||||||
|
duration.toMillis(),
|
||||||
TimeUnit.MILLISECONDS
|
TimeUnit.MILLISECONDS
|
||||||
);
|
));
|
||||||
scheduler.scheduleAtFixedRate(this::scheduledQueryRefresh,
|
|
||||||
queryRefreshDebounceTime.toMillis(),
|
|
||||||
queryRefreshDebounceTime.toMillis(),
|
|
||||||
TimeUnit.MILLISECONDS
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -368,17 +376,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
scheduler.shutdown();
|
scheduledTasksLifecycle.cancelAndWait();
|
||||||
try {
|
|
||||||
scheduler.awaitTermination(10, TimeUnit.MINUTES);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
if (!scheduler.isTerminated()) {
|
|
||||||
System.err.println("Terminating lucene scheduler");
|
|
||||||
scheduler.shutdownNow();
|
|
||||||
}
|
|
||||||
indexWriter.commit();
|
|
||||||
indexWriter.close();
|
indexWriter.close();
|
||||||
directory.close();
|
directory.close();
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user