andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Publish lucene 9 snapshots

This commit is contained in:
Andrea Cavalli 2021-10-16 14:59:38 +02:00
parent f77784fc50
commit e66efba834
8 changed files with 53 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.github/workflows/maven-publish.yml vendored
View File

@ -38,6 +38,24 @@ jobs:
env:
MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }}
- name: Deploy lucene to Maven (Release)
if: github.ref == 'refs/heads/master'
shell: bash
run: |
git clone https://github.com/apache/lucene --depth 1
cd lucene
./gradlew assemble -x test -x errorprone publishToMavenLocal -x signJarsPublication
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-core -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-join -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-analysis-common -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-analysis-icu -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-codecs -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-backward-codecs -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-queries -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-misc -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT-sources.jar
env:
MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }}
- name: Deploy to Maven (Release)
if: github.ref == 'refs/heads/master'
shell: bash

4
pom.xml
View File

@ -191,7 +191,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-icu</artifactId>
<artifactId>lucene-analysis-icu</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
@ -380,7 +380,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-icu</artifactId>
<artifactId>lucene-analysis-icu</artifactId>
<version>9.0.0-SNAPSHOT</version>
</dependency>
<dependency>

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java
View File

@ -119,7 +119,7 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
if (connected.compareAndSet(true, false)) {
var env = this.env.get();
if (env != null) {
env.close(Duration.ofSeconds(30));
env.close();
}
}
return null;

22
src/main/java/it/cavallium/dbengine/database/disk/LLTempLMDBEnv.java
View File

@ -20,8 +20,6 @@ public class LLTempLMDBEnv implements Closeable {
private static final long TWENTY_GIBIBYTES = 20L * 1024L * 1024L * 1024L;
private static final int MAX_DATABASES = 1024;
private final Phaser resources = new Phaser(1);
private final Path tempDirectory;
private final Env<ByteBuf> env;
private volatile boolean closed;
@ -35,33 +33,15 @@ public class LLTempLMDBEnv implements Closeable {
env = envBuilder.open(tempDirectory.toFile(), MDB_NOTLS, MDB_WRITEMAP, MDB_NORDAHEAD);
}
public Env<ByteBuf> getEnvAndIncrementRef() {
public Env<ByteBuf> getEnv() {
if (closed) {
throw new IllegalStateException("Environment closed");
}
resources.register();
return env;
}
public void decrementRef() {
resources.arriveAndDeregister();
}
@Override
public void close() throws IOException {
this.closed = true;
resources.arriveAndAwaitAdvance();
closeInternal();
}
public void close(Duration timeout) throws InterruptedException, TimeoutException, IOException {
this.closed = true;
int phase = resources.arrive();
resources.awaitAdvanceInterruptibly(phase, timeout.toMillis(), TimeUnit.MILLISECONDS);
closeInternal();
}
private void closeInternal() throws IOException {
this.closed = true;
env.close();
//noinspection ResultOfMethodCallIgnored

42
src/main/java/it/cavallium/dbengine/lucene/LMDBArray.java
View File

@ -8,24 +8,12 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.lmdbjava.CursorIterable;
import org.lmdbjava.CursorIterable.KeyVal;
import org.lmdbjava.Dbi;
import org.lmdbjava.Env;
import org.lmdbjava.GetOp;
import org.lmdbjava.Txn;
import reactor.core.publisher.Flux;
import reactor.core.scheduler.Scheduler;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuple3;
import reactor.util.function.Tuples;
public class LMDBArray<V> implements IArray<V>, Closeable {
@ -35,7 +23,6 @@ public class LMDBArray<V> implements IArray<V>, Closeable {
private static final AtomicLong NEXT_LMDB_ARRAY_ID = new AtomicLong(0);
private final AtomicBoolean closed = new AtomicBoolean();
private final Runnable onClose;
private final LMDBCodec<V> valueCodec;
private final Env<ByteBuf> env;
private final Dbi<ByteBuf> lmdb;
@ -49,10 +36,9 @@ public class LMDBArray<V> implements IArray<V>, Closeable {
private final long virtualSize;
public LMDBArray(LLTempLMDBEnv env, LMDBCodec<V> codec, long size, @Nullable V defaultValue) {
this.onClose = env::decrementRef;
var name = "$array_" + NEXT_LMDB_ARRAY_ID.getAndIncrement();
this.valueCodec = codec;
this.env = env.getEnvAndIncrementRef();
this.env = env.getEnv();
this.lmdb = this.env.openDbi(name, MDB_CREATE);
this.defaultValue = defaultValue;
@ -207,22 +193,18 @@ public class LMDBArray<V> implements IArray<V>, Closeable {
@Override
public void close() throws IOException {
if (closed.compareAndSet(false, true)) {
try {
ensureThread();
if (rwTxn != null) {
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
} finally {
onClose.run();
ensureThread();
if (rwTxn != null) {
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
}
}

40
src/main/java/it/cavallium/dbengine/lucene/LMDBPriorityQueue.java
View File

@ -9,12 +9,10 @@ import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
import java.io.IOException;
import java.util.Iterator;
import java.util.Objects;
import java.util.Optional;
import java.util.StringJoiner;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.lmdbjava.Cursor;
import org.lmdbjava.CursorIterable;
import org.lmdbjava.CursorIterable.KeyVal;
@ -23,8 +21,6 @@ import org.lmdbjava.Env;
import org.lmdbjava.GetOp;
import org.lmdbjava.Txn;
import reactor.core.publisher.Flux;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuple3;
import reactor.util.function.Tuples;
@ -38,7 +34,6 @@ public class LMDBPriorityQueue<T> implements PriorityQueue<T>, Reversable<Revers
private static final AtomicLong NEXT_ITEM_UID = new AtomicLong(0);
private final AtomicBoolean closed = new AtomicBoolean();
private final Runnable onClose;
private final LMDBSortedCodec<T> codec;
private final Env<ByteBuf> env;
private final Dbi<ByteBuf> lmdb;
@ -54,10 +49,9 @@ public class LMDBPriorityQueue<T> implements PriorityQueue<T>, Reversable<Revers
private long size = 0;
public LMDBPriorityQueue(LLTempLMDBEnv env, LMDBSortedCodec<T> codec) {
this.onClose = env::decrementRef;
var name = "$queue_" + NEXT_LMDB_QUEUE_ID.getAndIncrement();
this.codec = codec;
this.env = env.getEnvAndIncrementRef();
this.env = env.getEnv();
this.lmdb = this.env.openDbi(name, codec::compareDirect, MDB_CREATE, MDB_DUPSORT, MDB_DUPFIXED);
this.writing = true;
@ -467,25 +461,21 @@ public class LMDBPriorityQueue<T> implements PriorityQueue<T>, Reversable<Revers
@Override
public void close() throws IOException {
if (closed.compareAndSet(false, true)) {
try {
ensureThread();
if (cur != null) {
cur.close();
}
if (rwTxn != null) {
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
} finally {
onClose.run();
ensureThread();
if (cur != null) {
cur.close();
}
if (rwTxn != null) {
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
}
}

4
src/test/java/it/cavallium/dbengine/TestLuceneIndex.java
View File

@ -119,8 +119,8 @@ public class TestLuceneIndex {
}
@AfterAll
public static void afterAll() throws IOException, InterruptedException, TimeoutException {
ENV.close(Duration.ofSeconds(10));
public static void afterAll() throws IOException {
ENV.close();
}
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {

4
src/test/java/it/cavallium/dbengine/TestLuceneSearches.java
View File

@ -212,9 +212,9 @@ public class TestLuceneSearches {
}
@AfterAll
public static void afterAll() throws IOException, InterruptedException, TimeoutException {
public static void afterAll() throws IOException {
TEMP_DB_GENERATOR.closeTempDb(tempDb).block();
ENV.close(Duration.ofSeconds(10));
ENV.close();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
}