Publish lucene 9 snapshots

This commit is contained in:
Andrea Cavalli 2021-10-16 14:59:38 +02:00
parent f77784fc50
commit e66efba834
8 changed files with 53 additions and 83 deletions

View File

@ -38,6 +38,24 @@ jobs:
env: env:
MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }} MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }} MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }}
- name: Deploy lucene to Maven (Release)
if: github.ref == 'refs/heads/master'
shell: bash
run: |
git clone https://github.com/apache/lucene --depth 1
cd lucene
./gradlew assemble -x test -x errorprone publishToMavenLocal -x signJarsPublication
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-core -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-join -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-analysis-common -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-analysis-icu -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-codecs -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-backward-codecs -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-queries -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT-sources.jar
mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-misc -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-snapshot-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT-sources.jar
env:
MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }}
- name: Deploy to Maven (Release) - name: Deploy to Maven (Release)
if: github.ref == 'refs/heads/master' if: github.ref == 'refs/heads/master'
shell: bash shell: bash

View File

@ -191,7 +191,7 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-icu</artifactId> <artifactId>lucene-analysis-icu</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>
@ -380,7 +380,7 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-icu</artifactId> <artifactId>lucene-analysis-icu</artifactId>
<version>9.0.0-SNAPSHOT</version> <version>9.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>

View File

@ -119,7 +119,7 @@ public class LLLocalDatabaseConnection implements LLDatabaseConnection {
if (connected.compareAndSet(true, false)) { if (connected.compareAndSet(true, false)) {
var env = this.env.get(); var env = this.env.get();
if (env != null) { if (env != null) {
env.close(Duration.ofSeconds(30)); env.close();
} }
} }
return null; return null;

View File

@ -20,8 +20,6 @@ public class LLTempLMDBEnv implements Closeable {
private static final long TWENTY_GIBIBYTES = 20L * 1024L * 1024L * 1024L; private static final long TWENTY_GIBIBYTES = 20L * 1024L * 1024L * 1024L;
private static final int MAX_DATABASES = 1024; private static final int MAX_DATABASES = 1024;
private final Phaser resources = new Phaser(1);
private final Path tempDirectory; private final Path tempDirectory;
private final Env<ByteBuf> env; private final Env<ByteBuf> env;
private volatile boolean closed; private volatile boolean closed;
@ -35,33 +33,15 @@ public class LLTempLMDBEnv implements Closeable {
env = envBuilder.open(tempDirectory.toFile(), MDB_NOTLS, MDB_WRITEMAP, MDB_NORDAHEAD); env = envBuilder.open(tempDirectory.toFile(), MDB_NOTLS, MDB_WRITEMAP, MDB_NORDAHEAD);
} }
public Env<ByteBuf> getEnvAndIncrementRef() { public Env<ByteBuf> getEnv() {
if (closed) { if (closed) {
throw new IllegalStateException("Environment closed"); throw new IllegalStateException("Environment closed");
} }
resources.register();
return env; return env;
} }
public void decrementRef() {
resources.arriveAndDeregister();
}
@Override @Override
public void close() throws IOException { public void close() throws IOException {
this.closed = true;
resources.arriveAndAwaitAdvance();
closeInternal();
}
public void close(Duration timeout) throws InterruptedException, TimeoutException, IOException {
this.closed = true;
int phase = resources.arrive();
resources.awaitAdvanceInterruptibly(phase, timeout.toMillis(), TimeUnit.MILLISECONDS);
closeInternal();
}
private void closeInternal() throws IOException {
this.closed = true; this.closed = true;
env.close(); env.close();
//noinspection ResultOfMethodCallIgnored //noinspection ResultOfMethodCallIgnored

View File

@ -8,24 +8,12 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLTempLMDBEnv; import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.util.Iterator;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.lmdbjava.CursorIterable;
import org.lmdbjava.CursorIterable.KeyVal;
import org.lmdbjava.Dbi; import org.lmdbjava.Dbi;
import org.lmdbjava.Env; import org.lmdbjava.Env;
import org.lmdbjava.GetOp;
import org.lmdbjava.Txn; import org.lmdbjava.Txn;
import reactor.core.publisher.Flux;
import reactor.core.scheduler.Scheduler;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuple3;
import reactor.util.function.Tuples;
public class LMDBArray<V> implements IArray<V>, Closeable { public class LMDBArray<V> implements IArray<V>, Closeable {
@ -35,7 +23,6 @@ public class LMDBArray<V> implements IArray<V>, Closeable {
private static final AtomicLong NEXT_LMDB_ARRAY_ID = new AtomicLong(0); private static final AtomicLong NEXT_LMDB_ARRAY_ID = new AtomicLong(0);
private final AtomicBoolean closed = new AtomicBoolean(); private final AtomicBoolean closed = new AtomicBoolean();
private final Runnable onClose;
private final LMDBCodec<V> valueCodec; private final LMDBCodec<V> valueCodec;
private final Env<ByteBuf> env; private final Env<ByteBuf> env;
private final Dbi<ByteBuf> lmdb; private final Dbi<ByteBuf> lmdb;
@ -49,10 +36,9 @@ public class LMDBArray<V> implements IArray<V>, Closeable {
private final long virtualSize; private final long virtualSize;
public LMDBArray(LLTempLMDBEnv env, LMDBCodec<V> codec, long size, @Nullable V defaultValue) { public LMDBArray(LLTempLMDBEnv env, LMDBCodec<V> codec, long size, @Nullable V defaultValue) {
this.onClose = env::decrementRef;
var name = "$array_" + NEXT_LMDB_ARRAY_ID.getAndIncrement(); var name = "$array_" + NEXT_LMDB_ARRAY_ID.getAndIncrement();
this.valueCodec = codec; this.valueCodec = codec;
this.env = env.getEnvAndIncrementRef(); this.env = env.getEnv();
this.lmdb = this.env.openDbi(name, MDB_CREATE); this.lmdb = this.env.openDbi(name, MDB_CREATE);
this.defaultValue = defaultValue; this.defaultValue = defaultValue;
@ -207,22 +193,18 @@ public class LMDBArray<V> implements IArray<V>, Closeable {
@Override @Override
public void close() throws IOException { public void close() throws IOException {
if (closed.compareAndSet(false, true)) { if (closed.compareAndSet(false, true)) {
try { ensureThread();
ensureThread(); if (rwTxn != null) {
if (rwTxn != null) { rwTxn.close();
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
} finally {
onClose.run();
} }
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
} }
} }

View File

@ -9,12 +9,10 @@ import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
import java.io.IOException; import java.io.IOException;
import java.util.Iterator; import java.util.Iterator;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import java.util.StringJoiner; import java.util.StringJoiner;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.lmdbjava.Cursor; import org.lmdbjava.Cursor;
import org.lmdbjava.CursorIterable; import org.lmdbjava.CursorIterable;
import org.lmdbjava.CursorIterable.KeyVal; import org.lmdbjava.CursorIterable.KeyVal;
@ -23,8 +21,6 @@ import org.lmdbjava.Env;
import org.lmdbjava.GetOp; import org.lmdbjava.GetOp;
import org.lmdbjava.Txn; import org.lmdbjava.Txn;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2; import reactor.util.function.Tuple2;
import reactor.util.function.Tuple3; import reactor.util.function.Tuple3;
import reactor.util.function.Tuples; import reactor.util.function.Tuples;
@ -38,7 +34,6 @@ public class LMDBPriorityQueue<T> implements PriorityQueue<T>, Reversable<Revers
private static final AtomicLong NEXT_ITEM_UID = new AtomicLong(0); private static final AtomicLong NEXT_ITEM_UID = new AtomicLong(0);
private final AtomicBoolean closed = new AtomicBoolean(); private final AtomicBoolean closed = new AtomicBoolean();
private final Runnable onClose;
private final LMDBSortedCodec<T> codec; private final LMDBSortedCodec<T> codec;
private final Env<ByteBuf> env; private final Env<ByteBuf> env;
private final Dbi<ByteBuf> lmdb; private final Dbi<ByteBuf> lmdb;
@ -54,10 +49,9 @@ public class LMDBPriorityQueue<T> implements PriorityQueue<T>, Reversable<Revers
private long size = 0; private long size = 0;
public LMDBPriorityQueue(LLTempLMDBEnv env, LMDBSortedCodec<T> codec) { public LMDBPriorityQueue(LLTempLMDBEnv env, LMDBSortedCodec<T> codec) {
this.onClose = env::decrementRef;
var name = "$queue_" + NEXT_LMDB_QUEUE_ID.getAndIncrement(); var name = "$queue_" + NEXT_LMDB_QUEUE_ID.getAndIncrement();
this.codec = codec; this.codec = codec;
this.env = env.getEnvAndIncrementRef(); this.env = env.getEnv();
this.lmdb = this.env.openDbi(name, codec::compareDirect, MDB_CREATE, MDB_DUPSORT, MDB_DUPFIXED); this.lmdb = this.env.openDbi(name, codec::compareDirect, MDB_CREATE, MDB_DUPSORT, MDB_DUPFIXED);
this.writing = true; this.writing = true;
@ -467,25 +461,21 @@ public class LMDBPriorityQueue<T> implements PriorityQueue<T>, Reversable<Revers
@Override @Override
public void close() throws IOException { public void close() throws IOException {
if (closed.compareAndSet(false, true)) { if (closed.compareAndSet(false, true)) {
try { ensureThread();
ensureThread(); if (cur != null) {
if (cur != null) { cur.close();
cur.close();
}
if (rwTxn != null) {
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
} finally {
onClose.run();
} }
if (rwTxn != null) {
rwTxn.close();
}
if (readTxn != null) {
readTxn.close();
}
try (var txn = env.txnWrite()) {
lmdb.drop(txn, true);
txn.commit();
}
lmdb.close();
} }
} }

View File

@ -119,8 +119,8 @@ public class TestLuceneIndex {
} }
@AfterAll @AfterAll
public static void afterAll() throws IOException, InterruptedException, TimeoutException { public static void afterAll() throws IOException {
ENV.close(Duration.ofSeconds(10)); ENV.close();
} }
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) { private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {

View File

@ -212,9 +212,9 @@ public class TestLuceneSearches {
} }
@AfterAll @AfterAll
public static void afterAll() throws IOException, InterruptedException, TimeoutException { public static void afterAll() throws IOException {
TEMP_DB_GENERATOR.closeTempDb(tempDb).block(); TEMP_DB_GENERATOR.closeTempDb(tempDb).block();
ENV.close(Duration.ofSeconds(10)); ENV.close();
ensureNoLeaks(allocator.allocator(), true, false); ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator); destroyAllocator(allocator);
} }