From 665cd730fd29fabf192d1495c0390f1d0d442ae9 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Wed, 2 Mar 2022 12:37:23 +0100 Subject: [PATCH] Initial partial implementation --- .github/workflows/maven-publish.yml | 20 -- .gitignore | 2 + pom.xml | 211 ++++++++++++++++++ .../dbengine/database/remote/QuicServer.java | 96 ++++++-- src/main/resources/log4j2.xml | 22 ++ 5 files changed, 318 insertions(+), 33 deletions(-) create mode 100644 .gitignore diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 436c30b..9431137 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -1,6 +1,3 @@ -# This workflow will build a package using Maven and then publish it to GitHub packages when a release is created -# For more information see: https://github.com/actions/setup-java#apache-maven-with-a-settings-path - name: Maven Package on: @@ -38,23 +35,6 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }} MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }} -# - name: Deploy lucene to Maven (Release) -# if: github.ref == 'refs/heads/master' -# shell: bash -# run: | -# cd lucene -# ./gradlew assemble -x test -x errorprone publishToMavenLocal -x signJarsPublication -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-core -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/core/build/libs/lucene-core-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-join -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/join/build/libs/lucene-join-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-analysis-common -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/analysis/common/build/libs/lucene-analysis-common-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-analysis-icu -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/analysis/icu/build/libs/lucene-analysis-icu-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-codecs -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/codecs/build/libs/lucene-codecs-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-backward-codecs -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/backward-codecs/build/libs/lucene-backward-codecs-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-queries -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/queries/build/libs/lucene-queries-9.0.0-SNAPSHOT-sources.jar -# mvn deploy:deploy-file -DgroupId=org.lucene -DartifactId=lucene-misc -Dversion=9.0.0-SNAPSHOT -DgeneratePom=true -Dpackaging=jar -DrepositoryId=mchv-release-distribution -Durl=https://mvn.mchv.eu/repository/mchv-snapshot -Dfile=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT.jar -Djavadoc=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT-javadoc.jar -Dsources=lucene/misc/build/libs/lucene-misc-9.0.0-SNAPSHOT-sources.jar -# env: -# MAVEN_USERNAME: ${{ secrets.MCHV_USERNAME }} -# MAVEN_PASSWORD: ${{ secrets.MCHV_TOKEN }} - name: Deploy to Maven (Release) if: github.ref == 'refs/heads/master' shell: bash diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..86c056e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +target/ diff --git a/pom.xml b/pom.xml index e69de29..e8465e4 100644 --- a/pom.xml +++ b/pom.xml @@ -0,0 +1,211 @@ + + 4.0.0 + + CavalliumDBEngine Server + + it.cavallium + dbengine-server + 1.0.${revision} + + jar + + UTF-8 + 0-SNAPSHOT + false + + + + mchv-release + MCHV Release Apache Maven Packages + https://mvn.mchv.eu/repository/mchv + + + mchv-snapshot + MCHV Snapshot Apache Maven Packages + https://mvn.mchv.eu/repository/mchv-snapshot + + + mulesoft-public-snapshots + MuleSoft public snapshots + https://repository.mulesoft.org/nexus/content/repositories/public + false + true + + + netty5-snapshots + Netty 5 snapshots + https://oss.sonatype.org/content/repositories/snapshots + true + true + + + + + mchv-release-distribution + MCHV Release Apache Maven Packages Distribution + https://mvn.mchv.eu/repository/mchv + + + mchv-snapshot-distribution + MCHV Snapshot Apache Maven Packages Distribution + https://mvn.mchv.eu/repository/mchv-snapshot + + + + scm:git:https://git.ignuranza.net/andreacavalli/CavalliumDBEngine-Server.git + scm:git:https://git.ignuranza.net/andreacavalli/CavalliumDBEngine-Server.git + HEAD + + + + + io.projectreactor + reactor-bom + 2020.0.16 + pom + import + + + + + + com.lmax + disruptor + 3.4.4 + + + io.projectreactor.netty + reactor-netty-core + + + it.cavallium + dbengine + 3.0.0-SNAPSHOT + + + io.projectreactor.netty + reactor-netty-core + + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.17.1 + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-api + + + + + org.apache.logging.log4j + log4j-core + 2.17.1 + + + + src/test/java + + + kr.motd.maven + os-maven-plugin + 1.5.0.Final + + + + + org.apache.maven.plugins + maven-install-plugin + 3.0.0-M1 + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.2 + + + compile + + copy-dependencies + + + ${project.build.directory}/lib + false + false + true + true + true + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + + + io.soabase.record-builder + record-builder-processor + 1.19 + + + + io.soabase.recordbuilder.processor.RecordBuilderProcessor + + 17 + 17 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + org.junit.jupiter + junit-jupiter-engine + 5.8.0-M1 + + + + --add-modules jdk.incubator.foreign -Dforeign.restricted=permit --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED + + + ci + ${dbengine.ci} + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + enforce + + + + + + + enforce + + + + + + + diff --git a/src/main/java/it/cavallium/dbengine/database/remote/QuicServer.java b/src/main/java/it/cavallium/dbengine/database/remote/QuicServer.java index 996ce26..55ca4b2 100644 --- a/src/main/java/it/cavallium/dbengine/database/remote/QuicServer.java +++ b/src/main/java/it/cavallium/dbengine/database/remote/QuicServer.java @@ -1,18 +1,51 @@ package it.cavallium.dbengine.database.remote; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.core.instrument.noop.NoopMeter; +import io.net5.buffer.api.DefaultBufferAllocators; import io.netty.handler.ssl.ClientAuth; import io.netty.incubator.codec.quic.InsecureQuicTokenHandler; +import io.netty.incubator.codec.quic.QuicConnectionIdGenerator; import io.netty.incubator.codec.quic.QuicSslContext; import io.netty.incubator.codec.quic.QuicSslContextBuilder; +import it.cavallium.dbengine.database.LLKeyValueDatabase; +import it.cavallium.dbengine.database.LLLuceneIndex; +import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection; +import it.cavallium.dbengine.database.remote.RPCCodecs.RPCClientBoundResponseDecoder; +import it.cavallium.dbengine.database.remote.RPCCodecs.RPCServerBoundRequestDecoder; +import it.cavallium.dbengine.rpc.current.data.GeneratedEntityId; +import it.cavallium.dbengine.rpc.current.data.GetDatabase; import it.cavallium.dbengine.rpc.current.data.ServerBoundRequest; import java.io.File; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Path; +import java.time.Duration; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import reactor.core.publisher.Mono; +import reactor.core.publisher.SignalType; public class QuicServer { + private static final Logger LOG = LogManager.getLogger(QuicServer.class); + private static LLLocalDatabaseConnection localDb; + + private static final AtomicLong nextResourceId = new AtomicLong(1); + private static final ConcurrentHashMap dbToId = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap dbById = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap indexToId = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap indexById = new ConcurrentHashMap<>(); + public static void main(String[] args) throws URISyntaxException { + localDb = new LLLocalDatabaseConnection(DefaultBufferAllocators.preferredAllocator(), + new CompositeMeterRegistry(), Path.of("."), false); String keyFileLocation = System.getProperty("it.cavalliumdb.keyFile", null); String keyFilePassword = System.getProperty("it.cavalliumdb.keyPassword", null); String certFileLocation = System.getProperty("it.cavalliumdb.certFile", null); @@ -28,22 +61,59 @@ public class QuicServer { .build(); var qs = reactor.netty.incubator.quic.QuicServer .create() - .port(bindURI.getPort()) - .host(bindURI.getHost()) .tokenHandler(InsecureQuicTokenHandler.INSTANCE) + .bindAddress(() -> new InetSocketAddress(bindURI.getHost(), bindURI.getPort())) .secure(sslContext) - .handleStream((in, out) -> in - .withConnection(conn -> conn.addHandler(new RPCServerBoundRequestDecoder())) - .receiveObject() - .doFirst(() -> { - System.out.println("###################################Stream created"); - }) - .cast(ServerBoundRequest.class) - .log() - .then() - ); - qs.warmup().block(); + .idleTimeout(Duration.ofSeconds(30)) + .connectionIdAddressGenerator(QuicConnectionIdGenerator.randomGenerator()) + .initialSettings(spec -> spec + .maxData(10000000) + .maxStreamDataBidirectionalLocal(1000000) + .maxStreamDataBidirectionalRemote(1000000) + .maxStreamsBidirectional(100) + .maxStreamsUnidirectional(100) + ) + .handleStream((in, out) -> { + var inConn = in.withConnection(conn -> conn.addHandler(new RPCClientBoundResponseDecoder())); + var outConn = out.withConnection(conn -> conn.addHandler(new RPCServerBoundRequestDecoder())); + return inConn + .receiveObject() + .cast(ServerBoundRequest.class) + .log("req", Level.INFO, SignalType.ON_NEXT) + .switchOnFirst((first, flux) -> { + if (first.hasValue()) { + var value = first.get(); + if (value instanceof GetDatabase getDatabase) { + return handleGetDatabase(getDatabase); + } else { + return Mono.error(new UnsupportedOperationException("Unsupported request type: " + first)); + } + } else { + return flux; + } + }) + .doOnError(ex -> LOG.error("Failed to handle a request", ex)) + .onErrorResume(ex -> Mono.empty()) + .transform(outConn::sendObject); + }); var conn = qs.bindNow(); conn.onDispose().block(); } + + private static Mono handleGetDatabase(GetDatabase getDatabase) { + Mono dbCreationMono = localDb + .getDatabase(getDatabase.name(), getDatabase.columns(), getDatabase.databaseOptions()) + .flatMap(db -> Mono.fromCallable(() -> { + long id = nextResourceId.getAndIncrement(); + dbById.put(id, db); + dbToId.put(getDatabase.name(), id); + return GeneratedEntityId.of(id); + })); + + Mono existingDbMono = Mono + .fromSupplier(() -> dbToId.get(getDatabase.name())) + .map(GeneratedEntityId::of); + + return existingDbMono.switchIfEmpty(dbCreationMono); + } } diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index e69de29..5cfb8ca 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + +