andreacavalli/rockserver
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Native image fixes

Browse Source
This commit is contained in:
Andrea Cavalli 2024-09-13 10:52:34 +02:00
parent aeadadd7b4
commit 8d2a6f9118
12 changed files with 173 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
pom.xml
View File

@ -20,6 +20,7 @@
<protobuf-plugin.version>0.6.1</protobuf-plugin.version>
<protobuf.version>3.25.3</protobuf.version>
<grpc.version>1.65.0</grpc.version>
<rainbowgum.version>0.7.0</rainbowgum.version>
</properties>
<repositories>
@ -178,24 +179,6 @@
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.4.1</version>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
@ -230,11 +213,72 @@
</activation>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>${slf4j.version}</version>
<groupId>io.jstach.rainbowgum</groupId>
<artifactId>rainbowgum-core</artifactId>
<version>${rainbowgum.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.jstach.rainbowgum</groupId>
<artifactId>rainbowgum-pattern</artifactId>
<version>${rainbowgum.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.jstach.rainbowgum</groupId>
<artifactId>rainbowgum-slf4j</artifactId>
<version>${rainbowgum.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.4.0</version>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>src/standalone/java/</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>fatjar</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.4.1</version>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>native</id>
@ -279,8 +323,19 @@
<buildArg>-H:IncludeResources=it/cavallium/rockserver/core/resources/build.properties</buildArg>
<buildArg>-H:DynamicProxyConfigurationFiles=proxy-config.json</buildArg>
<buildArg>-H:ReflectionConfigurationFiles=reflect-config.json</buildArg>
<buildArg>--initialize-at-run-time=org.rocksdb.RocksDB,org.rocksdb.RocksObject,org.rocksdb.RocksDBException</buildArg>
<!--<buildArg>_-no-fallback</buildArg>-->
<buildArg>--initialize-at-build-time=io.jstach.rainbowgum</buildArg>
<buildArg>--initialize-at-build-time=io.jstach.rainbowgum.slf4j</buildArg>
<buildArg>--initialize-at-build-time=org.slf4j</buildArg>
<buildArg>--initialize-at-build-time=io.netty.util.internal.logging</buildArg>
<buildArg>--initialize-at-run-time=org.rocksdb.RocksDB</buildArg>
<buildArg>--initialize-at-run-time=org.rocksdb.RocksObject</buildArg>
<buildArg>--initialize-at-run-time=org.rocksdb.RocksDBException</buildArg>
<buildArg>--initialize-at-run-time=io.netty.util.AbstractReferenceCounted</buildArg>
<buildArg>-dsa</buildArg>
<buildArg>--no-fallback</buildArg>
<buildArg>--gc=G1</buildArg>
<buildArg>--link-at-build-time</buildArg>
</buildArgs>

8
src/main/java/it/cavallium/rockserver/core/Main.java
View File

@ -12,16 +12,16 @@ import java.net.URISyntaxException;
import java.net.UnixDomainSocketAddress;
import java.nio.file.Path;
import java.util.concurrent.CountDownLatch;
import java.util.logging.Level;
import java.util.logging.Logger;
import net.sourceforge.argparse4j.ArgumentParsers;
import net.sourceforge.argparse4j.inf.ArgumentParser;
import net.sourceforge.argparse4j.inf.ArgumentParserException;
import net.sourceforge.argparse4j.inf.Namespace;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Main {
private static final Logger LOG = Logger.getLogger("rockserver-core");
private static final Logger LOG = LoggerFactory.getLogger("rockserver-core");
public static void main(String[] args) throws IOException, URISyntaxException {
ArgumentParser parser = ArgumentParsers.newFor("rockserver-core").build()
@ -106,7 +106,7 @@ public class Main {
clientBuilder.setName(name);
try (var connection = clientBuilder.build()) {
LOG.log(Level.INFO, "Connected to {0}", connection);
LOG.info("Connected to {}", connection);
thriftServerBuilder.setClient(connection);
grpcServerBuilder.setClient(connection);

13
src/main/java/it/cavallium/rockserver/core/client/LoggingClient.java
View File

@ -8,7 +8,8 @@ import java.net.URI;
import java.util.concurrent.CompletionStage;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class LoggingClient implements RocksDBConnection {
@ -21,7 +22,7 @@ public class LoggingClient implements RocksDBConnection {
this.client = client;
this.syncApi = new LoggingSyncApi(client.getSyncApi());
this.asyncApi = new LoggingAsyncApi(client.getAsyncApi());
this.logger = Logger.getLogger("db.requests");
this.logger = LoggerFactory.getLogger("db.requests");
}
@Override
@ -58,10 +59,10 @@ public class LoggingClient implements RocksDBConnection {
try {
result = syncApi.requestSync(req);
} catch (Throwable e) {
logger.log(Level.FINEST, "Request failed: {0} Error: {1}", new Object[] {req, e.getMessage()});
logger.trace("Request failed: {} Error: {}", req, e.getMessage());
throw e;
}
logger.log(Level.FINEST, "Request executed: {0} Result: {1}", new Object[] {req, result});
logger.trace("Request executed: {} Result: {}", req, result);
return result;
}
}
@ -78,9 +79,9 @@ public class LoggingClient implements RocksDBConnection {
public <R> CompletionStage<R> requestAsync(RocksDBAPICommand<R> req) {
return asyncApi.requestAsync(req).whenComplete((result, e) -> {
if (e != null) {
logger.log(Level.FINEST, "Request failed: {0} Error: {1}", new Object[] {req, e.getMessage()});
logger.trace("Request failed: {} Error: {}", req, e.getMessage());
} else {
logger.log(Level.FINEST, "Request executed: {0} Result: {1}", new Object[] {req, result});
logger.trace("Request executed: {} Result: {}", req, result);
}
});
}

9
src/main/java/it/cavallium/rockserver/core/impl/EmbeddedDB.java
View File

@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.cliffc.high_scale_lib.NonBlockingHashMapLong;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@ -60,6 +59,8 @@ import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
import org.rocksdb.Status.Code;
import org.rocksdb.WriteOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class EmbeddedDB implements RocksDBSyncAPI, Closeable {
@ -80,7 +81,7 @@ public class EmbeddedDB implements RocksDBSyncAPI, Closeable {
public EmbeddedDB(@Nullable Path path, String name, @Nullable Path embeddedConfigPath) throws IOException {
this.path = path;
this.logger = Logger.getLogger("db." + name);
this.logger = LoggerFactory.getLogger("db." + name);
this.columns = new NonBlockingHashMapLong<>();
this.txs = new NonBlockingHashMapLong<>();
this.its = new NonBlockingHashMapLong<>();
@ -120,7 +121,7 @@ public class EmbeddedDB implements RocksDBSyncAPI, Closeable {
}
}
if (Boolean.parseBoolean(System.getProperty("rockserver.core.print-config", "true"))) {
logger.log(Level.INFO, "Database configuration: {0}", ConfigPrinter.stringify(config));
logger.info("Database configuration: {}", ConfigPrinter.stringify(config));
}
}
@ -220,7 +221,7 @@ public class EmbeddedDB implements RocksDBSyncAPI, Closeable {
Utils.deleteDirectory(db.getPath());
}
} catch (TimeoutException e) {
logger.log(Level.SEVERE, "Some operations lasted more than 10 seconds, forcing database shutdown...");
logger.error("Some operations lasted more than 10 seconds, forcing database shutdown...");
}
}

7
src/main/java/it/cavallium/rockserver/core/impl/rocksdb/DatabaseTasks.java
View File

@ -4,9 +4,10 @@ import java.io.Closeable;
import java.io.IOException;
import java.time.Duration;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DatabaseTasks implements Closeable {
@ -19,7 +20,7 @@ public class DatabaseTasks implements Closeable {
public DatabaseTasks(RocksDB db, boolean inMemory, Duration delayWalFlushConfig) {
this.db = db;
this.inMemory = inMemory;
this.logger = Logger.getLogger("db." + db.getName() + ".tasks");
this.logger = LoggerFactory.getLogger("db." + db.getName() + ".tasks");
this.delayWalFlushConfig = inMemory ? Duration.ZERO : delayWalFlushConfig;
}
@ -37,7 +38,7 @@ public class DatabaseTasks implements Closeable {
try {
db.flushWal(true);
} catch (RocksDBException e) {
logger.log(Level.SEVERE, "Failed to flush database \"%s\" wal".formatted(db.getName()), e);
logger.error("Failed to flush database \"%s\" wal".formatted(db.getName()), e);
}
}
});

25
src/main/java/it/cavallium/rockserver/core/impl/rocksdb/RocksDBLoader.java
View File

@ -5,6 +5,14 @@ import it.cavallium.rockserver.core.config.*;
import java.io.InputStream;
import java.nio.file.StandardCopyOption;
import java.time.Duration;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.SequencedMap;
import java.util.logging.Level;
import org.github.gestalt.config.exceptions.GestaltException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@ -16,9 +24,8 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.slf4j.Logger;
import static it.cavallium.rockserver.core.common.Utils.mapList;
import static java.lang.Boolean.parseBoolean;
@ -365,7 +372,7 @@ public class RocksDBLoader {
try {
columnFamilyOptions.setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY);
} catch (Throwable ex) {
logger.log(Level.SEVERE, "Failed to set prepopulate blob cache", ex);
logger.error("Failed to set prepopulate blob cache", ex);
}
// This option is not supported with multiple db paths
@ -374,6 +381,8 @@ public class RocksDBLoader {
boolean dynamicLevelBytes = volumeConfigs.size() <= 1;
if (dynamicLevelBytes) {
columnFamilyOptions.setLevelCompactionDynamicLevelBytes(true);
columnFamilyOptions.setMaxBytesForLevelBase(10 * SizeUnit.GB);
columnFamilyOptions.setMaxBytesForLevelMultiplier(10);
} else {
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
// https://nightlies.apache.org/flink/flink-docs-release-1.3/api/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.html
@ -565,13 +574,11 @@ public class RocksDBLoader {
try {
for (ColumnFamilyHandle cfh : handles) {
var props = db.getProperty(cfh, "rocksdb.stats");
logger.log(Level.FINEST, "Stats for database column {1}: {2}",
new Object[]{new String(cfh.getName(), StandardCharsets.UTF_8),
props}
);
logger.trace("Stats for database column {}: {}", new String(cfh.getName(), StandardCharsets.UTF_8),
props);
}
} catch (RocksDBException ex) {
logger.log(Level.FINE, "Failed to obtain stats", ex);
logger.debug("Failed to obtain stats", ex);
}
var delayWalFlushConfig = getWalFlushDelayConfig(databaseOptions);

3
src/main/java/it/cavallium/rockserver/core/impl/rocksdb/RocksLogger.java
View File

@ -1,8 +1,7 @@
package it.cavallium.rockserver.core.impl.rocksdb;
import org.rocksdb.DBOptions;
import java.util.logging.Logger;
import org.slf4j.Logger;
public class RocksLogger {
public RocksLogger(DBOptions rocksdbOptions, Logger logger) {

5
src/main/java/it/cavallium/rockserver/core/server/GrpcServer.java
View File

@ -57,11 +57,12 @@ import java.net.InetSocketAddress;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.logging.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GrpcServer extends Server {
private static final Logger LOG = Logger.getLogger(GrpcServer.class.getName());
private static final Logger LOG = LoggerFactory.getLogger(GrpcServer.class.getName());
private final GrpcServerImpl grpc;
private final io.grpc.Server server;

5
src/main/java/it/cavallium/rockserver/core/server/ThriftServer.java
View File

@ -22,16 +22,17 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;
import java.util.logging.Logger;
import org.apache.thrift.TException;
import org.apache.thrift.server.TThreadedSelectorServer;
import org.apache.thrift.transport.TNonblockingServerSocket;
import org.apache.thrift.transport.TTransportException;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ThriftServer extends Server {
private static final Logger LOG = Logger.getLogger(ThriftServer.class.getName());
private static final Logger LOG = LoggerFactory.getLogger(ThriftServer.class.getName());
private static final OfByte BYTE_BE = ValueLayout.JAVA_BYTE.withOrder(ByteOrder.BIG_ENDIAN);
private final Thread thriftThread;

4
src/main/java/module-info.java
View File

@ -16,6 +16,10 @@ module rockserver.core {
requires jsr305;
requires com.google.common;
requires io.grpc.netty;
requires io.jstach.rainbowgum;
requires io.jstach.rainbowgum.pattern;
requires org.graalvm.nativeimage;
requires io.netty.common;
exports it.cavallium.rockserver.core.client;
exports it.cavallium.rockserver.core.common;

19
src/standalone/java/it/cavallium/rockserver/core/logging/Log4J2LoggerSubstitutions.java Normal file
View File

@ -0,0 +1,19 @@
package it.cavallium.rockserver.core.logging;
import com.oracle.svm.core.annotate.Substitute;
import com.oracle.svm.core.annotate.TargetClass;
import io.netty.util.internal.logging.InternalLoggerFactory;
@TargetClass(value = InternalLoggerFactory.class)
public final class Log4J2LoggerSubstitutions {
@Substitute
private static InternalLoggerFactory useLog4J2LoggerFactory(String name) {
return null;
}
@Substitute
private static InternalLoggerFactory useLog4JLoggerFactory(String name) {
return null;
}
}

29
src/standalone/java/it/cavallium/rockserver/core/logging/LoggerConfig.java Normal file
View File

@ -0,0 +1,29 @@
package it.cavallium.rockserver.core.logging;
import io.jstach.rainbowgum.LogConfig;
import io.jstach.rainbowgum.RainbowGum;
import io.jstach.rainbowgum.spi.RainbowGumServiceProvider.RainbowGumProvider;
import java.lang.System.Logger.Level;
import java.util.Optional;
import io.jstach.rainbowgum.pattern.format.PatternEncoderBuilder;
public class LoggerConfig implements RainbowGumProvider {
@Override
public Optional<RainbowGum> provide(LogConfig config) {
return RainbowGum.builder(config) //
.route(r -> {
r.level(Level.DEBUG, "com.myapp");
r.appender("console", a -> {
a.encoder(new PatternEncoderBuilder("console")
// We use the pattern encoder which follows logback pattern
// syntax.
.pattern("[%thread] %-5level %logger{15} - %msg%n")
// We use properties to override the above pattern if set.
.fromProperties(config.properties())
.build());
});
}) //
.optional();
}
}