Improve loading, update configs
This commit is contained in:
parent
73c1fa3ea7
commit
0727be5866
6
.gitignore
vendored
6
.gitignore
vendored
@ -1,4 +1,6 @@
|
|||||||
|
|
||||||
.idea/
|
.idea/
|
||||||
|
|
||||||
target/
|
target/
|
||||||
|
tmp-rockserver-db/
|
||||||
|
.directory
|
||||||
|
*.iml
|
||||||
|
hs_err_pid*.log
|
||||||
|
15
pom.xml
15
pom.xml
@ -12,7 +12,8 @@
|
|||||||
<maven.compiler.source>21</maven.compiler.source>
|
<maven.compiler.source>21</maven.compiler.source>
|
||||||
<maven.compiler.target>21</maven.compiler.target>
|
<maven.compiler.target>21</maven.compiler.target>
|
||||||
<native.maven.plugin.version>0.9.28</native.maven.plugin.version>
|
<native.maven.plugin.version>0.9.28</native.maven.plugin.version>
|
||||||
<gestalt.version>0.22.0</gestalt.version>
|
<gestalt.version>0.24.1</gestalt.version>
|
||||||
|
<rocksdb.version>8.8.1</rocksdb.version>
|
||||||
<imageName>rockserver-core</imageName>
|
<imageName>rockserver-core</imageName>
|
||||||
<mainClass>it.cavallium.rockserver.core.Main</mainClass>
|
<mainClass>it.cavallium.rockserver.core.Main</mainClass>
|
||||||
</properties>
|
</properties>
|
||||||
@ -21,7 +22,7 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.rocksdb</groupId>
|
<groupId>org.rocksdb</groupId>
|
||||||
<artifactId>rocksdbjni</artifactId>
|
<artifactId>rocksdbjni</artifactId>
|
||||||
<version>8.8.1</version>
|
<version>${rocksdb.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>net.sourceforge.argparse4j</groupId>
|
<groupId>net.sourceforge.argparse4j</groupId>
|
||||||
@ -75,6 +76,16 @@
|
|||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
<resources>
|
||||||
|
<resource>
|
||||||
|
<directory>src/main/filtered-resources</directory>
|
||||||
|
<filtering>true</filtering>
|
||||||
|
</resource>
|
||||||
|
<resource>
|
||||||
|
<directory>src/main/resources</directory>
|
||||||
|
<filtering>false</filtering>
|
||||||
|
</resource>
|
||||||
|
</resources>
|
||||||
<plugins>
|
<plugins>
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.codehaus.mojo</groupId>
|
<groupId>org.codehaus.mojo</groupId>
|
||||||
|
@ -0,0 +1,2 @@
|
|||||||
|
rockserver.core.version=${project.version}
|
||||||
|
rockserver.core.rocksdb.version=${rocksdb.version}
|
@ -5,6 +5,7 @@ import static java.util.Objects.requireNonNull;
|
|||||||
|
|
||||||
import inet.ipaddr.HostName;
|
import inet.ipaddr.HostName;
|
||||||
|
|
||||||
|
import it.cavallium.rockserver.core.impl.rocksdb.RocksDBLoader;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
@ -60,7 +61,7 @@ public class Main {
|
|||||||
}
|
}
|
||||||
|
|
||||||
LOG.info("Starting...");
|
LOG.info("Starting...");
|
||||||
RocksDB.loadLibrary();
|
RocksDBLoader.loadLibrary();
|
||||||
|
|
||||||
var rawUrl = ns.getString("url");
|
var rawUrl = ns.getString("url");
|
||||||
var name = ns.getString("name");
|
var name = ns.getString("name");
|
||||||
|
@ -28,6 +28,10 @@ public class RocksDBException extends RuntimeException {
|
|||||||
this(errorUniqueId, ex.getMessage());
|
this(errorUniqueId, ex.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public RocksDBException(RocksDBErrorType errorUniqueId, String message, org.rocksdb.RocksDBException ex) {
|
||||||
|
this(errorUniqueId, message + ": " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
public RocksDBErrorType getErrorUniqueId() {
|
public RocksDBErrorType getErrorUniqueId() {
|
||||||
return errorUniqueId;
|
return errorUniqueId;
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,67 @@
|
|||||||
|
package it.cavallium.rockserver.core.config;
|
||||||
|
|
||||||
|
import it.cavallium.rockserver.core.common.RocksDBException;
|
||||||
|
import it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType;
|
||||||
|
import it.cavallium.rockserver.core.impl.DataSizeDecoder;
|
||||||
|
import it.cavallium.rockserver.core.impl.DbCompressionDecoder;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import org.github.gestalt.config.builder.GestaltBuilder;
|
||||||
|
import org.github.gestalt.config.builder.SourceBuilder;
|
||||||
|
import org.github.gestalt.config.exceptions.GestaltException;
|
||||||
|
import org.github.gestalt.config.source.ClassPathConfigSourceBuilder;
|
||||||
|
import org.github.gestalt.config.source.FileConfigSourceBuilder;
|
||||||
|
|
||||||
|
public class ConfigParser {
|
||||||
|
|
||||||
|
private final GestaltBuilder gsb;
|
||||||
|
private final List<SourceBuilder<?, ?>> sourceBuilders = new ArrayList<>();
|
||||||
|
|
||||||
|
public ConfigParser() {
|
||||||
|
gsb = new GestaltBuilder();
|
||||||
|
gsb
|
||||||
|
.setTreatMissingArrayIndexAsError(false)
|
||||||
|
.setTreatEmptyCollectionAsErrors(false)
|
||||||
|
.setTreatNullValuesInClassAsErrors(false)
|
||||||
|
.setTreatMissingValuesAsErrors(false)
|
||||||
|
.addDecoder(new DataSizeDecoder())
|
||||||
|
.addDecoder(new DbCompressionDecoder())
|
||||||
|
.addDefaultConfigLoaders()
|
||||||
|
.addDefaultDecoders();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DatabaseConfig parse(Path configPath) {
|
||||||
|
var parser = new ConfigParser();
|
||||||
|
parser.addSource(configPath);
|
||||||
|
return parser.parse();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DatabaseConfig parseDefault() {
|
||||||
|
var parser = new ConfigParser();
|
||||||
|
return parser.parse();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void addSource(Path path) {
|
||||||
|
if (path != null) {
|
||||||
|
sourceBuilders.add(FileConfigSourceBuilder.builder().setPath(path));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatabaseConfig parse() {
|
||||||
|
try {
|
||||||
|
gsb.addSource(ClassPathConfigSourceBuilder
|
||||||
|
.builder().setResource("it/cavallium/rockserver/core/resources/default.conf").build());
|
||||||
|
for (SourceBuilder<?, ?> sourceBuilder : sourceBuilders) {
|
||||||
|
gsb.addSource(sourceBuilder.build());
|
||||||
|
}
|
||||||
|
var gestalt = gsb.build();
|
||||||
|
gestalt.loadConfigs();
|
||||||
|
|
||||||
|
return gestalt.getConfig("database", DatabaseConfig.class);
|
||||||
|
} catch (GestaltException ex) {
|
||||||
|
throw new RocksDBException(RocksDBErrorType.CONFIG_ERROR, ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,5 +1,7 @@
|
|||||||
package it.cavallium.rockserver.core.config;
|
package it.cavallium.rockserver.core.config;
|
||||||
|
|
||||||
|
import it.cavallium.rockserver.core.common.RocksDBException;
|
||||||
|
import it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType;
|
||||||
import org.github.gestalt.config.exceptions.GestaltException;
|
import org.github.gestalt.config.exceptions.GestaltException;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
@ -15,6 +17,14 @@ public class ConfigPrinter {
|
|||||||
""".formatted(o.bitsPerKey(), o.optimizeForHits());
|
""".formatted(o.bitsPerKey(), o.optimizeForHits());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String stringify(DatabaseConfig config) {
|
||||||
|
try {
|
||||||
|
return stringifyDatabase(config);
|
||||||
|
} catch (GestaltException e) {
|
||||||
|
throw new RocksDBException(RocksDBErrorType.CONFIG_ERROR, "Can't stringify config", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static String stringifyDatabase(DatabaseConfig o) throws GestaltException {
|
public static String stringifyDatabase(DatabaseConfig o) throws GestaltException {
|
||||||
return """
|
return """
|
||||||
{
|
{
|
||||||
@ -31,11 +41,11 @@ public class ConfigPrinter {
|
|||||||
""".formatted(o.compression(), o.maxDictBytes());
|
""".formatted(o.compression(), o.maxDictBytes());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<VolumeConfig> getVolumeConfigs(GlobalDatabaseConfig g) throws GestaltException {
|
public static List<VolumeConfig> getVolumeConfigs(GlobalDatabaseConfig g) throws GestaltException {
|
||||||
try {
|
try {
|
||||||
return List.of(g.volumes());
|
return List.of(g.volumes());
|
||||||
} catch (GestaltException ex) {
|
} catch (GestaltException ex) {
|
||||||
if (ex.getMessage().equals("Failed to get proxy config while calling method: volumes in path: database.global.")) {
|
if (ex.getMessage().startsWith("Failed to get cached object from proxy config while calling method:")) {
|
||||||
return List.of();
|
return List.of();
|
||||||
} else {
|
} else {
|
||||||
throw ex;
|
throw ex;
|
||||||
@ -92,10 +102,10 @@ public class ConfigPrinter {
|
|||||||
return """
|
return """
|
||||||
{
|
{
|
||||||
"volume-path": "%s",
|
"volume-path": "%s",
|
||||||
"target-size-bytes": %b
|
"target-size-bytes": "%s"
|
||||||
}\
|
}\
|
||||||
""".formatted(o.volumePath(),
|
""".formatted(o.volumePath(),
|
||||||
o.targetSizeBytes()
|
o.targetSize()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8,5 +8,5 @@ public interface VolumeConfig {
|
|||||||
|
|
||||||
Path volumePath() throws GestaltException;
|
Path volumePath() throws GestaltException;
|
||||||
|
|
||||||
long targetSizeBytes() throws GestaltException;
|
DataSize targetSize() throws GestaltException;
|
||||||
}
|
}
|
||||||
|
@ -3,14 +3,16 @@ package it.cavallium.rockserver.core.impl;
|
|||||||
import it.cavallium.rockserver.core.config.DataSize;
|
import it.cavallium.rockserver.core.config.DataSize;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import org.github.gestalt.config.decoder.Decoder;
|
import org.github.gestalt.config.decoder.Decoder;
|
||||||
|
import org.github.gestalt.config.decoder.DecoderContext;
|
||||||
import org.github.gestalt.config.decoder.DecoderService;
|
import org.github.gestalt.config.decoder.DecoderService;
|
||||||
import org.github.gestalt.config.decoder.Priority;
|
import org.github.gestalt.config.decoder.Priority;
|
||||||
import org.github.gestalt.config.entity.ValidationError;
|
import org.github.gestalt.config.entity.ValidationError;
|
||||||
import org.github.gestalt.config.node.ConfigNode;
|
import org.github.gestalt.config.node.ConfigNode;
|
||||||
import org.github.gestalt.config.reflect.TypeCapture;
|
import org.github.gestalt.config.reflect.TypeCapture;
|
||||||
|
import org.github.gestalt.config.tag.Tags;
|
||||||
import org.github.gestalt.config.utils.ValidateOf;
|
import org.github.gestalt.config.utils.ValidateOf;
|
||||||
|
|
||||||
class DataSizeDecoder implements Decoder<DataSize> {
|
public class DataSizeDecoder implements Decoder<DataSize> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Priority priority() {
|
public Priority priority() {
|
||||||
@ -23,12 +25,16 @@ class DataSizeDecoder implements Decoder<DataSize> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean matches(TypeCapture klass) {
|
public boolean canDecode(String path, Tags tags, ConfigNode node, TypeCapture<?> type) {
|
||||||
return klass != null && klass.isAssignableFrom(DataSize.class);
|
return type != null && type.isAssignableFrom(DataSize.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ValidateOf<DataSize> decode(String path, ConfigNode node, TypeCapture type, DecoderService decoderService) {
|
public ValidateOf<DataSize> decode(String path,
|
||||||
|
Tags tags,
|
||||||
|
ConfigNode node,
|
||||||
|
TypeCapture<?> type,
|
||||||
|
DecoderContext decoderContext) {
|
||||||
try {
|
try {
|
||||||
return ValidateOf.validateOf(new DataSize(node.getValue().orElseThrow()), List.of());
|
return ValidateOf.validateOf(new DataSize(node.getValue().orElseThrow()), List.of());
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
|
@ -4,11 +4,13 @@ import it.cavallium.rockserver.core.config.DataSize;
|
|||||||
import it.cavallium.rockserver.core.config.DatabaseCompression;
|
import it.cavallium.rockserver.core.config.DatabaseCompression;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import org.github.gestalt.config.decoder.Decoder;
|
import org.github.gestalt.config.decoder.Decoder;
|
||||||
|
import org.github.gestalt.config.decoder.DecoderContext;
|
||||||
import org.github.gestalt.config.decoder.DecoderService;
|
import org.github.gestalt.config.decoder.DecoderService;
|
||||||
import org.github.gestalt.config.decoder.Priority;
|
import org.github.gestalt.config.decoder.Priority;
|
||||||
import org.github.gestalt.config.entity.ValidationError;
|
import org.github.gestalt.config.entity.ValidationError;
|
||||||
import org.github.gestalt.config.node.ConfigNode;
|
import org.github.gestalt.config.node.ConfigNode;
|
||||||
import org.github.gestalt.config.reflect.TypeCapture;
|
import org.github.gestalt.config.reflect.TypeCapture;
|
||||||
|
import org.github.gestalt.config.tag.Tags;
|
||||||
import org.github.gestalt.config.utils.ValidateOf;
|
import org.github.gestalt.config.utils.ValidateOf;
|
||||||
import org.rocksdb.CompressionType;
|
import org.rocksdb.CompressionType;
|
||||||
|
|
||||||
@ -25,15 +27,16 @@ public class DbCompressionDecoder implements Decoder<CompressionType> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean matches(TypeCapture<?> klass) {
|
public boolean canDecode(String path, Tags tags, ConfigNode node, TypeCapture<?> type) {
|
||||||
return klass != null && klass.isAssignableFrom(CompressionType.class);
|
return type != null && type.isAssignableFrom(CompressionType.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ValidateOf<CompressionType> decode(String path,
|
public ValidateOf<CompressionType> decode(String path,
|
||||||
|
Tags tags,
|
||||||
ConfigNode node,
|
ConfigNode node,
|
||||||
TypeCapture<?> type,
|
TypeCapture<?> type,
|
||||||
DecoderService decoderService) {
|
DecoderContext decoderContext) {
|
||||||
try {
|
try {
|
||||||
return ValidateOf.validateOf(DatabaseCompression.valueOf(node.getValue().orElseThrow()).compressionType(), List.of());
|
return ValidateOf.validateOf(DatabaseCompression.valueOf(node.getValue().orElseThrow()).compressionType(), List.of());
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
|
@ -19,6 +19,7 @@ import it.cavallium.rockserver.core.common.Delta;
|
|||||||
import it.cavallium.rockserver.core.common.RocksDBAPI;
|
import it.cavallium.rockserver.core.common.RocksDBAPI;
|
||||||
import it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType;
|
import it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType;
|
||||||
import it.cavallium.rockserver.core.common.Utils;
|
import it.cavallium.rockserver.core.common.Utils;
|
||||||
|
import it.cavallium.rockserver.core.config.ConfigParser;
|
||||||
import it.cavallium.rockserver.core.config.ConfigPrinter;
|
import it.cavallium.rockserver.core.config.ConfigPrinter;
|
||||||
import it.cavallium.rockserver.core.config.DatabaseConfig;
|
import it.cavallium.rockserver.core.config.DatabaseConfig;
|
||||||
import it.cavallium.rockserver.core.impl.rocksdb.REntry;
|
import it.cavallium.rockserver.core.impl.rocksdb.REntry;
|
||||||
@ -45,7 +46,10 @@ import org.cliffc.high_scale_lib.NonBlockingHashMapLong;
|
|||||||
import org.github.gestalt.config.builder.GestaltBuilder;
|
import org.github.gestalt.config.builder.GestaltBuilder;
|
||||||
import org.github.gestalt.config.exceptions.GestaltException;
|
import org.github.gestalt.config.exceptions.GestaltException;
|
||||||
import org.github.gestalt.config.source.ClassPathConfigSource;
|
import org.github.gestalt.config.source.ClassPathConfigSource;
|
||||||
|
import org.github.gestalt.config.source.ClassPathConfigSourceBuilder;
|
||||||
import org.github.gestalt.config.source.FileConfigSource;
|
import org.github.gestalt.config.source.FileConfigSource;
|
||||||
|
import org.github.gestalt.config.source.FileConfigSourceBuilder;
|
||||||
|
import org.github.gestalt.config.source.StringConfigSourceBuilder;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.rocksdb.*;
|
import org.rocksdb.*;
|
||||||
import org.rocksdb.Status.Code;
|
import org.rocksdb.Status.Code;
|
||||||
@ -75,28 +79,10 @@ public class EmbeddedDB implements RocksDBAPI, Closeable {
|
|||||||
this.its = new NonBlockingHashMapLong<>();
|
this.its = new NonBlockingHashMapLong<>();
|
||||||
this.columnNamesIndex = new ConcurrentHashMap<>();
|
this.columnNamesIndex = new ConcurrentHashMap<>();
|
||||||
this.ops = new SafeShutdown();
|
this.ops = new SafeShutdown();
|
||||||
|
this.config = ConfigParser.parse(this.embeddedConfigPath);
|
||||||
var gsb = new GestaltBuilder();
|
this.db = RocksDBLoader.load(path, config, logger);
|
||||||
try {
|
if (Boolean.parseBoolean(System.getProperty("rockserver.core.print-config", "true"))) {
|
||||||
gsb.addSource(new ClassPathConfigSource("it/cavallium/rockserver/core/resources/default.conf"));
|
logger.log(Level.INFO, "Database configuration: {0}", ConfigPrinter.stringify(this.config));
|
||||||
if (embeddedConfigPath != null) {
|
|
||||||
gsb.addSource(new FileConfigSource(this.embeddedConfigPath));
|
|
||||||
}
|
|
||||||
var gestalt = gsb
|
|
||||||
.addDecoder(new DataSizeDecoder())
|
|
||||||
.addDecoder(new DbCompressionDecoder())
|
|
||||||
.addDefaultConfigLoaders()
|
|
||||||
.addDefaultDecoders()
|
|
||||||
.build();
|
|
||||||
gestalt.loadConfigs();
|
|
||||||
|
|
||||||
this.config = gestalt.getConfig("database", DatabaseConfig.class);
|
|
||||||
this.db = RocksDBLoader.load(path, config, logger);
|
|
||||||
if (Boolean.parseBoolean(System.getProperty("rockserver.core.print-config", "true"))) {
|
|
||||||
logger.log(Level.INFO, "Database configuration: {0}", ConfigPrinter.stringifyDatabase(this.config));
|
|
||||||
}
|
|
||||||
} catch (GestaltException e) {
|
|
||||||
throw new it.cavallium.rockserver.core.common.RocksDBException(RocksDBErrorType.CONFIG_ERROR, e);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,17 +1,18 @@
|
|||||||
package it.cavallium.rockserver.core.impl.rocksdb;
|
package it.cavallium.rockserver.core.impl.rocksdb;
|
||||||
|
|
||||||
|
import it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType;
|
||||||
import it.cavallium.rockserver.core.config.*;
|
import it.cavallium.rockserver.core.config.*;
|
||||||
import org.github.gestalt.config.exceptions.GestaltException;
|
import org.github.gestalt.config.exceptions.GestaltException;
|
||||||
|
import org.jetbrains.annotations.NotNull;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.rocksdb.*;
|
import org.rocksdb.*;
|
||||||
|
import org.rocksdb.util.Environment;
|
||||||
import org.rocksdb.util.SizeUnit;
|
import org.rocksdb.util.SizeUnit;
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.Paths;
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.logging.Level;
|
import java.util.logging.Level;
|
||||||
import java.util.logging.Logger;
|
import java.util.logging.Logger;
|
||||||
@ -31,29 +32,56 @@ public class RocksDBLoader {
|
|||||||
= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.clockcache.enable", "false"));
|
= Boolean.parseBoolean(System.getProperty("it.cavallium.dbengine.clockcache.enable", "false"));
|
||||||
private static final CacheFactory CACHE_FACTORY = USE_CLOCK_CACHE ? new ClockCacheFactory() : new LRUCacheFactory();
|
private static final CacheFactory CACHE_FACTORY = USE_CLOCK_CACHE ? new ClockCacheFactory() : new LRUCacheFactory();
|
||||||
|
|
||||||
|
public static void loadLibrary() {
|
||||||
|
RocksDB.loadLibrary();
|
||||||
|
/* todo: rocksdb does not support loading the library outside of the default mechanism
|
||||||
|
try {
|
||||||
|
var jniPath = Path.of(".").resolve("jni").resolve(RocksDBMetadata.getRocksDBVersionHash());
|
||||||
|
if (Files.notExists(jniPath)) {
|
||||||
|
Files.createDirectories(jniPath);
|
||||||
|
}
|
||||||
|
// todo:
|
||||||
|
} catch (IOException e) {
|
||||||
|
RocksDB.loadLibrary();
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public static TransactionalDB load(@Nullable Path path, DatabaseConfig config, Logger logger) {
|
public static TransactionalDB load(@Nullable Path path, DatabaseConfig config, Logger logger) {
|
||||||
var refs = new RocksDBObjects();
|
var refs = new RocksDBObjects();
|
||||||
var optionsWithCache = makeRocksDBOptions(path, config, refs, logger);
|
// Get databases directory path
|
||||||
return loadDb(path, config, optionsWithCache, refs, logger);
|
Path definitiveDbPath;
|
||||||
|
if (path != null) {
|
||||||
|
definitiveDbPath = path.toAbsolutePath();
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
definitiveDbPath = Files.createTempDirectory("temp-rocksdb");
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var optionsWithCache = makeRocksDBOptions(path, definitiveDbPath, config, refs, logger);
|
||||||
|
return loadDb(path, definitiveDbPath, config, optionsWithCache, refs, logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
record OptionsWithCache(DBOptions options, @Nullable Cache standardCache) {}
|
record OptionsWithCache(DBOptions options, @Nullable Cache standardCache) {}
|
||||||
|
|
||||||
private static OptionsWithCache makeRocksDBOptions(@Nullable Path path, DatabaseConfig databaseOptions, RocksDBObjects refs, Logger logger) {
|
private static OptionsWithCache makeRocksDBOptions(@Nullable Path path,
|
||||||
|
Path definitiveDbPath,
|
||||||
|
DatabaseConfig databaseOptions,
|
||||||
|
RocksDBObjects refs,
|
||||||
|
Logger logger) {
|
||||||
try {
|
try {
|
||||||
// Get databases directory path
|
// Get databases directory path
|
||||||
Path databasesDirPath;
|
Path parentPath;
|
||||||
if (path != null) {
|
if (path != null) {
|
||||||
databasesDirPath = path.toAbsolutePath().getParent();
|
parentPath = path.toAbsolutePath().getParent();
|
||||||
// Create base directories
|
|
||||||
if (Files.notExists(databasesDirPath)) {
|
|
||||||
Files.createDirectories(databasesDirPath);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
databasesDirPath = null;
|
parentPath = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
List<VolumeConfig> volumeConfigs = getVolumeConfigs(databaseOptions);
|
List<DbPathRecord> volumeConfigs = getVolumeConfigs(definitiveDbPath, databaseOptions);
|
||||||
|
|
||||||
// the Options class contains a set of configurable DB options
|
// the Options class contains a set of configurable DB options
|
||||||
// that determines the behaviour of the database.
|
// that determines the behaviour of the database.
|
||||||
@ -71,14 +99,10 @@ public class RocksDBLoader {
|
|||||||
options.setDelayedWriteRate(customWriteRate);
|
options.setDelayedWriteRate(customWriteRate);
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional.ofNullable(databaseOptions.global().logPath())
|
getLogPath(definitiveDbPath, databaseOptions).map(Path::toString).ifPresent(options::setDbLogDir);
|
||||||
.map(Path::toString)
|
|
||||||
.ifPresent(options::setDbLogDir);
|
|
||||||
|
|
||||||
if (path != null) {
|
if (path != null) {
|
||||||
Optional.ofNullable(databaseOptions.global().walPath())
|
getWalDir(definitiveDbPath, databaseOptions).map(Path::toString).ifPresent(options::setWalDir);
|
||||||
.map(Path::toString)
|
|
||||||
.ifPresent(options::setWalDir);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
options.setCreateIfMissing(true);
|
options.setCreateIfMissing(true);
|
||||||
@ -97,14 +121,12 @@ public class RocksDBLoader {
|
|||||||
options.setDeleteObsoleteFilesPeriodMicros(20 * 1000000); // 20 seconds
|
options.setDeleteObsoleteFilesPeriodMicros(20 * 1000000); // 20 seconds
|
||||||
options.setKeepLogFileNum(10);
|
options.setKeepLogFileNum(10);
|
||||||
|
|
||||||
if (databasesDirPath != null) {
|
if (parentPath != null) {
|
||||||
requireNonNull(databasesDirPath);
|
requireNonNull(parentPath);
|
||||||
requireNonNull(path.getFileName());
|
requireNonNull(path.getFileName());
|
||||||
List<DbPath> paths = mapList(convertPaths(databasesDirPath, path.getFileName(), volumeConfigs),
|
List<DbPath> paths = mapList(volumeConfigs, p -> new DbPath(p.path, p.targetSize));
|
||||||
p -> new DbPath(p.path, p.targetSize)
|
|
||||||
);
|
|
||||||
options.setDbPaths(paths);
|
options.setDbPaths(paths);
|
||||||
} else if (!volumeConfigs.isEmpty()) {
|
} else if (!volumeConfigs.isEmpty() && (volumeConfigs.size() > 1 || definitiveDbPath.relativize(volumeConfigs.getFirst().path).isAbsolute())) {
|
||||||
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.CONFIG_ERROR, "in-memory databases should not have any volume configured");
|
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.CONFIG_ERROR, "in-memory databases should not have any volume configured");
|
||||||
}
|
}
|
||||||
options.setMaxOpenFiles(Optional.ofNullable(databaseOptions.global().maximumOpenFiles()).orElse(-1));
|
options.setMaxOpenFiles(Optional.ofNullable(databaseOptions.global().maximumOpenFiles()).orElse(-1));
|
||||||
@ -195,30 +217,63 @@ public class RocksDBLoader {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return new OptionsWithCache(options, blockCache);
|
return new OptionsWithCache(options, blockCache);
|
||||||
} catch (IOException e) {
|
|
||||||
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.ROCKSDB_LOAD_ERROR, e);
|
|
||||||
} catch (GestaltException e) {
|
} catch (GestaltException e) {
|
||||||
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.ROCKSDB_CONFIG_ERROR, e);
|
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.ROCKSDB_CONFIG_ERROR, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<VolumeConfig> getVolumeConfigs(DatabaseConfig databaseOptions) throws GestaltException {
|
private static Optional<Path> getWalDir(Path definitiveDbPath, DatabaseConfig databaseOptions)
|
||||||
try {
|
throws GestaltException {
|
||||||
return List.of(databaseOptions.global().volumes());
|
return Optional.ofNullable(databaseOptions.global().walPath())
|
||||||
} catch (GestaltException ex) {
|
.map(definitiveDbPath::resolve);
|
||||||
if (ex.getMessage().equals("Failed to get proxy config while calling method: volumes in path: database.global.")) {
|
|
||||||
return List.of();
|
|
||||||
} else {
|
|
||||||
throw ex;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static TransactionalDB loadDb(@Nullable Path path, DatabaseConfig databaseOptions, OptionsWithCache optionsWithCache, RocksDBObjects refs, Logger logger) {
|
private static Optional<Path> getLogPath(Path definitiveDbPath, DatabaseConfig databaseOptions)
|
||||||
|
throws GestaltException {
|
||||||
|
return Optional.ofNullable(databaseOptions.global().logPath())
|
||||||
|
.map(definitiveDbPath::resolve);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<DbPathRecord> getVolumeConfigs(@NotNull Path definitiveDbPath, DatabaseConfig databaseOptions)
|
||||||
|
throws GestaltException {
|
||||||
|
return ConfigPrinter
|
||||||
|
.getVolumeConfigs(databaseOptions.global())
|
||||||
|
.stream()
|
||||||
|
.map(volumeConfig -> {
|
||||||
|
try {
|
||||||
|
return new DbPathRecord(definitiveDbPath.resolve(volumeConfig.volumePath()), volumeConfig.targetSize().longValue());
|
||||||
|
} catch (GestaltException e) {
|
||||||
|
throw new it.cavallium.rockserver.core.common.RocksDBException(RocksDBErrorType.CONFIG_ERROR, "Failed to load volume configurations", e);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TransactionalDB loadDb(@Nullable Path path,
|
||||||
|
@NotNull Path definitiveDbPath,
|
||||||
|
DatabaseConfig databaseOptions, OptionsWithCache optionsWithCache, RocksDBObjects refs, Logger logger) {
|
||||||
var rocksdbOptions = optionsWithCache.options();
|
var rocksdbOptions = optionsWithCache.options();
|
||||||
try {
|
try {
|
||||||
List<VolumeConfig> volumeConfigs = getVolumeConfigs(databaseOptions);
|
List<DbPathRecord> volumeConfigs = getVolumeConfigs(definitiveDbPath, databaseOptions);
|
||||||
List<ColumnFamilyDescriptor> descriptors = new ArrayList<>();
|
List<ColumnFamilyDescriptor> descriptors = new ArrayList<>();
|
||||||
|
var walPath = getWalDir(definitiveDbPath, databaseOptions);
|
||||||
|
var logPath = getLogPath(definitiveDbPath, databaseOptions);
|
||||||
|
|
||||||
|
// Create base directories
|
||||||
|
if (Files.notExists(definitiveDbPath)) {
|
||||||
|
Files.createDirectories(definitiveDbPath);
|
||||||
|
}
|
||||||
|
for (DbPathRecord volumeConfig : volumeConfigs) {
|
||||||
|
if (Files.notExists(volumeConfig.path)) {
|
||||||
|
Files.createDirectories(volumeConfig.path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (walPath.isPresent() && Files.notExists(walPath.get())) {
|
||||||
|
Files.createDirectories(walPath.get());
|
||||||
|
}
|
||||||
|
if (logPath.isPresent() && Files.notExists(logPath.get())) {
|
||||||
|
Files.createDirectories(logPath.get());
|
||||||
|
}
|
||||||
|
|
||||||
var defaultColumnOptions = new ColumnFamilyOptions();
|
var defaultColumnOptions = new ColumnFamilyOptions();
|
||||||
refs.add(defaultColumnOptions);
|
refs.add(defaultColumnOptions);
|
||||||
@ -444,35 +499,22 @@ public class RocksDBLoader {
|
|||||||
descriptors.add(new ColumnFamilyDescriptor(name.getBytes(StandardCharsets.US_ASCII), columnFamilyOptions));
|
descriptors.add(new ColumnFamilyDescriptor(name.getBytes(StandardCharsets.US_ASCII), columnFamilyOptions));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get databases directory path
|
|
||||||
String definitiveDbPathString;
|
|
||||||
if (path != null) {
|
|
||||||
Path databasesDirPath = path.toAbsolutePath().getParent();
|
|
||||||
definitiveDbPathString = databasesDirPath.toString() + File.separatorChar + path.getFileName();
|
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
definitiveDbPathString = Files.createTempDirectory("temp-rocksdb").toString();
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var handles = new ArrayList<ColumnFamilyHandle>();
|
var handles = new ArrayList<ColumnFamilyHandle>();
|
||||||
RocksDB db;
|
RocksDB db;
|
||||||
// a factory method that returns a RocksDB instance
|
// a factory method that returns a RocksDB instance
|
||||||
if (databaseOptions.global().optimistic()) {
|
if (databaseOptions.global().optimistic()) {
|
||||||
db = OptimisticTransactionDB.open(rocksdbOptions, definitiveDbPathString, descriptors, handles);
|
db = OptimisticTransactionDB.open(rocksdbOptions, definitiveDbPath.toString(), descriptors, handles);
|
||||||
} else {
|
} else {
|
||||||
var transactionOptions = new TransactionDBOptions()
|
var transactionOptions = new TransactionDBOptions()
|
||||||
.setWritePolicy(TxnDBWritePolicy.WRITE_COMMITTED)
|
.setWritePolicy(TxnDBWritePolicy.WRITE_COMMITTED)
|
||||||
.setTransactionLockTimeout(5000)
|
.setTransactionLockTimeout(5000)
|
||||||
.setDefaultLockTimeout(5000);
|
.setDefaultLockTimeout(5000);
|
||||||
refs.add(transactionOptions);
|
refs.add(transactionOptions);
|
||||||
db = TransactionDB.open(rocksdbOptions,
|
db = TransactionDB.open(rocksdbOptions,
|
||||||
transactionOptions,
|
transactionOptions,
|
||||||
definitiveDbPathString,
|
definitiveDbPath.toString(),
|
||||||
descriptors,
|
descriptors,
|
||||||
handles
|
handles
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -489,7 +531,9 @@ public class RocksDBLoader {
|
|||||||
} catch (RocksDBException ex) {
|
} catch (RocksDBException ex) {
|
||||||
logger.log(Level.FINE, "Failed to obtain stats", ex);
|
logger.log(Level.FINE, "Failed to obtain stats", ex);
|
||||||
}
|
}
|
||||||
return TransactionalDB.create(definitiveDbPathString, db);
|
return TransactionalDB.create(definitiveDbPath.toString(), db);
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.ROCKSDB_LOAD_ERROR, "Failed to load rocksdb", ex);
|
||||||
} catch (RocksDBException ex) {
|
} catch (RocksDBException ex) {
|
||||||
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.ROCKSDB_LOAD_ERROR, "Failed to load rocksdb", ex);
|
throw new it.cavallium.rockserver.core.common.RocksDBException(it.cavallium.rockserver.core.common.RocksDBException.RocksDBErrorType.ROCKSDB_LOAD_ERROR, "Failed to load rocksdb", ex);
|
||||||
} catch (GestaltException e) {
|
} catch (GestaltException e) {
|
||||||
@ -499,29 +543,6 @@ public class RocksDBLoader {
|
|||||||
|
|
||||||
record DbPathRecord(Path path, long targetSize) {}
|
record DbPathRecord(Path path, long targetSize) {}
|
||||||
|
|
||||||
private static List<DbPathRecord> convertPaths(Path databasesDirPath, Path path, List<VolumeConfig> volumes) throws GestaltException {
|
|
||||||
var paths = new ArrayList<DbPathRecord>(volumes.size());
|
|
||||||
if (volumes.isEmpty()) {
|
|
||||||
return List.of(new DbPathRecord(databasesDirPath.resolve(path.getFileName() + "_hot"),
|
|
||||||
0), // Legacy
|
|
||||||
new DbPathRecord(databasesDirPath.resolve(path.getFileName() + "_cold"),
|
|
||||||
0), // Legacy
|
|
||||||
new DbPathRecord(databasesDirPath.resolve(path.getFileName() + "_colder"),
|
|
||||||
1000L * 1024L * 1024L * 1024L) // 1000GiB
|
|
||||||
); // Legacy
|
|
||||||
}
|
|
||||||
for (var volume : volumes) {
|
|
||||||
Path volumePath;
|
|
||||||
if (volume.volumePath().isAbsolute()) {
|
|
||||||
volumePath = volume.volumePath();
|
|
||||||
} else {
|
|
||||||
volumePath = databasesDirPath.resolve(volume.volumePath());
|
|
||||||
}
|
|
||||||
paths.add(new DbPathRecord(volumePath, volume.targetSizeBytes()));
|
|
||||||
}
|
|
||||||
return paths;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static boolean isDisableAutoCompactions() {
|
public static boolean isDisableAutoCompactions() {
|
||||||
return parseBoolean(System.getProperty("it.cavallium.dbengine.compactions.auto.disable", "false"));
|
return parseBoolean(System.getProperty("it.cavallium.dbengine.compactions.auto.disable", "false"));
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,31 @@
|
|||||||
|
package it.cavallium.rockserver.core.impl.rocksdb;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
public class RocksDBMetadata {
|
||||||
|
|
||||||
|
private static Exception LOAD_EXCEPTION = null;
|
||||||
|
private static String VERSION_HASH = null;
|
||||||
|
|
||||||
|
static {
|
||||||
|
try {
|
||||||
|
try (var is = RocksDBMetadata.class.getClassLoader().getResourceAsStream("it/cavallium/rockserver/core/resources/build.properties")) {
|
||||||
|
assert is != null;
|
||||||
|
var props = new Properties();
|
||||||
|
props.load(is);
|
||||||
|
VERSION_HASH = Objects.requireNonNull(props.getProperty("rockserver.core.rocksdb.version"));
|
||||||
|
}
|
||||||
|
} catch (IOException | NullPointerException e) {
|
||||||
|
LOAD_EXCEPTION = e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getRocksDBVersionHash() {
|
||||||
|
if (LOAD_EXCEPTION != null) {
|
||||||
|
throw new IllegalStateException(LOAD_EXCEPTION);
|
||||||
|
}
|
||||||
|
return VERSION_HASH;
|
||||||
|
}
|
||||||
|
}
|
@ -17,7 +17,14 @@ database: {
|
|||||||
# If "cacheIndexAndFilterBlocks" is false, the memory will rise when the number of open files rises.
|
# If "cacheIndexAndFilterBlocks" is false, the memory will rise when the number of open files rises.
|
||||||
maximum-open-files: -1
|
maximum-open-files: -1
|
||||||
# RocksDB data volumes.
|
# RocksDB data volumes.
|
||||||
volumes: []
|
volumes: [
|
||||||
|
{
|
||||||
|
# Path of the volume
|
||||||
|
volume-path: "./volume"
|
||||||
|
# Maximum size of the volume. This property is ignored on the last volume
|
||||||
|
target-size: "10TiB"
|
||||||
|
}
|
||||||
|
]
|
||||||
# Optimistic transactions
|
# Optimistic transactions
|
||||||
optimistic: true
|
optimistic: true
|
||||||
# Database block cache size
|
# Database block cache size
|
||||||
|
Loading…
Reference in New Issue
Block a user