Remove compressed cache, rocksdb 8.0.0
This commit is contained in:
parent
1aeb0c99d3
commit
468886d154
2
pom.xml
2
pom.xml
@ -14,7 +14,7 @@
|
|||||||
<dbengine.ci>false</dbengine.ci>
|
<dbengine.ci>false</dbengine.ci>
|
||||||
<micrometer.version>1.9.5</micrometer.version>
|
<micrometer.version>1.9.5</micrometer.version>
|
||||||
<lucene.version>9.5.0</lucene.version>
|
<lucene.version>9.5.0</lucene.version>
|
||||||
<rocksdb.version>7.10.2</rocksdb.version>
|
<rocksdb.version>8.0.0</rocksdb.version>
|
||||||
<junit.jupiter.version>5.9.0</junit.jupiter.version>
|
<junit.jupiter.version>5.9.0</junit.jupiter.version>
|
||||||
<data.generator.version>1.0.258</data.generator.version>
|
<data.generator.version>1.0.258</data.generator.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
@ -248,7 +248,6 @@ baseTypesData:
|
|||||||
optimistic: boolean
|
optimistic: boolean
|
||||||
maxOpenFiles: -int
|
maxOpenFiles: -int
|
||||||
blockCache: -long
|
blockCache: -long
|
||||||
compressedBlockCache: -long
|
|
||||||
persistentCaches: PersistentCache[]
|
persistentCaches: PersistentCache[]
|
||||||
writeBufferManager: -long
|
writeBufferManager: -long
|
||||||
spinning: boolean
|
spinning: boolean
|
||||||
|
@ -60,7 +60,6 @@ public class DefaultDatabaseOptions {
|
|||||||
true,
|
true,
|
||||||
Nullableint.empty(),
|
Nullableint.empty(),
|
||||||
Nullablelong.empty(),
|
Nullablelong.empty(),
|
||||||
Nullablelong.empty(),
|
|
||||||
Collections.emptyList(),
|
Collections.emptyList(),
|
||||||
Nullablelong.empty(),
|
Nullablelong.empty(),
|
||||||
false,
|
false,
|
||||||
|
@ -125,7 +125,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
private RocksDB db;
|
private RocksDB db;
|
||||||
private Statistics statistics;
|
private Statistics statistics;
|
||||||
private Cache standardCache;
|
private Cache standardCache;
|
||||||
private Cache compressedCache;
|
|
||||||
private final Map<Column, ColumnFamilyHandle> handles;
|
private final Map<Column, ColumnFamilyHandle> handles;
|
||||||
|
|
||||||
private final HashMap<String, PersistentCache> persistentCaches;
|
private final HashMap<String, PersistentCache> persistentCaches;
|
||||||
@ -363,7 +362,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
|
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
|
||||||
// https://nightlies.apache.org/flink/flink-docs-release-1.3/api/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.html
|
// https://nightlies.apache.org/flink/flink-docs-release-1.3/api/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.html
|
||||||
.setBlockSize(columnOptions.blockSize().orElse((databaseOptions.spinning() ? 128 : 16) * 1024))
|
.setBlockSize(columnOptions.blockSize().orElse((databaseOptions.spinning() ? 128 : 16) * 1024))
|
||||||
.setBlockCacheCompressed(optionsWithCache.compressedCache())
|
|
||||||
.setBlockCache(optionsWithCache.standardCache())
|
.setBlockCache(optionsWithCache.standardCache())
|
||||||
.setPersistentCache(resolvePersistentCache(persistentCaches,
|
.setPersistentCache(resolvePersistentCache(persistentCaches,
|
||||||
rocksdbOptions,
|
rocksdbOptions,
|
||||||
@ -440,7 +438,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
this.standardCache = optionsWithCache.standardCache;
|
this.standardCache = optionsWithCache.standardCache;
|
||||||
this.compressedCache = optionsWithCache.compressedCache;
|
|
||||||
break;
|
break;
|
||||||
} catch (RocksDBException ex) {
|
} catch (RocksDBException ex) {
|
||||||
switch (ex.getMessage()) {
|
switch (ex.getMessage()) {
|
||||||
@ -749,7 +746,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
return closeLock;
|
return closeLock;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void flushAndCloseDb(RocksDB db, Cache standardCache, Cache compressedCache, List<ColumnFamilyHandle> handles) {
|
private void flushAndCloseDb(RocksDB db, Cache standardCache, List<ColumnFamilyHandle> handles) {
|
||||||
var closeWriteLock = closeLock.writeLock();
|
var closeWriteLock = closeLock.writeLock();
|
||||||
try {
|
try {
|
||||||
if (closed) {
|
if (closed) {
|
||||||
@ -784,9 +781,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
logger.error("Can't close column family", ex);
|
logger.error("Can't close column family", ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (compressedCache != null) {
|
|
||||||
compressedCache.close();
|
|
||||||
}
|
|
||||||
if (standardCache != null) {
|
if (standardCache != null) {
|
||||||
standardCache.close();
|
standardCache.close();
|
||||||
}
|
}
|
||||||
@ -855,16 +849,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
record OptionsWithCache(DBOptions options, @Nullable Cache standardCache, @Nullable Cache compressedCache) {
|
record OptionsWithCache(DBOptions options, @Nullable Cache standardCache) {
|
||||||
|
|
||||||
/**
|
|
||||||
* SecondaryCache will replace compressed cache
|
|
||||||
*/
|
|
||||||
@Deprecated(forRemoval = true)
|
|
||||||
@Override
|
|
||||||
public Cache compressedCache() {
|
|
||||||
return compressedCache;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static OptionsWithCache openRocksDb(@Nullable Path path, DatabaseOptions databaseOptions, RocksDBRefs refs) {
|
private static OptionsWithCache openRocksDb(@Nullable Path path, DatabaseOptions databaseOptions, RocksDBRefs refs) {
|
||||||
@ -956,8 +941,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
}
|
}
|
||||||
|
|
||||||
Cache blockCache;
|
Cache blockCache;
|
||||||
//todo: compressed cache will be replaced with SecondaryCache in the future
|
|
||||||
Cache compressedCache;
|
|
||||||
final boolean useDirectIO = databaseOptions.useDirectIO();
|
final boolean useDirectIO = databaseOptions.useDirectIO();
|
||||||
final boolean allowMmapReads = !useDirectIO && databaseOptions.allowMemoryMapping();
|
final boolean allowMmapReads = !useDirectIO && databaseOptions.allowMemoryMapping();
|
||||||
final boolean allowMmapWrites = !useDirectIO && (databaseOptions.allowMemoryMapping()
|
final boolean allowMmapWrites = !useDirectIO && (databaseOptions.allowMemoryMapping()
|
||||||
@ -974,12 +957,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
;
|
;
|
||||||
blockCache = CACHE_FACTORY.newCache(writeBufferManagerSize + databaseOptions.blockCache().orElse(8L * SizeUnit.MB));
|
blockCache = CACHE_FACTORY.newCache(writeBufferManagerSize + databaseOptions.blockCache().orElse(8L * SizeUnit.MB));
|
||||||
refs.track(blockCache);
|
refs.track(blockCache);
|
||||||
if (databaseOptions.compressedBlockCache().isPresent()) {
|
|
||||||
compressedCache = CACHE_FACTORY.newCache(databaseOptions.compressedBlockCache().get());
|
|
||||||
refs.track(compressedCache);
|
|
||||||
} else {
|
|
||||||
compressedCache = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (useDirectIO) {
|
if (useDirectIO) {
|
||||||
options
|
options
|
||||||
@ -1010,12 +987,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
;
|
;
|
||||||
blockCache = CACHE_FACTORY.newCache(writeBufferManagerSize + databaseOptions.blockCache().orElse( 512 * SizeUnit.MB));
|
blockCache = CACHE_FACTORY.newCache(writeBufferManagerSize + databaseOptions.blockCache().orElse( 512 * SizeUnit.MB));
|
||||||
refs.track(blockCache);
|
refs.track(blockCache);
|
||||||
if (databaseOptions.compressedBlockCache().isPresent()) {
|
|
||||||
compressedCache = CACHE_FACTORY.newCache(databaseOptions.compressedBlockCache().get());
|
|
||||||
refs.track(compressedCache);
|
|
||||||
} else {
|
|
||||||
compressedCache = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (useDirectIO) {
|
if (useDirectIO) {
|
||||||
options
|
options
|
||||||
@ -1058,7 +1029,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
options.setUseDirectIoForFlushAndCompaction(true);
|
options.setUseDirectIoForFlushAndCompaction(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new OptionsWithCache(options, blockCache, compressedCache);
|
return new OptionsWithCache(options, blockCache);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new DBException(e);
|
throw new DBException(e);
|
||||||
}
|
}
|
||||||
@ -1525,7 +1496,6 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
|
|||||||
try {
|
try {
|
||||||
flushAndCloseDb(db,
|
flushAndCloseDb(db,
|
||||||
standardCache,
|
standardCache,
|
||||||
compressedCache,
|
|
||||||
new ArrayList<>(handles.values())
|
new ArrayList<>(handles.values())
|
||||||
);
|
);
|
||||||
handles.values().forEach(columnFamilyHandleRocksObj -> {
|
handles.values().forEach(columnFamilyHandleRocksObj -> {
|
||||||
|
Loading…
Reference in New Issue
Block a user