119 lines
4.0 KiB
Java
119 lines
4.0 KiB
Java
package it.cavallium.dbengine.database.disk;
|
|
|
|
import static com.google.common.collect.Lists.partition;
|
|
|
|
import it.cavallium.dbengine.database.LLUtils;
|
|
import it.cavallium.dbengine.utils.SimpleResource;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import org.apache.logging.log4j.Logger;
|
|
import org.jetbrains.annotations.Nullable;
|
|
import org.rocksdb.AbstractImmutableNativeReference;
|
|
import org.rocksdb.ColumnFamilyHandle;
|
|
import org.rocksdb.CompactionJobInfo;
|
|
import org.rocksdb.CompactionOptions;
|
|
import org.rocksdb.CompressionType;
|
|
import org.rocksdb.LevelMetaData;
|
|
import org.rocksdb.RocksDB;
|
|
import org.rocksdb.RocksDBException;
|
|
import org.rocksdb.SstFileMetaData;
|
|
import org.rocksdb.util.SizeUnit;
|
|
|
|
public class RocksDBUtils {
|
|
|
|
public static int getLevels(RocksDB db, ColumnFamilyHandle cfh) {
|
|
return db.numberLevels(cfh);
|
|
}
|
|
|
|
public static List<RocksDBFile> getColumnFiles(RocksDB db, ColumnFamilyHandle cfh, boolean excludeLastLevel) {
|
|
List<RocksDBFile> files = new ArrayList<>();
|
|
var meta = db.getColumnFamilyMetaData(cfh);
|
|
var lastLevelId = excludeLastLevel ? (getLevels(db, cfh) - 1) : -1;
|
|
for (LevelMetaData level : meta.levels()) {
|
|
if (!excludeLastLevel || level.level() < lastLevelId) {
|
|
for (SstFileMetaData file : level.files()) {
|
|
if (file.fileName().endsWith(".sst")) {
|
|
files.add(new RocksDBColumnFile(db, cfh, file, meta.name(), level.level()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
public static void forceCompaction(RocksDB db,
|
|
String logDbName,
|
|
ColumnFamilyHandle cfh,
|
|
int volumeId,
|
|
Logger logger) throws RocksDBException {
|
|
try (var co = new CompactionOptions()
|
|
.setCompression(CompressionType.LZ4_COMPRESSION)
|
|
.setMaxSubcompactions(0)
|
|
.setOutputFileSizeLimit(2 * SizeUnit.GB)) {
|
|
var filesToCompact = getColumnFiles(db, cfh, true);
|
|
|
|
if (!filesToCompact.isEmpty()) {
|
|
var partitionSize = filesToCompact.size() / Runtime.getRuntime().availableProcessors();
|
|
List<List<RocksDBFile>> partitions;
|
|
if (partitionSize > 0) {
|
|
partitions = partition(filesToCompact, partitionSize);
|
|
} else {
|
|
partitions = List.of(filesToCompact);
|
|
}
|
|
int finalBottommostLevelId = getLevels(db, cfh) - 1;
|
|
for (List<RocksDBFile> partition : partitions) {
|
|
logger.info("Compacting {} files in database {} in column family {} to level {}",
|
|
partition.size(),
|
|
logDbName,
|
|
new String(cfh.getName(), StandardCharsets.UTF_8),
|
|
finalBottommostLevelId
|
|
);
|
|
if (!partition.isEmpty()) {
|
|
var coi = new CompactionJobInfo();
|
|
try {
|
|
var partitionFileNames = partition.stream().map(x -> x.getMetadata().fileName()).toList();
|
|
db.compactFiles(co, cfh, partitionFileNames, finalBottommostLevelId, volumeId, coi);
|
|
logger.info("Compacted {} files in database {} in column family {} to level {}: {}",
|
|
partition.size(),
|
|
logDbName,
|
|
new String(cfh.getName(), StandardCharsets.UTF_8),
|
|
finalBottommostLevelId,
|
|
coi.status().getCodeString()
|
|
);
|
|
} catch (Throwable ex) {
|
|
logger.error("Failed to compact {} files in database {} in column family {} to level {}",
|
|
partition.size(),
|
|
logDbName,
|
|
new String(cfh.getName(), StandardCharsets.UTF_8),
|
|
finalBottommostLevelId,
|
|
ex
|
|
);
|
|
}
|
|
}
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
public static void ensureOpen(RocksDB db, @Nullable ColumnFamilyHandle cfh) {
|
|
if (LLUtils.isInNonBlockingThread()) {
|
|
throw new UnsupportedOperationException("Called in a nonblocking thread");
|
|
}
|
|
ensureOwned(db);
|
|
ensureOwned(cfh);
|
|
}
|
|
|
|
public static void ensureOwned(@Nullable AbstractImmutableNativeReference rocksObject) {
|
|
if (rocksObject != null && !rocksObject.isOwningHandle()) {
|
|
throw new IllegalStateException("Not owning handle");
|
|
}
|
|
}
|
|
|
|
public static void ensureOwned(@Nullable SimpleResource simpleResource) {
|
|
if (simpleResource != null && simpleResource.isClosed()) {
|
|
throw new IllegalStateException("Resource is closed");
|
|
}
|
|
}
|
|
}
|