Support ZSTD with finalized format
Summary: ZSTD 1.0.0 is coming. We can finally add a support of ZSTD without worrying about compatibility. Still keep ZSTDNotFinal for compatibility reason. Test Plan: Run all tests. Run db_bench with ZSTD version with RocksDB built with ZSTD 1.0 and older. Reviewers: andrewkr, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: cyan, igor, IslamAbdelRahman, leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D63141
This commit is contained in:
parent
ce1be2ce37
commit
607628d349
@ -4,6 +4,7 @@
|
||||
* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
|
||||
* Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
|
||||
* Remove ImmutableCFOptions.
|
||||
* Add a compression type ZSTD, which can work with ZSTD 0.8.0 or up. Still keep ZSTDNotFinal for compatibility reasons.
|
||||
|
||||
### New Features
|
||||
* Introduce NewClockCache, which is based on CLOCK algorithm with better concurrent performance in some cases. It can be used to replace the default LRU-based block cache and table cache. To use it, RocksDB need to be linked with TBB lib.
|
||||
|
@ -6,7 +6,7 @@ SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfc
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-5-glibc-2.23/9bc6787
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-5-glibc-2.23/9bc6787
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/bd13f49da8633897105b2b8541b106d487ce54ca/0.7.5/gcc-5-glibc-2.23/9bc6787
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-5-glibc-2.23/9bc6787
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-5-glibc-2.23/1c32b4b
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
|
||||
|
@ -6,7 +6,7 @@ SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfc
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/bd13f49da8633897105b2b8541b106d487ce54ca/0.7.5/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
|
@ -1193,7 +1193,7 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
|
||||
type = kXpressCompression;
|
||||
fprintf(stderr, "using xpress\n");
|
||||
} else if (ZSTD_Supported()) {
|
||||
type = kZSTDNotFinalCompression;
|
||||
type = kZSTD;
|
||||
fprintf(stderr, "using ZSTD\n");
|
||||
} else {
|
||||
fprintf(stderr, "skipping test, compression disabled\n");
|
||||
@ -4708,7 +4708,7 @@ TEST_F(DBTest, CompressionStatsTest) {
|
||||
type = kXpressCompression;
|
||||
fprintf(stderr, "using xpress\n");
|
||||
} else if (ZSTD_Supported()) {
|
||||
type = kZSTDNotFinalCompression;
|
||||
type = kZSTD;
|
||||
fprintf(stderr, "using ZSTD\n");
|
||||
} else {
|
||||
fprintf(stderr, "skipping test, compression disabled\n");
|
||||
|
@ -961,9 +961,9 @@ TEST_F(DBTest2, PresetCompressionDict) {
|
||||
compression_types.push_back(kLZ4Compression);
|
||||
compression_types.push_back(kLZ4HCCompression);
|
||||
#endif // LZ4_VERSION_NUMBER >= 10400
|
||||
#if ZSTD_VERSION_NUMBER >= 500 // v0.5.0+
|
||||
compression_types.push_back(kZSTDNotFinalCompression);
|
||||
#endif // ZSTD_VERSION_NUMBER >= 500
|
||||
if (ZSTD_Supported()) {
|
||||
compression_types.push_back(kZSTD);
|
||||
}
|
||||
|
||||
for (auto compression_type : compression_types) {
|
||||
options.compression = compression_type;
|
||||
|
@ -63,7 +63,13 @@ enum CompressionType : unsigned char {
|
||||
kLZ4Compression = 0x4,
|
||||
kLZ4HCCompression = 0x5,
|
||||
kXpressCompression = 0x6,
|
||||
// zstd format is not finalized yet so it's subject to changes.
|
||||
kZSTD = 0x7,
|
||||
|
||||
// Only use kZSTDNotFinalCompression if you have to use ZSTD lib older than
|
||||
// 0.8.0 or consider a possibility of downgrading the service or copying
|
||||
// the database files to another service running with an older version of
|
||||
// RocksDB that doesn't have kZSTD. Otherwise, you should use kZSTD. We will
|
||||
// eventually remove the option from the public API.
|
||||
kZSTDNotFinalCompression = 0x40,
|
||||
|
||||
// kDisableCompressionOption is used to disable some compression options.
|
||||
|
@ -377,6 +377,7 @@ Slice CompressBlock(const Slice& raw,
|
||||
return *compressed_output;
|
||||
}
|
||||
break;
|
||||
case kZSTD:
|
||||
case kZSTDNotFinalCompression:
|
||||
if (ZSTD_Compress(compression_options, raw.data(), raw.size(),
|
||||
compressed_output, compression_dict) &&
|
||||
|
@ -498,6 +498,7 @@ Status UncompressBlockContentsForCompressionType(
|
||||
*contents =
|
||||
BlockContents(std::move(ubuf), decompress_size, true, kNoCompression);
|
||||
break;
|
||||
case kZSTD:
|
||||
case kZSTDNotFinalCompression:
|
||||
ubuf.reset(ZSTD_Uncompress(data, n, &decompress_size, compression_dict));
|
||||
if (!ubuf) {
|
||||
|
@ -588,8 +588,8 @@ static std::vector<TestArgs> GenerateArgList() {
|
||||
compression_types.emplace_back(kXpressCompression, true);
|
||||
}
|
||||
if (ZSTD_Supported()) {
|
||||
compression_types.emplace_back(kZSTDNotFinalCompression, false);
|
||||
compression_types.emplace_back(kZSTDNotFinalCompression, true);
|
||||
compression_types.emplace_back(kZSTD, false);
|
||||
compression_types.emplace_back(kZSTD, true);
|
||||
}
|
||||
|
||||
for (auto test_type : test_types) {
|
||||
|
@ -597,7 +597,7 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
|
||||
else if (!strcasecmp(ctype, "xpress"))
|
||||
return rocksdb::kXpressCompression;
|
||||
else if (!strcasecmp(ctype, "zstd"))
|
||||
return rocksdb::kZSTDNotFinalCompression;
|
||||
return rocksdb::kZSTD;
|
||||
|
||||
fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
|
||||
return rocksdb::kSnappyCompression; // default value
|
||||
@ -1771,7 +1771,7 @@ class Benchmark {
|
||||
ok = XPRESS_Compress(input.data(),
|
||||
input.size(), compressed);
|
||||
break;
|
||||
case rocksdb::kZSTDNotFinalCompression:
|
||||
case rocksdb::kZSTD:
|
||||
ok = ZSTD_Compress(Options().compression_opts, input.data(),
|
||||
input.size(), compressed);
|
||||
break;
|
||||
@ -2606,7 +2606,7 @@ class Benchmark {
|
||||
&decompress_size);
|
||||
ok = uncompressed != nullptr;
|
||||
break;
|
||||
case rocksdb::kZSTDNotFinalCompression:
|
||||
case rocksdb::kZSTD:
|
||||
uncompressed = ZSTD_Uncompress(compressed.data(), compressed.size(),
|
||||
&decompress_size);
|
||||
ok = uncompressed != nullptr;
|
||||
|
@ -186,7 +186,7 @@ class SanityTestZSTDCompression : public SanityTest {
|
||||
public:
|
||||
explicit SanityTestZSTDCompression(const std::string& path)
|
||||
: SanityTest(path) {
|
||||
options_.compression = kZSTDNotFinalCompression;
|
||||
options_.compression = kZSTD;
|
||||
}
|
||||
virtual Options GetOptions() const override { return options_; }
|
||||
virtual std::string Name() const override { return "ZSTDCompression"; }
|
||||
|
@ -375,7 +375,7 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
|
||||
else if (!strcasecmp(ctype, "xpress"))
|
||||
return rocksdb::kXpressCompression;
|
||||
else if (!strcasecmp(ctype, "zstd"))
|
||||
return rocksdb::kZSTDNotFinalCompression;
|
||||
return rocksdb::kZSTD;
|
||||
|
||||
fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
|
||||
return rocksdb::kSnappyCompression; //default value
|
||||
|
@ -509,7 +509,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
|
||||
} else if (comp == "xpress") {
|
||||
opt.compression = kXpressCompression;
|
||||
} else if (comp == "zstd") {
|
||||
opt.compression = kZSTDNotFinalCompression;
|
||||
opt.compression = kZSTD;
|
||||
} else {
|
||||
// Unknown compression.
|
||||
exec_state_ =
|
||||
|
@ -185,16 +185,15 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) {
|
||||
|
||||
fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
|
||||
|
||||
std::pair<CompressionType,const char*> compressions[] = {
|
||||
{ CompressionType::kNoCompression, "kNoCompression" },
|
||||
{ CompressionType::kSnappyCompression, "kSnappyCompression" },
|
||||
{ CompressionType::kZlibCompression, "kZlibCompression" },
|
||||
{ CompressionType::kBZip2Compression, "kBZip2Compression" },
|
||||
{ CompressionType::kLZ4Compression, "kLZ4Compression" },
|
||||
{ CompressionType::kLZ4HCCompression, "kLZ4HCCompression" },
|
||||
{ CompressionType::kXpressCompression, "kXpressCompression" },
|
||||
{ CompressionType::kZSTDNotFinalCompression, "kZSTDNotFinalCompression" }
|
||||
};
|
||||
std::pair<CompressionType, const char*> compressions[] = {
|
||||
{CompressionType::kNoCompression, "kNoCompression"},
|
||||
{CompressionType::kSnappyCompression, "kSnappyCompression"},
|
||||
{CompressionType::kZlibCompression, "kZlibCompression"},
|
||||
{CompressionType::kBZip2Compression, "kBZip2Compression"},
|
||||
{CompressionType::kLZ4Compression, "kLZ4Compression"},
|
||||
{CompressionType::kLZ4HCCompression, "kLZ4HCCompression"},
|
||||
{CompressionType::kXpressCompression, "kXpressCompression"},
|
||||
{CompressionType::kZSTD, "kZSTD"}};
|
||||
|
||||
for (auto& i : compressions) {
|
||||
if (CompressionTypeSupported(i.first)) {
|
||||
|
@ -79,6 +79,14 @@ inline bool XPRESS_Supported() {
|
||||
}
|
||||
|
||||
inline bool ZSTD_Supported() {
|
||||
#ifdef ZSTD
|
||||
// ZSTD format is finalized since version 0.8.0.
|
||||
return (ZSTD_versionNumber() >= 800);
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
inline bool ZSTDNotFinal_Supported() {
|
||||
#ifdef ZSTD
|
||||
return true;
|
||||
#endif
|
||||
@ -102,6 +110,8 @@ inline bool CompressionTypeSupported(CompressionType compression_type) {
|
||||
case kXpressCompression:
|
||||
return XPRESS_Supported();
|
||||
case kZSTDNotFinalCompression:
|
||||
return ZSTDNotFinal_Supported();
|
||||
case kZSTD:
|
||||
return ZSTD_Supported();
|
||||
default:
|
||||
assert(false);
|
||||
@ -125,6 +135,7 @@ inline std::string CompressionTypeToString(CompressionType compression_type) {
|
||||
return "LZ4HC";
|
||||
case kXpressCompression:
|
||||
return "Xpress";
|
||||
case kZSTD:
|
||||
case kZSTDNotFinalCompression:
|
||||
return "ZSTD";
|
||||
default:
|
||||
|
@ -598,6 +598,7 @@ static std::unordered_map<std::string, CompressionType>
|
||||
{"kLZ4Compression", kLZ4Compression},
|
||||
{"kLZ4HCCompression", kLZ4HCCompression},
|
||||
{"kXpressCompression", kXpressCompression},
|
||||
{"kZSTD", kZSTD},
|
||||
{"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
|
||||
{"kDisableCompressionOption", kDisableCompressionOption}};
|
||||
|
||||
|
@ -56,6 +56,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
|
||||
"kLZ4Compression:"
|
||||
"kLZ4HCCompression:"
|
||||
"kXpressCompression:"
|
||||
"kZSTD:"
|
||||
"kZSTDNotFinalCompression"},
|
||||
{"bottommost_compression", "kLZ4Compression"},
|
||||
{"compression_opts", "4:5:6:7"},
|
||||
@ -140,7 +141,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
|
||||
ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
|
||||
ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
|
||||
ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level.size(), 8U);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
|
||||
@ -148,7 +149,8 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTDNotFinalCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
|
||||
ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
|
||||
ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
|
||||
ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
|
||||
ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
|
||||
|
@ -80,8 +80,7 @@ class ColumnAwareEncodingExp {
|
||||
std::unordered_map<std::string, CompressionType> compressions = {
|
||||
{"kNoCompression", CompressionType::kNoCompression},
|
||||
{"kZlibCompression", CompressionType::kZlibCompression},
|
||||
{"kZSTDNotFinalCompression",
|
||||
CompressionType::kZSTDNotFinalCompression}};
|
||||
{"kZSTD", CompressionType::kZSTD}};
|
||||
|
||||
// Find Compression
|
||||
CompressionType compression_type = compressions[FLAGS_compression_type];
|
||||
|
Loading…
Reference in New Issue
Block a user