From b6280d01f9f9c4305c536dfb804775fce3956280 Mon Sep 17 00:00:00 2001 From: Andrew Kryczka Date: Wed, 22 Aug 2018 18:22:10 -0700 Subject: [PATCH] Require ZSTD 1.1.3+ to use dictionary trainer (#4295) Summary: ZSTD's dynamic library exports `ZDICT_trainFromBuffer` symbol since v1.1.3, and its static library exports it since v0.6.1. We don't know whether linkage is static or dynamic, so just require v1.1.3 to use dictionary trainer. Fixes the issue reported here: https://jira.mariadb.org/browse/MDEV-16525. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4295 Differential Revision: D9417183 Pulled By: ajkr fbshipit-source-id: 0e89d2f48d9e7f6eee73e7f4572660a9f7122db8 --- HISTORY.md | 1 + util/compression.h | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 1c3378221..fd18c3f88 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -8,6 +8,7 @@ ### Public API Change * The merge operands are passed to `MergeOperator::ShouldMerge` in the reversed order relative to how they were merged (passed to FullMerge or FullMergeV2) for performance reasons * GetAllKeyVersions() to take an extra argument of `max_num_ikeys`. +* Using ZSTD dictionary trainer (i.e., setting `CompressionOptions::zstd_max_train_bytes` to a nonzero value) now requires ZSTD version 1.1.3 or later. ### New Features * Changes the format of index blocks by delta encoding the index values, which are the block handles. This saves the encoding of BlockHandle::offset of the non-head index entries in each restart interval. The feature is backward compatible but not forward compatible. It is disabled by default unless format_version 4 or above is used. diff --git a/util/compression.h b/util/compression.h index 3a980a5d8..e918e14fb 100644 --- a/util/compression.h +++ b/util/compression.h @@ -36,9 +36,9 @@ #if defined(ZSTD) #include -#if ZSTD_VERSION_NUMBER >= 800 // v0.8.0+ +#if ZSTD_VERSION_NUMBER >= 10103 // v1.1.3+ #include -#endif // ZSTD_VERSION_NUMBER >= 800 +#endif // ZSTD_VERSION_NUMBER >= 10103 namespace rocksdb { // Need this for the context allocation override // On windows we need to do this explicitly @@ -1065,9 +1065,10 @@ inline char* ZSTD_Uncompress(const UncompressionContext& ctx, inline std::string ZSTD_TrainDictionary(const std::string& samples, const std::vector& sample_lens, size_t max_dict_bytes) { - // Dictionary trainer is available since v0.6.1, but ZSTD was marked stable - // only since v0.8.0. For now we enable the feature in stable versions only. -#if ZSTD_VERSION_NUMBER >= 800 // v0.8.0+ + // Dictionary trainer is available since v0.6.1 for static linking, but not + // available for dynamic linking until v1.1.3. For now we enable the feature + // in v1.1.3+ only. +#if ZSTD_VERSION_NUMBER >= 10103 // v1.1.3+ std::string dict_data(max_dict_bytes, '\0'); size_t dict_len = ZDICT_trainFromBuffer( &dict_data[0], max_dict_bytes, &samples[0], &sample_lens[0], @@ -1078,13 +1079,13 @@ inline std::string ZSTD_TrainDictionary(const std::string& samples, assert(dict_len <= max_dict_bytes); dict_data.resize(dict_len); return dict_data; -#else // up to v0.7.x +#else // up to v1.1.2 assert(false); (void)samples; (void)sample_lens; (void)max_dict_bytes; return ""; -#endif // ZSTD_VERSION_NUMBER >= 800 +#endif // ZSTD_VERSION_NUMBER >= 10103 } inline std::string ZSTD_TrainDictionary(const std::string& samples, @@ -1092,18 +1093,18 @@ inline std::string ZSTD_TrainDictionary(const std::string& samples, size_t max_dict_bytes) { // Dictionary trainer is available since v0.6.1, but ZSTD was marked stable // only since v0.8.0. For now we enable the feature in stable versions only. -#if ZSTD_VERSION_NUMBER >= 800 // v0.8.0+ +#if ZSTD_VERSION_NUMBER >= 10103 // v1.1.3+ // skips potential partial sample at the end of "samples" size_t num_samples = samples.size() >> sample_len_shift; std::vector sample_lens(num_samples, size_t(1) << sample_len_shift); return ZSTD_TrainDictionary(samples, sample_lens, max_dict_bytes); -#else // up to v0.7.x +#else // up to v1.1.2 assert(false); (void)samples; (void)sample_len_shift; (void)max_dict_bytes; return ""; -#endif // ZSTD_VERSION_NUMBER >= 800 +#endif // ZSTD_VERSION_NUMBER >= 10103 } } // namespace rocksdb