andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix PlainTableReader not to crash sst_dump (#5940)

Browse Source 
Summary:
Plain table SSTs could crash sst_dump because of a bug in
PlainTableReader that can leave table_properties_ as null. Even if it
was intended not to keep the table properties in some cases, they were
leaked on the offending code path.

Steps to reproduce:

    $ db_bench --benchmarks=fillrandom --num=2000000 --use_plain_table --prefix-size=12
    $ sst_dump --file=0000xx.sst --show_properties
    from [] to []
    Process /dev/shm/dbbench/000014.sst
    Sst file format: plain table
    Raw user collected properties
    ------------------------------
    Segmentation fault (core dumped)

Also added missing unit testing of plain table full_scan_mode, and
an assertion in NewIterator to check for regression.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5940

Test Plan: new unit test, manual, make check

Differential Revision: D18018145

Pulled By: pdillinger

fbshipit-source-id: 4310c755e824c4cd6f3f86a3abc20dfa417c5e07
This commit is contained in:
Peter Dillinger 2019-10-18 14:43:17 -07:00 committed by Facebook Github Bot
parent 526e3b9763
commit fe464bca5c
4 changed files with 54 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
db/plain_table_db_test.cc
View File

@ -302,6 +302,7 @@ class TestPlainTableReader : public PlainTableReader {
EXPECT_TRUE(num_blocks_ptr != props->user_collected_properties.end()); EXPECT_TRUE(num_blocks_ptr != props->user_collected_properties.end());
} }
} }
table_properties_.reset(props);
} }
~TestPlainTableReader() override {} ~TestPlainTableReader() override {}
@ -396,7 +397,9 @@ TEST_P(PlainTableDBTest, Flush) {
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
huge_page_tlb_size += 2 * 1024 * 1024) { huge_page_tlb_size += 2 * 1024 * 1024) {
for (EncodingType encoding_type : {kPlain, kPrefix}) { for (EncodingType encoding_type : {kPlain, kPrefix}) {
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) { for (int bloom = -1; bloom <= 117; bloom += 117) {
const int bloom_bits = std::max(bloom, 0);
const bool full_scan_mode = bloom < 0;
for (int total_order = 0; total_order <= 1; total_order++) { for (int total_order = 0; total_order <= 1; total_order++) {
for (int store_index_in_file = 0; store_index_in_file <= 1; for (int store_index_in_file = 0; store_index_in_file <= 1;
++store_index_in_file) { ++store_index_in_file) {
@ -414,7 +417,7 @@ TEST_P(PlainTableDBTest, Flush) {
plain_table_options.index_sparseness = 2; plain_table_options.index_sparseness = 2;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
plain_table_options.full_scan_mode = false; plain_table_options.full_scan_mode = full_scan_mode;
plain_table_options.store_index_in_file = store_index_in_file; plain_table_options.store_index_in_file = store_index_in_file;
options.table_factory.reset( options.table_factory.reset(
@ -427,7 +430,7 @@ TEST_P(PlainTableDBTest, Flush) {
plain_table_options.index_sparseness = 16; plain_table_options.index_sparseness = 16;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
plain_table_options.full_scan_mode = false; plain_table_options.full_scan_mode = full_scan_mode;
plain_table_options.store_index_in_file = store_index_in_file; plain_table_options.store_index_in_file = store_index_in_file;
options.table_factory.reset( options.table_factory.reset(
@ -454,20 +457,36 @@ TEST_P(PlainTableDBTest, Flush) {
auto row = ptc.begin(); auto row = ptc.begin();
auto tp = row->second; auto tp = row->second;
if (!store_index_in_file) { if (full_scan_mode) {
ASSERT_EQ(total_order ? "4" : "12", // Does not support Get/Seek
(tp->user_collected_properties) std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
.at("plain_table_hash_table_size")); iter->SeekToFirst();
ASSERT_EQ("0", (tp->user_collected_properties) ASSERT_TRUE(iter->Valid());
.at("plain_table_sub_index_size")); ASSERT_EQ("0000000000000bar", iter->key().ToString());
ASSERT_EQ("v2", iter->value().ToString());
iter->Next();
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000000foo", iter->key().ToString());
ASSERT_EQ("v3", iter->value().ToString());
iter->Next();
ASSERT_TRUE(!iter->Valid());
ASSERT_TRUE(iter->status().ok());
} else { } else {
ASSERT_EQ("0", (tp->user_collected_properties) if (!store_index_in_file) {
.at("plain_table_hash_table_size")); ASSERT_EQ(total_order ? "4" : "12",
ASSERT_EQ("0", (tp->user_collected_properties) (tp->user_collected_properties)
.at("plain_table_sub_index_size")); .at("plain_table_hash_table_size"));
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_sub_index_size"));
} else {
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_hash_table_size"));
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_sub_index_size"));
}
ASSERT_EQ("v3", Get("1000000000000foo"));
ASSERT_EQ("v2", Get("0000000000000bar"));
} }
ASSERT_EQ("v3", Get("1000000000000foo"));
ASSERT_EQ("v2", Get("0000000000000bar"));
} }
} }
} }

8
table/plain/plain_table_reader.cc
View File

@ -183,6 +183,8 @@ Status PlainTableReader::Open(
// can be used. // can be used.
new_reader->full_scan_mode_ = true; new_reader->full_scan_mode_ = true;
} }
// PopulateIndex can add to the props, so don't store them until now
new_reader->table_properties_.reset(props);
if (immortal_table && new_reader->file_info_.is_mmap_mode) { if (immortal_table && new_reader->file_info_.is_mmap_mode) {
new_reader->dummy_cleanable_.reset(new Cleanable()); new_reader->dummy_cleanable_.reset(new Cleanable());
@ -199,6 +201,9 @@ InternalIterator* PlainTableReader::NewIterator(
const ReadOptions& options, const SliceTransform* /* prefix_extractor */, const ReadOptions& options, const SliceTransform* /* prefix_extractor */,
Arena* arena, bool /*skip_filters*/, TableReaderCaller /*caller*/, Arena* arena, bool /*skip_filters*/, TableReaderCaller /*caller*/,
size_t /*compaction_readahead_size*/) { size_t /*compaction_readahead_size*/) {
// Not necessarily used here, but make sure this has been initialized
assert(table_properties_);
bool use_prefix_seek = !IsTotalOrderMode() && !options.total_order_seek; bool use_prefix_seek = !IsTotalOrderMode() && !options.total_order_seek;
if (arena == nullptr) { if (arena == nullptr) {
return new PlainTableIterator(this, use_prefix_seek); return new PlainTableIterator(this, use_prefix_seek);
@ -291,7 +296,6 @@ Status PlainTableReader::PopulateIndex(TableProperties* props,
size_t index_sparseness, size_t index_sparseness,
size_t huge_page_tlb_size) { size_t huge_page_tlb_size) {
assert(props != nullptr); assert(props != nullptr);
table_properties_.reset(props);
BlockContents index_block_contents; BlockContents index_block_contents;
Status s = ReadMetaBlock(file_info_.file.get(), nullptr /* prefetch_buffer */, Status s = ReadMetaBlock(file_info_.file.get(), nullptr /* prefetch_buffer */,
@ -351,7 +355,7 @@ Status PlainTableReader::PopulateIndex(TableProperties* props,
// Allocate bloom filter here for total order mode. // Allocate bloom filter here for total order mode.
if (IsTotalOrderMode()) { if (IsTotalOrderMode()) {
AllocateBloom(bloom_bits_per_key, AllocateBloom(bloom_bits_per_key,
static_cast<uint32_t>(table_properties_->num_entries), static_cast<uint32_t>(props->num_entries),
huge_page_tlb_size); huge_page_tlb_size);
} }
} else if (bloom_in_file) { } else if (bloom_in_file) {

2
table/plain/plain_table_reader.h
View File

@ -165,7 +165,9 @@ class PlainTableReader: public TableReader {
const ImmutableCFOptions& ioptions_; const ImmutableCFOptions& ioptions_;
std::unique_ptr<Cleanable> dummy_cleanable_; std::unique_ptr<Cleanable> dummy_cleanable_;
uint64_t file_size_; uint64_t file_size_;
protected: // for testing
std::shared_ptr<const TableProperties> table_properties_; std::shared_ptr<const TableProperties> table_properties_;
private:
bool IsFixedLength() const { bool IsFixedLength() const {
return user_key_len_ != kPlainTableVariableLength; return user_key_len_ != kPlainTableVariableLength;

22
tools/sst_dump_tool.cc
View File

@ -740,17 +740,19 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) {
total_data_block_size += table_properties->data_size; total_data_block_size += table_properties->data_size;
total_index_block_size += table_properties->index_size; total_index_block_size += table_properties->index_size;
total_filter_block_size += table_properties->filter_size; total_filter_block_size += table_properties->filter_size;
} if (show_properties) {
if (show_properties) { fprintf(stdout,
fprintf(stdout, "Raw user collected properties\n"
"Raw user collected properties\n" "------------------------------\n");
"------------------------------\n"); for (const auto& kv : table_properties->user_collected_properties) {
for (const auto& kv : table_properties->user_collected_properties) { std::string prop_name = kv.first;
std::string prop_name = kv.first; std::string prop_val = Slice(kv.second).ToString(true);
std::string prop_val = Slice(kv.second).ToString(true); fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(),
fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(), prop_val.c_str());
prop_val.c_str()); }
} }
} else {
fprintf(stderr, "Reader unexpectedly returned null properties\n");
} }
} }
} }