Fix compilation errors and add fuzzers to CircleCI (#9420)
Summary: This PR does the following: - Fix compilation and linking errors when building fuzzer - Add the above to CircleCI - Update documentation Pull Request resolved: https://github.com/facebook/rocksdb/pull/9420 Test Plan: CI Reviewed By: jay-zhuang Differential Revision: D33849452 Pulled By: riversand963 fbshipit-source-id: 0794e5d04a3f53bfd2216fe2b3cd827ca2083ac3
This commit is contained in:
Yanqin Jin
Facebook GitHub Bot
parent
980b9ff385
commit
c58c5596e7
@ -136,6 +136,21 @@ commands:
|
||||
command: |
|
||||
sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
|
||||
|
||||
install-libprotobuf-mutator:
|
||||
steps:
|
||||
- run:
|
||||
name: Install libprotobuf-mutator libs
|
||||
command: |
|
||||
git clone --single-branch --branch master --depth 1 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator
|
||||
cd ~/libprotobuf-mutator && mkdir build && cd build
|
||||
cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
|
||||
ninja && sudo ninja install
|
||||
- run:
|
||||
name: Setup environment variables
|
||||
command: |
|
||||
echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV
|
||||
echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV
|
||||
|
||||
executors:
|
||||
windows-2xlarge:
|
||||
machine:
|
||||
@ -732,6 +747,23 @@ jobs:
|
||||
tools/check_format_compatible.sh
|
||||
- post-steps
|
||||
|
||||
build-fuzzers:
|
||||
machine:
|
||||
image: ubuntu-2004:202010-01
|
||||
resource_class: large
|
||||
steps:
|
||||
- pre-steps
|
||||
- install-clang-13
|
||||
- run: sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
|
||||
- install-libprotobuf-mutator
|
||||
- run:
|
||||
name: "Build rocksdb lib"
|
||||
command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
|
||||
- run:
|
||||
name: "Build fuzzers"
|
||||
command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
|
||||
- post-steps
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build-linux:
|
||||
@ -838,6 +870,9 @@ workflows:
|
||||
build-microbench:
|
||||
jobs:
|
||||
- build-linux-microbench
|
||||
build-fuzzers:
|
||||
jobs:
|
||||
- build-fuzzers
|
||||
nightly:
|
||||
triggers:
|
||||
- schedule:
|
||||
|
||||
5
fuzz/.gitignore
vendored
Normal file
5
fuzz/.gitignore
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
db_fuzzer
|
||||
db_map_fuzzer
|
||||
sst_file_writer_fuzzer
|
||||
|
||||
proto/gen/*
|
||||
@ -20,10 +20,10 @@ PROTO_IN = $(ROOT_DIR)/fuzz/proto
|
||||
PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen
|
||||
|
||||
ifneq ($(FUZZ_ENV), ossfuzz)
|
||||
CC = clang++
|
||||
CC = $(CXX)
|
||||
CCFLAGS += -Wall -fsanitize=address,fuzzer
|
||||
CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
|
||||
LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
|
||||
LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
|
||||
else
|
||||
# OSS-Fuzz sets various environment flags that are used for compilation.
|
||||
# These environment flags depend on which type of sanitizer build is being
|
||||
@ -42,15 +42,21 @@ CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKS
|
||||
LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
|
||||
endif
|
||||
|
||||
.PHONY: gen_proto
|
||||
.PHONY: gen_proto clean
|
||||
|
||||
# Set PROTOC_BIN when invoking `make` if a custom protoc is required.
|
||||
PROTOC_BIN ?= protoc
|
||||
|
||||
gen_proto:
|
||||
mkdir -p $(PROTO_OUT)
|
||||
protoc \
|
||||
$(PROTOC_BIN) \
|
||||
--proto_path=$(PROTO_IN) \
|
||||
--cpp_out=$(PROTO_OUT) \
|
||||
$(PROTO_IN)/*.proto
|
||||
|
||||
clean:
|
||||
rm -rf db_fuzzer db_map_fuzzer sst_file_writer_fuzzer $(PROTO_OUT)
|
||||
|
||||
db_fuzzer: db_fuzzer.cc
|
||||
$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
|
||||
@ -15,6 +15,11 @@ Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/b
|
||||
We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer,
|
||||
and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator.
|
||||
So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them.
|
||||
On some systems, there are both protobuf2 and protobuf3 in the package management system,
|
||||
make sure protobuf3 is installed.
|
||||
|
||||
If you do not want to install protobuf library yourself, you can rely on libprotobuf-mutator to download protobuf
|
||||
for you. For details about installation, please refer to [libprotobuf-mutator README](https://github.com/google/libprotobuf-mutator#readme)
|
||||
|
||||
## Example
|
||||
|
||||
|
||||
@ -153,6 +153,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||
delete iter;
|
||||
break;
|
||||
}
|
||||
case OP_COUNT:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -12,9 +12,33 @@
|
||||
#include "rocksdb/file_system.h"
|
||||
#include "rocksdb/sst_file_writer.h"
|
||||
#include "src/libfuzzer/libfuzzer_macro.h"
|
||||
#include "table/table_builder.h"
|
||||
#include "table/table_reader.h"
|
||||
#include "util.h"
|
||||
|
||||
using ROCKSDB_NAMESPACE::BytewiseComparator;
|
||||
using ROCKSDB_NAMESPACE::Comparator;
|
||||
using ROCKSDB_NAMESPACE::EnvOptions;
|
||||
using ROCKSDB_NAMESPACE::ExternalSstFileInfo;
|
||||
using ROCKSDB_NAMESPACE::FileOptions;
|
||||
using ROCKSDB_NAMESPACE::FileSystem;
|
||||
using ROCKSDB_NAMESPACE::ImmutableCFOptions;
|
||||
using ROCKSDB_NAMESPACE::ImmutableOptions;
|
||||
using ROCKSDB_NAMESPACE::InternalIterator;
|
||||
using ROCKSDB_NAMESPACE::IOOptions;
|
||||
using ROCKSDB_NAMESPACE::kMaxSequenceNumber;
|
||||
using ROCKSDB_NAMESPACE::Options;
|
||||
using ROCKSDB_NAMESPACE::ParsedInternalKey;
|
||||
using ROCKSDB_NAMESPACE::ParseInternalKey;
|
||||
using ROCKSDB_NAMESPACE::RandomAccessFileReader;
|
||||
using ROCKSDB_NAMESPACE::ReadOptions;
|
||||
using ROCKSDB_NAMESPACE::SstFileWriter;
|
||||
using ROCKSDB_NAMESPACE::Status;
|
||||
using ROCKSDB_NAMESPACE::TableReader;
|
||||
using ROCKSDB_NAMESPACE::TableReaderCaller;
|
||||
using ROCKSDB_NAMESPACE::TableReaderOptions;
|
||||
using ROCKSDB_NAMESPACE::ValueType;
|
||||
|
||||
// Keys in SST file writer operations must be unique and in ascending order.
|
||||
// For each DBOperation generated by the fuzzer, this function is called on
|
||||
// it to deduplicate and sort the keys in the DBOperations.
|
||||
@ -60,15 +84,15 @@ TableReader* NewTableReader(const std::string& sst_file_path,
|
||||
std::unique_ptr<TableReader> table_reader;
|
||||
const auto& fs = options.env->GetFileSystem();
|
||||
FileOptions fopts(env_options);
|
||||
Status s = options.env->GetFileSize(sst_file_path, fopts.io_options,
|
||||
&file_size, nullptr);
|
||||
Status s = options.env->GetFileSize(sst_file_path, &file_size);
|
||||
if (s.ok()) {
|
||||
s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader,
|
||||
nullptr);
|
||||
}
|
||||
if (s.ok()) {
|
||||
TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr,
|
||||
env_options, cf_ioptions.internal_comparator);
|
||||
ImmutableOptions iopts(options, cf_ioptions);
|
||||
TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options,
|
||||
cf_ioptions.internal_comparator);
|
||||
t_opt.largest_seqno = kMaxSequenceNumber;
|
||||
s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
|
||||
file_size, &table_reader,
|
||||
@ -153,7 +177,7 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
|
||||
|
||||
// Iterate and verify key-value pairs.
|
||||
std::unique_ptr<TableReader> table_reader(
|
||||
NewTableReader(sstfile, options, env_options, cf_ioptions));
|
||||
::NewTableReader(sstfile, options, env_options, cf_ioptions));
|
||||
ReadOptions roptions;
|
||||
CHECK_OK(table_reader->VerifyChecksum(roptions,
|
||||
TableReaderCaller::kUncategorized));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user