From c58c5596e778e04d444fba4aac411349a8e9acdc Mon Sep 17 00:00:00 2001 From: Yanqin Jin Date: Tue, 1 Feb 2022 10:29:56 -0800 Subject: [PATCH] Fix compilation errors and add fuzzers to CircleCI (#9420) Summary: This PR does the following: - Fix compilation and linking errors when building fuzzer - Add the above to CircleCI - Update documentation Pull Request resolved: https://github.com/facebook/rocksdb/pull/9420 Test Plan: CI Reviewed By: jay-zhuang Differential Revision: D33849452 Pulled By: riversand963 fbshipit-source-id: 0794e5d04a3f53bfd2216fe2b3cd827ca2083ac3 --- .circleci/config.yml | 35 ++++++++++++++++++++++++++++++++++ fuzz/.gitignore | 5 +++++ fuzz/Makefile | 14 ++++++++++---- fuzz/README.md | 5 +++++ fuzz/db_fuzzer.cc | 2 ++ fuzz/sst_file_writer_fuzzer.cc | 34 ++++++++++++++++++++++++++++----- 6 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 fuzz/.gitignore diff --git a/.circleci/config.yml b/.circleci/config.yml index ca98058b5..2f2a1be81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,6 +136,21 @@ commands: command: | sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev + install-libprotobuf-mutator: + steps: + - run: + name: Install libprotobuf-mutator libs + command: | + git clone --single-branch --branch master --depth 1 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator + cd ~/libprotobuf-mutator && mkdir build && cd build + cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON + ninja && sudo ninja install + - run: + name: Setup environment variables + command: | + echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV + echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV + executors: windows-2xlarge: machine: @@ -732,6 +747,23 @@ jobs: tools/check_format_compatible.sh - post-steps + build-fuzzers: + machine: + image: ubuntu-2004:202010-01 + resource_class: large + steps: + - pre-steps + - install-clang-13 + - run: sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool + - install-libprotobuf-mutator + - run: + name: "Build rocksdb lib" + command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib + - run: + name: "Build fuzzers" + command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer + - post-steps + workflows: version: 2 build-linux: @@ -838,6 +870,9 @@ workflows: build-microbench: jobs: - build-linux-microbench + build-fuzzers: + jobs: + - build-fuzzers nightly: triggers: - schedule: diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 000000000..9dab42105 --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1,5 @@ +db_fuzzer +db_map_fuzzer +sst_file_writer_fuzzer + +proto/gen/* diff --git a/fuzz/Makefile b/fuzz/Makefile index fa45b9e78..b83040504 100644 --- a/fuzz/Makefile +++ b/fuzz/Makefile @@ -20,10 +20,10 @@ PROTO_IN = $(ROOT_DIR)/fuzz/proto PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen ifneq ($(FUZZ_ENV), ossfuzz) -CC = clang++ +CC = $(CXX) CCFLAGS += -Wall -fsanitize=address,fuzzer CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR) -LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb +LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb else # OSS-Fuzz sets various environment flags that are used for compilation. # These environment flags depend on which type of sanitizer build is being @@ -42,15 +42,21 @@ CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKS LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb endif -.PHONY: gen_proto +.PHONY: gen_proto clean + +# Set PROTOC_BIN when invoking `make` if a custom protoc is required. +PROTOC_BIN ?= protoc gen_proto: mkdir -p $(PROTO_OUT) - protoc \ + $(PROTOC_BIN) \ --proto_path=$(PROTO_IN) \ --cpp_out=$(PROTO_OUT) \ $(PROTO_IN)/*.proto +clean: + rm -rf db_fuzzer db_map_fuzzer sst_file_writer_fuzzer $(PROTO_OUT) + db_fuzzer: db_fuzzer.cc $(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS) diff --git a/fuzz/README.md b/fuzz/README.md index 79b89bbc3..238b283a2 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -15,6 +15,11 @@ Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/b We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer, and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator. So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them. +On some systems, there are both protobuf2 and protobuf3 in the package management system, +make sure protobuf3 is installed. + +If you do not want to install protobuf library yourself, you can rely on libprotobuf-mutator to download protobuf +for you. For details about installation, please refer to [libprotobuf-mutator README](https://github.com/google/libprotobuf-mutator#readme) ## Example diff --git a/fuzz/db_fuzzer.cc b/fuzz/db_fuzzer.cc index 3881cab26..383a95096 100644 --- a/fuzz/db_fuzzer.cc +++ b/fuzz/db_fuzzer.cc @@ -153,6 +153,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { delete iter; break; } + case OP_COUNT: + break; } } diff --git a/fuzz/sst_file_writer_fuzzer.cc b/fuzz/sst_file_writer_fuzzer.cc index ab0af8872..e93b9a3f5 100644 --- a/fuzz/sst_file_writer_fuzzer.cc +++ b/fuzz/sst_file_writer_fuzzer.cc @@ -12,9 +12,33 @@ #include "rocksdb/file_system.h" #include "rocksdb/sst_file_writer.h" #include "src/libfuzzer/libfuzzer_macro.h" +#include "table/table_builder.h" #include "table/table_reader.h" #include "util.h" +using ROCKSDB_NAMESPACE::BytewiseComparator; +using ROCKSDB_NAMESPACE::Comparator; +using ROCKSDB_NAMESPACE::EnvOptions; +using ROCKSDB_NAMESPACE::ExternalSstFileInfo; +using ROCKSDB_NAMESPACE::FileOptions; +using ROCKSDB_NAMESPACE::FileSystem; +using ROCKSDB_NAMESPACE::ImmutableCFOptions; +using ROCKSDB_NAMESPACE::ImmutableOptions; +using ROCKSDB_NAMESPACE::InternalIterator; +using ROCKSDB_NAMESPACE::IOOptions; +using ROCKSDB_NAMESPACE::kMaxSequenceNumber; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ParsedInternalKey; +using ROCKSDB_NAMESPACE::ParseInternalKey; +using ROCKSDB_NAMESPACE::RandomAccessFileReader; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::SstFileWriter; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::TableReader; +using ROCKSDB_NAMESPACE::TableReaderCaller; +using ROCKSDB_NAMESPACE::TableReaderOptions; +using ROCKSDB_NAMESPACE::ValueType; + // Keys in SST file writer operations must be unique and in ascending order. // For each DBOperation generated by the fuzzer, this function is called on // it to deduplicate and sort the keys in the DBOperations. @@ -60,15 +84,15 @@ TableReader* NewTableReader(const std::string& sst_file_path, std::unique_ptr table_reader; const auto& fs = options.env->GetFileSystem(); FileOptions fopts(env_options); - Status s = options.env->GetFileSize(sst_file_path, fopts.io_options, - &file_size, nullptr); + Status s = options.env->GetFileSize(sst_file_path, &file_size); if (s.ok()) { s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader, nullptr); } if (s.ok()) { - TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr, - env_options, cf_ioptions.internal_comparator); + ImmutableOptions iopts(options, cf_ioptions); + TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options, + cf_ioptions.internal_comparator); t_opt.largest_seqno = kMaxSequenceNumber; s = options.table_factory->NewTableReader(t_opt, std::move(file_reader), file_size, &table_reader, @@ -153,7 +177,7 @@ DEFINE_PROTO_FUZZER(DBOperations& input) { // Iterate and verify key-value pairs. std::unique_ptr table_reader( - NewTableReader(sstfile, options, env_options, cf_ioptions)); + ::NewTableReader(sstfile, options, env_options, cf_ioptions)); ReadOptions roptions; CHECK_OK(table_reader->VerifyChecksum(roptions, TableReaderCaller::kUncategorized));