andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update buckifier, add support for microbenchmarks (#9598)

Browse Source 
Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/9598

Reviewed By: jay-zhuang, hodgesds

Differential Revision: D34130191

fbshipit-source-id: e5413f7d6af70a66940022d153b64a3383eccff1
This commit is contained in:
Patrick Somaru 2022-02-18 10:59:57 -08:00 committed by Facebook GitHub Bot
parent 2fbc672732
commit f066b5cecb
7 changed files with 1239 additions and 1915 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2731
TARGETS
View File

File diff suppressed because it is too large Load Diff

53
buckifier/buckify_rocksdb.py
View File

@ -163,9 +163,8 @@ def generate_targets(repo_path, deps_map):
src_mk.get("EXP_LIB_SOURCES", []) +
src_mk.get("ANALYZER_LIB_SOURCES", []),
[":rocksdb_lib"],
extra_external_deps=""" + [
("googletest", None, "gtest"),
]""")
extra_test_libs=True
)
# rocksdb_tools_lib
TARGETS.add_library(
"rocksdb_tools_lib",
@ -184,12 +183,19 @@ def generate_targets(repo_path, deps_map):
src_mk.get("ANALYZER_LIB_SOURCES", [])
+ src_mk.get('STRESS_LIB_SOURCES', [])
+ ["test_util/testutil.cc"])
# bench binaries
for src in src_mk.get("MICROBENCH_SOURCES", []):
name = src.rsplit('/',1)[1].split('.')[0] if '/' in src else src.split('.')[0]
TARGETS.add_binary(
name,
[src],
[],
extra_bench_libs=True
)
print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
# Dictionary test executable name -> relative source file path
test_source_map = {}
print(src_mk)
# c_test.c is added through TARGETS.add_c_test(). If there
# are more than one .c test file, we need to extend
@ -200,6 +206,23 @@ def generate_targets(repo_path, deps_map):
return False
TARGETS.add_c_test()
try:
with open(f"{repo_path}/buckifier/bench.json") as json_file:
fast_fancy_bench_config_list = json.load(json_file)
for config_dict in fast_fancy_bench_config_list:
TARGETS.add_fancy_bench_config(config_dict['name'],config_dict['benchmarks'], False, config_dict['expected_runtime'])
with open(f"{repo_path}/buckifier/bench-slow.json") as json_file:
slow_fancy_bench_config_list = json.load(json_file)
for config_dict in slow_fancy_bench_config_list:
TARGETS.add_fancy_bench_config(config_dict['name']+"_slow",config_dict['benchmarks'], True, config_dict['expected_runtime'])
except (FileNotFoundError, KeyError):
print(ColorString.warning("Failed to process bench config jsons"))
pass
TARGETS.add_test_header()
for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
test = test_src.split('.c')[0].strip().split('/')[-1].strip()
test_source_map[test] = test_src
@ -213,17 +236,21 @@ def generate_targets(repo_path, deps_map):
test_target_name = \
test if not target_alias else test + "_" + target_alias
TARGETS.register_test(
test_target_name,
test_src,
test not in non_parallel_tests,
json.dumps(deps['extra_deps']),
json.dumps(deps['extra_compiler_flags']))
if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test_target_name
TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
TARGETS.flush_tests()
TARGETS.add_library(test_library, [test_src], deps=[":rocksdb_test_lib"], extra_test_libs=True)
TARGETS.register_test(
test_target_name,
test_src,
deps = json.dumps(deps['extra_deps'] + [':'+test_library]),
extra_compiler_flags = json.dumps(deps['extra_compiler_flags']))
else:
TARGETS.register_test(
test_target_name,
test_src,
deps = json.dumps(deps['extra_deps'] + [":rocksdb_test_lib"] ),
extra_compiler_flags = json.dumps(deps['extra_compiler_flags']))
print(ColorString.info("Generated TARGETS Summary:"))
print(ColorString.info("- %d libs" % TARGETS.total_lib))

91
buckifier/targets_builder.py
View File

@ -10,6 +10,7 @@ except ImportError:
from __builtin__ import object
from __builtin__ import str
import targets_cfg
import pprint
def pretty_list(lst, indent=8):
if lst is None or len(lst) == 0:
@ -40,85 +41,71 @@ class TARGETSBuilder(object):
self.targets_file.close()
def add_library(self, name, srcs, deps=None, headers=None,
extra_external_deps="", link_whole=False):
headers_attr_prefix = ""
if headers is None:
headers_attr_prefix = "auto_"
headers = "AutoHeaders.RECURSIVE_GLOB"
else:
extra_external_deps="", link_whole=False,
external_dependencies=None, extra_test_libs=False):
if headers is not None:
headers = "[" + pretty_list(headers) + "]"
self.targets_file.write(targets_cfg.library_template.format(
name=name,
srcs=pretty_list(srcs),
headers_attr_prefix=headers_attr_prefix,
headers=headers,
deps=pretty_list(deps),
extra_external_deps=extra_external_deps,
link_whole=link_whole).encode("utf-8"))
link_whole=link_whole,
external_dependencies=pretty_list(external_dependencies),
extra_test_libs=extra_test_libs
).encode("utf-8"))
self.total_lib = self.total_lib + 1
def add_rocksdb_library(self, name, srcs, headers=None):
headers_attr_prefix = ""
if headers is None:
headers_attr_prefix = "auto_"
headers = "AutoHeaders.RECURSIVE_GLOB"
else:
def add_rocksdb_library(self, name, srcs, headers=None,
external_dependencies=None):
if headers is not None:
headers = "[" + pretty_list(headers) + "]"
self.targets_file.write(targets_cfg.rocksdb_library_template.format(
name=name,
srcs=pretty_list(srcs),
headers_attr_prefix=headers_attr_prefix,
headers=headers).encode("utf-8"))
headers=headers,
external_dependencies=pretty_list(external_dependencies)
).encode("utf-8")
)
self.total_lib = self.total_lib + 1
def add_binary(self, name, srcs, deps=None):
def add_binary(self, name, srcs, deps=None, extra_preprocessor_flags=None,extra_bench_libs=False):
self.targets_file.write(targets_cfg.binary_template.format(
name=name,
srcs=pretty_list(srcs),
deps=pretty_list(deps)).encode("utf-8"))
deps=pretty_list(deps),
extra_preprocessor_flags=pretty_list(extra_preprocessor_flags),
extra_bench_libs=extra_bench_libs,
).encode("utf-8"))
self.total_bin = self.total_bin + 1
def add_c_test(self):
self.targets_file.write(b"""
cpp_binary(
name = "c_test_bin",
srcs = ["db/c_test.c"],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
deps = [":rocksdb_test_lib"],
) if not is_opt_mode else None
custom_unittest(
name = "c_test",
command = [
native.package_name() + "/buckifier/rocks_test_runner.sh",
"$(location :{})".format("c_test_bin"),
],
type = "simple",
) if not is_opt_mode else None
add_c_test_wrapper()
""")
def add_test_header(self):
self.targets_file.write(b"""
# Generate a test rule for each entry in ROCKS_TESTS
# Do not build the tests in opt mode, since SyncPoint and other test code
# will not be included.
""")
def add_fancy_bench_config(self, name, bench_config, slow, expected_runtime):
self.targets_file.write(targets_cfg.fancy_bench_template.format(
name=name,
bench_config=pprint.pformat(bench_config),
slow=slow,
expected_runtime=expected_runtime,
).encode("utf-8"))
def register_test(self,
test_name,
src,
is_parallel,
extra_deps,
deps,
extra_compiler_flags):
exec_mode = "serial"
if is_parallel:
exec_mode = "parallel"
self.tests_cfg += targets_cfg.test_cfg_template % (
test_name,
str(src),
str(exec_mode),
extra_deps,
extra_compiler_flags)
self.targets_file.write(targets_cfg.unittests_template.format(test_name=test_name,test_cc=str(src),deps=deps,
extra_compiler_flags=extra_compiler_flags).encode("utf-8"))
self.total_test = self.total_test + 1
def flush_tests(self):
self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
self.tests_cfg = ""

220
buckifier/targets_cfg.py
View File

@ -12,223 +12,35 @@ rocksdb_target_header_template = \
# only be validated by Facebook employees.
#
# @noautodeps @nocodemods
load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")
load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
load(":defs.bzl", "test_binary")
REPO_PATH = package_name() + "/"
ROCKSDB_COMPILER_FLAGS_0 = [
"-fno-builtin-memcmp",
# Allow offsetof to work on non-standard layout types. Some compiler could
# completely reject our usage of offsetof, but we will solve that when it
# happens.
"-Wno-invalid-offsetof",
# Added missing flags from output of build_detect_platform
"-Wnarrowing",
"-DROCKSDB_NO_DYNAMIC_EXTENSION",
]
ROCKSDB_EXTERNAL_DEPS = [
("bzip2", None, "bz2"),
("snappy", None, "snappy"),
("zlib", None, "z"),
("gflags", None, "gflags"),
("lz4", None, "lz4"),
("zstd", None, "zstd"),
]
ROCKSDB_OS_DEPS_0 = [
(
"linux",
[
"third-party//numa:numa",
"third-party//liburing:uring",
"third-party//tbb:tbb",
],
),
(
"macos",
["third-party//tbb:tbb"],
),
]
ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
(
"linux",
[
"-DOS_LINUX",
"-DROCKSDB_FALLOCATE_PRESENT",
"-DROCKSDB_MALLOC_USABLE_SIZE",
"-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
"-DROCKSDB_RANGESYNC_PRESENT",
"-DROCKSDB_SCHED_GETCPU_PRESENT",
"-DROCKSDB_IOURING_PRESENT",
"-DHAVE_SSE42",
"-DLIBURING",
"-DNUMA",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
],
),
(
"macos",
[
"-DOS_MACOSX",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
],
),
(
"windows",
[
"-DOS_WIN",
"-DWIN32",
"-D_MBCS",
"-DWIN64",
"-DNOMINMAX",
],
),
]
ROCKSDB_PREPROCESSOR_FLAGS = [
"-DROCKSDB_SUPPORT_THREAD_LOCAL",
# Flags to enable libs we include
"-DSNAPPY",
"-DZLIB",
"-DBZIP2",
"-DLZ4",
"-DZSTD",
"-DZSTD_STATIC_LINKING_ONLY",
"-DGFLAGS=gflags",
# Added missing flags from output of build_detect_platform
"-DROCKSDB_BACKTRACE",
]
# Directories with files for #include
ROCKSDB_INCLUDE_PATHS = [
"",
"include",
]
ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
"x86_64": [
"-DHAVE_PCLMUL",
],
}}
build_mode = read_config("fbcode", "build_mode")
is_opt_mode = build_mode.startswith("opt")
# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
# doesn't harm and avoid forgetting to add it.
ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])
sanitizer = read_config("fbcode", "sanitizer")
# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
# whether the binary is linked with jemalloc at runtime.
ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
"linux",
["-DROCKSDB_JEMALLOC"],
)] if sanitizer == "" else [])
ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
"linux",
["third-party//jemalloc:headers"],
)] if sanitizer == "" else [])
ROCKSDB_LIB_DEPS = [
":rocksdb_lib",
":rocksdb_test_lib",
] if not is_opt_mode else [":rocksdb_lib"]
"""
library_template = """
cpp_library(
name = "{name}",
srcs = [{srcs}],
{headers_attr_prefix}headers = {headers},
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
link_whole = {link_whole},
os_deps = ROCKSDB_OS_DEPS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
exported_deps = [{deps}],
exported_external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
)
cpp_library_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], headers={headers}, link_whole={link_whole}, extra_test_libs={extra_test_libs})
"""
rocksdb_library_template = """
cpp_library(
name = "{name}",
srcs = [{srcs}],
{headers_attr_prefix}headers = {headers},
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
os_deps = ROCKSDB_OS_DEPS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
exported_deps = ROCKSDB_LIB_DEPS,
exported_external_deps = ROCKSDB_EXTERNAL_DEPS,
)
rocks_cpp_library_wrapper(name="{name}", srcs=[{srcs}], headers={headers})
"""
binary_template = """
cpp_binary(
name = "{name}",
srcs = [{srcs}],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
deps = [{deps}],
external_deps = ROCKSDB_EXTERNAL_DEPS,
)
"""
test_cfg_template = """ [
"%s",
"%s",
"%s",
%s,
%s,
],
cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocessor_flags=[{extra_preprocessor_flags}], extra_bench_libs={extra_bench_libs})
"""
unittests_template = """
# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
ROCKS_TESTS = [
{tests}]
cpp_unittest_wrapper(name="{test_name}",
srcs=["{test_cc}"],
deps={deps},
extra_compiler_flags={extra_compiler_flags})
"""
fancy_bench_template = """
fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime})
# Generate a test rule for each entry in ROCKS_TESTS
# Do not build the tests in opt mode, since SyncPoint and other test code
# will not be included.
[
cpp_unittest(
name = test_name,
srcs = [test_cc],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS + extra_compiler_flags,
include_paths = ROCKSDB_INCLUDE_PATHS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
deps = [":rocksdb_test_lib"] + extra_deps,
external_deps = ROCKSDB_EXTERNAL_DEPS + [
("googletest", None, "gtest"),
],
)
for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
if not is_opt_mode
]
"""

56
defs.bzl
View File

@ -1,56 +0,0 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# defs.bzl - Definitions for Facebook-specific buck build integration
# in TARGETS
load("@fbcode_macros//build_defs:coverage.bzl", "coverage")
load("@fbcode_macros//build_defs:cpp_binary.bzl", "cpp_binary")
load("@fbcode_macros//build_defs:custom_unittest.bzl", "custom_unittest")
def test_binary(
test_name,
test_cc,
parallelism,
rocksdb_arch_preprocessor_flags,
rocksdb_os_preprocessor_flags,
rocksdb_compiler_flags,
rocksdb_preprocessor_flags,
rocksdb_external_deps,
rocksdb_os_deps,
extra_deps,
extra_compiler_flags):
TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
ttype = "gtest" if parallelism == "parallel" else "simple"
test_bin = test_name + "_bin"
cpp_binary(
name = test_bin,
srcs = [test_cc],
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
os_preprocessor_flags = rocksdb_os_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
preprocessor_flags = rocksdb_preprocessor_flags,
deps = [":rocksdb_test_lib"] + extra_deps,
os_deps = rocksdb_os_deps,
external_deps = rocksdb_external_deps,
)
binary_path = "$(location :{})".format(test_bin)
base_path = native.package_name()
tags = []
if coverage.is_coverage_enabled(base_path):
# This tag instructs testpilot to use
# the lower-memory coverage runner
# (e.g. it tells testpilot that the binary
# is actually instrumented with coverage info)
tags = ["coverage"]
custom_unittest(
name = test_name,
command = [TEST_RUNNER, binary_path],
type = ttype,
env = {"BUCK_BASE_BINARY": binary_path},
tags = tags,
)

2
microbench/ribbon_bench.cc
View File

@ -143,7 +143,7 @@ static void FilterQueryNegative(benchmark::State &state) {
fp_cnt++;
}
}
state.counters["FP %"] =
state.counters["fp_pct"] =
benchmark::Counter(fp_cnt * 100, benchmark::Counter::kAvgIterations);
}
BENCHMARK(FilterQueryNegative)->Apply(CustomArguments);

1
src.mk
View File

@ -590,6 +590,7 @@ TEST_MAIN_SOURCES_C = \
MICROBENCH_SOURCES = \
microbench/ribbon_bench.cc \
microbench/db_basic_bench.cc \
JNI_NATIVE_SOURCES = \
java/rocksjni/backupenginejni.cc \