update buckifier, add support for microbenchmarks (#9598)

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/9598

Reviewed By: jay-zhuang, hodgesds

Differential Revision: D34130191

fbshipit-source-id: e5413f7d6af70a66940022d153b64a3383eccff1

buckifier/buckify_rocksdb.py

@@ -163,9 +163,8 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("EXP_LIB_SOURCES", []) +
         src_mk.get("ANALYZER_LIB_SOURCES", []),
         [":rocksdb_lib"],
-        extra_external_deps=""" + [
+        extra_test_libs=True
-        ("googletest", None, "gtest"),
+        )
-    ]""")
     # rocksdb_tools_lib
     TARGETS.add_library(
         "rocksdb_tools_lib",
@@ -184,12 +183,19 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("ANALYZER_LIB_SOURCES", [])
         + src_mk.get('STRESS_LIB_SOURCES', [])
         + ["test_util/testutil.cc"])
-
+    # bench binaries
+    for src in src_mk.get("MICROBENCH_SOURCES", []):
+        name =  src.rsplit('/',1)[1].split('.')[0] if '/' in src else src.split('.')[0]
+        TARGETS.add_binary(
+            name,
+            [src],
+            [],
+            extra_bench_libs=True
+            )
     print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
 
     # Dictionary test executable name -> relative source file path
     test_source_map = {}
-    print(src_mk)
 
     # c_test.c is added through TARGETS.add_c_test(). If there
     # are more than one .c test file, we need to extend
@@ -200,6 +206,23 @@ def generate_targets(repo_path, deps_map):
             return False
     TARGETS.add_c_test()
 
+    try:
+        with open(f"{repo_path}/buckifier/bench.json") as json_file:
+            fast_fancy_bench_config_list = json.load(json_file)
+            for config_dict in fast_fancy_bench_config_list:
+                TARGETS.add_fancy_bench_config(config_dict['name'],config_dict['benchmarks'], False, config_dict['expected_runtime'])
+
+        with open(f"{repo_path}/buckifier/bench-slow.json") as json_file:
+            slow_fancy_bench_config_list = json.load(json_file)
+            for config_dict in slow_fancy_bench_config_list:
+                TARGETS.add_fancy_bench_config(config_dict['name']+"_slow",config_dict['benchmarks'], True, config_dict['expected_runtime'])
+
+    except (FileNotFoundError, KeyError):
+        print(ColorString.warning("Failed to process bench config jsons"))
+        pass
+
+    TARGETS.add_test_header()
+
     for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
         test = test_src.split('.c')[0].strip().split('/')[-1].strip()
         test_source_map[test] = test_src
@@ -213,17 +236,21 @@ def generate_targets(repo_path, deps_map):
 
             test_target_name = \
                 test if not target_alias else test + "_" + target_alias
-            TARGETS.register_test(
-                test_target_name,
-                test_src,
-                test not in non_parallel_tests,
-                json.dumps(deps['extra_deps']),
-                json.dumps(deps['extra_compiler_flags']))
 
             if test in _EXPORTED_TEST_LIBS:
                 test_library = "%s_lib" % test_target_name
-                TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
+                TARGETS.add_library(test_library, [test_src], deps=[":rocksdb_test_lib"], extra_test_libs=True)
-    TARGETS.flush_tests()
+                TARGETS.register_test(
+                    test_target_name,
+                    test_src,
+                    deps = json.dumps(deps['extra_deps'] + [':'+test_library]),
+                    extra_compiler_flags = json.dumps(deps['extra_compiler_flags']))
+            else:
+                TARGETS.register_test(
+                    test_target_name,
+                    test_src,
+                    deps = json.dumps(deps['extra_deps'] + [":rocksdb_test_lib"] ),
+                    extra_compiler_flags = json.dumps(deps['extra_compiler_flags']))
 
     print(ColorString.info("Generated TARGETS Summary:"))
     print(ColorString.info("- %d libs" % TARGETS.total_lib))

buckifier/targets_builder.py

@@ -10,6 +10,7 @@ except ImportError:
     from __builtin__ import object
     from __builtin__ import str
 import targets_cfg
+import pprint
 
 def pretty_list(lst, indent=8):
     if lst is None or len(lst) == 0:
@@ -40,85 +41,71 @@ class TARGETSBuilder(object):
         self.targets_file.close()
 
     def add_library(self, name, srcs, deps=None, headers=None,
-                    extra_external_deps="", link_whole=False):
+                    extra_external_deps="", link_whole=False,
-        headers_attr_prefix = ""
+                     external_dependencies=None, extra_test_libs=False):
-        if headers is None:
+        if headers is not None:
-            headers_attr_prefix = "auto_"
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
             headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.library_template.format(
             name=name,
             srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
             headers=headers,
             deps=pretty_list(deps),
             extra_external_deps=extra_external_deps,
-            link_whole=link_whole).encode("utf-8"))
+            link_whole=link_whole,
+            external_dependencies=pretty_list(external_dependencies),
+            extra_test_libs=extra_test_libs
+            ).encode("utf-8"))
         self.total_lib = self.total_lib + 1
 
-    def add_rocksdb_library(self, name, srcs, headers=None):
+    def add_rocksdb_library(self, name, srcs, headers=None,
-        headers_attr_prefix = ""
+                            external_dependencies=None):
-        if headers is None:
+        if headers is not None:
-            headers_attr_prefix = "auto_"
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
             headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.rocksdb_library_template.format(
             name=name,
             srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
+            headers=headers,
-            headers=headers).encode("utf-8"))
+            external_dependencies=pretty_list(external_dependencies)
+            ).encode("utf-8")
+            )
         self.total_lib = self.total_lib + 1
 
-    def add_binary(self, name, srcs, deps=None):
+    def add_binary(self, name, srcs, deps=None, extra_preprocessor_flags=None,extra_bench_libs=False):
         self.targets_file.write(targets_cfg.binary_template.format(
             name=name,
             srcs=pretty_list(srcs),
-            deps=pretty_list(deps)).encode("utf-8"))
+            deps=pretty_list(deps),
+            extra_preprocessor_flags=pretty_list(extra_preprocessor_flags),
+            extra_bench_libs=extra_bench_libs,
+            ).encode("utf-8"))
         self.total_bin = self.total_bin + 1
 
     def add_c_test(self):
         self.targets_file.write(b"""
-cpp_binary(
+add_c_test_wrapper()
-    name = "c_test_bin",
-    srcs = ["db/c_test.c"],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    deps = [":rocksdb_test_lib"],
-) if not is_opt_mode else None
-
-custom_unittest(
-    name = "c_test",
-    command = [
-        native.package_name() + "/buckifier/rocks_test_runner.sh",
-        "$(location :{})".format("c_test_bin"),
-    ],
-    type = "simple",
-) if not is_opt_mode else None
 """)
 
+    def add_test_header(self):
+        self.targets_file.write(b"""
+        # Generate a test rule for each entry in ROCKS_TESTS
+        # Do not build the tests in opt mode, since SyncPoint and other test code
+        # will not be included.
+""")
+
+    def add_fancy_bench_config(self, name, bench_config, slow, expected_runtime):
+        self.targets_file.write(targets_cfg.fancy_bench_template.format(
+                    name=name,
+                    bench_config=pprint.pformat(bench_config),
+                    slow=slow,
+                    expected_runtime=expected_runtime,
+                    ).encode("utf-8"))
+
     def register_test(self,
                       test_name,
                       src,
-                      is_parallel,
+                      deps,
-                      extra_deps,
                       extra_compiler_flags):
-        exec_mode = "serial"
-        if is_parallel:
-            exec_mode = "parallel"
-        self.tests_cfg += targets_cfg.test_cfg_template % (
-            test_name,
-            str(src),
-            str(exec_mode),
-            extra_deps,
-            extra_compiler_flags)
 
+        self.targets_file.write(targets_cfg.unittests_template.format(test_name=test_name,test_cc=str(src),deps=deps,
+            extra_compiler_flags=extra_compiler_flags).encode("utf-8"))
         self.total_test = self.total_test + 1
-
-    def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
-        self.tests_cfg = ""

buckifier/targets_cfg.py

@@ -12,223 +12,35 @@ rocksdb_target_header_template = \
 # only be validated by Facebook employees.
 #
 # @noautodeps @nocodemods
+load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")
 
-load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
-load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
-load(":defs.bzl", "test_binary")
-
-REPO_PATH = package_name() + "/"
-
-ROCKSDB_COMPILER_FLAGS_0 = [
-    "-fno-builtin-memcmp",
-    # Allow offsetof to work on non-standard layout types. Some compiler could
-    # completely reject our usage of offsetof, but we will solve that when it
-    # happens.
-    "-Wno-invalid-offsetof",
-    # Added missing flags from output of build_detect_platform
-    "-Wnarrowing",
-    "-DROCKSDB_NO_DYNAMIC_EXTENSION",
-]
-
-ROCKSDB_EXTERNAL_DEPS = [
-    ("bzip2", None, "bz2"),
-    ("snappy", None, "snappy"),
-    ("zlib", None, "z"),
-    ("gflags", None, "gflags"),
-    ("lz4", None, "lz4"),
-    ("zstd", None, "zstd"),
-]
-
-ROCKSDB_OS_DEPS_0 = [
-    (
-        "linux",
-        [
-            "third-party//numa:numa",
-            "third-party//liburing:uring",
-            "third-party//tbb:tbb",
-        ],
-    ),
-    (
-        "macos",
-        ["third-party//tbb:tbb"],
-    ),
-]
-
-ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
-    (
-        "linux",
-        [
-            "-DOS_LINUX",
-            "-DROCKSDB_FALLOCATE_PRESENT",
-            "-DROCKSDB_MALLOC_USABLE_SIZE",
-            "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
-            "-DROCKSDB_RANGESYNC_PRESENT",
-            "-DROCKSDB_SCHED_GETCPU_PRESENT",
-            "-DROCKSDB_IOURING_PRESENT",
-            "-DHAVE_SSE42",
-            "-DLIBURING",
-            "-DNUMA",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "macos",
-        [
-            "-DOS_MACOSX",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "windows",
-        [
-            "-DOS_WIN",
-            "-DWIN32",
-            "-D_MBCS",
-            "-DWIN64",
-            "-DNOMINMAX",
-        ],
-    ),
-]
-
-ROCKSDB_PREPROCESSOR_FLAGS = [
-    "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-
-    # Flags to enable libs we include
-    "-DSNAPPY",
-    "-DZLIB",
-    "-DBZIP2",
-    "-DLZ4",
-    "-DZSTD",
-    "-DZSTD_STATIC_LINKING_ONLY",
-    "-DGFLAGS=gflags",
-
-    # Added missing flags from output of build_detect_platform
-    "-DROCKSDB_BACKTRACE",
-]
-
-# Directories with files for #include
-ROCKSDB_INCLUDE_PATHS = [
-    "",
-    "include",
-]
-
-ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
-    "x86_64": [
-        "-DHAVE_PCLMUL",
-    ],
-}}
-
-build_mode = read_config("fbcode", "build_mode")
-
-is_opt_mode = build_mode.startswith("opt")
-
-# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
-# doesn't harm and avoid forgetting to add it.
-ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])
-
-sanitizer = read_config("fbcode", "sanitizer")
-
-# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
-# whether the binary is linked with jemalloc at runtime.
-ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
-    "linux",
-    ["-DROCKSDB_JEMALLOC"],
-)] if sanitizer == "" else [])
-
-ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
-    "linux",
-    ["third-party//jemalloc:headers"],
-)] if sanitizer == "" else [])
-
-ROCKSDB_LIB_DEPS = [
-    ":rocksdb_lib",
-    ":rocksdb_test_lib",
-] if not is_opt_mode else [":rocksdb_lib"]
 """
 
 
 library_template = """
-cpp_library(
+cpp_library_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], headers={headers}, link_whole={link_whole}, extra_test_libs={extra_test_libs})
-    name = "{name}",
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    link_whole = {link_whole},
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    exported_deps = [{deps}],
-    exported_external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
-)
 """
 
 rocksdb_library_template = """
-cpp_library(
+rocks_cpp_library_wrapper(name="{name}", srcs=[{srcs}], headers={headers})
-    name = "{name}",
+
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    exported_deps = ROCKSDB_LIB_DEPS,
-    exported_external_deps = ROCKSDB_EXTERNAL_DEPS,
-)
 """
 
+
+
 binary_template = """
-cpp_binary(
+cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocessor_flags=[{extra_preprocessor_flags}], extra_bench_libs={extra_bench_libs})
-    name = "{name}",
-    srcs = [{srcs}],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
-)
-"""
-
-test_cfg_template = """    [
-        "%s",
-        "%s",
-        "%s",
-        %s,
-        %s,
-    ],
 """
 
 unittests_template = """
-# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
+cpp_unittest_wrapper(name="{test_name}",
-ROCKS_TESTS = [
+            srcs=["{test_cc}"],
-{tests}]
+            deps={deps},
+            extra_compiler_flags={extra_compiler_flags})
+
+"""
+
+fancy_bench_template = """
+fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime})
 
-# Generate a test rule for each entry in ROCKS_TESTS
-# Do not build the tests in opt mode, since SyncPoint and other test code
-# will not be included.
-[
-    cpp_unittest(
-        name = test_name,
-        srcs = [test_cc],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS + extra_compiler_flags,
-        include_paths = ROCKSDB_INCLUDE_PATHS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        external_deps = ROCKSDB_EXTERNAL_DEPS + [
-            ("googletest", None, "gtest"),
-        ],
-    )
-    for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
-    if not is_opt_mode
-]
 """

defs.bzl

@@ -1,56 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# defs.bzl - Definitions for Facebook-specific buck build integration
-# in TARGETS
-
-load("@fbcode_macros//build_defs:coverage.bzl", "coverage")
-load("@fbcode_macros//build_defs:cpp_binary.bzl", "cpp_binary")
-load("@fbcode_macros//build_defs:custom_unittest.bzl", "custom_unittest")
-
-def test_binary(
-        test_name,
-        test_cc,
-        parallelism,
-        rocksdb_arch_preprocessor_flags,
-        rocksdb_os_preprocessor_flags,
-        rocksdb_compiler_flags,
-        rocksdb_preprocessor_flags,
-        rocksdb_external_deps,
-        rocksdb_os_deps,
-        extra_deps,
-        extra_compiler_flags):
-    TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
-
-    ttype = "gtest" if parallelism == "parallel" else "simple"
-    test_bin = test_name + "_bin"
-
-    cpp_binary(
-        name = test_bin,
-        srcs = [test_cc],
-        arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
-        os_preprocessor_flags = rocksdb_os_preprocessor_flags,
-        compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
-        preprocessor_flags = rocksdb_preprocessor_flags,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        os_deps = rocksdb_os_deps,
-        external_deps = rocksdb_external_deps,
-    )
-
-    binary_path = "$(location :{})".format(test_bin)
-
-    base_path = native.package_name()
-    tags = []
-    if coverage.is_coverage_enabled(base_path):
-        # This tag instructs testpilot to use
-        # the lower-memory coverage runner
-        # (e.g. it tells testpilot that the binary
-        # is actually instrumented with coverage info)
-        tags = ["coverage"]
-
-    custom_unittest(
-        name = test_name,
-        command = [TEST_RUNNER, binary_path],
-        type = ttype,
-        env = {"BUCK_BASE_BINARY": binary_path},
-        tags = tags,
-    )

microbench/ribbon_bench.cc

@@ -143,7 +143,7 @@ static void FilterQueryNegative(benchmark::State &state) {
       fp_cnt++;
     }
   }
-  state.counters["FP %"] =
+  state.counters["fp_pct"] =
       benchmark::Counter(fp_cnt * 100, benchmark::Counter::kAvgIterations);
 }
 BENCHMARK(FilterQueryNegative)->Apply(CustomArguments);

src.mk

@@ -590,6 +590,7 @@ TEST_MAIN_SOURCES_C = \
 
 MICROBENCH_SOURCES =                                          \
   microbench/ribbon_bench.cc                                  \
+  microbench/db_basic_bench.cc                                  \
 
 JNI_NATIVE_SOURCES =                                          \
   java/rocksjni/backupenginejni.cc                            \