Use in-repo gtest in buck build (#6858)

Summary:
... so that we have freedom to upgrade it (see https://github.com/facebook/rocksdb/issues/6808).

As a side benefit, gtest will no longer be linked into main library in
buck build.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6858

Test Plan: fb internal build & link

Reviewed By: riversand963

Differential Revision: D21652061

Pulled By: pdillinger

fbshipit-source-id: 6018104af944debde576b5beda6c134e737acedb

Makefile

@@ -327,7 +327,7 @@ endif
 
 export GTEST_THROW_ON_FAILURE=1
 export GTEST_HAS_EXCEPTIONS=1
-GTEST_DIR = ./third-party/gtest-1.8.1/fused-src
+GTEST_DIR = third-party/gtest-1.8.1/fused-src
 # AIX: pre-defined system headers are surrounded by an extern "C" block
 ifeq ($(PLATFORM), OS_AIX)
 	PLATFORM_CCFLAGS += -I$(GTEST_DIR)
@@ -2219,6 +2219,7 @@ endif
 #  	Source files dependencies detection
 # ---------------------------------------------------------------------------
 
+# FIXME: nothing checks that entries in MAIN_SOURCES actually exist
 all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 DEPFILES = $(all_sources:.cc=.cc.d)
 

TARGETS

@@ -26,7 +26,6 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("lz4", None, "lz4"),
     ("zstd", None),
     ("tbb", None),
-    ("googletest", None, "gtest"),
 ]
 
 ROCKSDB_OS_DEPS = [
@@ -79,6 +78,7 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
     # Directories with files for #include
     "-I" + REPO_PATH + "include/",
     "-I" + REPO_PATH,
+    "-I" + REPO_PATH + "third-party/gtest-1.8.1/fused-src/",
 ]
 
 ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
@@ -398,7 +398,10 @@ cpp_library(
     os_deps = ROCKSDB_OS_DEPS,
     os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
     preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    deps = [":rocksdb_lib"],
+    deps = [
+        ":rocksdb_lib",
+        ":rocksdb_third_party_gtest",
+    ],
     external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 
@@ -446,6 +449,19 @@ cpp_library(
     external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 
+cpp_library(
+    name = "rocksdb_third_party_gtest",
+    srcs = ["third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc"],
+    headers = ["third-party/gtest-1.8.1/fused-src/gtest/gtest.h"],
+    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
+    compiler_flags = ROCKSDB_COMPILER_FLAGS,
+    os_deps = ROCKSDB_OS_DEPS,
+    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
+    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
+    deps = [],
+    external_deps = ROCKSDB_EXTERNAL_DEPS,
+)
+
 cpp_library(
     name = "env_basic_test_lib",
     srcs = ["env/env_basic_test.cc"],

buckifier/buckify_rocksdb.py

@@ -109,6 +109,15 @@ def get_tests(repo_path):
     return tests
 
 
+# Get gtest dir from Makefile
+def get_gtest_dir(repo_path):
+    for line in open(repo_path + "/Makefile"):
+        if line.strip().startswith("GTEST_DIR ="):
+            return line.split("=")[1].strip()
+    # if not found
+    exit_with_error("Unable to find GTEST_DIR in Makefile")
+
+
 # Parse extra dependencies passed by user from command line
 def get_dependencies():
     deps_map = {
@@ -142,11 +151,14 @@ def generate_targets(repo_path, deps_map):
     cc_files = get_cc_files(repo_path)
     # get tests from Makefile
     tests = get_tests(repo_path)
+    # get gtest dir
+    gtest_dir = get_gtest_dir(repo_path) + "/"
 
     if src_mk is None or cc_files is None or tests is None:
         return False
 
-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
+    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, gtest_dir)
+
     # rocksdb_lib
     TARGETS.add_library(
         "rocksdb_lib",
@@ -159,7 +171,7 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("TEST_LIB_SOURCES", []) +
         src_mk.get("EXP_LIB_SOURCES", []) +
         src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib"])
+        [":rocksdb_lib", ":rocksdb_third_party_gtest"])
     # rocksdb_tools_lib
     TARGETS.add_library(
         "rocksdb_tools_lib",
@@ -173,6 +185,12 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("ANALYZER_LIB_SOURCES", [])
         + src_mk.get('STRESS_LIB_SOURCES', [])
         + ["test_util/testutil.cc"])
+    # rocksdb_third_party_gtest
+    TARGETS.add_library(
+        "rocksdb_third_party_gtest",
+        [gtest_dir + "gtest/gtest-all.cc"],
+        [],
+        [gtest_dir + "gtest/gtest.h"])
 
     print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
     # test for every test we found in the Makefile
@@ -219,6 +237,7 @@ def get_rocksdb_path():
 
     return rocksdb_path
 
+
 def exit_with_error(msg):
     print(ColorString.error(msg))
     sys.exit(1)

buckifier/check_buck_targets.sh

@@ -26,6 +26,7 @@ then
 else
   echo "Please run 'python buckifier/buckify_rocksdb.py' to update TARGETS file."
   echo "Do not manually update TARGETS file."
+  python --version
   mv TARGETS.bkp TARGETS
   exit 1
 fi

buckifier/targets_builder.py

@@ -25,10 +25,12 @@ def pretty_list(lst, indent=8):
 
 
 class TARGETSBuilder(object):
-    def __init__(self, path):
+    def __init__(self, path, gtest_dir):
         self.path = path
         self.targets_file = open(path, 'w')
-        self.targets_file.write(targets_cfg.rocksdb_target_header)
+        header = targets_cfg.rocksdb_target_header_template.format(
+            gtest_dir=gtest_dir)
+        self.targets_file.write(header)
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
@@ -42,6 +44,8 @@ class TARGETSBuilder(object):
         if headers is None:
             headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
+        else:
+            headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.library_template.format(
             name=name,
             srcs=pretty_list(srcs),
@@ -55,6 +59,8 @@ class TARGETSBuilder(object):
         if headers is None:
             headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
+        else:
+            headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.rocksdb_library_template.format(
             name=name,
             srcs=pretty_list(srcs),

buckifier/targets_cfg.py

@@ -4,7 +4,8 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-rocksdb_target_header = """# This file \100generated by `python buckifier/buckify_rocksdb.py`
+rocksdb_target_header_template = \
+    """# This file \100generated by `python buckifier/buckify_rocksdb.py`
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
@@ -32,7 +33,6 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("lz4", None, "lz4"),
     ("zstd", None),
     ("tbb", None),
-    ("googletest", None, "gtest"),
 ]
 
 ROCKSDB_OS_DEPS = [
@@ -85,13 +85,14 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
     # Directories with files for #include
     "-I" + REPO_PATH + "include/",
     "-I" + REPO_PATH,
+    "-I" + REPO_PATH + "{gtest_dir}",
 ]
 
-ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
+ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
     "x86_64": [
         "-DHAVE_PCLMUL",
     ],
-}
+}}
 
 build_mode = read_config("fbcode", "build_mode")
 

src.mk

@@ -428,7 +428,7 @@ MAIN_SOURCES =                                                          \
   table/table_reader_bench.cc                                           \
   table/table_test.cc                                                   \
   table/block_fetcher_test.cc                                           \
-  third-party/gtest-1.7.0/fused-src/gtest/gtest-all.cc                  \
+  third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc                  \
   tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc         \
   tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc         \
   tools/db_bench.cc                                                     \