Change buckifier to support parameterized dependencies (#5648)

Summary:
Users may desire to specify extra dependencies via buck. This PR allows users to pass additional dependencies as a JSON object so that the buckifier script can generate TARGETS file with desired extra dependencies.

Test plan (on dev server)
```
$python buckifier/buckify_rocksdb.py '{"fake": {"extra_deps": [":test_dep", "//fakes/module:mock1"], "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]}}'
Generating TARGETS
Extra dependencies:
{'': {'extra_compiler_flags': [], 'extra_deps': []}, 'test_dep1': {'extra_compiler_flags': ['-O2', '-DROCKSDB_LITE'], 'extra_deps': [':fake', '//dep1/mock']}}
Generated TARGETS Summary:
- 5 libs
- 0 binarys
- 296 tests
```
Verify the TARGETS file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5648

Differential Revision: D16565043

Pulled By: riversand963

fbshipit-source-id: a6ef02274174fcf159692d7b846e828454d01e89

buckifier/buckify_rocksdb.py

@@ -4,12 +4,31 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 from targets_builder import TARGETSBuilder
+import json
 import os
 import fnmatch
 import sys
 
 from util import ColorString
 
+# This script generates TARGETS file for Buck.
+# Buck is a build tool specifying dependencies among different build targets.
+# User can pass extra dependencies as a JSON object via command line, and this
+# script can include these dependencies in the generate TARGETS file.
+# Usage:
+# $python buckifier/buckify_rocksdb.py
+# (This generates a TARGET file without user-specified dependency for unit
+# tests.)
+# $python buckifier/buckify_rocksdb.py \
+#        '{"fake": { \
+#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],  \
+#                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
+#                  } \
+#         }'
+# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
+# unit tests, and will use the extra_compiler_flags to compile the unit test
+# source.)
+
 # tests to export as libraries for inclusion in other projects
 _EXPORTED_TEST_LIBS = ["env_basic_test"]
 
@@ -86,8 +105,38 @@ def get_tests(repo_path):
     return tests
 
 
+# Parse extra dependencies passed by user from command line
+def get_dependencies():
+    deps_map = {
+        ''.encode('ascii'): {
+            'extra_deps'.encode('ascii'): [],
+            'extra_compiler_flags'.encode('ascii'): []
+        }
+    }
+    if len(sys.argv) < 2:
+        return deps_map
+
+    def encode_dict(data):
+        rv = {}
+        for k, v in data.items():
+            if isinstance(k, unicode):
+                k = k.encode('ascii')
+            if isinstance(v, unicode):
+                v = v.encode('ascii')
+            elif isinstance(v, list):
+                v = [x.encode('ascii') for x in v]
+            elif isinstance(v, dict):
+                v = encode_dict(v)
+            rv[k] = v
+        return rv
+    extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
+    for target_alias, deps in extra_deps.items():
+        deps_map[target_alias] = deps
+    return deps_map
+
+
 # Prepare TARGETS file for buck
-def generate_targets(repo_path):
+def generate_targets(repo_path, deps_map):
     print(ColorString.info("Generating TARGETS"))
     # parsed src.mk file
     src_mk = parse_src_mk(repo_path)
@@ -121,24 +170,33 @@ def generate_targets(repo_path):
         ["test_util/testutil.cc"],
         [":rocksdb_lib"])
 
+    print("Extra dependencies:\n{0}".format(str(deps_map)))
     # test for every test we found in the Makefile
-    for test in sorted(tests):
+    for target_alias, deps in deps_map.items():
-        match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+        for test in sorted(tests):
-        if len(match_src) == 0:
+            match_src = [src for src in cc_files if ("/%s.c" % test) in src]
-            print(ColorString.warning("Cannot find .cc file for %s" % test))
+            if len(match_src) == 0:
-            continue
+                print(ColorString.warning("Cannot find .cc file for %s" % test))
-        elif len(match_src) > 1:
+                continue
-            print(ColorString.warning("Found more than one .cc for %s" % test))
+            elif len(match_src) > 1:
-            print(match_src)
+                print(ColorString.warning("Found more than one .cc for %s" % test))
-            continue
+                print(match_src)
+                continue
 
-        assert(len(match_src) == 1)
+            assert(len(match_src) == 1)
-        is_parallel = tests[test]
+            is_parallel = tests[test]
-        TARGETS.register_test(test, match_src[0], is_parallel)
+            test_target_name = \
+                test if not target_alias else test + "_" + target_alias
+            TARGETS.register_test(
+                test_target_name,
+                match_src[0],
+                is_parallel,
+                deps['extra_deps'],
+                deps['extra_compiler_flags'])
 
-        if test in _EXPORTED_TEST_LIBS:
+            if test in _EXPORTED_TEST_LIBS:
-            test_library = "%s_lib" % test
+                test_library = "%s_lib" % test_target_name
-            TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
+                TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
     TARGETS.flush_tests()
 
     print(ColorString.info("Generated TARGETS Summary:"))
@@ -163,8 +221,9 @@ def exit_with_error(msg):
 
 
 def main():
+    deps_map = get_dependencies()
     # Generate TARGETS file for buck
-    ok = generate_targets(get_rocksdb_path())
+    ok = generate_targets(get_rocksdb_path(), deps_map)
     if not ok:
         exit_with_error("Failed to generate TARGETS files")
 

buckifier/targets_builder.py

@@ -51,14 +51,21 @@ class TARGETSBuilder:
             pretty_list(deps)))
         self.total_bin = self.total_bin + 1
 
-    def register_test(self, test_name, src, is_parallel):
+    def register_test(self,
+                      test_name,
+                      src,
+                      is_parallel,
+                      extra_deps,
+                      extra_compiler_flags):
         exec_mode = "serial"
         if is_parallel:
             exec_mode = "parallel"
         self.tests_cfg += targets_cfg.test_cfg_template % (
             test_name,
             str(src),
-            str(exec_mode))
+            str(exec_mode),
+            extra_deps,
+            extra_compiler_flags)
 
         self.total_test = self.total_test + 1
 

buckifier/targets_cfg.py

@@ -140,11 +140,13 @@ test_cfg_template = """    [
         "%s",
         "%s",
         "%s",
+        %s,
+        %s,
     ],
 """
 
 unittests_template = """
-# [test_name, test_src, test_type]
+# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
 ROCKS_TESTS = [
 %s]
 
@@ -153,6 +155,8 @@ ROCKS_TESTS = [
 # will not be included.
 [
     test_binary(
+        extra_compiler_flags = extra_compiler_flags,
+        extra_deps = extra_deps,
         parallelism = parallelism,
         rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
         rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
@@ -163,7 +167,7 @@ ROCKS_TESTS = [
         test_cc = test_cc,
         test_name = test_name,
     )
-    for test_name, test_cc, parallelism in ROCKS_TESTS
+    for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
     if not is_opt_mode
 ]
 """

defs.bzl

@@ -12,7 +12,9 @@ def test_binary(
         rocksdb_compiler_flags,
         rocksdb_preprocessor_flags,
         rocksdb_external_deps,
-        rocksdb_os_deps):
+        rocksdb_os_deps,
+        extra_deps,
+        extra_compiler_flags):
     TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
 
     ttype = "gtest" if parallelism == "parallel" else "simple"
@@ -23,9 +25,9 @@ def test_binary(
         srcs = [test_cc],
         arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
         os_preprocessor_flags = rocksdb_os_preprocessor_flags,
-        compiler_flags = rocksdb_compiler_flags,
+        compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
         preprocessor_flags = rocksdb_preprocessor_flags,
-        deps = [":rocksdb_test_lib"],
+        deps = [":rocksdb_test_lib"] + extra_deps,
         os_deps = rocksdb_os_deps,
         external_deps = rocksdb_external_deps,
     )