Change buckifier to support parameterized dependencies (#5648)

Summary:
Users may desire to specify extra dependencies via buck. This PR allows users to pass additional dependencies as a JSON object so that the buckifier script can generate TARGETS file with desired extra dependencies.

Test plan (on dev server)
```
$python buckifier/buckify_rocksdb.py '{"fake": {"extra_deps": [":test_dep", "//fakes/module:mock1"], "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]}}'
Generating TARGETS
Extra dependencies:
{'': {'extra_compiler_flags': [], 'extra_deps': []}, 'test_dep1': {'extra_compiler_flags': ['-O2', '-DROCKSDB_LITE'], 'extra_deps': [':fake', '//dep1/mock']}}
Generated TARGETS Summary:
- 5 libs
- 0 binarys
- 296 tests
```
Verify the TARGETS file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5648

Differential Revision: D16565043

Pulled By: riversand963

fbshipit-source-id: a6ef02274174fcf159692d7b846e828454d01e89
This commit is contained in:
Yanqin Jin 2019-08-02 10:40:32 -07:00 committed by Facebook Github Bot
parent d1c9ede195
commit 30edf1874c
5 changed files with 396 additions and 26 deletions

302
TARGETS

File diff suppressed because it is too large Load Diff

View File

@ -4,12 +4,31 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from targets_builder import TARGETSBuilder
import json
import os
import fnmatch
import sys
from util import ColorString
# This script generates TARGETS file for Buck.
# Buck is a build tool specifying dependencies among different build targets.
# User can pass extra dependencies as a JSON object via command line, and this
# script can include these dependencies in the generate TARGETS file.
# Usage:
# $python buckifier/buckify_rocksdb.py
# (This generates a TARGET file without user-specified dependency for unit
# tests.)
# $python buckifier/buckify_rocksdb.py \
# '{"fake": { \
# "extra_deps": [":test_dep", "//fakes/module:mock1"], \
# "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
# } \
# }'
# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
# unit tests, and will use the extra_compiler_flags to compile the unit test
# source.)
# tests to export as libraries for inclusion in other projects
_EXPORTED_TEST_LIBS = ["env_basic_test"]
@ -86,8 +105,38 @@ def get_tests(repo_path):
return tests
# Parse extra dependencies passed by user from command line
def get_dependencies():
deps_map = {
''.encode('ascii'): {
'extra_deps'.encode('ascii'): [],
'extra_compiler_flags'.encode('ascii'): []
}
}
if len(sys.argv) < 2:
return deps_map
def encode_dict(data):
rv = {}
for k, v in data.items():
if isinstance(k, unicode):
k = k.encode('ascii')
if isinstance(v, unicode):
v = v.encode('ascii')
elif isinstance(v, list):
v = [x.encode('ascii') for x in v]
elif isinstance(v, dict):
v = encode_dict(v)
rv[k] = v
return rv
extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
for target_alias, deps in extra_deps.items():
deps_map[target_alias] = deps
return deps_map
# Prepare TARGETS file for buck
def generate_targets(repo_path):
def generate_targets(repo_path, deps_map):
print(ColorString.info("Generating TARGETS"))
# parsed src.mk file
src_mk = parse_src_mk(repo_path)
@ -121,24 +170,33 @@ def generate_targets(repo_path):
["test_util/testutil.cc"],
[":rocksdb_lib"])
print("Extra dependencies:\n{0}".format(str(deps_map)))
# test for every test we found in the Makefile
for test in sorted(tests):
match_src = [src for src in cc_files if ("/%s.c" % test) in src]
if len(match_src) == 0:
print(ColorString.warning("Cannot find .cc file for %s" % test))
continue
elif len(match_src) > 1:
print(ColorString.warning("Found more than one .cc for %s" % test))
print(match_src)
continue
for target_alias, deps in deps_map.items():
for test in sorted(tests):
match_src = [src for src in cc_files if ("/%s.c" % test) in src]
if len(match_src) == 0:
print(ColorString.warning("Cannot find .cc file for %s" % test))
continue
elif len(match_src) > 1:
print(ColorString.warning("Found more than one .cc for %s" % test))
print(match_src)
continue
assert(len(match_src) == 1)
is_parallel = tests[test]
TARGETS.register_test(test, match_src[0], is_parallel)
assert(len(match_src) == 1)
is_parallel = tests[test]
test_target_name = \
test if not target_alias else test + "_" + target_alias
TARGETS.register_test(
test_target_name,
match_src[0],
is_parallel,
deps['extra_deps'],
deps['extra_compiler_flags'])
if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test_target_name
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
TARGETS.flush_tests()
print(ColorString.info("Generated TARGETS Summary:"))
@ -163,8 +221,9 @@ def exit_with_error(msg):
def main():
deps_map = get_dependencies()
# Generate TARGETS file for buck
ok = generate_targets(get_rocksdb_path())
ok = generate_targets(get_rocksdb_path(), deps_map)
if not ok:
exit_with_error("Failed to generate TARGETS files")

View File

@ -51,14 +51,21 @@ class TARGETSBuilder:
pretty_list(deps)))
self.total_bin = self.total_bin + 1
def register_test(self, test_name, src, is_parallel):
def register_test(self,
test_name,
src,
is_parallel,
extra_deps,
extra_compiler_flags):
exec_mode = "serial"
if is_parallel:
exec_mode = "parallel"
self.tests_cfg += targets_cfg.test_cfg_template % (
test_name,
str(src),
str(exec_mode))
str(exec_mode),
extra_deps,
extra_compiler_flags)
self.total_test = self.total_test + 1

View File

@ -140,11 +140,13 @@ test_cfg_template = """ [
"%s",
"%s",
"%s",
%s,
%s,
],
"""
unittests_template = """
# [test_name, test_src, test_type]
# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
ROCKS_TESTS = [
%s]
@ -153,6 +155,8 @@ ROCKS_TESTS = [
# will not be included.
[
test_binary(
extra_compiler_flags = extra_compiler_flags,
extra_deps = extra_deps,
parallelism = parallelism,
rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
@ -163,7 +167,7 @@ ROCKS_TESTS = [
test_cc = test_cc,
test_name = test_name,
)
for test_name, test_cc, parallelism in ROCKS_TESTS
for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
if not is_opt_mode
]
"""

View File

@ -12,7 +12,9 @@ def test_binary(
rocksdb_compiler_flags,
rocksdb_preprocessor_flags,
rocksdb_external_deps,
rocksdb_os_deps):
rocksdb_os_deps,
extra_deps,
extra_compiler_flags):
TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
ttype = "gtest" if parallelism == "parallel" else "simple"
@ -23,9 +25,9 @@ def test_binary(
srcs = [test_cc],
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
os_preprocessor_flags = rocksdb_os_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
preprocessor_flags = rocksdb_preprocessor_flags,
deps = [":rocksdb_test_lib"],
deps = [":rocksdb_test_lib"] + extra_deps,
os_deps = rocksdb_os_deps,
external_deps = rocksdb_external_deps,
)