Change buckifier to support parameterized dependencies (#5648)
Summary:
Users may desire to specify extra dependencies via buck. This PR allows users to pass additional dependencies as a JSON object so that the buckifier script can generate TARGETS file with desired extra dependencies.
Test plan (on dev server)
```
$python buckifier/buckify_rocksdb.py '{"fake": {"extra_deps": [":test_dep", "//fakes/module:mock1"], "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]}}'
Generating TARGETS
Extra dependencies:
{'': {'extra_compiler_flags': [], 'extra_deps': []}, 'test_dep1': {'extra_compiler_flags': ['-O2', '-DROCKSDB_LITE'], 'extra_deps': [':fake', '//dep1/mock']}}
Generated TARGETS Summary:
- 5 libs
- 0 binarys
- 296 tests
```
Verify the TARGETS file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5648
Differential Revision: D16565043
Pulled By: riversand963
fbshipit-source-id: a6ef02274174fcf159692d7b846e828454d01e89
This commit is contained in:
Yanqin Jin
parent
ab3e6f6476
commit
08d5a83499
@ -4,12 +4,31 @@ from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
from targets_builder import TARGETSBuilder
|
||||
import json
|
||||
import os
|
||||
import fnmatch
|
||||
import sys
|
||||
|
||||
from util import ColorString
|
||||
|
||||
# This script generates TARGETS file for Buck.
|
||||
# Buck is a build tool specifying dependencies among different build targets.
|
||||
# User can pass extra dependencies as a JSON object via command line, and this
|
||||
# script can include these dependencies in the generate TARGETS file.
|
||||
# Usage:
|
||||
# $python buckifier/buckify_rocksdb.py
|
||||
# (This generates a TARGET file without user-specified dependency for unit
|
||||
# tests.)
|
||||
# $python buckifier/buckify_rocksdb.py \
|
||||
# '{"fake": { \
|
||||
# "extra_deps": [":test_dep", "//fakes/module:mock1"], \
|
||||
# "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
|
||||
# } \
|
||||
# }'
|
||||
# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
|
||||
# unit tests, and will use the extra_compiler_flags to compile the unit test
|
||||
# source.)
|
||||
|
||||
# tests to export as libraries for inclusion in other projects
|
||||
_EXPORTED_TEST_LIBS = ["env_basic_test"]
|
||||
|
||||
@ -86,8 +105,38 @@ def get_tests(repo_path):
|
||||
return tests
|
||||
|
||||
|
||||
# Parse extra dependencies passed by user from command line
|
||||
def get_dependencies():
|
||||
deps_map = {
|
||||
''.encode('ascii'): {
|
||||
'extra_deps'.encode('ascii'): [],
|
||||
'extra_compiler_flags'.encode('ascii'): []
|
||||
}
|
||||
}
|
||||
if len(sys.argv) < 2:
|
||||
return deps_map
|
||||
|
||||
def encode_dict(data):
|
||||
rv = {}
|
||||
for k, v in data.items():
|
||||
if isinstance(k, unicode):
|
||||
k = k.encode('ascii')
|
||||
if isinstance(v, unicode):
|
||||
v = v.encode('ascii')
|
||||
elif isinstance(v, list):
|
||||
v = [x.encode('ascii') for x in v]
|
||||
elif isinstance(v, dict):
|
||||
v = encode_dict(v)
|
||||
rv[k] = v
|
||||
return rv
|
||||
extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
|
||||
for target_alias, deps in extra_deps.items():
|
||||
deps_map[target_alias] = deps
|
||||
return deps_map
|
||||
|
||||
|
||||
# Prepare TARGETS file for buck
|
||||
def generate_targets(repo_path):
|
||||
def generate_targets(repo_path, deps_map):
|
||||
print(ColorString.info("Generating TARGETS"))
|
||||
# parsed src.mk file
|
||||
src_mk = parse_src_mk(repo_path)
|
||||
@ -121,24 +170,33 @@ def generate_targets(repo_path):
|
||||
["test_util/testutil.cc"],
|
||||
[":rocksdb_lib"])
|
||||
|
||||
print("Extra dependencies:\n{0}".format(str(deps_map)))
|
||||
# test for every test we found in the Makefile
|
||||
for test in sorted(tests):
|
||||
match_src = [src for src in cc_files if ("/%s.c" % test) in src]
|
||||
if len(match_src) == 0:
|
||||
print(ColorString.warning("Cannot find .cc file for %s" % test))
|
||||
continue
|
||||
elif len(match_src) > 1:
|
||||
print(ColorString.warning("Found more than one .cc for %s" % test))
|
||||
print(match_src)
|
||||
continue
|
||||
for target_alias, deps in deps_map.items():
|
||||
for test in sorted(tests):
|
||||
match_src = [src for src in cc_files if ("/%s.c" % test) in src]
|
||||
if len(match_src) == 0:
|
||||
print(ColorString.warning("Cannot find .cc file for %s" % test))
|
||||
continue
|
||||
elif len(match_src) > 1:
|
||||
print(ColorString.warning("Found more than one .cc for %s" % test))
|
||||
print(match_src)
|
||||
continue
|
||||
|
||||
assert(len(match_src) == 1)
|
||||
is_parallel = tests[test]
|
||||
TARGETS.register_test(test, match_src[0], is_parallel)
|
||||
assert(len(match_src) == 1)
|
||||
is_parallel = tests[test]
|
||||
test_target_name = \
|
||||
test if not target_alias else test + "_" + target_alias
|
||||
TARGETS.register_test(
|
||||
test_target_name,
|
||||
match_src[0],
|
||||
is_parallel,
|
||||
deps['extra_deps'],
|
||||
deps['extra_compiler_flags'])
|
||||
|
||||
if test in _EXPORTED_TEST_LIBS:
|
||||
test_library = "%s_lib" % test
|
||||
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
|
||||
if test in _EXPORTED_TEST_LIBS:
|
||||
test_library = "%s_lib" % test_target_name
|
||||
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
|
||||
TARGETS.flush_tests()
|
||||
|
||||
print(ColorString.info("Generated TARGETS Summary:"))
|
||||
@ -163,8 +221,9 @@ def exit_with_error(msg):
|
||||
|
||||
|
||||
def main():
|
||||
deps_map = get_dependencies()
|
||||
# Generate TARGETS file for buck
|
||||
ok = generate_targets(get_rocksdb_path())
|
||||
ok = generate_targets(get_rocksdb_path(), deps_map)
|
||||
if not ok:
|
||||
exit_with_error("Failed to generate TARGETS files")
|
||||
|
||||
|
||||
@ -51,14 +51,21 @@ class TARGETSBuilder:
|
||||
pretty_list(deps)))
|
||||
self.total_bin = self.total_bin + 1
|
||||
|
||||
def register_test(self, test_name, src, is_parallel):
|
||||
def register_test(self,
|
||||
test_name,
|
||||
src,
|
||||
is_parallel,
|
||||
extra_deps,
|
||||
extra_compiler_flags):
|
||||
exec_mode = "serial"
|
||||
if is_parallel:
|
||||
exec_mode = "parallel"
|
||||
self.tests_cfg += targets_cfg.test_cfg_template % (
|
||||
test_name,
|
||||
str(src),
|
||||
str(exec_mode))
|
||||
str(exec_mode),
|
||||
extra_deps,
|
||||
extra_compiler_flags)
|
||||
|
||||
self.total_test = self.total_test + 1
|
||||
|
||||
|
||||
@ -140,11 +140,13 @@ test_cfg_template = """ [
|
||||
"%s",
|
||||
"%s",
|
||||
"%s",
|
||||
%s,
|
||||
%s,
|
||||
],
|
||||
"""
|
||||
|
||||
unittests_template = """
|
||||
# [test_name, test_src, test_type]
|
||||
# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
|
||||
ROCKS_TESTS = [
|
||||
%s]
|
||||
|
||||
@ -153,6 +155,8 @@ ROCKS_TESTS = [
|
||||
# will not be included.
|
||||
[
|
||||
test_binary(
|
||||
extra_compiler_flags = extra_compiler_flags,
|
||||
extra_deps = extra_deps,
|
||||
parallelism = parallelism,
|
||||
rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
@ -163,7 +167,7 @@ ROCKS_TESTS = [
|
||||
test_cc = test_cc,
|
||||
test_name = test_name,
|
||||
)
|
||||
for test_name, test_cc, parallelism in ROCKS_TESTS
|
||||
for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
|
||||
if not is_opt_mode
|
||||
]
|
||||
"""
|
||||
|
||||
8
defs.bzl
8
defs.bzl
@ -12,7 +12,9 @@ def test_binary(
|
||||
rocksdb_compiler_flags,
|
||||
rocksdb_preprocessor_flags,
|
||||
rocksdb_external_deps,
|
||||
rocksdb_os_deps):
|
||||
rocksdb_os_deps,
|
||||
extra_deps,
|
||||
extra_compiler_flags):
|
||||
TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
|
||||
|
||||
ttype = "gtest" if parallelism == "parallel" else "simple"
|
||||
@ -23,9 +25,9 @@ def test_binary(
|
||||
srcs = [test_cc],
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
os_preprocessor_flags = rocksdb_os_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [":rocksdb_test_lib"],
|
||||
deps = [":rocksdb_test_lib"] + extra_deps,
|
||||
os_deps = rocksdb_os_deps,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user