From 30edf1874c11762a6cacf4434112ce34d13100d3 Mon Sep 17 00:00:00 2001 From: Yanqin Jin Date: Fri, 2 Aug 2019 10:40:32 -0700 Subject: [PATCH] Change buckifier to support parameterized dependencies (#5648) Summary: Users may desire to specify extra dependencies via buck. This PR allows users to pass additional dependencies as a JSON object so that the buckifier script can generate TARGETS file with desired extra dependencies. Test plan (on dev server) ``` $python buckifier/buckify_rocksdb.py '{"fake": {"extra_deps": [":test_dep", "//fakes/module:mock1"], "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]}}' Generating TARGETS Extra dependencies: {'': {'extra_compiler_flags': [], 'extra_deps': []}, 'test_dep1': {'extra_compiler_flags': ['-O2', '-DROCKSDB_LITE'], 'extra_deps': [':fake', '//dep1/mock']}} Generated TARGETS Summary: - 5 libs - 0 binarys - 296 tests ``` Verify the TARGETS file. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5648 Differential Revision: D16565043 Pulled By: riversand963 fbshipit-source-id: a6ef02274174fcf159692d7b846e828454d01e89 --- TARGETS | 302 ++++++++++++++++++++++++++++++++++- buckifier/buckify_rocksdb.py | 93 +++++++++-- buckifier/targets_builder.py | 11 +- buckifier/targets_cfg.py | 8 +- defs.bzl | 8 +- 5 files changed, 396 insertions(+), 26 deletions(-) diff --git a/TARGETS b/TARGETS index 884d69b14..25d7ff667 100644 --- a/TARGETS +++ b/TARGETS @@ -396,747 +396,1043 @@ cpp_library( external_deps = ROCKSDB_EXTERNAL_DEPS, ) -# [test_name, test_src, test_type] +# [test_name, test_src, test_type, extra_deps, extra_compiler_flags] ROCKS_TESTS = [ [ "arena_test", "memory/arena_test.cc", "serial", + [], + [], ], [ "auto_roll_logger_test", "logging/auto_roll_logger_test.cc", "serial", + [], + [], ], [ "autovector_test", "util/autovector_test.cc", "serial", + [], + [], ], [ "backupable_db_test", "utilities/backupable/backupable_db_test.cc", "parallel", + [], + [], ], [ "blob_db_test", "utilities/blob_db/blob_db_test.cc", "serial", + [], + [], ], [ "block_based_filter_block_test", "table/block_based/block_based_filter_block_test.cc", "serial", + [], + [], ], [ "block_cache_trace_analyzer_test", "tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc", "serial", + [], + [], ], [ "block_cache_tracer_test", "trace_replay/block_cache_tracer_test.cc", "serial", + [], + [], ], [ "block_test", "table/block_based/block_test.cc", "serial", + [], + [], ], [ "bloom_test", "util/bloom_test.cc", "serial", + [], + [], ], [ "c_test", "db/c_test.c", "serial", + [], + [], ], [ "cache_simulator_test", "utilities/simulator_cache/cache_simulator_test.cc", "serial", + [], + [], ], [ "cache_test", "cache/cache_test.cc", "serial", + [], + [], ], [ "cassandra_format_test", "utilities/cassandra/cassandra_format_test.cc", "serial", + [], + [], ], [ "cassandra_functional_test", "utilities/cassandra/cassandra_functional_test.cc", "serial", + [], + [], ], [ "cassandra_row_merge_test", "utilities/cassandra/cassandra_row_merge_test.cc", "serial", + [], + [], ], [ "cassandra_serialize_test", "utilities/cassandra/cassandra_serialize_test.cc", "serial", + [], + [], ], [ "checkpoint_test", "utilities/checkpoint/checkpoint_test.cc", "serial", + [], + [], ], [ "cleanable_test", "table/cleanable_test.cc", "serial", + [], + [], ], [ "coding_test", "util/coding_test.cc", "serial", + [], + [], ], [ "column_family_test", "db/column_family_test.cc", "serial", + [], + [], ], [ "compact_files_test", "db/compact_files_test.cc", "serial", + [], + [], ], [ "compact_on_deletion_collector_test", "utilities/table_properties_collectors/compact_on_deletion_collector_test.cc", "serial", + [], + [], ], [ "compaction_iterator_test", "db/compaction/compaction_iterator_test.cc", "serial", + [], + [], ], [ "compaction_job_stats_test", "db/compaction/compaction_job_stats_test.cc", "serial", + [], + [], ], [ "compaction_job_test", "db/compaction/compaction_job_test.cc", "serial", + [], + [], ], [ "compaction_picker_test", "db/compaction/compaction_picker_test.cc", "serial", + [], + [], ], [ "comparator_db_test", "db/comparator_db_test.cc", "serial", + [], + [], ], [ "corruption_test", "db/corruption_test.cc", "serial", + [], + [], ], [ "crc32c_test", "util/crc32c_test.cc", "serial", + [], + [], ], [ "cuckoo_table_builder_test", "table/cuckoo/cuckoo_table_builder_test.cc", "serial", + [], + [], ], [ "cuckoo_table_db_test", "db/cuckoo_table_db_test.cc", "serial", + [], + [], ], [ "cuckoo_table_reader_test", "table/cuckoo/cuckoo_table_reader_test.cc", "serial", + [], + [], ], [ "data_block_hash_index_test", "table/block_based/data_block_hash_index_test.cc", "serial", + [], + [], ], [ "db_basic_test", "db/db_basic_test.cc", "serial", + [], + [], ], [ "db_blob_index_test", "db/db_blob_index_test.cc", "serial", + [], + [], ], [ "db_block_cache_test", "db/db_block_cache_test.cc", "serial", + [], + [], ], [ "db_bloom_filter_test", "db/db_bloom_filter_test.cc", "parallel", + [], + [], ], [ "db_compaction_filter_test", "db/db_compaction_filter_test.cc", "parallel", + [], + [], ], [ "db_compaction_test", "db/db_compaction_test.cc", "parallel", + [], + [], ], [ "db_dynamic_level_test", "db/db_dynamic_level_test.cc", "serial", + [], + [], ], [ "db_encryption_test", "db/db_encryption_test.cc", "serial", + [], + [], ], [ "db_flush_test", "db/db_flush_test.cc", "serial", + [], + [], ], [ "db_inplace_update_test", "db/db_inplace_update_test.cc", "serial", + [], + [], ], [ "db_io_failure_test", "db/db_io_failure_test.cc", "serial", + [], + [], ], [ "db_iter_stress_test", "db/db_iter_stress_test.cc", "serial", + [], + [], ], [ "db_iter_test", "db/db_iter_test.cc", "serial", + [], + [], ], [ "db_iterator_test", "db/db_iterator_test.cc", "serial", + [], + [], ], [ "db_log_iter_test", "db/db_log_iter_test.cc", "serial", + [], + [], ], [ "db_memtable_test", "db/db_memtable_test.cc", "serial", + [], + [], ], [ "db_merge_operator_test", "db/db_merge_operator_test.cc", "parallel", + [], + [], ], [ "db_options_test", "db/db_options_test.cc", "serial", + [], + [], ], [ "db_properties_test", "db/db_properties_test.cc", "serial", + [], + [], ], [ "db_range_del_test", "db/db_range_del_test.cc", "serial", + [], + [], ], [ "db_secondary_test", "db/db_impl/db_secondary_test.cc", "serial", + [], + [], ], [ "db_sst_test", "db/db_sst_test.cc", "parallel", + [], + [], ], [ "db_statistics_test", "db/db_statistics_test.cc", "serial", + [], + [], ], [ "db_table_properties_test", "db/db_table_properties_test.cc", "serial", + [], + [], ], [ "db_tailing_iter_test", "db/db_tailing_iter_test.cc", "serial", + [], + [], ], [ "db_test", "db/db_test.cc", "parallel", + [], + [], ], [ "db_test2", "db/db_test2.cc", "serial", + [], + [], ], [ "db_universal_compaction_test", "db/db_universal_compaction_test.cc", "parallel", + [], + [], ], [ "db_wal_test", "db/db_wal_test.cc", "parallel", + [], + [], ], [ "db_write_test", "db/db_write_test.cc", "serial", + [], + [], ], [ "dbformat_test", "db/dbformat_test.cc", "serial", + [], + [], ], [ "delete_scheduler_test", "file/delete_scheduler_test.cc", "serial", + [], + [], ], [ "deletefile_test", "db/deletefile_test.cc", "serial", + [], + [], ], [ "dynamic_bloom_test", "util/dynamic_bloom_test.cc", "serial", + [], + [], ], [ "env_basic_test", "env/env_basic_test.cc", "serial", + [], + [], ], [ "env_logger_test", "logging/env_logger_test.cc", "serial", + [], + [], ], [ "env_test", "env/env_test.cc", "serial", + [], + [], ], [ "env_timed_test", "utilities/env_timed_test.cc", "serial", + [], + [], ], [ "error_handler_test", "db/error_handler_test.cc", "serial", + [], + [], ], [ "event_logger_test", "logging/event_logger_test.cc", "serial", + [], + [], ], [ "external_sst_file_basic_test", "db/external_sst_file_basic_test.cc", "serial", + [], + [], ], [ "external_sst_file_test", "db/external_sst_file_test.cc", "parallel", + [], + [], ], [ "fault_injection_test", "db/fault_injection_test.cc", "parallel", + [], + [], ], [ "file_indexer_test", "db/file_indexer_test.cc", "serial", + [], + [], ], [ "file_reader_writer_test", "util/file_reader_writer_test.cc", "parallel", + [], + [], ], [ "filelock_test", "util/filelock_test.cc", "serial", + [], + [], ], [ "filename_test", "db/filename_test.cc", "serial", + [], + [], ], [ "flush_job_test", "db/flush_job_test.cc", "serial", + [], + [], ], [ "full_filter_block_test", "table/block_based/full_filter_block_test.cc", "serial", + [], + [], ], [ "hash_table_test", "utilities/persistent_cache/hash_table_test.cc", "serial", + [], + [], ], [ "hash_test", "util/hash_test.cc", "serial", + [], + [], ], [ "heap_test", "util/heap_test.cc", "serial", + [], + [], ], [ "histogram_test", "monitoring/histogram_test.cc", "serial", + [], + [], ], [ "import_column_family_test", "db/import_column_family_test.cc", "parallel", + [], + [], ], [ "inlineskiplist_test", "memtable/inlineskiplist_test.cc", "parallel", + [], + [], ], [ "iostats_context_test", "monitoring/iostats_context_test.cc", "serial", + [], + [], ], [ "ldb_cmd_test", "tools/ldb_cmd_test.cc", "serial", + [], + [], ], [ "listener_test", "db/listener_test.cc", "serial", + [], + [], ], [ "log_test", "db/log_test.cc", "serial", + [], + [], ], [ "lru_cache_test", "cache/lru_cache_test.cc", "serial", + [], + [], ], [ "manual_compaction_test", "db/manual_compaction_test.cc", "parallel", + [], + [], ], [ "memory_test", "utilities/memory/memory_test.cc", "serial", + [], + [], ], [ "memtable_list_test", "db/memtable_list_test.cc", "serial", + [], + [], ], [ "merge_helper_test", "db/merge_helper_test.cc", "serial", + [], + [], ], [ "merge_test", "db/merge_test.cc", "serial", + [], + [], ], [ "merger_test", "table/merger_test.cc", "serial", + [], + [], ], [ "mock_env_test", "env/mock_env_test.cc", "serial", + [], + [], ], [ "object_registry_test", "utilities/object_registry_test.cc", "serial", + [], + [], ], [ "obsolete_files_test", "db/obsolete_files_test.cc", "serial", + [], + [], ], [ "optimistic_transaction_test", "utilities/transactions/optimistic_transaction_test.cc", "serial", + [], + [], ], [ "option_change_migration_test", "utilities/option_change_migration/option_change_migration_test.cc", "serial", + [], + [], ], [ "options_file_test", "db/options_file_test.cc", "serial", + [], + [], ], [ "options_settable_test", "options/options_settable_test.cc", "serial", + [], + [], ], [ "options_test", "options/options_test.cc", "serial", + [], + [], ], [ "options_util_test", "utilities/options/options_util_test.cc", "serial", + [], + [], ], [ "partitioned_filter_block_test", "table/block_based/partitioned_filter_block_test.cc", "serial", + [], + [], ], [ "perf_context_test", "db/perf_context_test.cc", "serial", + [], + [], ], [ "persistent_cache_test", "utilities/persistent_cache/persistent_cache_test.cc", "parallel", + [], + [], ], [ "plain_table_db_test", "db/plain_table_db_test.cc", "serial", + [], + [], ], [ "prefix_test", "db/prefix_test.cc", "serial", + [], + [], ], [ "range_del_aggregator_test", "db/range_del_aggregator_test.cc", "serial", + [], + [], ], [ "range_tombstone_fragmenter_test", "db/range_tombstone_fragmenter_test.cc", "serial", + [], + [], ], [ "rate_limiter_test", "util/rate_limiter_test.cc", "serial", + [], + [], ], [ "reduce_levels_test", "tools/reduce_levels_test.cc", "serial", + [], + [], ], [ "repair_test", "db/repair_test.cc", "serial", + [], + [], ], [ "repeatable_thread_test", "util/repeatable_thread_test.cc", "serial", + [], + [], ], [ "sim_cache_test", "utilities/simulator_cache/sim_cache_test.cc", "serial", + [], + [], ], [ "skiplist_test", "memtable/skiplist_test.cc", "serial", + [], + [], ], [ "slice_transform_test", "util/slice_transform_test.cc", "serial", + [], + [], ], [ "sst_dump_test", "tools/sst_dump_test.cc", "serial", + [], + [], ], [ "sst_file_reader_test", "table/sst_file_reader_test.cc", "serial", + [], + [], ], [ "statistics_test", "monitoring/statistics_test.cc", "serial", + [], + [], ], [ "stats_history_test", "monitoring/stats_history_test.cc", "serial", + [], + [], ], [ "stringappend_test", "utilities/merge_operators/string_append/stringappend_test.cc", "serial", + [], + [], ], [ "table_properties_collector_test", "db/table_properties_collector_test.cc", "serial", + [], + [], ], [ "table_test", "table/table_test.cc", "parallel", + [], + [], ], [ "thread_list_test", "util/thread_list_test.cc", "serial", + [], + [], ], [ "thread_local_test", "util/thread_local_test.cc", "serial", + [], + [], ], [ "timer_queue_test", "util/timer_queue_test.cc", "serial", + [], + [], ], [ "trace_analyzer_test", "tools/trace_analyzer_test.cc", "serial", + [], + [], ], [ "transaction_test", "utilities/transactions/transaction_test.cc", "parallel", + [], + [], ], [ "ttl_test", "utilities/ttl/ttl_test.cc", "serial", + [], + [], ], [ "util_merge_operators_test", "utilities/util_merge_operators_test.cc", "serial", + [], + [], ], [ "version_builder_test", "db/version_builder_test.cc", "serial", + [], + [], ], [ "version_edit_test", "db/version_edit_test.cc", "serial", + [], + [], ], [ "version_set_test", "db/version_set_test.cc", "serial", + [], + [], ], [ "wal_manager_test", "db/wal_manager_test.cc", "serial", + [], + [], ], [ "write_batch_test", "db/write_batch_test.cc", "serial", + [], + [], ], [ "write_batch_with_index_test", "utilities/write_batch_with_index/write_batch_with_index_test.cc", "serial", + [], + [], ], [ "write_buffer_manager_test", "memtable/write_buffer_manager_test.cc", "serial", + [], + [], ], [ "write_callback_test", "db/write_callback_test.cc", "serial", + [], + [], ], [ "write_controller_test", "db/write_controller_test.cc", "serial", + [], + [], ], [ "write_prepared_transaction_test", "utilities/transactions/write_prepared_transaction_test.cc", "parallel", + [], + [], ], [ "write_unprepared_transaction_test", "utilities/transactions/write_unprepared_transaction_test.cc", "parallel", + [], + [], ], ] @@ -1145,6 +1441,8 @@ ROCKS_TESTS = [ # will not be included. [ test_binary( + extra_compiler_flags = extra_compiler_flags, + extra_deps = extra_deps, parallelism = parallelism, rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS, rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS, @@ -1155,6 +1453,6 @@ ROCKS_TESTS = [ test_cc = test_cc, test_name = test_name, ) - for test_name, test_cc, parallelism in ROCKS_TESTS + for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS if not is_opt_mode ] diff --git a/buckifier/buckify_rocksdb.py b/buckifier/buckify_rocksdb.py index 94b63a4e8..fc59cf583 100644 --- a/buckifier/buckify_rocksdb.py +++ b/buckifier/buckify_rocksdb.py @@ -4,12 +4,31 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals from targets_builder import TARGETSBuilder +import json import os import fnmatch import sys from util import ColorString +# This script generates TARGETS file for Buck. +# Buck is a build tool specifying dependencies among different build targets. +# User can pass extra dependencies as a JSON object via command line, and this +# script can include these dependencies in the generate TARGETS file. +# Usage: +# $python buckifier/buckify_rocksdb.py +# (This generates a TARGET file without user-specified dependency for unit +# tests.) +# $python buckifier/buckify_rocksdb.py \ +# '{"fake": { \ +# "extra_deps": [":test_dep", "//fakes/module:mock1"], \ +# "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \ +# } \ +# }' +# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB +# unit tests, and will use the extra_compiler_flags to compile the unit test +# source.) + # tests to export as libraries for inclusion in other projects _EXPORTED_TEST_LIBS = ["env_basic_test"] @@ -86,8 +105,38 @@ def get_tests(repo_path): return tests +# Parse extra dependencies passed by user from command line +def get_dependencies(): + deps_map = { + ''.encode('ascii'): { + 'extra_deps'.encode('ascii'): [], + 'extra_compiler_flags'.encode('ascii'): [] + } + } + if len(sys.argv) < 2: + return deps_map + + def encode_dict(data): + rv = {} + for k, v in data.items(): + if isinstance(k, unicode): + k = k.encode('ascii') + if isinstance(v, unicode): + v = v.encode('ascii') + elif isinstance(v, list): + v = [x.encode('ascii') for x in v] + elif isinstance(v, dict): + v = encode_dict(v) + rv[k] = v + return rv + extra_deps = json.loads(sys.argv[1], object_hook=encode_dict) + for target_alias, deps in extra_deps.items(): + deps_map[target_alias] = deps + return deps_map + + # Prepare TARGETS file for buck -def generate_targets(repo_path): +def generate_targets(repo_path, deps_map): print(ColorString.info("Generating TARGETS")) # parsed src.mk file src_mk = parse_src_mk(repo_path) @@ -121,24 +170,33 @@ def generate_targets(repo_path): ["test_util/testutil.cc"], [":rocksdb_lib"]) + print("Extra dependencies:\n{0}".format(str(deps_map))) # test for every test we found in the Makefile - for test in sorted(tests): - match_src = [src for src in cc_files if ("/%s.c" % test) in src] - if len(match_src) == 0: - print(ColorString.warning("Cannot find .cc file for %s" % test)) - continue - elif len(match_src) > 1: - print(ColorString.warning("Found more than one .cc for %s" % test)) - print(match_src) - continue + for target_alias, deps in deps_map.items(): + for test in sorted(tests): + match_src = [src for src in cc_files if ("/%s.c" % test) in src] + if len(match_src) == 0: + print(ColorString.warning("Cannot find .cc file for %s" % test)) + continue + elif len(match_src) > 1: + print(ColorString.warning("Found more than one .cc for %s" % test)) + print(match_src) + continue - assert(len(match_src) == 1) - is_parallel = tests[test] - TARGETS.register_test(test, match_src[0], is_parallel) + assert(len(match_src) == 1) + is_parallel = tests[test] + test_target_name = \ + test if not target_alias else test + "_" + target_alias + TARGETS.register_test( + test_target_name, + match_src[0], + is_parallel, + deps['extra_deps'], + deps['extra_compiler_flags']) - if test in _EXPORTED_TEST_LIBS: - test_library = "%s_lib" % test - TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) + if test in _EXPORTED_TEST_LIBS: + test_library = "%s_lib" % test_target_name + TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) TARGETS.flush_tests() print(ColorString.info("Generated TARGETS Summary:")) @@ -163,8 +221,9 @@ def exit_with_error(msg): def main(): + deps_map = get_dependencies() # Generate TARGETS file for buck - ok = generate_targets(get_rocksdb_path()) + ok = generate_targets(get_rocksdb_path(), deps_map) if not ok: exit_with_error("Failed to generate TARGETS files") diff --git a/buckifier/targets_builder.py b/buckifier/targets_builder.py index 493cd8a8a..78db6a169 100644 --- a/buckifier/targets_builder.py +++ b/buckifier/targets_builder.py @@ -51,14 +51,21 @@ class TARGETSBuilder: pretty_list(deps))) self.total_bin = self.total_bin + 1 - def register_test(self, test_name, src, is_parallel): + def register_test(self, + test_name, + src, + is_parallel, + extra_deps, + extra_compiler_flags): exec_mode = "serial" if is_parallel: exec_mode = "parallel" self.tests_cfg += targets_cfg.test_cfg_template % ( test_name, str(src), - str(exec_mode)) + str(exec_mode), + extra_deps, + extra_compiler_flags) self.total_test = self.total_test + 1 diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py index 0ebd6d942..19ea77727 100644 --- a/buckifier/targets_cfg.py +++ b/buckifier/targets_cfg.py @@ -140,11 +140,13 @@ test_cfg_template = """ [ "%s", "%s", "%s", + %s, + %s, ], """ unittests_template = """ -# [test_name, test_src, test_type] +# [test_name, test_src, test_type, extra_deps, extra_compiler_flags] ROCKS_TESTS = [ %s] @@ -153,6 +155,8 @@ ROCKS_TESTS = [ # will not be included. [ test_binary( + extra_compiler_flags = extra_compiler_flags, + extra_deps = extra_deps, parallelism = parallelism, rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS, rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS, @@ -163,7 +167,7 @@ ROCKS_TESTS = [ test_cc = test_cc, test_name = test_name, ) - for test_name, test_cc, parallelism in ROCKS_TESTS + for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS if not is_opt_mode ] """ diff --git a/defs.bzl b/defs.bzl index a9f25ebcc..d5b7b6af7 100644 --- a/defs.bzl +++ b/defs.bzl @@ -12,7 +12,9 @@ def test_binary( rocksdb_compiler_flags, rocksdb_preprocessor_flags, rocksdb_external_deps, - rocksdb_os_deps): + rocksdb_os_deps, + extra_deps, + extra_compiler_flags): TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh" ttype = "gtest" if parallelism == "parallel" else "simple" @@ -23,9 +25,9 @@ def test_binary( srcs = [test_cc], arch_preprocessor_flags = rocksdb_arch_preprocessor_flags, os_preprocessor_flags = rocksdb_os_preprocessor_flags, - compiler_flags = rocksdb_compiler_flags, + compiler_flags = rocksdb_compiler_flags + extra_compiler_flags, preprocessor_flags = rocksdb_preprocessor_flags, - deps = [":rocksdb_test_lib"], + deps = [":rocksdb_test_lib"] + extra_deps, os_deps = rocksdb_os_deps, external_deps = rocksdb_external_deps, )