diff --git a/tools/advisor/advisor/config_optimizer_example.py b/tools/advisor/advisor/config_optimizer_example.py index 0c0b74fc0..e3736387e 100644 --- a/tools/advisor/advisor/config_optimizer_example.py +++ b/tools/advisor/advisor/config_optimizer_example.py @@ -78,16 +78,16 @@ if __name__ == '__main__': ) # these are options that are column-family agnostic and are not yet # supported by the Rocksdb Options file: eg. bloom_bits=2 - parser.add_argument( - '--base_db_path', required=True, type=str, - help='path for the Rocksdb database' - ) parser.add_argument( '--misc_options', nargs='*', help='whitespace-separated list of options that are not supported ' + 'by the Rocksdb OPTIONS file, given in the ' + '= format eg. "bloom_bits=2 ' + 'rate_limiter_bytes_per_sec=128000000"') + parser.add_argument( + '--base_db_path', required=True, type=str, + help='path for the Rocksdb database' + ) parser.add_argument( '--rules_spec', required=True, type=str, help='path of the file containing the expert-specified Rules' diff --git a/tools/advisor/advisor/db_bench_runner.py b/tools/advisor/advisor/db_bench_runner.py index 9c99b10ef..54424440b 100644 --- a/tools/advisor/advisor/db_bench_runner.py +++ b/tools/advisor/advisor/db_bench_runner.py @@ -1,3 +1,8 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + from advisor.bench_runner import BenchmarkRunner from advisor.db_log_parser import DataSource, DatabaseLogs, NO_COL_FAMILY from advisor.db_stats_fetcher import ( @@ -196,8 +201,10 @@ class DBBenchRunner(BenchmarkRunner): self._setup_db_before_experiment(db_options, db_path) # get the command to run the experiment command = self._build_experiment_command(db_options, db_path) + experiment_start_time = int(time.time()) # run experiment self._run_command(command) + experiment_end_time = int(time.time()) # parse the db_bench experiment output parsed_output = self._parse_output(get_perf_context=True) @@ -224,10 +231,15 @@ class DBBenchRunner(BenchmarkRunner): } # Create the ODS STATS object if self.ods_args: + key_prefix = '' + if 'key_prefix' in self.ods_args: + key_prefix = self.ods_args['key_prefix'] data_sources[DataSource.Type.TIME_SERIES].append(OdsStatsFetcher( self.ods_args['client_script'], self.ods_args['entity'], - self.ods_args['key_prefix'] + experiment_start_time, + experiment_end_time, + key_prefix )) # return the experiment's data-sources and throughput return data_sources, parsed_output[self.THROUGHPUT] diff --git a/tools/advisor/advisor/db_stats_fetcher.py b/tools/advisor/advisor/db_stats_fetcher.py index 35cef14ba..cf497cf1f 100755 --- a/tools/advisor/advisor/db_stats_fetcher.py +++ b/tools/advisor/advisor/db_stats_fetcher.py @@ -174,7 +174,6 @@ class OdsStatsFetcher(TimeSeriesData): OUTPUT_FILE = 'temp/stats_out.tmp' ERROR_FILE = 'temp/stats_err.tmp' RAPIDO_COMMAND = "%s --entity=%s --key=%s --tstart=%s --tend=%s --showtime" - ODS_COMMAND = '%s %s %s' # client, entities, keys # static methods @staticmethod @@ -191,16 +190,23 @@ class OdsStatsFetcher(TimeSeriesData): second = float(pair[1].strip()) return [first, second] - def __init__(self, client, entities, key_prefix=None): + @staticmethod + def _get_ods_cli_stime(start_time): + diff = int(time.time() - int(start_time)) + stime = str(diff) + '_s' + return stime + + def __init__( + self, client, entities, start_time, end_time, key_prefix=None + ): super().__init__() self.client = client self.entities = entities + self.start_time = start_time + self.end_time = end_time self.key_prefix = key_prefix self.stats_freq_sec = 60 self.duration_sec = 60 - # Fetch last 3 hours data by default - self.end_time = int(time.time()) - self.start_time = self.end_time - (3 * 60 * 60) def execute_script(self, command): print('executing...') @@ -265,9 +271,10 @@ class OdsStatsFetcher(TimeSeriesData): # Parse output and populate the 'keys_ts' map self.parse_rapido_output() elif re.search('ods', self.client, re.IGNORECASE): - command = self.ODS_COMMAND % ( - self.client, - self._get_string_in_quotes(self.entities), + command = ( + self.client + ' ' + + '--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' + + self._get_string_in_quotes(self.entities) + ' ' + self._get_string_in_quotes(','.join(statistics)) ) # Run the tool and fetch the time-series data @@ -304,17 +311,26 @@ class OdsStatsFetcher(TimeSeriesData): transform_desc = transform_desc + ",%)" else: transform_desc = transform_desc + ")" - - command = self.RAPIDO_COMMAND + " --transform=%s --url=%s" - command = command % ( - self.client, - self._get_string_in_quotes(','.join(entities)), - self._get_string_in_quotes(','.join(keys)), - self._get_string_in_quotes(self.start_time), - self._get_string_in_quotes(self.end_time), - self._get_string_in_quotes(transform_desc), - self._get_string_in_quotes(display) - ) + if re.search('rapido', self.client, re.IGNORECASE): + command = self.RAPIDO_COMMAND + " --transform=%s --url=%s" + command = command % ( + self.client, + self._get_string_in_quotes(','.join(entities)), + self._get_string_in_quotes(','.join(keys)), + self._get_string_in_quotes(self.start_time), + self._get_string_in_quotes(self.end_time), + self._get_string_in_quotes(transform_desc), + self._get_string_in_quotes(display) + ) + elif re.search('ods', self.client, re.IGNORECASE): + command = ( + self.client + ' ' + + '--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' + + '--fburlonly ' + + self._get_string_in_quotes(entities) + ' ' + + self._get_string_in_quotes(','.join(keys)) + ' ' + + self._get_string_in_quotes(transform_desc) + ) self.execute_script(command) url = "" with open(self.OUTPUT_FILE, 'r') as fp: