Rules Advisor: some fixes to support fetching stats from ODS (#4223)
Summary: This PR includes fixes for some bugs that I encountered while testing the Optimizer with ODS stats support. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4223 Differential Revision: D9140786 Pulled By: poojam23 fbshipit-source-id: 045cb3f27d075c2042040ac2d561938349419516
This commit is contained in:
Pooja Malik
Facebook Github Bot
parent
892a156267
commit
9dbf39399e
@ -78,16 +78,16 @@ if __name__ == '__main__':
|
||||
)
|
||||
# these are options that are column-family agnostic and are not yet
|
||||
# supported by the Rocksdb Options file: eg. bloom_bits=2
|
||||
parser.add_argument(
|
||||
'--base_db_path', required=True, type=str,
|
||||
help='path for the Rocksdb database'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--misc_options', nargs='*',
|
||||
help='whitespace-separated list of options that are not supported ' +
|
||||
'by the Rocksdb OPTIONS file, given in the ' +
|
||||
'<option_name>=<option_value> format eg. "bloom_bits=2 ' +
|
||||
'rate_limiter_bytes_per_sec=128000000"')
|
||||
parser.add_argument(
|
||||
'--base_db_path', required=True, type=str,
|
||||
help='path for the Rocksdb database'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--rules_spec', required=True, type=str,
|
||||
help='path of the file containing the expert-specified Rules'
|
||||
|
||||
@ -1,3 +1,8 @@
|
||||
# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
# This source code is licensed under both the GPLv2 (found in the
|
||||
# COPYING file in the root directory) and Apache 2.0 License
|
||||
# (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
from advisor.bench_runner import BenchmarkRunner
|
||||
from advisor.db_log_parser import DataSource, DatabaseLogs, NO_COL_FAMILY
|
||||
from advisor.db_stats_fetcher import (
|
||||
@ -196,8 +201,10 @@ class DBBenchRunner(BenchmarkRunner):
|
||||
self._setup_db_before_experiment(db_options, db_path)
|
||||
# get the command to run the experiment
|
||||
command = self._build_experiment_command(db_options, db_path)
|
||||
experiment_start_time = int(time.time())
|
||||
# run experiment
|
||||
self._run_command(command)
|
||||
experiment_end_time = int(time.time())
|
||||
# parse the db_bench experiment output
|
||||
parsed_output = self._parse_output(get_perf_context=True)
|
||||
|
||||
@ -224,10 +231,15 @@ class DBBenchRunner(BenchmarkRunner):
|
||||
}
|
||||
# Create the ODS STATS object
|
||||
if self.ods_args:
|
||||
key_prefix = ''
|
||||
if 'key_prefix' in self.ods_args:
|
||||
key_prefix = self.ods_args['key_prefix']
|
||||
data_sources[DataSource.Type.TIME_SERIES].append(OdsStatsFetcher(
|
||||
self.ods_args['client_script'],
|
||||
self.ods_args['entity'],
|
||||
self.ods_args['key_prefix']
|
||||
experiment_start_time,
|
||||
experiment_end_time,
|
||||
key_prefix
|
||||
))
|
||||
# return the experiment's data-sources and throughput
|
||||
return data_sources, parsed_output[self.THROUGHPUT]
|
||||
|
||||
@ -174,7 +174,6 @@ class OdsStatsFetcher(TimeSeriesData):
|
||||
OUTPUT_FILE = 'temp/stats_out.tmp'
|
||||
ERROR_FILE = 'temp/stats_err.tmp'
|
||||
RAPIDO_COMMAND = "%s --entity=%s --key=%s --tstart=%s --tend=%s --showtime"
|
||||
ODS_COMMAND = '%s %s %s' # client, entities, keys
|
||||
|
||||
# static methods
|
||||
@staticmethod
|
||||
@ -191,16 +190,23 @@ class OdsStatsFetcher(TimeSeriesData):
|
||||
second = float(pair[1].strip())
|
||||
return [first, second]
|
||||
|
||||
def __init__(self, client, entities, key_prefix=None):
|
||||
@staticmethod
|
||||
def _get_ods_cli_stime(start_time):
|
||||
diff = int(time.time() - int(start_time))
|
||||
stime = str(diff) + '_s'
|
||||
return stime
|
||||
|
||||
def __init__(
|
||||
self, client, entities, start_time, end_time, key_prefix=None
|
||||
):
|
||||
super().__init__()
|
||||
self.client = client
|
||||
self.entities = entities
|
||||
self.start_time = start_time
|
||||
self.end_time = end_time
|
||||
self.key_prefix = key_prefix
|
||||
self.stats_freq_sec = 60
|
||||
self.duration_sec = 60
|
||||
# Fetch last 3 hours data by default
|
||||
self.end_time = int(time.time())
|
||||
self.start_time = self.end_time - (3 * 60 * 60)
|
||||
|
||||
def execute_script(self, command):
|
||||
print('executing...')
|
||||
@ -265,9 +271,10 @@ class OdsStatsFetcher(TimeSeriesData):
|
||||
# Parse output and populate the 'keys_ts' map
|
||||
self.parse_rapido_output()
|
||||
elif re.search('ods', self.client, re.IGNORECASE):
|
||||
command = self.ODS_COMMAND % (
|
||||
self.client,
|
||||
self._get_string_in_quotes(self.entities),
|
||||
command = (
|
||||
self.client + ' ' +
|
||||
'--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
|
||||
self._get_string_in_quotes(self.entities) + ' ' +
|
||||
self._get_string_in_quotes(','.join(statistics))
|
||||
)
|
||||
# Run the tool and fetch the time-series data
|
||||
@ -304,7 +311,7 @@ class OdsStatsFetcher(TimeSeriesData):
|
||||
transform_desc = transform_desc + ",%)"
|
||||
else:
|
||||
transform_desc = transform_desc + ")"
|
||||
|
||||
if re.search('rapido', self.client, re.IGNORECASE):
|
||||
command = self.RAPIDO_COMMAND + " --transform=%s --url=%s"
|
||||
command = command % (
|
||||
self.client,
|
||||
@ -315,6 +322,15 @@ class OdsStatsFetcher(TimeSeriesData):
|
||||
self._get_string_in_quotes(transform_desc),
|
||||
self._get_string_in_quotes(display)
|
||||
)
|
||||
elif re.search('ods', self.client, re.IGNORECASE):
|
||||
command = (
|
||||
self.client + ' ' +
|
||||
'--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
|
||||
'--fburlonly ' +
|
||||
self._get_string_in_quotes(entities) + ' ' +
|
||||
self._get_string_in_quotes(','.join(keys)) + ' ' +
|
||||
self._get_string_in_quotes(transform_desc)
|
||||
)
|
||||
self.execute_script(command)
|
||||
url = ""
|
||||
with open(self.OUTPUT_FILE, 'r') as fp:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user