Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 126 insertions, 0 deletions

diff --git a/src/rocksdb/tools/advisor/test/test_db_stats_fetcher.py b/src/rocksdb/tools/advisor/test/test_db_stats_fetcher.py
new file mode 100644
index 000000000..afbbe8339
--- /dev/null
+++ b/src/rocksdb/tools/advisor/test/test_db_stats_fetcher.py
@@ -0,0 +1,126 @@
+# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  This source code is licensed under both the GPLv2 (found in the
+#  COPYING file in the root directory) and Apache 2.0 License
+#  (found in the LICENSE.Apache file in the root directory).
+
+from advisor.db_stats_fetcher import LogStatsParser, DatabasePerfContext
+from advisor.db_timeseries_parser import NO_ENTITY
+from advisor.rule_parser import Condition, TimeSeriesCondition
+import os
+import time
+import unittest
+from unittest.mock import MagicMock
+
+
+class TestLogStatsParser(unittest.TestCase):
+    def setUp(self):
+        this_path = os.path.abspath(os.path.dirname(__file__))
+        stats_file = os.path.join(
+            this_path, 'input_files/log_stats_parser_keys_ts'
+        )
+        # populate the keys_ts dictionary of LogStatsParser
+        self.stats_dict = {NO_ENTITY: {}}
+        with open(stats_file, 'r') as fp:
+            for line in fp:
+                stat_name = line.split(':')[0].strip()
+                self.stats_dict[NO_ENTITY][stat_name] = {}
+                token_list = line.split(':')[1].strip().split(',')
+                for token in token_list:
+                    timestamp = int(token.split()[0])
+                    value = float(token.split()[1])
+                    self.stats_dict[NO_ENTITY][stat_name][timestamp] = value
+        self.log_stats_parser = LogStatsParser('dummy_log_file', 20)
+        self.log_stats_parser.keys_ts = self.stats_dict
+
+    def test_check_and_trigger_conditions_bursty(self):
+        # mock fetch_timeseries() because 'keys_ts' has been pre-populated
+        self.log_stats_parser.fetch_timeseries = MagicMock()
+        # condition: bursty
+        cond1 = Condition('cond-1')
+        cond1 = TimeSeriesCondition.create(cond1)
+        cond1.set_parameter('keys', 'rocksdb.db.get.micros.p50')
+        cond1.set_parameter('behavior', 'bursty')
+        cond1.set_parameter('window_sec', 40)
+        cond1.set_parameter('rate_threshold', 0)
+        self.log_stats_parser.check_and_trigger_conditions([cond1])
+        expected_cond_trigger = {
+            NO_ENTITY: {1530896440: 0.9767546362322214}
+        }
+        self.assertDictEqual(expected_cond_trigger, cond1.get_trigger())
+        # ensure that fetch_timeseries() was called once
+        self.log_stats_parser.fetch_timeseries.assert_called_once()
+
+    def test_check_and_trigger_conditions_eval_agg(self):
+        # mock fetch_timeseries() because 'keys_ts' has been pre-populated
+        self.log_stats_parser.fetch_timeseries = MagicMock()
+        # condition: evaluate_expression
+        cond1 = Condition('cond-1')
+        cond1 = TimeSeriesCondition.create(cond1)
+        cond1.set_parameter('keys', 'rocksdb.db.get.micros.p50')
+        cond1.set_parameter('behavior', 'evaluate_expression')
+        keys = [
+            'rocksdb.manifest.file.sync.micros.p99',
+            'rocksdb.db.get.micros.p50'
+        ]
+        cond1.set_parameter('keys', keys)
+        cond1.set_parameter('aggregation_op', 'latest')
+        # condition evaluates to FALSE
+        cond1.set_parameter('evaluate', 'keys[0]-(keys[1]*100)>200')
+        self.log_stats_parser.check_and_trigger_conditions([cond1])
+        expected_cond_trigger = {NO_ENTITY: [1792.0, 15.9638]}
+        self.assertIsNone(cond1.get_trigger())
+        # condition evaluates to TRUE
+        cond1.set_parameter('evaluate', 'keys[0]-(keys[1]*100)<200')
+        self.log_stats_parser.check_and_trigger_conditions([cond1])
+        expected_cond_trigger = {NO_ENTITY: [1792.0, 15.9638]}
+        self.assertDictEqual(expected_cond_trigger, cond1.get_trigger())
+        # ensure that fetch_timeseries() was called
+        self.log_stats_parser.fetch_timeseries.assert_called()
+
+    def test_check_and_trigger_conditions_eval(self):
+        # mock fetch_timeseries() because 'keys_ts' has been pre-populated
+        self.log_stats_parser.fetch_timeseries = MagicMock()
+        # condition: evaluate_expression
+        cond1 = Condition('cond-1')
+        cond1 = TimeSeriesCondition.create(cond1)
+        cond1.set_parameter('keys', 'rocksdb.db.get.micros.p50')
+        cond1.set_parameter('behavior', 'evaluate_expression')
+        keys = [
+            'rocksdb.manifest.file.sync.micros.p99',
+            'rocksdb.db.get.micros.p50'
+        ]
+        cond1.set_parameter('keys', keys)
+        cond1.set_parameter('evaluate', 'keys[0]-(keys[1]*100)>500')
+        self.log_stats_parser.check_and_trigger_conditions([cond1])
+        expected_trigger = {NO_ENTITY: {
+            1530896414: [9938.0, 16.31508],
+            1530896440: [9938.0, 16.346602],
+            1530896466: [9938.0, 16.284669],
+            1530896492: [9938.0, 16.16005]
+        }}
+        self.assertDictEqual(expected_trigger, cond1.get_trigger())
+        self.log_stats_parser.fetch_timeseries.assert_called_once()
+
+
+class TestDatabasePerfContext(unittest.TestCase):
+    def test_unaccumulate_metrics(self):
+        perf_dict = {
+            "user_key_comparison_count": 675903942,
+            "block_cache_hit_count": 830086,
+        }
+        timestamp = int(time.time())
+        perf_ts = {}
+        for key in perf_dict:
+            perf_ts[key] = {}
+            start_val = perf_dict[key]
+            for ix in range(5):
+                perf_ts[key][timestamp+(ix*10)] = start_val + (2 * ix * ix)
+        db_perf_context = DatabasePerfContext(perf_ts, 10, True)
+        timestamps = [timestamp+(ix*10) for ix in range(1, 5, 1)]
+        values = [val for val in range(2, 15, 4)]
+        inner_dict = {timestamps[ix]: values[ix] for ix in range(4)}
+        expected_keys_ts = {NO_ENTITY: {
+            'user_key_comparison_count': inner_dict,
+            'block_cache_hit_count': inner_dict
+        }}
+        self.assertDictEqual(expected_keys_ts, db_perf_context.keys_ts)