diff options
Diffstat (limited to '')
| -rwxr-xr-x | src/VBox/ValidationKit/analysis/analyze.py | 447 |
1 files changed, 447 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/analysis/analyze.py b/src/VBox/ValidationKit/analysis/analyze.py new file mode 100755 index 00000000..ec2b3cc1 --- /dev/null +++ b/src/VBox/ValidationKit/analysis/analyze.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: analyze.py $ + +""" +Analyzer CLI. +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports. +import re; +import os; +import textwrap; +import sys; + +# Only the main script needs to modify the path. +try: __file__ +except: __file__ = sys.argv[0]; +g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); +sys.path.append(g_ksValidationKitDir); + +# Validation Kit imports. +from analysis import reader +from analysis import reporting + + +def usage(): + """ + Display usage. + """ + # Set up the output wrapper. + try: cCols = os.get_terminal_size()[0] # since 3.3 + except: cCols = 79; + oWrapper = textwrap.TextWrapper(width = cCols); + + # Do the outputting. + print('Tool for comparing test results.'); + print(''); + oWrapper.subsequent_indent = ' ' * (len('usage: ') + 4); + print(oWrapper.fill('usage: analyze.py [options] [collection-1] -- [collection-2] [-- [collection3] [..]])')) + oWrapper.subsequent_indent = ''; + print(''); + print(oWrapper.fill('This tool compares two or more result collections, using one as a baseline (first by default) ' + 'and showing how the results in others differs from it.')); + print(''); + print(oWrapper.fill('The results (XML file) from one or more test runs makes up a collection. A collection can be ' + 'named using the --name <name> option, or will get a sequential name automatically. The baseline ' + 'collection will have "(baseline)" appended to its name.')); + print(''); + print(oWrapper.fill('A test run produces one XML file, either via the testdriver/reporter.py machinery or via the IPRT ' + 'test.cpp code. In the latter case it can be enabled and controlled via IPRT_TEST_FILE. A collection ' + 'consists of one or more of test runs (i.e. XML result files). These are combined (aka distilled) ' + 'into a single set of results before comparing them with the others. The --best and --avg options ' + 'controls how this combining is done. The need for this is mainly to try counteract some of the ' + 'instability typically found in the restuls. Just because one test run produces a better result ' + 'after a change does not necessarily mean this will always be the case and that the change was to ' + 'the better, it might just have been regular fluctuations in the test results.')); + + oWrapper.initial_indent = ' '; + oWrapper.subsequent_indent = ' '; + print(''); + print('Options governing combining (distillation):'); + print(' --avg, --average'); + print(oWrapper.fill('Picks the best result by calculating the average values across all the runs.')); + print(''); + print(' --best'); + print(oWrapper.fill('Picks the best result from all the runs. For values, this means making guessing what result is ' + 'better based on the unit. This may not always lead to the right choices.')); + print(oWrapper.initial_indent + 'Default: --best'); + + print(''); + print('Options relating to collections:'); + print(' --name <name>'); + print(oWrapper.fill('Sets the name of the current collection. By default a collection gets a sequential number.')); + print(''); + print(' --baseline <num>'); + print(oWrapper.fill('Sets collection given by <num> (0-based) as the baseline collection.')); + print(oWrapper.initial_indent + 'Default: --baseline 0') + + print(''); + print('Filtering options:'); + print(' --filter-test <substring>'); + print(oWrapper.fill('Exclude tests not containing any of the substrings given via the --filter-test option. The ' + 'matching is done with full test name, i.e. all parent names are prepended with ", " as separator ' + '(for example "tstIOInstr, CPUID EAX=1").')); + print(''); + print(' --filter-test-out <substring>'); + print(oWrapper.fill('Exclude tests containing the given substring. As with --filter-test, the matching is done against ' + 'the full test name.')); + print(''); + print(' --filter-value <substring>'); + print(oWrapper.fill('Exclude values not containing any of the substrings given via the --filter-value option. The ' + 'matching is done against the value name prefixed by the full test name and ": " ' + '(for example "tstIOInstr, CPUID EAX=1: real mode, CPUID").')); + print(''); + print(' --filter-value-out <substring>'); + print(oWrapper.fill('Exclude value containing the given substring. As with --filter-value, the matching is done against ' + 'the value name prefixed by the full test name.')); + + print(''); + print(' --regex-test <expr>'); + print(oWrapper.fill('Same as --filter-test except the substring matching is done via a regular expression.')); + print(''); + print(' --regex-test-out <expr>'); + print(oWrapper.fill('Same as --filter-test-out except the substring matching is done via a regular expression.')); + print(''); + print(' --regex-value <expr>'); + print(oWrapper.fill('Same as --filter-value except the substring matching is done via a regular expression.')); + print(''); + print(' --regex-value-out <expr>'); + print(oWrapper.fill('Same as --filter-value-out except the substring matching is done via a regular expression.')); + print(''); + print(' --filter-out-empty-leaf-tests'); + print(oWrapper.fill('Removes any leaf tests that are without any values or sub-tests. This is useful when ' + 'only considering values, especially when doing additional value filtering.')); + + print(''); + print('Analysis options:'); + print(' --pct-same-value <float>'); + print(oWrapper.fill('The threshold at which the percent difference between two values are considered the same ' + 'during analysis.')); + print(oWrapper.initial_indent + 'Default: --pct-same-value 0.10'); + + print(''); + print('Output options:'); + print(' --brief, --verbose'); + print(oWrapper.fill('Whether to omit (--brief) the value for non-baseline runs and just get along with the difference.')); + print(oWrapper.initial_indent + 'Default: --brief'); + print(''); + print(' --pct <num>, --pct-precision <num>'); + print(oWrapper.fill('Specifies the number of decimal place to use when formatting the difference as percent.')); + print(oWrapper.initial_indent + 'Default: --pct 2'); + return 1; + + +class ResultCollection(object): + """ + One or more test runs that should be merged before comparison. + """ + + def __init__(self, sName): + self.sName = sName; + self.aoTestTrees = [] # type: [Test] + self.asTestFiles = [] # type: [str] - runs parallel to aoTestTrees + self.oDistilled = None # type: Test + + def append(self, sFilename): + """ + Loads sFilename and appends the result. + Returns True on success, False on failure. + """ + oTestTree = reader.parseTestResult(sFilename); + if oTestTree: + self.aoTestTrees.append(oTestTree); + self.asTestFiles.append(sFilename); + return True; + return False; + + def isEmpty(self): + """ Checks if the result is empty. """ + return len(self.aoTestTrees) == 0; + + def filterTests(self, asFilters): + """ + Keeps all the tests in the test trees sub-string matching asFilters (str or re). + """ + for oTestTree in self.aoTestTrees: + oTestTree.filterTests(asFilters); + return self; + + def filterOutTests(self, asFilters): + """ + Removes all the tests in the test trees sub-string matching asFilters (str or re). + """ + for oTestTree in self.aoTestTrees: + oTestTree.filterOutTests(asFilters); + return self; + + def filterValues(self, asFilters): + """ + Keeps all the tests in the test trees sub-string matching asFilters (str or re). + """ + for oTestTree in self.aoTestTrees: + oTestTree.filterValues(asFilters); + return self; + + def filterOutValues(self, asFilters): + """ + Removes all the tests in the test trees sub-string matching asFilters (str or re). + """ + for oTestTree in self.aoTestTrees: + oTestTree.filterOutValues(asFilters); + return self; + + def filterOutEmptyLeafTests(self): + """ + Removes all the tests in the test trees that have neither child tests nor values. + """ + for oTestTree in self.aoTestTrees: + oTestTree.filterOutEmptyLeafTests(); + return self; + + def distill(self, sMethod, fDropLoners = False): + """ + Distills the set of test results into a single one by the given method. + + Valid sMethod values: + - 'best': Pick the best result for each test and value among all the test runs. + - 'avg': Calculate the average value among all the test runs. + + When fDropLoners is True, tests and values that only appear in a single test run + will be discarded. When False (the default), the lone result will be used. + """ + assert sMethod in ['best', 'avg']; + assert not self.oDistilled; + + # If empty, nothing to do. + if self.isEmpty(): + return None; + + # If there is only a single tree, make a deep copy of it. + if len(self.aoTestTrees) == 1: + oDistilled = self.aoTestTrees[0].clone(); + else: + + # Since we don't know if the test runs are all from the same test, we create + # dummy root tests for each run and use these are the start for the distillation. + aoDummyInputTests = []; + for oRun in self.aoTestTrees: + oDummy = reader.Test(); + oDummy.aoChildren = [oRun,]; + aoDummyInputTests.append(oDummy); + + # Similarly, we end up with a "dummy" root test for the result. + oDistilled = reader.Test(); + oDistilled.distill(aoDummyInputTests, sMethod, fDropLoners); + + # We can drop this if there is only a single child, i.e. if all runs are for + # the same test. + if len(oDistilled.aoChildren) == 1: + oDistilled = oDistilled.aoChildren[0]; + + self.oDistilled = oDistilled; + return oDistilled; + + + +# matchWithValue hacks. +g_asOptions = []; +g_iOptInd = 1; +g_sOptArg = ''; + +def matchWithValue(sOption): + """ Matches an option with a value, placing the value in g_sOptArg if it matches. """ + global g_asOptions, g_iOptInd, g_sOptArg; + sArg = g_asOptions[g_iOptInd]; + if sArg.startswith(sOption): + if len(sArg) == len(sOption): + if g_iOptInd + 1 < len(g_asOptions): + g_iOptInd += 1; + g_sOptArg = g_asOptions[g_iOptInd]; + return True; + + print('syntax error: Option %s takes a value!' % (sOption,)); + raise Exception('syntax error: Option %s takes a value!' % (sOption,)); + + if sArg[len(sOption)] in ('=', ':'): + g_sOptArg = sArg[len(sOption) + 1:]; + return True; + return False; + + +def main(asArgs): + """ C style main(). """ + # + # Parse arguments + # + oCurCollection = ResultCollection('#0'); + aoCollections = [ oCurCollection, ]; + iBaseline = 0; + sDistillationMethod = 'best'; + fBrief = True; + cPctPrecision = 2; + rdPctSameValue = 0.1; + asTestFilters = []; + asTestOutFilters = []; + asValueFilters = []; + asValueOutFilters = []; + fFilterOutEmptyLeafTest = True; + + global g_asOptions, g_iOptInd, g_sOptArg; + g_asOptions = asArgs; + g_iOptInd = 1; + while g_iOptInd < len(asArgs): + sArg = asArgs[g_iOptInd]; + g_sOptArg = ''; + #print("dbg: g_iOptInd=%s '%s'" % (g_iOptInd, sArg,)); + + if sArg.startswith('--help'): + return usage(); + + if matchWithValue('--filter-test'): + asTestFilters.append(g_sOptArg); + elif matchWithValue('--filter-test-out'): + asTestOutFilters.append(g_sOptArg); + elif matchWithValue('--filter-value'): + asValueFilters.append(g_sOptArg); + elif matchWithValue('--filter-value-out'): + asValueOutFilters.append(g_sOptArg); + + elif matchWithValue('--regex-test'): + asTestFilters.append(re.compile(g_sOptArg)); + elif matchWithValue('--regex-test-out'): + asTestOutFilters.append(re.compile(g_sOptArg)); + elif matchWithValue('--regex-value'): + asValueFilters.append(re.compile(g_sOptArg)); + elif matchWithValue('--regex-value-out'): + asValueOutFilters.append(re.compile(g_sOptArg)); + + elif sArg == '--filter-out-empty-leaf-tests': + fFilterOutEmptyLeafTest = True; + elif sArg == '--no-filter-out-empty-leaf-tests': + fFilterOutEmptyLeafTest = False; + + elif sArg == '--best': + sDistillationMethod = 'best'; + elif sArg in ('--avg', '--average'): + sDistillationMethod = 'avg'; + + elif sArg == '--brief': + fBrief = True; + elif sArg == '--verbose': + fBrief = False; + + elif matchWithValue('--pct') or matchWithValue('--pct-precision'): + cPctPrecision = int(g_sOptArg); + elif matchWithValue('--base') or matchWithValue('--baseline'): + iBaseline = int(g_sOptArg); + + elif matchWithValue('--pct-same-value'): + rdPctSameValue = float(g_sOptArg); + + # '--' starts a new collection. If current one is empty, drop it. + elif sArg == '--': + print("dbg: new collection"); + #if oCurCollection.isEmpty(): + # del aoCollections[-1]; + oCurCollection = ResultCollection("#%s" % (len(aoCollections),)); + aoCollections.append(oCurCollection); + + # Name the current result collection. + elif matchWithValue('--name'): + oCurCollection.sName = g_sOptArg; + + # Read in a file and add it to the current data set. + else: + if not oCurCollection.append(sArg): + return 1; + g_iOptInd += 1; + + # + # Post argument parsing processing. + # + + # Drop the last collection if empty. + if oCurCollection.isEmpty(): + del aoCollections[-1]; + if not aoCollections: + print("error: No input files given!"); + return 1; + + # Check the baseline value and mark the column as such. + if iBaseline < 0 or iBaseline > len(aoCollections): + print("error: specified baseline is out of range: %s, valid range 0 <= baseline < %s" + % (iBaseline, len(aoCollections),)); + return 1; + aoCollections[iBaseline].sName += ' (baseline)'; + + # + # Apply filtering before distilling each collection into a single result tree. + # + if asTestFilters: + for oCollection in aoCollections: + oCollection.filterTests(asTestFilters); + if asTestOutFilters: + for oCollection in aoCollections: + oCollection.filterOutTests(asTestOutFilters); + + if asValueFilters: + for oCollection in aoCollections: + oCollection.filterValues(asValueFilters); + if asValueOutFilters: + for oCollection in aoCollections: + oCollection.filterOutValues(asValueOutFilters); + + if fFilterOutEmptyLeafTest: + for oCollection in aoCollections: + oCollection.filterOutEmptyLeafTests(); + + # Distillation. + for oCollection in aoCollections: + oCollection.distill(sDistillationMethod); + + # + # Produce the report. + # + oTable = reporting.RunTable(iBaseline, fBrief, cPctPrecision, rdPctSameValue); + oTable.populateFromRuns([oCollection.oDistilled for oCollection in aoCollections], + [oCollection.sName for oCollection in aoCollections]); + print('\n'.join(oTable.formatAsText())); + return 0; + +if __name__ == '__main__': + sys.exit(main(sys.argv)); + |