path: root/testing/raptor/raptor/output.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# some parts of this originally taken from /testing/talos/talos/output.py

"""output raptor test results"""
import copy
import json
import os
import warnings
from abc import ABCMeta, abstractmethod
from collections.abc import Iterable

import filters
import six
from logger.logger import RaptorLogger
from utils import flatten

LOG = RaptorLogger(component="perftest-output")

VISUAL_METRICS = [
    "SpeedIndex",
    "ContentfulSpeedIndex",
    "PerceptualSpeedIndex",
    "FirstVisualChange",
    "LastVisualChange",
    "VisualReadiness",
    "VisualComplete85",
    "VisualComplete95",
    "VisualComplete99",
]

METRIC_BLOCKLIST = [
    "mean",
    "median",
    "geomean",
]


@six.add_metaclass(ABCMeta)
class PerftestOutput(object):
    """Abstract base class to handle output of perftest results"""

    def __init__(
        self, results, supporting_data, subtest_alert_on, app, extra_summary_methods=[]
    ):
        """
        - results : list of RaptorTestResult instances
        """
        self.app = app
        self.results = results
        self.summarized_results = {}
        self.supporting_data = supporting_data
        self.summarized_supporting_data = []
        self.summarized_screenshots = []
        self.subtest_alert_on = subtest_alert_on
        self.browser_name = None
        self.browser_version = None
        self.extra_summary_methods = extra_summary_methods

    @abstractmethod
    def summarize(self, test_names):
        raise NotImplementedError()

    def set_browser_meta(self, browser_name, browser_version):
        # sets the browser metadata for the perfherder data
        self.browser_name = browser_name
        self.browser_version = browser_version

    def summarize_supporting_data(self):
        """
        Supporting data was gathered outside of the main raptor test; it will be kept
        separate from the main raptor test results. Summarize it appropriately.

        supporting_data = {
            'type': 'data-type',
            'test': 'raptor-test-ran-when-data-was-gathered',
            'unit': 'unit that the values are in',
            'summarize-values': True/False,
            'suite-suffix-type': True/False,
            'values': {
                'name': value_dict,
                'nameN': value_dictN
            }
        }

        More specifically, subtest supporting data will look like this:

        supporting_data = {
            'type': 'power',
            'test': 'raptor-speedometer-geckoview',
            'unit': 'mAh',
            'values': {
                'cpu': {
                    'values': val,
                    'lowerIsBetter': True/False,
                    'alertThreshold': 2.0,
                    'subtest-prefix-type': True/False,
                    'unit': 'mWh'
                },
                'wifi': ...
            }
        }

        We want to treat each value as a 'subtest'; and for the overall aggregated
        test result the summary value is dependent on the unit. An exception is
        raised in case we don't know about the specified unit.
        """
        if self.supporting_data is None:
            return

        self.summarized_supporting_data = []
        support_data_by_type = {}

        for data_set in self.supporting_data:

            data_type = data_set["type"]
            LOG.info("summarizing %s data" % data_type)

            if data_type not in support_data_by_type:
                support_data_by_type[data_type] = {
                    "framework": {"name": "raptor"},
                    "suites": [],
                }

            # suite name will be name of the actual raptor test that ran, plus the type of
            # supporting data i.e. 'raptor-speedometer-geckoview-power'
            vals = []
            subtests = []

            suite_name = data_set["test"]
            if data_set.get("suite-suffix-type", True):
                suite_name = "%s-%s" % (data_set["test"], data_set["type"])

            suite = {
                "name": suite_name,
                "type": data_set["type"],
                "subtests": subtests,
            }
            if data_set.get("summarize-values", True):
                suite.update(
                    {
                        "lowerIsBetter": True,
                        "unit": data_set["unit"],
                        "alertThreshold": 2.0,
                    }
                )

            for result in self.results:
                if result["name"] == data_set["test"]:
                    suite["extraOptions"] = result["extra_options"]
                    break

            support_data_by_type[data_type]["suites"].append(suite)
            for measurement_name, value_info in data_set["values"].items():
                # Subtests are expected to be specified in a dictionary, this
                # provides backwards compatibility with the old method
                if not isinstance(value_info, dict):
                    value_info = {"values": value_info}

                new_subtest = {}
                if value_info.get("subtest-prefix-type", True):
                    new_subtest["name"] = data_type + "-" + measurement_name
                else:
                    new_subtest["name"] = measurement_name

                new_subtest["value"] = value_info["values"]
                new_subtest["lowerIsBetter"] = value_info.get("lowerIsBetter", True)
                new_subtest["alertThreshold"] = value_info.get("alertThreshold", 2.0)
                new_subtest["unit"] = value_info.get("unit", data_set["unit"])

                if "shouldAlert" in value_info:
                    new_subtest["shouldAlert"] = value_info.get("shouldAlert")

                subtests.append(new_subtest)
                vals.append([new_subtest["value"], new_subtest["name"]])

            if len(subtests) >= 1 and data_set.get("summarize-values", True):
                suite["value"] = self.construct_summary(
                    vals, testname="supporting_data", unit=data_set["unit"]
                )

        # split the supporting data by type, there will be one
        # perfherder output per type
        for data_type in support_data_by_type:
            data = support_data_by_type[data_type]
            if self.browser_name:
                data["application"] = {"name": self.browser_name}
                if self.browser_version:
                    data["application"]["version"] = self.browser_version
            self.summarized_supporting_data.append(data)

        return

    def output(self, test_names):
        """output to file and perfherder data json"""
        if os.getenv("MOZ_UPLOAD_DIR"):
            # i.e. testing/mozharness/build/raptor.json locally; in production it will
            # be at /tasks/task_*/build/ (where it will be picked up by mozharness later
            # and made into a tc artifact accessible in treeherder as perfherder-data.json)
            results_path = os.path.join(
                os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]), "raptor.json"
            )
            screenshot_path = os.path.join(
                os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]), "screenshots.html"
            )
        else:
            results_path = os.path.join(os.getcwd(), "raptor.json")
            screenshot_path = os.path.join(os.getcwd(), "screenshots.html")

        success = True
        if self.summarized_results == {}:
            success = False
            LOG.error(
                "no summarized raptor results found for any of %s"
                % ", ".join(test_names)
            )
        else:
            for suite in self.summarized_results["suites"]:
                gecko_profiling_enabled = "gecko-profile" in suite.get(
                    "extraOptions", []
                )
                if gecko_profiling_enabled:
                    LOG.info("gecko profiling enabled")
                    suite["shouldAlert"] = False

                # as we do navigation, tname could end in .<alias>
                # test_names doesn't have tname, so either add it to test_names,
                # or strip it
                tname = suite["name"]
                parts = tname.split(".")
                try:
                    tname = ".".join(parts[:-1])
                except Exception as e:
                    LOG.info("no alias found on test, ignoring: %s" % e)
                    pass

                # Since test names might have been modified, check if
                # part of the test name exists in the test_names list entries
                found = False
                for test in test_names:
                    if tname in test:
                        found = True
                        break
                if not found:
                    success = False
                    LOG.error("no summarized raptor results found for %s" % (tname))

            with open(results_path, "w") as f:
                for result in self.summarized_results:
                    f.write("%s\n" % result)

        if len(self.summarized_screenshots) > 0:
            with open(screenshot_path, "w") as f:
                for result in self.summarized_screenshots:
                    f.write("%s\n" % result)
            LOG.info("screen captures can be found locally at: %s" % screenshot_path)

        # now that we've checked for screen captures too, if there were no actual
        # test results we can bail out here
        if self.summarized_results == {}:
            return success, 0

        test_type = self.summarized_results["suites"][0].get("type", "")
        output_perf_data = True
        not_posting = "- not posting regular test results for perfherder"
        if test_type == "scenario":
            # if a resource-usage flag was supplied the perfherder data
            # will still be output from output_supporting_data
            LOG.info("scenario test type was run %s" % not_posting)
            output_perf_data = False

        if self.browser_name:
            self.summarized_results["application"] = {"name": self.browser_name}
            if self.browser_version:
                self.summarized_results["application"]["version"] = self.browser_version

        total_perfdata = 0
        if output_perf_data:
            # if we have supporting data i.e. power, we ONLY want those measurements
            # dumped out. TODO: Bug 1515406 - Add option to output both supplementary
            # data (i.e. power) and the regular Raptor test result
            # Both are already available as separate PERFHERDER_DATA json blobs
            if len(self.summarized_supporting_data) == 0:
                LOG.info("PERFHERDER_DATA: %s" % json.dumps(self.summarized_results))
                total_perfdata = 1
            else:
                LOG.info(
                    "supporting data measurements exist - only posting those to perfherder"
                )

        json.dump(
            self.summarized_results, open(results_path, "w"), indent=2, sort_keys=True
        )
        LOG.info("results can also be found locally at: %s" % results_path)

        return success, total_perfdata

    def output_supporting_data(self, test_names):
        """
        Supporting data was gathered outside of the main raptor test; it has already
        been summarized, now output it appropriately.

        We want to output supporting data in a completely separate perfherder json blob and
        in a corresponding file artifact. This way, supporting data can be ingested as its own
        test suite in perfherder and alerted upon if desired; kept outside of the test results
        from the actual Raptor test which was run when the supporting data was gathered.
        """
        if len(self.summarized_supporting_data) == 0:
            LOG.error(
                "no summarized supporting data found for %s" % ", ".join(test_names)
            )
            return False, 0

        total_perfdata = 0
        for next_data_set in self.summarized_supporting_data:
            data_type = next_data_set["suites"][0]["type"]

            if os.environ["MOZ_UPLOAD_DIR"]:
                # i.e. testing/mozharness/build/raptor.json locally; in production it will
                # be at /tasks/task_*/build/ (where it will be picked up by mozharness later
                # and made into a tc artifact accessible in treeherder as perfherder-data.json)
                results_path = os.path.join(
                    os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]),
                    "raptor-%s.json" % data_type,
                )
            else:
                results_path = os.path.join(os.getcwd(), "raptor-%s.json" % data_type)

            # dump data to raptor-data.json artifact
            json.dump(next_data_set, open(results_path, "w"), indent=2, sort_keys=True)

            # the output that treeherder expects to find
            LOG.info("PERFHERDER_DATA: %s" % json.dumps(next_data_set))
            LOG.info(
                "%s results can also be found locally at: %s"
                % (data_type, results_path)
            )
            total_perfdata += 1

        return True, total_perfdata

    def construct_summary(self, vals, testname, unit=None):
        def _filter(vals, value=None):
            if value is None:
                return [i for i, j in vals]
            return [i for i, j in vals if j == value]

        if testname.startswith("raptor-v8_7"):
            return 100 * filters.geometric_mean(_filter(vals))

        if testname == "speedometer3":
            score = None
            for val, name in vals:
                if name == "score":
                    score = val
            if score is None:
                raise Exception("Unable to find score for Speedometer 3")
            return score

        if "speedometer" in testname:
            correctionFactor = 3
            results = _filter(vals)
            # speedometer has 16 tests, each of these are made of up 9 subtests
            # and a sum of the 9 values.  We receive 160 values, and want to use
            # the 16 test values, not the sub test values.
            if len(results) != 160:
                raise Exception(
                    "Speedometer has 160 subtests, found: %s instead" % len(results)
                )

            results = results[9::10]
            # pylint --py3k W1619
            score = 60 * 1000 / filters.geometric_mean(results) / correctionFactor
            return score

        if "stylebench" in testname:
            # see https://bug-172968-attachments.webkit.org/attachment.cgi?id=319888
            correctionFactor = 3
            results = _filter(vals)

            # stylebench has 5 tests, each of these are made of up 5 subtests
            #
            #   * Adding classes.
            #   * Removing classes.
            #   * Mutating attributes.
            #   * Adding leaf elements.
            #   * Removing leaf elements.
            #
            # which are made of two subtests each (sync/async) and repeated 5 times
            # each, thus, the list here looks like:
            #
            #   [Test name/Adding classes - 0/ Sync; <x>]
            #   [Test name/Adding classes - 0/ Async; <y>]
            #   [Test name/Adding classes - 0; <x> + <y>]
            #   [Test name/Removing classes - 0/ Sync; <x>]
            #   [Test name/Removing classes - 0/ Async; <y>]
            #   [Test name/Removing classes - 0; <x> + <y>]
            #   ...
            #   [Test name/Adding classes - 1 / Sync; <x>]
            #   [Test name/Adding classes - 1 / Async; <y>]
            #   [Test name/Adding classes - 1 ; <x> + <y>]
            #   ...
            #   [Test name/Removing leaf elements - 4; <x> + <y>]
            #   [Test name; <sum>] <- This is what we want.
            #
            # So, 5 (subtests) *
            #     5 (repetitions) *
            #     3 (entries per repetition (sync/async/sum)) =
            #     75 entries for test before the sum.
            #
            # We receive 76 entries per test, which ads up to 380. We want to use
            # the 5 test entries, not the rest.
            if len(results) != 380:
                raise Exception(
                    "StyleBench requires 380 entries, found: %s instead" % len(results)
                )
            results = results[75::76]
            # pylint --py3k W1619
            return 60 * 1000 / filters.geometric_mean(results) / correctionFactor

        if testname.startswith("raptor-kraken") or "sunspider" in testname:
            return sum(_filter(vals))

        if "unity-webgl" in testname or "webaudio" in testname:
            # webaudio_score and unity_webgl_score: self reported as 'Geometric Mean'
            return filters.mean(_filter(vals, "Geometric Mean"))

        if "assorted-dom" in testname:
            # pylint: disable=W1633
            return round(filters.geometric_mean(_filter(vals)), 2)

        if "wasm-misc" in testname:
            # wasm_misc_score: self reported as '__total__'
            return filters.mean(_filter(vals, "__total__"))

        if "wasm-godot" in testname:
            # wasm_godot_score: first-interactive mean
            return filters.mean(_filter(vals, "first-interactive"))

        if "youtube-playback" in testname:
            # pylint: disable=W1633
            return round(filters.mean(_filter(vals)), 2)

        if "twitch-animation" in testname:
            return round(filters.geometric_mean(_filter(vals, "run")), 2)

        if testname.startswith("supporting_data"):
            if not unit:
                return sum(_filter(vals))

            if unit == "%":
                return filters.mean(_filter(vals))

            if unit in ("W", "MHz"):
                # For power in Watts and clock frequencies,
                # summarize with the sum of the averages
                allavgs = []
                for val, subtest in vals:
                    if "avg" in subtest:
                        allavgs.append(val)
                if allavgs:
                    return sum(allavgs)

                raise Exception(
                    "No average measurements found for supporting data with W, or MHz unit ."
                )

            if unit in ["KB", "mAh", "mWh"]:
                return sum(_filter(vals))

            raise NotImplementedError("Unit %s not suported" % unit)

        if len(vals) > 1:
            # pylint: disable=W1633
            return round(filters.geometric_mean(_filter(vals)), 2)

        # pylint: disable=W1633
        return round(filters.mean(_filter(vals)), 2)

    def parseUnknown(self, test):
        # Attempt to flatten whatever we've been given
        # Dictionary keys will be joined by dashes, arrays represent
        # represent "iterations"
        _subtests = {}

        if not isinstance(test["measurements"], dict):
            raise Exception(
                "Expected a dictionary with a single entry as the name of the test. "
                "The value of this key should be the data."
            )
        if test.get("custom_data", False):
            # If custom_data is true it means that the data was already flattened
            # and the test name is included in the keys (the test might have
            # also removed it if it's in the subtest_name_filters option). Handle this
            # exception by wrapping it
            test["measurements"] = {test["name"]: [test["measurements"]]}

        for iteration in test["measurements"][list(test["measurements"].keys())[0]]:
            flattened_metrics = None
            if not test.get("custom_data", False):
                flattened_metrics = flatten(iteration, ())

            for metric, value in (flattened_metrics or iteration).items():
                if metric in METRIC_BLOCKLIST:
                    # TODO: Add an option in the test manifest for this
                    continue
                if metric not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[metric] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": metric,
                        "replicates": [],
                    }
                if not isinstance(value, Iterable):
                    value = [value]
                # pylint: disable=W1633
                _subtests[metric]["replicates"].extend([round(x, 3) for x in value])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        summaries = {
            "median": filters.median,
            "mean": filters.mean,
            "geomean": filters.geometric_mean,
        }
        for name in names:
            summary_method = test.get("submetric_summary_method", "median")
            _subtests[name]["value"] = round(
                summaries[summary_method](_subtests[name]["replicates"]), 3
            )
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseSpeedometerOutput(self, test):
        # each benchmark 'index' becomes a subtest; each pagecycle / iteration
        # of the test has multiple values per index/subtest

        # this is the format we receive the results in from the benchmark
        # i.e. this is ONE pagecycle of speedometer:

        # {u'name': u'raptor-speedometer', u'type': u'benchmark', u'measurements':
        # {u'speedometer': [[{u'AngularJS-TodoMVC/DeletingAllItems': [147.3000000000011,
        # 149.95999999999913, 143.29999999999927, 150.34000000000378, 257.6999999999971],
        # u'Inferno-TodoMVC/CompletingAllItems/Sync': [88.03999999999996,#
        # 85.60000000000036, 94.18000000000029, 95.19999999999709, 86.47999999999593],
        # u'AngularJS-TodoMVC': [518.2400000000016, 525.8199999999997, 610.5199999999968,
        # 532.8200000000215, 640.1800000000003], ...(repeated for each index/subtest)}]]},
        # u'browser': u'Firefox 62.0a1 20180528123052', u'lower_is_better': False, u'page':
        # u'http://localhost:55019/Speedometer/index.html?raptor', u'unit': u'score',
        # u'alert_threshold': 2}

        _subtests = {}
        data = test["measurements"]["speedometer"]
        for page_cycle in data:
            for sub, replicates in page_cycle[0].items():
                # for each pagecycle, build a list of subtests and append all related replicates
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                # pylint: disable=W1633
                _subtests[sub]["replicates"].extend([round(x, 3) for x in replicates])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseAresSixOutput(self, test):
        """
        https://browserbench.org/ARES-6/
        Every pagecycle will perform the tests from the index page
        We have 4 main tests per index page:
        - Air, Basic, Babylon, ML
        - and from these 4 above, ares6 generates the Overall results
        Each test has 3 subtests (firstIteration, steadyState, averageWorstCase):
        - _steadyState
        - _firstIteration
        - _averageWorstCase
        Each index page will run 5 cycles, this is set in glue.js

        {
            'expected_browser_cycles': 1,
            'subtest_unit': 'ms',
            'name': 'raptor-ares6-firefox',
            'lower_is_better': False,
            'browser_cycle': '1',
            'subtest_lower_is_better': True,
            'cold': False,
            'browser': 'Firefox 69.0a1 20190531035909',
            'type': 'benchmark',
            'page': 'http://127.0.0.1:35369/ARES-6/index.html?raptor',
            'unit': 'ms',
            'alert_threshold': 2
            'measurements': {
                'ares6': [[{
                    'Babylon_firstIteration': [
                        123.68,
                        168.21999999999997,
                        127.34000000000003,
                        113.56,
                        128.78,
                        169.44000000000003
                    ],
                    'Air_steadyState': [
                        21.184723618090434,
                        22.906331658291457,
                        19.939396984924624,
                        20.572462311557775,
                        20.790452261306534,
                        18.378693467336696
                    ],
                    etc.
                }]]
            }
        }

        Details on how /ARES6/index.html is showing the mean on subsequent test results:

        I selected just a small part from the metrics just to be easier to explain
        what is going on.

        After the raptor GeckoView test finishes, we have these results in the logs:

        Extracted from "INFO - raptor-control-server Info: received webext_results:"
        'Air_firstIteration': [660.8000000000002, 626.4599999999999, 655.6199999999999,
        635.9000000000001, 636.4000000000001]

        Extracted from "INFO - raptor-output Info: PERFHERDER_DATA:"
        {"name": "Air_firstIteration", "lowerIsBetter": true, "alertThreshold": 2.0,
        "replicates": [660.8, 626.46, 655.62, 635.9, 636.4], "value": 636.4, "unit": "ms"}

        On GeckoView's /ARES6/index.html this is what we see for Air - First Iteration:

        - on 1st test cycle : 660.80 (rounded from 660.8000000000002)

        - on 2nd test cycle : 643.63 , this is coming from
          (660.8000000000002 + 626.4599999999999) / 2 ,
          then rounded up to a precision of 2 decimals

        - on 3rd test cycle : 647.63 this is coming from
          (660.8000000000002 + 626.4599999999999 + 655.6199999999999) / 3 ,
          then rounded up to a precision of 2 decimals

        - and so on
        """

        _subtests = {}
        data = test["measurements"]["ares6"]

        for page_cycle in data:
            for sub, replicates in page_cycle[0].items():
                # for each pagecycle, build a list of subtests and append all related replicates
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                # pylint: disable=W1633
                _subtests[sub]["replicates"].extend(
                    [float(round(x, 3)) for x in replicates]
                )

        vals = []
        for name, test in _subtests.items():
            test["value"] = filters.mean(test["replicates"])
            vals.append([test["value"], name])

        # pylint W1656
        return list(_subtests.values()), sorted(vals, reverse=True)

    def parseMotionmarkOutput(self, test):
        # for motionmark we want the frameLength:average value for each test

        # this is the format we receive the results in from the benchmark
        # i.e. this is ONE pagecycle of motionmark htmlsuite test:composited Transforms:

        # {u'name': u'raptor-motionmark-firefox',
        #  u'type': u'benchmark',
        #  u'measurements': {
        #    u'motionmark':
        #      [[{u'HTMLsuite':
        #        {u'Composited Transforms':
        #          {u'scoreLowerBound': 272.9947975553528,
        #           u'frameLength': {u'average': 25.2, u'stdev': 27.0,
        #                            u'percent': 68.2, u'concern': 39.5},
        #           u'controller': {u'average': 300, u'stdev': 0, u'percent': 0, u'concern': 3},
        #           u'scoreUpperBound': 327.0052024446473,
        #           u'complexity': {u'segment1': [[300, 16.6], [300, 16.6]], u'complexity': 300,
        #                           u'segment2': [[300, None], [300, None]], u'stdev': 6.8},
        #           u'score': 300.00000000000006,
        #           u'complexityAverage': {u'segment1': [[30, 30], [30, 30]], u'complexity': 30,
        #                                  u'segment2': [[300, 300], [300, 300]], u'stdev': None}
        #  }}}]]}}

        _subtests = {}
        data = test["measurements"]["motionmark"]
        for page_cycle in data:
            page_cycle_results = page_cycle[0]

            # TODO: this assumes a single suite is run
            suite = list(page_cycle_results)[0]
            for sub in page_cycle_results[suite].keys():
                try:
                    # pylint: disable=W1633
                    replicate = round(
                        float(page_cycle_results[suite][sub]["frameLength"]["average"]),
                        3,
                    )
                except TypeError as e:
                    LOG.warning(
                        "[{}][{}] : {} - {}".format(suite, sub, e.__class__.__name__, e)
                    )

                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                _subtests[sub]["replicates"].extend([replicate])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseYoutubePlaybackPerformanceOutput(self, test):
        """Parse the metrics for the Youtube playback performance test.

        For each video measured values for dropped and decoded frames will be
        available from the benchmark site.

        {u'PlaybackPerf.VP9.2160p60@2X': {u'droppedFrames': 1, u'decodedFrames': 796}

        With each page cycle / iteration of the test multiple values can be present.

        Raptor will calculate the percentage of dropped frames to decoded frames.
        All those three values will then be emitted as separate sub tests.
        """
        _subtests = {}
        test_name = [
            measurement
            for measurement in test["measurements"].keys()
            if "youtube-playback" in measurement
        ]
        if len(test_name) > 0:
            data = test["measurements"].get(test_name[0])
        else:
            raise Exception("No measurements found for youtube test!")

        def create_subtest_entry(
            name,
            value,
            unit=test["subtest_unit"],
            lower_is_better=test["subtest_lower_is_better"],
        ):
            # build a list of subtests and append all related replicates
            if name not in _subtests:
                # subtest not added yet, first pagecycle, so add new one
                _subtests[name] = {
                    "name": name,
                    "unit": unit,
                    "lowerIsBetter": lower_is_better,
                    "replicates": [],
                }

            _subtests[name]["replicates"].append(value)
            if self.subtest_alert_on is not None:
                if name in self.subtest_alert_on:
                    LOG.info(
                        "turning on subtest alerting for measurement type: %s" % name
                    )
                    _subtests[name]["shouldAlert"] = True

        failed_tests = []
        for pagecycle in data:
            for _sub, _value in six.iteritems(pagecycle[0]):
                if _value["decodedFrames"] == 0:
                    failed_tests.append(
                        "%s test Failed. decodedFrames %s droppedFrames %s."
                        % (_sub, _value["decodedFrames"], _value["droppedFrames"])
                    )

                try:
                    percent_dropped = (
                        float(_value["droppedFrames"]) / _value["decodedFrames"] * 100.0
                    )
                except ZeroDivisionError:
                    # if no frames have been decoded the playback failed completely
                    percent_dropped = 100.0

                # Remove the not needed "PlaybackPerf." prefix from each test
                _sub = _sub.split("PlaybackPerf", 1)[-1]
                if _sub.startswith("."):
                    _sub = _sub[1:]

                # build a list of subtests and append all related replicates
                create_subtest_entry(
                    "{}_decoded_frames".format(_sub),
                    _value["decodedFrames"],
                    lower_is_better=False,
                )
                create_subtest_entry(
                    "{}_dropped_frames".format(_sub), _value["droppedFrames"]
                )
                create_subtest_entry(
                    "{}_%_dropped_frames".format(_sub), percent_dropped
                )

        # Check if any youtube test failed and generate exception
        if len(failed_tests) > 0:
            [LOG.warning("Youtube sub-test FAILED: %s" % test) for test in failed_tests]
            # TODO: Change this to raise Exception after we figure out the failing tests
            LOG.warning(
                "Youtube playback sub-tests failed!!! "
                "Not submitting results to perfherder!"
            )
        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            # pylint: disable=W1633
            _subtests[name]["value"] = round(
                float(filters.median(_subtests[name]["replicates"])), 2
            )
            subtests.append(_subtests[name])
            # only include dropped_frames values, without the %_dropped_frames values
            if name.endswith("X_dropped_frames"):
                vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseUnityWebGLOutput(self, test):
        """
        Example output (this is one page cycle):

        {'name': 'raptor-unity-webgl-firefox',
         'type': 'benchmark',
         'measurements': {
            'unity-webgl': [
                [
                    '[{"benchmark":"Mandelbrot GPU","result":1035361},...}]'
                ]
            ]
         },
         'lower_is_better': False,
         'unit': 'score'
        }
        """
        _subtests = {}
        data = test["measurements"]["unity-webgl"]
        for page_cycle in data:
            data = json.loads(page_cycle[0])
            for item in data:
                # for each pagecycle, build a list of subtests and append all related replicates
                sub = item["benchmark"]
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                _subtests[sub]["replicates"].append(item["result"])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseWebaudioOutput(self, test):
        # each benchmark 'index' becomes a subtest; each pagecycle / iteration
        # of the test has multiple values per index/subtest

        # this is the format we receive the results in from the benchmark
        # i.e. this is ONE pagecycle of speedometer:

        # {u'name': u'raptor-webaudio-firefox', u'type': u'benchmark', u'measurements':
        # {u'webaudio': [[u'[{"name":"Empty testcase","duration":26,"buffer":{}},{"name"
        # :"Simple gain test without resampling","duration":66,"buffer":{}},{"name":"Simple
        # gain test without resampling (Stereo)","duration":71,"buffer":{}},{"name":"Simple
        # gain test without resampling (Stereo and positional)","duration":67,"buffer":{}},
        # {"name":"Simple gain test","duration":41,"buffer":{}},{"name":"Simple gain test
        # (Stereo)","duration":59,"buffer":{}},{"name":"Simple gain test (Stereo and positional)",
        # "duration":68,"buffer":{}},{"name":"Upmix without resampling (Mono -> Stereo)",
        # "duration":53,"buffer":{}},{"name":"Downmix without resampling (Mono -> Stereo)",
        # "duration":44,"buffer":{}},{"name":"Simple mixing (same buffer)",
        # "duration":288,"buffer":{}}

        _subtests = {}
        data = test["measurements"]["webaudio"]
        for page_cycle in data:
            data = json.loads(page_cycle[0])
            for item in data:
                # for each pagecycle, build a list of subtests and append all related replicates
                sub = item["name"]
                replicates = [item["duration"]]
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                # pylint: disable=W1633
                _subtests[sub]["replicates"].extend(
                    [float(round(x, 3)) for x in replicates]
                )

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        print(subtests)
        return subtests, vals

    def parseWASMGodotOutput(self, test):
        """
        {u'wasm-godot': [
            {
              "name": "wasm-instantiate",
              "time": 349
            },{
              "name": "engine-instantiate",
              "time": 1263
            ...
            }]}
        """
        _subtests = {}
        data = test["measurements"]["wasm-godot"]
        print(data)
        for page_cycle in data:
            for item in page_cycle[0]:
                # for each pagecycle, build a list of subtests and append all related replicates
                sub = item["name"]
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                _subtests[sub]["replicates"].append(item["time"])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseSunspiderOutput(self, test):
        _subtests = {}
        data = test["measurements"]["sunspider"]
        for page_cycle in data:
            for sub, replicates in page_cycle[0].items():
                # for each pagecycle, build a list of subtests and append all related replicates
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                # pylint: disable=W1633
                _subtests[sub]["replicates"].extend(
                    [float(round(x, 3)) for x in replicates]
                )

        subtests = []
        vals = []

        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"])
            subtests.append(_subtests[name])

            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseAssortedDomOutput(self, test):
        # each benchmark 'index' becomes a subtest; each pagecycle / iteration
        # of the test has multiple values

        # this is the format we receive the results in from the benchmark
        # i.e. this is ONE pagecycle of assorted-dom ('test' is a valid subtest name btw):

        # {u'worker-getname-performance-getter': 5.9, u'window-getname-performance-getter': 6.1,
        # u'window-getprop-performance-getter': 6.1, u'worker-getprop-performance-getter': 6.1,
        # u'test': 5.8, u'total': 30}

        # the 'total' is provided for us from the benchmark; the overall score will be the mean of
        # the totals from all pagecycles; but keep all the subtest values for the logs/json

        _subtests = {}
        data = test["measurements"]["assorted-dom"]
        for pagecycle in data:
            for _sub, _value in pagecycle[0].items():
                # build a list of subtests and append all related replicates
                if _sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[_sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": _sub,
                        "replicates": [],
                    }
                _subtests[_sub]["replicates"].extend([_value])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            # pylint: disable=W1633
            _subtests[name]["value"] = float(
                round(filters.median(_subtests[name]["replicates"]), 2)
            )
            subtests.append(_subtests[name])
            # only use the 'total's to compute the overall result
            if name == "total":
                vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseJetstreamTwoOutput(self, test):
        # https://browserbench.org/JetStream/

        _subtests = {}
        data = test["measurements"]["jetstream2"]
        for page_cycle in data:
            for sub, replicates in page_cycle[0].items():
                # for each pagecycle, build a list of subtests and append all related replicates
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                # pylint: disable=W1633
                _subtests[sub]["replicates"].extend(
                    [float(round(x, 3)) for x in replicates]
                )

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseWASMMiscOutput(self, test):
        """
        {u'wasm-misc': [
          [[{u'name': u'validate', u'time': 163.44000000000005},
            ...
            {u'name': u'__total__', u'time': 63308.434904788155}]],
          ...
          [[{u'name': u'validate', u'time': 129.42000000000002},
            {u'name': u'__total__', u'time': 63181.24089257814}]]
         ]}
        """
        _subtests = {}
        data = test["measurements"]["wasm-misc"]
        for page_cycle in data:
            for item in page_cycle[0]:
                # for each pagecycle, build a list of subtests and append all related replicates
                sub = item["name"]
                if sub not in _subtests:
                    # subtest not added yet, first pagecycle, so add new one
                    _subtests[sub] = {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    }
                _subtests[sub]["replicates"].append(item["time"])

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseMatrixReactBenchOutput(self, test):
        # https://github.com/jandem/matrix-react-bench

        _subtests = {}
        data = test["measurements"]["matrix-react-bench"]
        for page_cycle in data:
            # Each cycle is formatted like `[[iterations, val], [iterations, val2], ...]`
            for iteration, val in page_cycle:
                sub = f"{iteration}-iterations"
                _subtests.setdefault(
                    sub,
                    {
                        "unit": test["subtest_unit"],
                        "alertThreshold": float(test["alert_threshold"]),
                        "lowerIsBetter": test["subtest_lower_is_better"],
                        "name": sub,
                        "replicates": [],
                    },
                )

                # The values produced are far too large for perfherder
                _subtests[sub]["replicates"].append(val)

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals

    def parseTwitchAnimationOutput(self, test):
        _subtests = {}

        for metric, data in test["measurements"].items():
            if "perfstat-" not in metric and metric != "twitch-animation":
                # Only keep perfstats or the run metric
                continue
            if metric == "twitch-animation":
                metric = "run"

            # data is just an array with a single number
            for page_cycle in data:
                # Each benchmark cycle is formatted like `[val]`, perfstats
                # are not
                if not isinstance(page_cycle, list):
                    page_cycle = [page_cycle]
                for val in page_cycle:
                    _subtests.setdefault(
                        metric,
                        {
                            "unit": test["subtest_unit"],
                            "alertThreshold": float(test["alert_threshold"]),
                            "lowerIsBetter": test["subtest_lower_is_better"],
                            "name": metric,
                            "replicates": [],
                        },
                    )

                    # The values produced are far too large for perfherder
                    _subtests[metric]["replicates"].append(val)

        vals = []
        subtests = []
        names = list(_subtests)
        names.sort(reverse=True)
        for name in names:
            _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"])
            subtests.append(_subtests[name])
            vals.append([_subtests[name]["value"], name])

        return subtests, vals


class RaptorOutput(PerftestOutput):
    """class for raptor output"""

    def summarize(self, test_names):
        suites = []
        test_results = {"framework": {"name": "raptor"}, "suites": suites}

        # check if we actually have any results
        if len(self.results) == 0:
            LOG.error("no raptor test results found for %s" % ", ".join(test_names))
            return

        for test in self.results:
            vals = []
            subtests = []
            suite = {
                "name": test["name"],
                "type": test["type"],
                "tags": test.get("tags", []),
                "extraOptions": test["extra_options"],
                "subtests": subtests,
                "lowerIsBetter": test["lower_is_better"],
                "unit": test["unit"],
                "alertThreshold": float(test["alert_threshold"]),
            }

            # Check if optional properties have been set by the test
            if hasattr(test, "alert_change_type"):
                suite["alertChangeType"] = test["alert_change_type"]

            # if cold load add that info to the suite result dict; this will be used later
            # when combining the results from multiple browser cycles into one overall result
            if test["cold"] is True:
                suite["cold"] = True
                suite["browser_cycle"] = int(test["browser_cycle"])
                suite["expected_browser_cycles"] = int(test["expected_browser_cycles"])
                suite["tags"].append("cold")
            else:
                suite["tags"].append("warm")

            suites.append(suite)

            # process results for pageloader type of tests
            if test["type"] in ("pageload", "scenario"):
                # each test can report multiple measurements per pageload
                # each measurement becomes a subtest inside the 'suite'

                # this is the format we receive the results in from the pageload test
                # i.e. one test (subtest) in raptor-firefox-tp6:

                # {u'name': u'raptor-firefox-tp6-amazon', u'type': u'pageload', u'measurements':
                # {u'fnbpaint': [788, 315, 334, 286, 318, 276, 296, 296, 292, 285, 268, 277, 274,
                # 328, 295, 290, 286, 270, 279, 280, 346, 303, 308, 398, 281]}, u'browser':
                # u'Firefox 62.0a1 20180528123052', u'lower_is_better': True, u'page':
                # u'https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop',
                # u'unit': u'ms', u'alert_threshold': 2}

                for measurement_name, replicates in test["measurements"].items():
                    new_subtest = {}
                    new_subtest["name"] = measurement_name
                    new_subtest["replicates"] = replicates
                    new_subtest["lowerIsBetter"] = test["subtest_lower_is_better"]
                    new_subtest["alertThreshold"] = float(test["alert_threshold"])
                    new_subtest["value"] = 0
                    new_subtest["unit"] = test["subtest_unit"]

                    if test["cold"] is False:
                        # for warm page-load, ignore first value due to 1st pageload noise
                        LOG.info(
                            "ignoring the first %s value due to initial pageload noise"
                            % measurement_name
                        )
                        filtered_values = filters.ignore_first(
                            new_subtest["replicates"], 1
                        )
                    else:
                        # for cold-load we want all the values
                        filtered_values = new_subtest["replicates"]

                    # for pageload tests that measure TTFI: TTFI is not guaranteed to be available
                    # everytime; the raptor measure.js webext will substitute a '-1' value in the
                    # cases where TTFI is not available, which is acceptable; however we don't want
                    # to include those '-1' TTFI values in our final results calculations
                    if measurement_name == "ttfi":
                        filtered_values = filters.ignore_negative(filtered_values)
                        # we've already removed the first pageload value; if there aren't any more
                        # valid TTFI values available for this pageload just remove it from results
                        if len(filtered_values) < 1:
                            continue

                    # if 'alert_on' is set for this particular measurement, then we want to set the
                    # flag in the perfherder output to turn on alerting for this subtest
                    if self.subtest_alert_on is not None:
                        if measurement_name in self.subtest_alert_on:
                            LOG.info(
                                "turning on subtest alerting for measurement type: %s"
                                % measurement_name
                            )
                            new_subtest["shouldAlert"] = True
                        else:
                            # Explicitly set `shouldAlert` to False so that the measurement
                            # is not alerted on. Otherwise Perfherder defaults to alerting
                            LOG.info(
                                "turning off subtest alerting for measurement type: %s"
                                % measurement_name
                            )
                            new_subtest["shouldAlert"] = False

                    new_subtest["value"] = filters.median(filtered_values)

                    vals.append([new_subtest["value"], new_subtest["name"]])
                    subtests.append(new_subtest)

            elif test["type"] == "benchmark":

                if any(
                    [
                        "youtube-playback" in measurement
                        for measurement in test["measurements"].keys()
                    ]
                ):
                    subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test)
                elif "assorted-dom" in test["measurements"]:
                    subtests, vals = self.parseAssortedDomOutput(test)
                elif "ares6" in test["measurements"]:
                    subtests, vals = self.parseAresSixOutput(test)
                elif "jetstream2" in test["measurements"]:
                    subtests, vals = self.parseJetstreamTwoOutput(test)
                elif "motionmark" in test["measurements"]:
                    subtests, vals = self.parseMotionmarkOutput(test)
                elif "speedometer" in test["measurements"]:
                    # this includes stylebench
                    subtests, vals = self.parseSpeedometerOutput(test)
                elif "sunspider" in test["measurements"]:
                    subtests, vals = self.parseSunspiderOutput(test)
                elif "unity-webgl" in test["measurements"]:
                    subtests, vals = self.parseUnityWebGLOutput(test)
                elif "wasm-godot" in test["measurements"]:
                    subtests, vals = self.parseWASMGodotOutput(test)
                elif "wasm-misc" in test["measurements"]:
                    subtests, vals = self.parseWASMMiscOutput(test)
                elif "webaudio" in test["measurements"]:
                    subtests, vals = self.parseWebaudioOutput(test)
                else:
                    subtests, vals in self.parseUnknown(test)

                suite["subtests"] = subtests

            else:
                LOG.error(
                    "output.summarize received unsupported test results type for %s"
                    % test["name"]
                )
                return

            suite["tags"].append(test["type"])

            # for benchmarks there is generally more than one subtest in each cycle
            # and a benchmark-specific formula is needed to calculate the final score
            # we no longer summarise the page load as we alert on individual subtests
            # and the geometric mean was found to be of little value
            if len(subtests) > 1 and test["type"] != "pageload":
                suite["value"] = self.construct_summary(vals, testname=test["name"])

            subtests.sort(key=lambda subtest: subtest["name"])
            suite["tags"].sort()

        suites.sort(key=lambda suite: suite["name"])

        self.summarized_results = test_results

    def combine_browser_cycles(self):
        """
        At this point the results have been summarized; however there may have been multiple
        browser cycles (i.e. cold load). In which case the results have one entry for each
        test for each browser cycle. For each test we need to combine the results for all
        browser cycles into one results entry.

        For example, this is what the summarized results suites list looks like from a test that
        was run with multiple (two) browser cycles:

        [{'expected_browser_cycles': 2, 'extraOptions': [],
            'name': u'raptor-tp6m-amazon-geckoview-cold', 'lowerIsBetter': True,
            'alertThreshold': 2.0, 'value': 1776.94, 'browser_cycle': 1,
            'subtests': [{'name': u'dcf', 'lowerIsBetter': True, 'alertThreshold': 2.0,
                'value': 818, 'replicates': [818], 'unit': u'ms'}, {'name': u'fcp',
                'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 1131, 'shouldAlert': True,
                'replicates': [1131], 'unit': u'ms'}, {'name': u'fnbpaint', 'lowerIsBetter': True,
                'alertThreshold': 2.0, 'value': 1056, 'replicates': [1056], 'unit': u'ms'},
                {'name': u'ttfi', 'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 18074,
                'replicates': [18074], 'unit': u'ms'}, {'name': u'loadtime', 'lowerIsBetter': True,
                'alertThreshold': 2.0, 'value': 1002, 'shouldAlert': True, 'replicates': [1002],
                'unit': u'ms'}],
            'cold': True, 'type': u'pageload', 'unit': u'ms'},
        {'expected_browser_cycles': 2, 'extraOptions': [],
            'name': u'raptor-tp6m-amazon-geckoview-cold', 'lowerIsBetter': True,
            'alertThreshold': 2.0, 'value': 840.25, 'browser_cycle': 2,
            'subtests': [{'name': u'dcf', 'lowerIsBetter': True, 'alertThreshold': 2.0,
                'value': 462, 'replicates': [462], 'unit': u'ms'}, {'name': u'fcp',
                'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 718, 'shouldAlert': True,
                'replicates': [718], 'unit': u'ms'}, {'name': u'fnbpaint', 'lowerIsBetter': True,
                'alertThreshold': 2.0, 'value': 676, 'replicates': [676], 'unit': u'ms'},
                {'name': u'ttfi', 'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 3084,
                'replicates': [3084], 'unit': u'ms'}, {'name': u'loadtime', 'lowerIsBetter': True,
                'alertThreshold': 2.0, 'value': 605, 'shouldAlert': True, 'replicates': [605],
                'unit': u'ms'}],
            'cold': True, 'type': u'pageload', 'unit': u'ms'}]

        Need to combine those into a single entry.
        """
        # check if we actually have any results
        if len(self.results) == 0:
            LOG.info(
                "error: no raptor test results found, so no need to combine browser cycles"
            )
            return

        # first build a list of entries that need to be combined; and as we do that, mark the
        # original suite entry as up for deletion, so once combined we know which ones to del
        # note that summarized results are for all tests that were ran in the session, which
        # could include cold and / or warm page-load and / or benchnarks combined
        suites_to_be_combined = []
        combined_suites = []

        for _index, suite in enumerate(self.summarized_results.get("suites", [])):
            if suite.get("cold") is None:
                continue

            if suite["expected_browser_cycles"] > 1:
                _name = suite["name"]
                _details = suite.copy()
                suites_to_be_combined.append({"name": _name, "details": _details})
                suite["to_be_deleted"] = True

        # now create a new suite entry that will have all the results from
        # all of the browser cycles, but in one result entry for each test
        combined_suites = {}

        for next_suite in suites_to_be_combined:
            suite_name = next_suite["details"]["name"]
            browser_cycle = next_suite["details"]["browser_cycle"]
            LOG.info(
                "combining results from browser cycle %d for %s"
                % (browser_cycle, suite_name)
            )
            if suite_name not in combined_suites:
                # first browser cycle so just take entire entry to start with
                combined_suites[suite_name] = next_suite["details"]
                LOG.info("created new combined result with intial cycle replicates")
                # remove the 'cold', 'browser_cycle', and 'expected_browser_cycles' info
                # as we don't want that showing up in perfherder data output
                del combined_suites[suite_name]["cold"]
                del combined_suites[suite_name]["browser_cycle"]
                del combined_suites[suite_name]["expected_browser_cycles"]
            else:
                # subsequent browser cycles, already have an entry; just add subtest replicates
                for next_subtest in next_suite["details"]["subtests"]:
                    # find the existing entry for that subtest in our new combined test entry
                    found_subtest = False
                    for combined_subtest in combined_suites[suite_name]["subtests"]:
                        if combined_subtest["name"] == next_subtest["name"]:
                            # add subtest (measurement type) replicates to the combined entry
                            LOG.info("adding replicates for %s" % next_subtest["name"])
                            combined_subtest["replicates"].extend(
                                next_subtest["replicates"]
                            )
                            found_subtest = True
                    # the subtest / measurement type wasn't found in our existing combined
                    # result entry; if it is for the same suite name add it - this could happen
                    # as ttfi may not be available in every browser cycle
                    if not found_subtest:
                        LOG.info("adding replicates for %s" % next_subtest["name"])
                        combined_suites[next_suite["details"]["name"]][
                            "subtests"
                        ].append(next_subtest)

        # now we have a single entry for each test; with all replicates from all browser cycles
        for i, name in enumerate(combined_suites):
            vals = []
            for next_sub in combined_suites[name]["subtests"]:
                # calculate sub-test results (i.e. each measurement type)
                next_sub["value"] = filters.median(next_sub["replicates"])
                # add to vals; vals is used to calculate overall suite result i.e. the
                # geomean of all of the subtests / measurement types
                vals.append([next_sub["value"], next_sub["name"]])

            # calculate overall suite result ('value') which is geomean of all measures
            if len(combined_suites[name]["subtests"]) > 1:
                combined_suites[name]["value"] = self.construct_summary(
                    vals, testname=name
                )

            # now add the combined suite entry to our overall summarized results!
            self.summarized_results["suites"].append(combined_suites[name])

        # now it is safe to delete the original entries that were made by each cycle
        self.summarized_results["suites"] = [
            item
            for item in self.summarized_results["suites"]
            if item.get("to_be_deleted") is not True
        ]

    def summarize_screenshots(self, screenshots):
        if len(screenshots) == 0:
            return

        self.summarized_screenshots.append(
            """<!DOCTYPE html>
        <head>
        <style>
            table, th, td {
              border: 1px solid black;
              border-collapse: collapse;
            }
        </style>
        </head>
        <html> <body>
        <h1>Captured screenshots!</h1>
        <table style="width:100%">
          <tr>
            <th>Test Name</th>
            <th>Pagecycle</th>
            <th>Screenshot</th>
          </tr>"""
        )

        for screenshot in screenshots:
            self.summarized_screenshots.append(
                """<tr>
            <th>%s</th>
            <th> %s</th>
            <th>
                <img src="%s" alt="%s %s" width="320" height="240">
            </th>
            </tr>"""
                % (
                    screenshot["test_name"],
                    screenshot["page_cycle"],
                    screenshot["screenshot"],
                    screenshot["test_name"],
                    screenshot["page_cycle"],
                )
            )

        self.summarized_screenshots.append("""</table></body> </html>""")


class BrowsertimeOutput(PerftestOutput):
    """class for browsertime output"""

    def summarize(self, test_names):
        """
        Summarize the parsed browsertime test output, and format accordingly so the output can
        be ingested by Perfherder.

        At this point each entry in self.results for browsertime-pageload tests is in this format:

        {'statistics':{'fcp': {u'p99': 932, u'mdev': 10.0941, u'min': 712, u'p90': 810, u'max':
        932, u'median': 758, u'p10': 728, u'stddev': 50, u'mean': 769}, 'dcf': {u'p99': 864,
        u'mdev': 11.6768, u'min': 614, u'p90': 738, u'max': 864, u'median': 670, u'p10': 632,
        u'stddev': 58, u'mean': 684}, 'fnbpaint': {u'p99': 830, u'mdev': 9.6851, u'min': 616,
        u'p90': 719, u'max': 830, u'median': 668, u'p10': 642, u'stddev': 48, u'mean': 680},
        'loadtime': {u'p99': 5818, u'mdev': 111.7028, u'min': 3220, u'p90': 4450, u'max': 5818,
        u'median': 3476, u'p10': 3241, u'stddev': 559, u'mean': 3642}}, 'name':
        'raptor-tp6-guardian-firefox', 'url': 'https://www.theguardian.co.uk', 'lower_is_better':
        True, 'measurements': {'fcp': [932, 744, 744, 810, 712, 775, 759, 744, 777, 739, 809, 906,
        734, 742, 760, 758, 728, 792, 757, 759, 742, 759, 775, 726, 730], 'dcf': [864, 679, 637,
        662, 652, 651, 710, 679, 646, 689, 686, 845, 670, 694, 632, 703, 670, 738, 633, 703, 614,
        703, 650, 622, 670], 'fnbpaint': [830, 648, 666, 704, 616, 683, 678, 650, 685, 651, 719,
        820, 634, 664, 681, 664, 642, 703, 668, 670, 669, 668, 681, 652, 642], 'loadtime': [4450,
        3592, 3770, 3345, 3453, 3220, 3434, 3621, 3511, 3416, 3430, 5818, 4729, 3406, 3506, 3588,
        3245, 3381, 3707, 3241, 3595, 3483, 3236, 3390, 3476]}, 'subtest_unit': 'ms', 'bt_ver':
        '4.9.2-android', 'alert_threshold': 2, 'cold': True, 'type': 'browsertime-pageload',
        'unit': 'ms', 'browser': "{u'userAgent': u'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13;
        rv:70.0) Gecko/20100101 Firefox/70.0', u'windowSize': u'1366x694'}"}

        Now we must process this further and prepare the result for output suitable for perfherder
        ingestion.

        Note: For the overall subtest values/results (i.e. for each measurement type) we will use
        the Browsertime-provided statistics, instead of calcuating our own geomeans from the
        replicates.
        """

        def _filter_data(data, method, subtest_name):
            import numpy as np
            from scipy.cluster.vq import kmeans2, whiten

            """
            Take the kmeans of the data, and attempt to filter this way.
            We'll use hard-coded values to get rid of data that is 2x
            smaller/larger than the majority of the data. If the data approaches
            a 35%/65% split, then it won't be filtered as we can't figure
            out which one is the right mean to take.

            The way that this will work for multi-modal data (more than 2 modes)
            is that the majority of the modes will end up in either one bin or
            the other. Taking the group with the most points lets us
            consistently remove very large outliers out of the data and target the
            modes with the largest prominence.

            TODO: The seed exists because using a randomized one often gives us
            multiple results on the same dataset. This should keep things more
            consistent from one task to the next. We should also look into playing
            with iterations, but that comes at the cost of processing time (this
            might not be a valid concern).
            """
            data = np.asarray(data)

            # Disable kmeans2 empty cluster warnings
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                kmeans, result = kmeans2(
                    whiten(np.asarray([float(d) for d in data])), 2, seed=1000
                )

            if len(kmeans) < 2:
                # Default to a gaussian filter if we didn't get 2 means
                summary_method = np.mean
                if method == "geomean":
                    filters.geometric_mean

                # Apply a gaussian filter
                data = data[
                    np.where(data > summary_method(data) - (np.std(data) * 2))[0]
                ]
                data = list(
                    data[np.where(data < summary_method(data) + (np.std(data) * 2))[0]]
                )
            else:
                first_group = data[np.where(result == 0)]
                secnd_group = data[np.where(result == 1)]

                total_len = len(data)
                first_len = len(first_group)
                secnd_len = len(secnd_group)

                ratio = np.ceil((min(first_len, secnd_len) / total_len) * 100)
                if ratio <= 35:
                    # If one of the groups are less than 35% of the total
                    # size, then filter it out if the difference in the
                    # k-means are large enough (200% difference).
                    max_mean = max(kmeans)
                    min_mean = min(kmeans)
                    if abs(max_mean / min_mean) > 2:
                        major_group = first_group
                        major_mean = np.mean(first_group) if first_len > 0 else 0
                        minor_mean = np.mean(secnd_group) if secnd_len > 0 else 0
                        if first_len < secnd_len:
                            major_group = secnd_group
                            tmp = major_mean
                            major_mean = minor_mean
                            minor_mean = tmp

                        LOG.info(
                            f"{subtest_name}: Filtering out {total_len - len(major_group)} "
                            f"data points found in minor_group of data with "
                            f"mean {minor_mean} vs. {major_mean} in major group"
                        )
                        data = major_group

            return data

        def _process_alt_method(subtest, alternative_method):
            # Don't filter with less than 10 data points
            data = subtest["replicates"]
            if len(subtest["replicates"]) > 10:
                data = _filter_data(data, alternative_method, subtest["name"])
            if alternative_method == "geomean":
                subtest["value"] = round(filters.geometric_mean(data), 1)
            elif alternative_method == "mean":
                subtest["value"] = round(filters.mean(data), 1)

        # converting suites and subtests into lists, and sorting them
        def _process(subtest, alternative_method=""):
            if test["type"] == "power":
                subtest["value"] = filters.mean(subtest["replicates"])
            elif (
                subtest["name"] in VISUAL_METRICS
                or subtest["name"].startswith("perfstat")
                or subtest["name"] == "cpuTime"
            ):
                if alternative_method in ("geomean", "mean"):
                    _process_alt_method(subtest, alternative_method)
                else:
                    subtest["value"] = filters.median(subtest["replicates"])
            else:
                if alternative_method in ("geomean", "mean"):
                    _process_alt_method(subtest, alternative_method)
                else:
                    subtest["value"] = filters.median(
                        filters.ignore_first(subtest["replicates"], 1)
                    )
            return subtest

        def _process_suite(suite):
            suite["subtests"] = [
                _process(subtest)
                for subtest in suite["subtests"].values()
                if subtest["replicates"]
            ]

            # Duplicate for different summary values if needed
            if self.extra_summary_methods:
                new_subtests = []
                for subtest in suite["subtests"]:
                    try:
                        for alternative_method in self.extra_summary_methods:
                            new_subtest = copy.deepcopy(subtest)
                            new_subtest[
                                "name"
                            ] = f"{new_subtest['name']} ({alternative_method})"
                            _process(new_subtest, alternative_method)
                            new_subtests.append(new_subtest)
                    except Exception as e:
                        # Ignore failures here
                        LOG.info(f"Failed to summarize with alternative methods: {e}")
                        pass
                suite["subtests"].extend(new_subtests)

            suite["subtests"].sort(key=lambda subtest: subtest["name"])

            # for benchmarks there is generally more than one subtest in each cycle
            # and a benchmark-specific formula is needed to calculate the final score
            # we no longer summarise the page load as we alert on individual subtests
            # and the geometric mean was found to be of little value
            if len(suite["subtests"]) > 1 and suite["type"] != "pageload":
                vals = [
                    [subtest["value"], subtest["name"]] for subtest in suite["subtests"]
                ]
                testname = suite["name"]
                if suite["type"] == "power":
                    testname = "supporting_data"
                suite["value"] = self.construct_summary(vals, testname=testname)
            return suite

        LOG.info("preparing browsertime results for output")

        # check if we actually have any results
        if len(self.results) == 0:
            LOG.error(
                "no browsertime test results found for %s" % ", ".join(test_names)
            )
            return

        test_results = {"framework": {"name": "browsertime"}}

        # using a mapping so we can have a unique set of results given a name
        suites = {}

        for test in self.results:
            test_name = test["name"]
            extra_options = test["extra_options"]

            # If a test with the same name has different extra options, handle it
            # by appending the difference in options to the key used in `suites`. We
            # need to do a loop here in case we get a conflicting test name again.
            prev_name = test_name
            while (
                test_name in suites
                and suites[test_name]["extraOptions"] != extra_options
            ):
                missing = set(extra_options) - set(suites[test_name]["extraOptions"])
                if len(missing) == 0:
                    missing = set(suites[test_name]["extraOptions"]) - set(
                        extra_options
                    )
                test_name = test_name + "-".join(list(missing))

                if prev_name == test_name:
                    # Kill the loop if we get the same name again
                    break
                else:
                    prev_name = test_name

            suite = suites.setdefault(
                test_name,
                {
                    "name": test["name"],
                    "type": test["type"],
                    "extraOptions": extra_options,
                    # There may be unique options in tags now, but we don't want to remove the
                    # previous behaviour which includes the extra options in the tags.
                    "tags": test.get("tags", []) + extra_options,
                    "lowerIsBetter": test["lower_is_better"],
                    "unit": test["unit"],
                    "alertThreshold": float(test["alert_threshold"]),
                    # like suites, subtests are identified by names
                    "subtests": {},
                },
            )
            # Add the alert window settings if needed
            for alert_option, schema_name in (
                ("min_back_window", "minBackWindow"),
                ("max_back_window", "maxBackWindow"),
                ("fore_window", "foreWindow"),
            ):
                if test.get(alert_option, None) is not None:
                    suite[schema_name] = int(test[alert_option])

            # Setting shouldAlert to False whenever self.app is either chrome, chrome-m, chromium, chromium-as-release
            if self.app in ("chrome", "chrome-m", "chromium", "custom-car"):
                suite["shouldAlert"] = False
            # Check if the test has set optional properties
            if "alert_change_type" in test and "alertChangeType" not in suite:
                suite["alertChangeType"] = test["alert_change_type"]

            def _process_measurements(measurement_name, replicates):
                subtest = {}
                subtest["name"] = measurement_name
                subtest["lowerIsBetter"] = test["subtest_lower_is_better"]
                subtest["alertThreshold"] = float(test["alert_threshold"])
                subtest["unit"] = (
                    "ms" if measurement_name == "cpuTime" else test["subtest_unit"]
                )

                # Add the alert window settings if needed here too in case
                # there is no summary value in the test
                for schema_name in (
                    "minBackWindow",
                    "maxBackWindow",
                    "foreWindow",
                ):
                    if suite.get(schema_name, None) is not None:
                        subtest[schema_name] = suite[schema_name]

                # if 'alert_on' is set for this particular measurement, then we want to set
                # the flag in the perfherder output to turn on alerting for this subtest
                if self.subtest_alert_on is not None:
                    if measurement_name in self.subtest_alert_on:
                        LOG.info(
                            "turning on subtest alerting for measurement type: %s"
                            % measurement_name
                        )
                        subtest["shouldAlert"] = True
                        if self.app in ("chrome", "chrome-m", "chromium", "custom-car"):
                            subtest["shouldAlert"] = False
                    else:
                        # Explicitly set `shouldAlert` to False so that the measurement
                        # is not alerted on. Otherwise Perfherder defaults to alerting.
                        LOG.info(
                            "turning off subtest alerting for measurement type: %s"
                            % measurement_name
                        )
                        subtest["shouldAlert"] = False
                subtest["replicates"] = replicates
                return subtest

            if test["type"] in ["pageload", "scenario", "power"]:
                for measurement_name, replicates in test["measurements"].items():
                    new_subtest = _process_measurements(measurement_name, replicates)
                    if measurement_name not in suite["subtests"]:
                        suite["subtests"][measurement_name] = new_subtest
                    else:
                        suite["subtests"][measurement_name]["replicates"].extend(
                            new_subtest["replicates"]
                        )

            elif "benchmark" in test["type"]:
                subtests = None

                if "speedometer" in test["measurements"]:
                    # this includes stylebench
                    subtests, vals = self.parseSpeedometerOutput(test)
                elif "ares6" in test["name"]:
                    subtests, vals = self.parseAresSixOutput(test)
                elif "motionmark" in test["measurements"]:
                    subtests, vals = self.parseMotionmarkOutput(test)
                elif "youtube-playback" in test["name"]:
                    subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test)
                elif "unity-webgl" in test["name"]:
                    subtests, vals = self.parseUnityWebGLOutput(test)
                elif "webaudio" in test["measurements"]:
                    subtests, vals = self.parseWebaudioOutput(test)
                elif "wasm-godot" in test["measurements"]:
                    subtests, vals = self.parseWASMGodotOutput(test)
                elif "wasm-misc" in test["measurements"]:
                    subtests, vals = self.parseWASMMiscOutput(test)
                elif "sunspider" in test["measurements"]:
                    subtests, vals = self.parseSunspiderOutput(test)
                elif "assorted-dom" in test["measurements"]:
                    subtests, vals = self.parseAssortedDomOutput(test)
                elif "jetstream2" in test["measurements"]:
                    subtests, vals = self.parseJetstreamTwoOutput(test)
                elif "matrix-react-bench" in test["name"]:
                    subtests, vals = self.parseMatrixReactBenchOutput(test)
                elif "twitch-animation" in test["name"]:
                    subtests, vals = self.parseTwitchAnimationOutput(test)
                else:
                    # Attempt to parse the unknown benchmark by flattening the
                    # given data and merging all the arrays of non-iterable
                    # data that fall under the same key.
                    # XXX Note that this is not fully implemented for the summary
                    # of the metric or test as we don't have a use case for that yet.
                    subtests, vals = self.parseUnknown(test)

                if subtests is None:
                    raise Exception("No benchmark metrics found in browsertime results")

                suite["subtests"] = subtests

                if "cpuTime" in test["measurements"] and test.get(
                    "gather_cpuTime", None
                ):
                    replicates = test["measurements"]["cpuTime"]
                    cpu_subtest = _process_measurements("cpuTime", replicates)
                    _process(cpu_subtest)
                    suite["subtests"].append(cpu_subtest)

                # summarize results for both benchmark type tests
                if len(subtests) > 1:
                    suite["value"] = self.construct_summary(vals, testname=test["name"])
                subtests.sort(key=lambda subtest: subtest["name"])

        # convert suites to list
        suites = [
            s if "benchmark" in s["type"] else _process_suite(s)
            for s in suites.values()
        ]

        suites.sort(key=lambda suite: suite["name"])

        test_results["suites"] = suites
        self.summarized_results = test_results