diff options
Diffstat (limited to 'python/mozperftest/mozperftest/metrics/notebook')
13 files changed, 903 insertions, 0 deletions
diff --git a/python/mozperftest/mozperftest/metrics/notebook/__init__.py b/python/mozperftest/mozperftest/metrics/notebook/__init__.py new file mode 100644 index 0000000000..8d69182664 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/__init__.py @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from .perftestetl import PerftestETL +from .perftestnotebook import PerftestNotebook + +__all__ = ["PerftestETL", "PerftestNotebook"] diff --git a/python/mozperftest/mozperftest/metrics/notebook/constant.py b/python/mozperftest/mozperftest/metrics/notebook/constant.py new file mode 100644 index 0000000000..ca40d289d4 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/constant.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import pathlib +from types import MappingProxyType + +from .transformer import get_transformers + + +class Constant(object): + """A singleton class to store all constants.""" + + __instance = None + + def __new__(cls, *args, **kw): + if cls.__instance is None: + cls.__instance = object.__new__(cls, *args, **kw) + return cls.__instance + + def __init__(self): + self.__here = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + self.__predefined_transformers = get_transformers(self.__here / "transforms") + + @property + def predefined_transformers(self): + return MappingProxyType(self.__predefined_transformers).copy() + + @property + def here(self): + return self.__here diff --git a/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/compare b/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/compare new file mode 100644 index 0000000000..f6870f0246 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/compare @@ -0,0 +1,85 @@ +%% md +<div id="table-wrapper"> + <table id="compareTable" border="1"></table> +</div> + +%% py +from js import document, data_object +import json +import numpy as np + +split_data = {} +dir_names = set() +subtests = set() +newest_run_name = "" +for element in data_object: + name = element["name"] + if "- newest run" in name: + newest_run_name = name + subtest = element["subtest"] + dir_names.add(name) + subtests.add(subtest) + + data = [p["value"] for p in element["data"]] + split_data.setdefault(name, {}).update({ + subtest:{ + "data":data, + "stats":{ + "Mean": np.round(np.mean(data),2), + "Median": np.median(data), + "Std. Dev.": np.round(np.std(data),2) + } + } + }) + +table = document.getElementById("compareTable") +table.innerHTML='' + +# build table head +thead = table.createTHead() +throw = thead.insertRow() +for name in ["Metrics", "Statistics"] + list(dir_names): + th = document.createElement("th") + th.appendChild(document.createTextNode(name)) + throw.appendChild(th) + +def fillRow(row, subtest, stat): + row.insertCell().appendChild(document.createTextNode(stat)) + newest_run_val = split_data[newest_run_name][subtest]["stats"][stat] + for name in dir_names: + cell_val = split_data[name][subtest]["stats"][stat] + diff = np.round((cell_val - newest_run_val * 1.0)/newest_run_val * 100, 2) + color = "red" if diff>0 else "green" + row.insertCell().innerHTML = f"{cell_val}\n(<span style=\"color:{color}\">{diff}</span>%)" + +# build table body +tbody = document.createElement("tbody") +for subtest in subtests: + row1 = tbody.insertRow() + cell0 = row1.insertCell() + cell0.appendChild(document.createTextNode(subtest)) + cell0.rowSpan = 3; + a = split_data + fillRow(row1, subtest, "Mean") + + row2 = tbody.insertRow() + fillRow(row2, subtest, "Median") + + row3 = tbody.insertRow() + fillRow(row3, subtest, "Std. Dev.") + +table.appendChild(tbody) + +%% css +#table-wrapper { + height: 600px; + overflow: auto; +} + +#table { + display: table; +} + +td { + white-space:pre-line; +} diff --git a/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/header b/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/header new file mode 100644 index 0000000000..1a0f659e54 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/header @@ -0,0 +1,12 @@ +%% md +# Welcome to PerftestNotebook + +press the :fast_forward: button on your top left corner to run whole notebook + +%% fetch + +text: data_string = http://127.0.0.1:5000/data + +%% js + +var data_object = JSON.parse(data_string); diff --git a/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/scatterplot b/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/scatterplot new file mode 100644 index 0000000000..f68b540236 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/notebook-sections/scatterplot @@ -0,0 +1,15 @@ +%% py +from js import data_object +import matplotlib.pyplot as plt + +plt.figure() + +for element in data_object: + data_array = element["data"] + x = [x["xaxis"] for x in data_array] + y = [x["value"] for x in data_array] + label = element["name"]+"\n"+element["subtest"] + plt.scatter(x,y,label=label) + +plt.legend() +plt.show() diff --git a/python/mozperftest/mozperftest/metrics/notebook/perftestetl.py b/python/mozperftest/mozperftest/metrics/notebook/perftestetl.py new file mode 100644 index 0000000000..bd28d9be6d --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/perftestetl.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json +import os +import pathlib +from collections import OrderedDict + +from .constant import Constant +from .transformer import SimplePerfherderTransformer, Transformer, get_transformer + + +class PerftestETL(object): + """Controller class for the PerftestETL.""" + + def __init__( + self, + file_groups, + config, + prefix, + logger, + custom_transform=None, + sort_files=False, + ): + """Initializes PerftestETL. + + :param dict file_groups: A dict of file groupings. The value + of each of the dict entries is the name of the data that + will be produced. + :param str custom_transform: The class name of a custom transformer. + """ + self.fmt_data = {} + self.file_groups = file_groups + self.config = config + self.sort_files = sort_files + self.const = Constant() + self.prefix = prefix + self.logger = logger + + # Gather the available transformers + tfms_dict = self.const.predefined_transformers + + # XXX NOTEBOOK_PLUGIN functionality is broken at the moment. + # This code block will raise an exception if it detects it in + # the environment. + plugin_path = os.getenv("NOTEBOOK_PLUGIN") + if plugin_path: + raise Exception("NOTEBOOK_PLUGIN is currently broken.") + + # Initialize the requested transformer + if custom_transform: + # try to load it directly, and fallback to registry + try: + tfm_cls = get_transformer(custom_transform) + except ImportError: + tfm_cls = tfms_dict.get(custom_transform) + + if tfm_cls: + self.transformer = Transformer( + files=[], + custom_transformer=tfm_cls(), + logger=self.logger, + prefix=self.prefix, + ) + self.logger.info(f"Found {custom_transform} transformer", self.prefix) + else: + raise Exception(f"Could not get a {custom_transform} transformer.") + else: + self.transformer = Transformer( + files=[], + custom_transformer=SimplePerfherderTransformer(), + logger=self.logger, + prefix=self.prefix, + ) + + def parse_file_grouping(self, file_grouping): + """Handles differences in the file_grouping definitions. + + It can either be a path to a folder containing the files, a list of files, + or it can contain settings from an artifact_downloader instance. + + :param file_grouping: A file grouping entry. + :return: A list of files to process. + """ + files = [] + if isinstance(file_grouping, list): + # A list of files was provided + files = file_grouping + elif isinstance(file_grouping, dict): + # A dictionary of settings from an artifact_downloader instance + # was provided here + raise Exception( + "Artifact downloader tooling is disabled for the time being." + ) + elif isinstance(file_grouping, str): + # Assume a path to files was given + filepath = file_grouping + newf = [f.resolve().as_posix() for f in pathlib.Path(filepath).rglob("*")] + files = newf + else: + raise Exception( + "Unknown file grouping type provided here: %s" % file_grouping + ) + + if self.sort_files: + if isinstance(files, list): + files.sort() + else: + for _, file_list in files.items(): + file_list.sort() + files = OrderedDict(sorted(files.items(), key=lambda entry: entry[0])) + + if not files: + raise Exception( + "Could not find any files in this configuration: %s" % file_grouping + ) + + return files + + def parse_output(self): + # XXX Fix up this function, it should only return a directory for output + # not a directory or a file. Or remove it completely, it's not very useful. + prefix = "" if "prefix" not in self.config else self.config["prefix"] + filepath = f"{prefix}std-output.json" + + if "output" in self.config: + filepath = self.config["output"] + if os.path.isdir(filepath): + filepath = os.path.join(filepath, f"{prefix}std-output.json") + + return filepath + + def process(self, **kwargs): + """Process the file groups and return the results of the requested analyses. + + :return: All the results in a dictionary. The field names are the Analyzer + funtions that were called. + """ + fmt_data = [] + + for name, files in self.file_groups.items(): + files = self.parse_file_grouping(files) + if isinstance(files, dict): + raise Exception( + "Artifact downloader tooling is disabled for the time being." + ) + else: + # Transform the data + self.transformer.files = files + trfm_data = self.transformer.process(name, **kwargs) + + if isinstance(trfm_data, list): + fmt_data.extend(trfm_data) + else: + fmt_data.append(trfm_data) + + self.fmt_data = fmt_data + + # Write formatted data output to filepath + output_data_filepath = self.parse_output() + + print("Writing results to %s" % output_data_filepath) + with open(output_data_filepath, "w") as f: + json.dump(self.fmt_data, f, indent=4, sort_keys=True) + + return {"data": self.fmt_data, "file-output": output_data_filepath} diff --git a/python/mozperftest/mozperftest/metrics/notebook/perftestnotebook.py b/python/mozperftest/mozperftest/metrics/notebook/perftestnotebook.py new file mode 100644 index 0000000000..99c3766b42 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/perftestnotebook.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json +import webbrowser +from http.server import BaseHTTPRequestHandler, HTTPServer + +from .constant import Constant + + +class PerftestNotebook(object): + """Controller class for PerftestNotebook.""" + + def __init__(self, data, logger, prefix): + """Initialize the PerftestNotebook. + + :param dict data: Standardized data, post-transformation. + """ + self.data = data + self.logger = logger + self.prefix = prefix + self.const = Constant() + + def get_notebook_section(self, func): + """Fetch notebook content based on analysis name. + + :param str func: analysis or notebook section name + """ + template_path = self.const.here / "notebook-sections" / func + if not template_path.exists(): + self.logger.warning( + f"Could not find the notebook-section called {func}", self.prefix + ) + return "" + with template_path.open() as f: + return f.read() + + def post_to_iodide(self, analysis=None, start_local_server=True): + """Build notebook and post it to iodide. + + :param list analysis: notebook section names, analysis to perform in iodide + """ + data = self.data + notebook_sections = "" + + template_header_path = self.const.here / "notebook-sections" / "header" + with template_header_path.open() as f: + notebook_sections += f.read() + + if analysis: + for func in analysis: + notebook_sections += self.get_notebook_section(func) + + template_upload_file_path = self.const.here / "template_upload_file.html" + with template_upload_file_path.open() as f: + html = f.read().replace("replace_me", repr(notebook_sections)) + + upload_file_path = self.const.here / "upload_file.html" + with upload_file_path.open("w") as f: + f.write(html) + + # set up local server. Iodide will fetch data from localhost:5000/data + class DataRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/data": + self.send_response(200) + self.send_header("Content-type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(bytes(json.dumps(data).encode("utf-8"))) + + PORT_NUMBER = 5000 + server = HTTPServer(("", PORT_NUMBER), DataRequestHandler) + if start_local_server: + webbrowser.open_new_tab(str(upload_file_path)) + try: + server.serve_forever() + finally: + server.server_close() diff --git a/python/mozperftest/mozperftest/metrics/notebook/template_upload_file.html b/python/mozperftest/mozperftest/metrics/notebook/template_upload_file.html new file mode 100644 index 0000000000..2400be4e87 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/template_upload_file.html @@ -0,0 +1,39 @@ +<!DOCTYPE html> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> +<html> + <body> + Redirecting to Iodide... + <script> + function post(path, params, method='post') { + const form = document.createElement('form'); + form.method = method; + form.action = path; + form.id = 'uploadform'; + + for (const key in params) { + if (params.hasOwnProperty(key)) { + const textarea = document.createElement('textarea'); + textarea.name = key; + textarea.value = params[key]; + textarea.style.display = "none"; + form.appendChild(textarea); + } + } + + + document.body.appendChild(form); + form.submit(); + } + + // TODO Need to escape all `'`, + // Otherwsie, this will result in javascript failures. + var template = replace_me + + // Create a form object, and send it + // after release, change back to https://alpha.iodide.io/from-template/ + post("https://alpha.iodide.io/from-template/", {"iomd": template}) + </script> + </body> +</html> diff --git a/python/mozperftest/mozperftest/metrics/notebook/transformer.py b/python/mozperftest/mozperftest/metrics/notebook/transformer.py new file mode 100644 index 0000000000..7ecbc40d89 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/transformer.py @@ -0,0 +1,228 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import importlib.util +import inspect +import json +import pathlib + +from jsonschema import validate + +from mozperftest.metrics.exceptions import ( + NotebookDuplicateTransformsError, + NotebookInvalidPathError, + NotebookInvalidTransformError, +) +from mozperftest.runner import HERE +from mozperftest.utils import load_class + + +class Transformer(object): + """Abstract class for data transformers.""" + + def __init__(self, files=None, custom_transformer=None, logger=None, prefix=None): + """Initialize the transformer with files. + + :param list files: A list of files containing data to transform. + :param object custom_transformer: A custom transformer instance. + Must implement `transform` and `merge` methods. + """ + self._files = files + self.logger = logger + self.prefix = prefix + + if custom_transformer: + valid = ( + hasattr(custom_transformer, "transform") + and hasattr(custom_transformer, "merge") + and callable(custom_transformer.transform) + and callable(custom_transformer.merge) + ) + + if not valid: + raise NotebookInvalidTransformError( + "The custom transformer must contain `transform` and `merge` methods." + ) + + self._custom_transformer = custom_transformer + + with pathlib.Path(HERE, "schemas", "transformer_schema.json").open() as f: + self.schema = json.load(f) + + @property + def files(self): + return self._files + + @files.setter + def files(self, val): + if not isinstance(val, list): + self.logger.warning( + "`files` must be a list, got %s" % type(val), self.prefix + ) + return + self._files = val + + @property + def custom_transformer(self): + return self._custom_transformer + + def open_data(self, file): + """Opens a file of data. + + If it's not a JSON file, then the data + will be opened as a text file. + + :param str file: Path to the data file. + :return: Data contained in the file. + """ + with open(file) as f: + if file.endswith(".json"): + return json.load(f) + return f.readlines() + + def process(self, name, **kwargs): + """Process all the known data into a merged, and standardized data format. + + :param str name: Name of the merged data. + :return dict: Merged data. + """ + trfmdata = [] + + for file in self.files: + data = {} + + # Open data + try: + if hasattr(self._custom_transformer, "open_data"): + data = self._custom_transformer.open_data(file) + else: + data = self.open_data(file) + except Exception as e: + self.logger.warning( + "Failed to open file %s, skipping" % file, self.prefix + ) + self.logger.warning("%s %s" % (e.__class__.__name__, e), self.prefix) + + # Transform data + try: + data = self._custom_transformer.transform(data, **kwargs) + if not isinstance(data, list): + data = [data] + for entry in data: + for ele in entry["data"]: + if "file" not in ele: + ele.update({"file": file}) + trfmdata.extend(data) + except Exception as e: + self.logger.warning( + "Failed to transform file %s, skipping" % file, self.prefix + ) + self.logger.warning("%s %s" % (e.__class__.__name__, e), self.prefix) + + merged = self._custom_transformer.merge(trfmdata) + + if isinstance(merged, dict): + merged["name"] = name + else: + for e in merged: + e["name"] = name + + validate(instance=merged, schema=self.schema) + return merged + + +class SimplePerfherderTransformer: + """Transforms perfherder data into the standardized data format.""" + + entry_number = 0 + + def transform(self, data): + self.entry_number += 1 + return { + "data": [{"value": data["suites"][0]["value"], "xaxis": self.entry_number}] + } + + def merge(self, sde): + merged = {"data": []} + for entry in sde: + if isinstance(entry["data"], list): + merged["data"].extend(entry["data"]) + else: + merged["data"].append(entry["data"]) + + self.entry_number = 0 + return merged + + +def get_transformer(path, ret_members=False): + """This function returns a Transformer class with the given path. + + :param str path: The path points to the custom transformer. + :param bool ret_members: If true then return inspect.getmembers(). + :return Transformer if not ret_members else inspect.getmembers(). + """ + file = pathlib.Path(path) + + if file.suffix != ".py": + return load_class(path) + + if not file.exists(): + raise NotebookInvalidPathError(f"The path {path} does not exist.") + + # Importing a source file directly + spec = importlib.util.spec_from_file_location(name=file.name, location=path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + members = inspect.getmembers( + module, + lambda c: inspect.isclass(c) + and hasattr(c, "transform") + and hasattr(c, "merge") + and callable(c.transform) + and callable(c.merge), + ) + + if not members and not ret_members: + raise NotebookInvalidTransformError( + f"The path {path} was found but it was not a valid transformer." + ) + + return members if ret_members else members[0][-1] + + +def get_transformers(dirpath=None): + """This function returns a dict of transformers under the given path. + + If more than one transformers have the same class name, an exception will be raised. + + :param pathlib.Path dirpath: Path to a directory containing the transformers. + :return dict: {"Transformer class name": Transformer class}. + """ + + ret = {} + + if not dirpath.exists(): + raise NotebookInvalidPathError(f"The path {dirpath.as_posix()} does not exist.") + + if not dirpath.is_dir(): + raise NotebookInvalidPathError( + f"Path given is not a directory: {dirpath.as_posix()}" + ) + + tfm_files = list(dirpath.glob("*.py")) + importlib.machinery.SOURCE_SUFFIXES.append("") + + for file in tfm_files: + members = get_transformer(file.resolve().as_posix(), True) + + for (name, tfm_class) in members: + if name in ret: + raise NotebookDuplicateTransformsError( + f"Duplicated transformer {name} " + + f"is found in the directory {dirpath.as_posix()}." + + "Please define each transformer class with a unique class name.", + ) + ret.update({name: tfm_class}) + + return ret diff --git a/python/mozperftest/mozperftest/metrics/notebook/transforms/__init__.py b/python/mozperftest/mozperftest/metrics/notebook/transforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/transforms/__init__.py diff --git a/python/mozperftest/mozperftest/metrics/notebook/transforms/logcattime.py b/python/mozperftest/mozperftest/metrics/notebook/transforms/logcattime.py new file mode 100644 index 0000000000..184b327540 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/transforms/logcattime.py @@ -0,0 +1,121 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import re +from datetime import datetime, timedelta + +from mozperftest.metrics.exceptions import ( + NotebookTransformError, + NotebookTransformOptionsError, +) + +TIME_MATCHER = re.compile(r"(\s+[\d.:]+\s+)") + + +class LogCatTimeTransformer: + """Used for parsing times/durations from logcat logs.""" + + def open_data(self, file): + with open(file) as f: + return f.read() + + def _get_duration(self, startline, endline): + """Parse duration between two logcat lines. + + Expecting lines with a prefix like: + 05-26 11:45:41.226 ... + + We only parse the hours, minutes, seconds, and milliseconds here + because we have no use for the days and other times. + """ + match = TIME_MATCHER.search(startline) + if not match: + return None + start = match.group(1).strip() + + match = TIME_MATCHER.search(endline) + if not match: + return None + end = match.group(1).strip() + + sdt = datetime.strptime(start, "%H:%M:%S.%f") + edt = datetime.strptime(end, "%H:%M:%S.%f") + + # If the ending is less than the start, we rolled into a new + # day, so we add 1 day to the end time to handle this + if sdt > edt: + edt += timedelta(1) + + return (edt - sdt).total_seconds() * 1000 + + def _parse_logcat(self, logcat, first_ts, second_ts=None, processor=None): + """Parse data from logcat lines. + + If two regexes are provided (first_ts, and second_ts), then the elapsed + time between those lines will be measured. Otherwise, if only `first_ts` + is defined then, we expect a number as the first group from the + match. Optionally, a `processor` function can be provided to process + all the groups that were obtained from the match, allowing users to + customize what the result is. + + :param list logcat: The logcat lines to parse. + :param str first_ts: Regular expression for the first matching line. + :param str second_ts: Regular expression for the second matching line. + :param func processor: Function to process the groups from the first_ts + regular expression. + :return list: Returns a list of durations/times parsed. + """ + full_re = r"(" + first_ts + r"\n)" + if second_ts: + full_re += r".+(?:\n.+)+?(\n" + second_ts + r"\n)" + + durations = [] + for match in re.findall(full_re, logcat, re.MULTILINE): + if isinstance(match, str): + raise NotebookTransformOptionsError( + "Only one regex was provided, and it has no groups to process." + ) + + if second_ts is not None: + if len(match) != 2: + raise NotebookTransformError( + "More than 2 groups found. It's unclear which " + "to use for calculating the durations." + ) + val = self._get_duration(match[0], match[1]) + elif processor is not None: + # Ignore the first match (that is the full line) + val = processor(match[1:]) + else: + val = match[1] + + if val is not None: + durations.append(float(val)) + + return durations + + def transform(self, data, **kwargs): + alltimes = self._parse_logcat( + data, + kwargs.get("first-timestamp"), + second_ts=kwargs.get("second-timestamp"), + processor=kwargs.get("processor"), + ) + subtest = kwargs.get("transform-subtest-name") + return [ + { + "data": [{"value": val, "xaxis": c} for c, val in enumerate(alltimes)], + "subtest": subtest if subtest else "logcat-metric", + } + ] + + def merge(self, sde): + grouped_data = {} + + for entry in sde: + subtest = entry["subtest"] + data = grouped_data.get(subtest, []) + data.extend(entry["data"]) + grouped_data.update({subtest: data}) + + return [{"data": v, "subtest": k} for k, v in grouped_data.items()] diff --git a/python/mozperftest/mozperftest/metrics/notebook/transforms/single_json.py b/python/mozperftest/mozperftest/metrics/notebook/transforms/single_json.py new file mode 100644 index 0000000000..375615fb23 --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/transforms/single_json.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mozperftest.metrics.notebook.utilities import flat + + +class SingleJsonRetriever: + """Transforms perfherder data into the standardized data format.""" + + entry_number = 0 + + def transform(self, data): + self.entry_number += 1 + + # flat(data, ()) returns a dict that have one key per dictionary path + # in the original data. + return [ + { + "data": [{"value": i, "xaxis": self.entry_number} for i in v], + "subtest": k, + } + for k, v in flat(data, ()).items() + ] + + def merge(self, sde): + grouped_data = {} + for entry in sde: + subtest = entry["subtest"] + data = grouped_data.get(subtest, []) + data.extend(entry["data"]) + grouped_data.update({subtest: data}) + + merged_data = [{"data": v, "subtest": k} for k, v in grouped_data.items()] + + self.entry_number = 0 + return merged_data + + def summary(self, suite): + """Summarize a suite of perfherder data into a single value. + + Returning None means that there's no summary. Otherwise, an integer + or float must be returned. + + Only available in the Perfherder layer. + """ + return None + + def subtest_summary(self, subtest): + """Summarize a set of replicates for a given subtest. + + By default, it returns a None so we fall back to using the + average of the replicates which is the default. + + Only available in the Perfherder layer. + """ + return None diff --git a/python/mozperftest/mozperftest/metrics/notebook/utilities.py b/python/mozperftest/mozperftest/metrics/notebook/utilities.py new file mode 100644 index 0000000000..7fd97fa3fa --- /dev/null +++ b/python/mozperftest/mozperftest/metrics/notebook/utilities.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from collections.abc import Iterable + + +def flat(data, parent_dir): + """ + Converts a dictionary with nested entries like this + { + "dict1": { + "dict2": { + "key1": value1, + "key2": value2, + ... + }, + ... + }, + ... + "dict3": { + "key3": value3, + "key4": value4, + ... + } + ... + } + + to a "flattened" dictionary like this that has no nested entries: + { + "dict1.dict2.key1": value1, + "dict1.dict2.key2": value2, + ... + "dict3.key3": value3, + "dict3.key4": value4, + ... + } + + :param Iterable data : json data. + :param tuple parent_dir: json fields. + + :return dict: {subtest: value} + """ + result = {} + + if not data: + return result + + if isinstance(data, list): + for item in data: + for k, v in flat(item, parent_dir).items(): + result.setdefault(k, []).extend(v) + + if isinstance(data, dict): + for k, v in data.items(): + current_dir = parent_dir + (k,) + subtest = ".".join(current_dir) + if isinstance(v, Iterable) and not isinstance(v, str): + for x, y in flat(v, current_dir).items(): + result.setdefault(x, []).extend(y) + elif v or v == 0: + result.setdefault(subtest, []).append(v) + + return result |