diff options
Diffstat (limited to 'third_party/libwebrtc/video/full_stack_tests_plot.py')
| -rwxr-xr-x | third_party/libwebrtc/video/full_stack_tests_plot.py | 469 |
1 files changed, 469 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/full_stack_tests_plot.py b/third_party/libwebrtc/video/full_stack_tests_plot.py new file mode 100755 index 0000000000..c195b72a54 --- /dev/null +++ b/third_party/libwebrtc/video/full_stack_tests_plot.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Generate graphs for data generated by loopback tests. + +Usage examples: + Show end to end time for a single full stack test. + ./full_stack_tests_plot.py -df end_to_end -o 600 --frames 1000 vp9_data.txt + + Show simultaneously PSNR and encoded frame size for two different runs of + full stack test. Averaged over a cycle of 200 frames. Used e.g. for + screenshare slide test. + ./full_stack_tests_plot.py -c 200 -df psnr -drf encoded_frame_size \\ + before.txt after.txt + + Similar to the previous test, but multiple graphs. + ./full_stack_tests_plot.py -c 200 -df psnr vp8.txt vp9.txt --next \\ + -c 200 -df sender_time vp8.txt vp9.txt --next \\ + -c 200 -df end_to_end vp8.txt vp9.txt +""" + +import argparse +from collections import defaultdict +import itertools +import sys +import matplotlib.pyplot as plt +import numpy + +# Fields +DROPPED = 0 +INPUT_TIME = 1 # ms (timestamp) +SEND_TIME = 2 # ms (timestamp) +RECV_TIME = 3 # ms (timestamp) +RENDER_TIME = 4 # ms (timestamp) +ENCODED_FRAME_SIZE = 5 # bytes +PSNR = 6 +SSIM = 7 +ENCODE_TIME = 8 # ms (time interval) + +TOTAL_RAW_FIELDS = 9 + +SENDER_TIME = TOTAL_RAW_FIELDS + 0 +RECEIVER_TIME = TOTAL_RAW_FIELDS + 1 +END_TO_END = TOTAL_RAW_FIELDS + 2 +RENDERED_DELTA = TOTAL_RAW_FIELDS + 3 + +FIELD_MASK = 255 + +# Options +HIDE_DROPPED = 256 +RIGHT_Y_AXIS = 512 + +# internal field id, field name, title +_FIELDS = [ + # Raw + (DROPPED, "dropped", "dropped"), + (INPUT_TIME, "input_time_ms", "input time"), + (SEND_TIME, "send_time_ms", "send time"), + (RECV_TIME, "recv_time_ms", "recv time"), + (ENCODED_FRAME_SIZE, "encoded_frame_size", "encoded frame size"), + (PSNR, "psnr", "PSNR"), + (SSIM, "ssim", "SSIM"), + (RENDER_TIME, "render_time_ms", "render time"), + (ENCODE_TIME, "encode_time_ms", "encode time"), + # Auto-generated + (SENDER_TIME, "sender_time", "sender time"), + (RECEIVER_TIME, "receiver_time", "receiver time"), + (END_TO_END, "end_to_end", "end to end"), + (RENDERED_DELTA, "rendered_delta", "rendered delta"), +] + +NAME_TO_ID = {field[1]: field[0] for field in _FIELDS} +ID_TO_TITLE = {field[0]: field[2] for field in _FIELDS} + + +def FieldArgToId(arg): + if arg == "none": + return None + if arg in NAME_TO_ID: + return NAME_TO_ID[arg] + if arg + "_ms" in NAME_TO_ID: + return NAME_TO_ID[arg + "_ms"] + raise Exception("Unrecognized field name \"{}\"".format(arg)) + + +class PlotLine(object): + """Data for a single graph line.""" + + def __init__(self, label, values, flags): + self.label = label + self.values = values + self.flags = flags + + +class Data(object): + """Object representing one full stack test.""" + + def __init__(self, filename): + self.title = "" + self.length = 0 + self.samples = defaultdict(list) + + self._ReadSamples(filename) + + def _ReadSamples(self, filename): + """Reads graph data from the given file.""" + f = open(filename) + it = iter(f) + + self.title = it.next().strip() + self.length = int(it.next()) + field_names = [name.strip() for name in it.next().split()] + field_ids = [NAME_TO_ID[name] for name in field_names] + + for field_id in field_ids: + self.samples[field_id] = [0.0] * self.length + + for sample_id in xrange(self.length): + for col, value in enumerate(it.next().split()): + self.samples[field_ids[col]][sample_id] = float(value) + + self._SubtractFirstInputTime() + self._GenerateAdditionalData() + + f.close() + + def _SubtractFirstInputTime(self): + offset = self.samples[INPUT_TIME][0] + for field in [INPUT_TIME, SEND_TIME, RECV_TIME, RENDER_TIME]: + if field in self.samples: + self.samples[field] = [x - offset for x in self.samples[field]] + + def _GenerateAdditionalData(self): + """Calculates sender time, receiver time etc. from the raw data.""" + s = self.samples + last_render_time = 0 + for field_id in [ + SENDER_TIME, RECEIVER_TIME, END_TO_END, RENDERED_DELTA + ]: + s[field_id] = [0] * self.length + + for k in range(self.length): + s[SENDER_TIME][k] = s[SEND_TIME][k] - s[INPUT_TIME][k] + + decoded_time = s[RENDER_TIME][k] + s[RECEIVER_TIME][k] = decoded_time - s[RECV_TIME][k] + s[END_TO_END][k] = decoded_time - s[INPUT_TIME][k] + if not s[DROPPED][k]: + if k > 0: + s[RENDERED_DELTA][k] = decoded_time - last_render_time + last_render_time = decoded_time + + def _Hide(self, values): + """ + Replaces values for dropped frames with None. + These values are then skipped by the Plot() method. + """ + + return [ + None if self.samples[DROPPED][k] else values[k] + for k in range(len(values)) + ] + + def AddSamples(self, config, target_lines_list): + """Creates graph lines from the current data set with given config.""" + for field in config.fields: + # field is None means the user wants just to skip the color. + if field is None: + target_lines_list.append(None) + continue + + field_id = field & FIELD_MASK + values = self.samples[field_id] + + if field & HIDE_DROPPED: + values = self._Hide(values) + + target_lines_list.append( + PlotLine(self.title + " " + ID_TO_TITLE[field_id], values, + field & ~FIELD_MASK)) + + +def AverageOverCycle(values, length): + """ + Returns the list: + [ + avg(values[0], values[length], ...), + avg(values[1], values[length + 1], ...), + ... + avg(values[length - 1], values[2 * length - 1], ...), + ] + + Skips None values when calculating the average value. + """ + + total = [0.0] * length + count = [0] * length + for k, val in enumerate(values): + if val is not None: + total[k % length] += val + count[k % length] += 1 + + result = [0.0] * length + for k in range(length): + result[k] = total[k] / count[k] if count[k] else None + return result + + +class PlotConfig(object): + """Object representing a single graph.""" + + def __init__(self, + fields, + data_list, + cycle_length=None, + frames=None, + offset=0, + output_filename=None, + title="Graph"): + self.fields = fields + self.data_list = data_list + self.cycle_length = cycle_length + self.frames = frames + self.offset = offset + self.output_filename = output_filename + self.title = title + + def Plot(self, ax1): + lines = [] + for data in self.data_list: + if not data: + # Add None lines to skip the colors. + lines.extend([None] * len(self.fields)) + else: + data.AddSamples(self, lines) + + def _SliceValues(values): + if self.offset: + values = values[self.offset:] + if self.frames: + values = values[:self.frames] + return values + + length = None + for line in lines: + if line is None: + continue + + line.values = _SliceValues(line.values) + if self.cycle_length: + line.values = AverageOverCycle(line.values, self.cycle_length) + + if length is None: + length = len(line.values) + elif length != len(line.values): + raise Exception("All arrays should have the same length!") + + ax1.set_xlabel("Frame", fontsize="large") + if any(line.flags & RIGHT_Y_AXIS for line in lines if line): + ax2 = ax1.twinx() + ax2.set_xlabel("Frame", fontsize="large") + else: + ax2 = None + + # Have to implement color_cycle manually, due to two scales in a graph. + color_cycle = ["b", "r", "g", "c", "m", "y", "k"] + color_iter = itertools.cycle(color_cycle) + + for line in lines: + if not line: + color_iter.next() + continue + + if self.cycle_length: + x = numpy.array(range(self.cycle_length)) + else: + x = numpy.array( + range(self.offset, self.offset + len(line.values))) + y = numpy.array(line.values) + ax = ax2 if line.flags & RIGHT_Y_AXIS else ax1 + ax.Plot(x, + y, + "o-", + label=line.label, + markersize=3.0, + linewidth=1.0, + color=color_iter.next()) + + ax1.grid(True) + if ax2: + ax1.legend(loc="upper left", shadow=True, fontsize="large") + ax2.legend(loc="upper right", shadow=True, fontsize="large") + else: + ax1.legend(loc="best", shadow=True, fontsize="large") + + +def LoadFiles(filenames): + result = [] + for filename in filenames: + if filename in LoadFiles.cache: + result.append(LoadFiles.cache[filename]) + else: + data = Data(filename) + LoadFiles.cache[filename] = data + result.append(data) + return result + + +LoadFiles.cache = {} + + +def GetParser(): + class CustomAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + if "ordered_args" not in namespace: + namespace.ordered_args = [] + namespace.ordered_args.append((self.dest, values)) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument("-c", + "--cycle_length", + nargs=1, + action=CustomAction, + type=int, + help="Cycle length over which to average the values.") + parser.add_argument( + "-f", + "--field", + nargs=1, + action=CustomAction, + help="Name of the field to show. Use 'none' to skip a color.") + parser.add_argument("-r", + "--right", + nargs=0, + action=CustomAction, + help="Use right Y axis for given field.") + parser.add_argument("-d", + "--drop", + nargs=0, + action=CustomAction, + help="Hide values for dropped frames.") + parser.add_argument("-o", + "--offset", + nargs=1, + action=CustomAction, + type=int, + help="Frame offset.") + parser.add_argument("-n", + "--next", + nargs=0, + action=CustomAction, + help="Separator for multiple graphs.") + parser.add_argument( + "--frames", + nargs=1, + action=CustomAction, + type=int, + help="Frame count to show or take into account while averaging.") + parser.add_argument("-t", + "--title", + nargs=1, + action=CustomAction, + help="Title of the graph.") + parser.add_argument("-O", + "--output_filename", + nargs=1, + action=CustomAction, + help="Use to save the graph into a file. " + "Otherwise, a window will be shown.") + parser.add_argument( + "files", + nargs="+", + action=CustomAction, + help="List of text-based files generated by loopback tests.") + return parser + + +def _PlotConfigFromArgs(args, graph_num): + # Pylint complains about using kwargs, so have to do it this way. + cycle_length = None + frames = None + offset = 0 + output_filename = None + title = "Graph" + + fields = [] + files = [] + mask = 0 + for key, values in args: + if key == "cycle_length": + cycle_length = values[0] + elif key == "frames": + frames = values[0] + elif key == "offset": + offset = values[0] + elif key == "output_filename": + output_filename = values[0] + elif key == "title": + title = values[0] + elif key == "drop": + mask |= HIDE_DROPPED + elif key == "right": + mask |= RIGHT_Y_AXIS + elif key == "field": + field_id = FieldArgToId(values[0]) + fields.append(field_id | mask if field_id is not None else None) + mask = 0 # Reset mask after the field argument. + elif key == "files": + files.extend(values) + + if not files: + raise Exception( + "Missing file argument(s) for graph #{}".format(graph_num)) + if not fields: + raise Exception( + "Missing field argument(s) for graph #{}".format(graph_num)) + + return PlotConfig(fields, + LoadFiles(files), + cycle_length=cycle_length, + frames=frames, + offset=offset, + output_filename=output_filename, + title=title) + + +def PlotConfigsFromArgs(args): + """Generates plot configs for given command line arguments.""" + # The way it works: + # First we detect separators -n/--next and split arguments into groups, one + # for each plot. For each group, we partially parse it with + # argparse.ArgumentParser, modified to remember the order of arguments. + # Then we traverse the argument list and fill the PlotConfig. + args = itertools.groupby(args, lambda x: x in ["-n", "--next"]) + prep_args = list(list(group) for match, group in args if not match) + + parser = GetParser() + plot_configs = [] + for index, raw_args in enumerate(prep_args): + graph_args = parser.parse_args(raw_args).ordered_args + plot_configs.append(_PlotConfigFromArgs(graph_args, index)) + return plot_configs + + +def ShowOrSavePlots(plot_configs): + for config in plot_configs: + fig = plt.figure(figsize=(14.0, 10.0)) + ax = fig.add_subPlot(1, 1, 1) + + plt.title(config.title) + config.Plot(ax) + if config.output_filename: + print "Saving to", config.output_filename + fig.savefig(config.output_filename) + plt.close(fig) + + plt.show() + + +if __name__ == "__main__": + ShowOrSavePlots(PlotConfigsFromArgs(sys.argv[1:])) |