ResultsDashboard = Utilities.createClass( function(options, testData) { this._iterationsSamplers = []; this._options = options; this._results = null; if (testData) { this._iterationsSamplers = testData; this._processData(); } }, { push: function(suitesSamplers) { this._iterationsSamplers.push(suitesSamplers); }, _processData: function() { this._results = {}; this._results[Strings.json.results.iterations] = []; var iterationsScores = []; this._iterationsSamplers.forEach(function(iteration, index) { var testsScores = []; var testsLowerBoundScores = []; var testsUpperBoundScores = []; var result = {}; this._results[Strings.json.results.iterations][index] = result; var suitesResult = {}; result[Strings.json.results.tests] = suitesResult; for (var suiteName in iteration) { var suiteData = iteration[suiteName]; var suiteResult = {}; suitesResult[suiteName] = suiteResult; for (var testName in suiteData) { if (!suiteData[testName][Strings.json.result]) this.calculateScore(suiteData[testName]); suiteResult[testName] = suiteData[testName][Strings.json.result]; delete suiteData[testName][Strings.json.result]; testsScores.push(suiteResult[testName][Strings.json.score]); testsLowerBoundScores.push(suiteResult[testName][Strings.json.scoreLowerBound]); testsUpperBoundScores.push(suiteResult[testName][Strings.json.scoreUpperBound]); } } result[Strings.json.score] = Statistics.geometricMean(testsScores); result[Strings.json.scoreLowerBound] = Statistics.geometricMean(testsLowerBoundScores); result[Strings.json.scoreUpperBound] = Statistics.geometricMean(testsUpperBoundScores); iterationsScores.push(result[Strings.json.score]); }, this); this._results[Strings.json.score] = Statistics.sampleMean(iterationsScores.length, iterationsScores.reduce(function(a, b) { return a + b; })); this._results[Strings.json.scoreLowerBound] = this._results[Strings.json.results.iterations][0][Strings.json.scoreLowerBound]; this._results[Strings.json.scoreUpperBound] = this._results[Strings.json.results.iterations][0][Strings.json.scoreUpperBound]; }, calculateScore: function(data) { var result = {}; data[Strings.json.result] = result; var samples = data[Strings.json.samples]; var desiredFrameLength = 1000/60; if (this._options["controller"] == "ramp30") desiredFrameLength = 1000/30; function findRegression(series, profile) { var minIndex = Math.round(.025 * series.length); var maxIndex = Math.round(.975 * (series.length - 1)); var minComplexity = series.getFieldInDatum(minIndex, Strings.json.complexity); var maxComplexity = series.getFieldInDatum(maxIndex, Strings.json.complexity); if (Math.abs(maxComplexity - minComplexity) < 20 && maxIndex - minIndex < 20) { minIndex = 0; maxIndex = series.length - 1; minComplexity = series.getFieldInDatum(minIndex, Strings.json.complexity); maxComplexity = series.getFieldInDatum(maxIndex, Strings.json.complexity); } var complexityIndex = series.fieldMap[Strings.json.complexity]; var frameLengthIndex = series.fieldMap[Strings.json.frameLength]; var regressionOptions = { desiredFrameLength: desiredFrameLength }; if (profile) regressionOptions.preferredProfile = profile; return { minComplexity: minComplexity, maxComplexity: maxComplexity, samples: series.slice(minIndex, maxIndex + 1), regression: new Regression( series.data, function (data, i) { return data[i][complexityIndex]; }, function (data, i) { return data[i][frameLengthIndex]; }, minIndex, maxIndex, regressionOptions) }; } var complexitySamples; // Convert these samples into SampleData objects if needed [Strings.json.complexity, Strings.json.complexityAverage, Strings.json.controller].forEach(function(seriesName) { var series = samples[seriesName]; if (series && !(series instanceof SampleData)) samples[seriesName] = new SampleData(series.fieldMap, series.data); }); var isRampController = ["ramp", "ramp30"].indexOf(this._options["controller"]) != -1; var predominantProfile = ""; if (isRampController) { var profiles = {}; data[Strings.json.controller].forEach(function(regression) { if (regression[Strings.json.regressions.profile]) { var profile = regression[Strings.json.regressions.profile]; profiles[profile] = (profiles[profile] || 0) + 1; } }); var maxProfileCount = 0; for (var profile in profiles) { if (profiles[profile] > maxProfileCount) { predominantProfile = profile; maxProfileCount = profiles[profile]; } } } [Strings.json.complexity, Strings.json.complexityAverage].forEach(function(seriesName) { if (!(seriesName in samples)) return; var regression = {}; result[seriesName] = regression; var regressionResult = findRegression(samples[seriesName], predominantProfile); if (seriesName == Strings.json.complexity) complexitySamples = regressionResult.samples; var calculation = regressionResult.regression; regression[Strings.json.regressions.segment1] = [ [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity], [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity] ]; regression[Strings.json.regressions.segment2] = [ [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity], [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity] ]; regression[Strings.json.complexity] = calculation.complexity; regression[Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[seriesName].length); }); if (isRampController) { var timeComplexity = new Experiment; data[Strings.json.controller].forEach(function(regression) { timeComplexity.sample(regression[Strings.json.complexity]); }); var experimentResult = {}; result[Strings.json.controller] = experimentResult; experimentResult[Strings.json.score] = timeComplexity.mean(); experimentResult[Strings.json.measurements.average] = timeComplexity.mean(); experimentResult[Strings.json.measurements.stdev] = timeComplexity.standardDeviation(); experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage(); const bootstrapIterations = 2500; var bootstrapResult = Regression.bootstrap(complexitySamples.data, bootstrapIterations, function(resampleData) { var complexityIndex = complexitySamples.fieldMap[Strings.json.complexity]; resampleData.sort(function(a, b) { return a[complexityIndex] - b[complexityIndex]; }); var resample = new SampleData(complexitySamples.fieldMap, resampleData); var regressionResult = findRegression(resample, predominantProfile); return regressionResult.regression.complexity; }, .8); result[Strings.json.complexity][Strings.json.bootstrap] = bootstrapResult; result[Strings.json.score] = bootstrapResult.median; result[Strings.json.scoreLowerBound] = bootstrapResult.confidenceLow; result[Strings.json.scoreUpperBound] = bootstrapResult.confidenceHigh; } else { var marks = data[Strings.json.marks]; var samplingStartIndex = 0, samplingEndIndex = -1; if (Strings.json.samplingStartTimeOffset in marks) samplingStartIndex = marks[Strings.json.samplingStartTimeOffset].index; if (Strings.json.samplingEndTimeOffset in marks) samplingEndIndex = marks[Strings.json.samplingEndTimeOffset].index; var averageComplexity = new Experiment; var averageFrameLength = new Experiment; var controllerSamples = samples[Strings.json.controller]; controllerSamples.forEach(function (sample, i) { if (i >= samplingStartIndex && (samplingEndIndex == -1 || i < samplingEndIndex)) { averageComplexity.sample(controllerSamples.getFieldInDatum(sample, Strings.json.complexity)); var smoothedFrameLength = controllerSamples.getFieldInDatum(sample, Strings.json.smoothedFrameLength); if (smoothedFrameLength && smoothedFrameLength != -1) averageFrameLength.sample(smoothedFrameLength); } }); var experimentResult = {}; result[Strings.json.controller] = experimentResult; experimentResult[Strings.json.measurements.average] = averageComplexity.mean(); experimentResult[Strings.json.measurements.concern] = averageComplexity.concern(Experiment.defaults.CONCERN); experimentResult[Strings.json.measurements.stdev] = averageComplexity.standardDeviation(); experimentResult[Strings.json.measurements.percent] = averageComplexity.percentage(); experimentResult = {}; result[Strings.json.frameLength] = experimentResult; experimentResult[Strings.json.measurements.average] = 1000 / averageFrameLength.mean(); experimentResult[Strings.json.measurements.concern] = averageFrameLength.concern(Experiment.defaults.CONCERN); experimentResult[Strings.json.measurements.stdev] = averageFrameLength.standardDeviation(); experimentResult[Strings.json.measurements.percent] = averageFrameLength.percentage(); result[Strings.json.score] = averageComplexity.score(Experiment.defaults.CONCERN); result[Strings.json.scoreLowerBound] = result[Strings.json.score] - averageFrameLength.standardDeviation(); result[Strings.json.scoreUpperBound] = result[Strings.json.score] + averageFrameLength.standardDeviation(); } }, get data() { return this._iterationsSamplers; }, get results() { if (this._results) return this._results[Strings.json.results.iterations]; this._processData(); return this._results[Strings.json.results.iterations]; }, get options() { return this._options; }, _getResultsProperty: function(property) { if (this._results) return this._results[property]; this._processData(); return this._results[property]; }, get score() { return this._getResultsProperty(Strings.json.score); }, get scoreLowerBound() { return this._getResultsProperty(Strings.json.scoreLowerBound); }, get scoreUpperBound() { return this._getResultsProperty(Strings.json.scoreUpperBound); } }); ResultsTable = Utilities.createClass( function(element, headers) { this.element = element; this._headers = headers; this._flattenedHeaders = []; this._headers.forEach(function(header) { if (header.disabled) return; if (header.children) this._flattenedHeaders = this._flattenedHeaders.concat(header.children); else this._flattenedHeaders.push(header); }, this); this._flattenedHeaders = this._flattenedHeaders.filter(function (header) { return !header.disabled; }); this.clear(); }, { clear: function() { this.element.textContent = ""; }, _addHeader: function() { var thead = Utilities.createElement("thead", {}, this.element); var row = Utilities.createElement("tr", {}, thead); this._headers.forEach(function (header) { if (header.disabled) return; var th = Utilities.createElement("th", {}, row); if (header.title != Strings.text.graph) th.innerHTML = header.title; if (header.children) th.colSpan = header.children.length; }); }, _addBody: function() { this.tbody = Utilities.createElement("tbody", {}, this.element); }, _addEmptyRow: function() { var row = Utilities.createElement("tr", {}, this.tbody); this._flattenedHeaders.forEach(function (header) { return Utilities.createElement("td", { class: "suites-separator" }, row); }); }, _addTest: function(testName, testResult, options) { var row = Utilities.createElement("tr", {}, this.tbody); this._flattenedHeaders.forEach(function (header) { var td = Utilities.createElement("td", {}, row); if (header.text == Strings.text.testName) { td.textContent = testName; } else if (typeof header.text == "string") { var data = testResult[header.text]; if (typeof data == "number") data = data.toFixed(2); td.innerHTML = data; } else td.innerHTML = header.text(testResult); }, this); }, _addIteration: function(iterationResult, iterationData, options) { var testsResults = iterationResult[Strings.json.results.tests]; for (var suiteName in testsResults) { this._addEmptyRow(); var suiteResult = testsResults[suiteName]; var suiteData = iterationData[suiteName]; for (var testName in suiteResult) this._addTest(testName, suiteResult[testName], options, suiteData[testName]); } }, showIterations: function(dashboard) { this.clear(); this._addHeader(); this._addBody(); var iterationsResults = dashboard.results; iterationsResults.forEach(function(iterationResult, index) { this._addIteration(iterationResult, dashboard.data[index], dashboard.options); }, this); } }); window.benchmarkRunnerClient = { iterationCount: 1, options: null, results: null, initialize: function(suites, options) { this.options = options; }, willStartFirstIteration: function() { this.results = new ResultsDashboard(this.options); }, didRunSuites: function(suitesSamplers) { this.results.push(suitesSamplers); }, didRunTest: function(testData) { this.results.calculateScore(testData); }, didFinishLastIteration: function() { benchmarkController.showResults(); } }; window.sectionsManager = { showSection: function(sectionIdentifier, pushState) { var sections = document.querySelectorAll("main > section"); for (var i = 0; i < sections.length; ++i) { document.body.classList.remove("showing-" + sections[i].id); } document.body.classList.add("showing-" + sectionIdentifier); var currentSectionElement = document.querySelector("section.selected"); console.assert(currentSectionElement); var newSectionElement = document.getElementById(sectionIdentifier); console.assert(newSectionElement); currentSectionElement.classList.remove("selected"); newSectionElement.classList.add("selected"); if (pushState) history.pushState({section: sectionIdentifier}, document.title); }, setSectionScore: function(sectionIdentifier, score, confidence) { document.querySelector("#" + sectionIdentifier + " .score").textContent = score; if (confidence) document.querySelector("#" + sectionIdentifier + " .confidence").textContent = confidence; }, populateTable: function(tableIdentifier, headers, dashboard) { var table = new ResultsTable(document.getElementById(tableIdentifier), headers); table.showIterations(dashboard); } }; window.benchmarkController = { initialize: function() { benchmarkController.addOrientationListenerIfNecessary(); }, determineCanvasSize: function() { var match = window.matchMedia("(max-device-width: 760px)"); if (match.matches) { document.body.classList.add("small"); return; } match = window.matchMedia("(max-device-width: 1600px)"); if (match.matches) { document.body.classList.add("medium"); return; } match = window.matchMedia("(max-width: 1600px)"); if (match.matches) { document.body.classList.add("medium"); return; } document.body.classList.add("large"); }, addOrientationListenerIfNecessary: function() { if (!("orientation" in window)) return; this.orientationQuery = window.matchMedia("(orientation: landscape)"); this._orientationChanged(this.orientationQuery); this.orientationQuery.addListener(this._orientationChanged); }, _orientationChanged: function(match) { benchmarkController.isInLandscapeOrientation = match.matches; if (match.matches) document.querySelector(".start-benchmark p").classList.add("hidden"); else document.querySelector(".start-benchmark p").classList.remove("hidden"); benchmarkController.updateStartButtonState(); }, updateStartButtonState: function() { document.getElementById("run-benchmark").disabled = !this.isInLandscapeOrientation; }, _startBenchmark: function(suites, options, frameContainerID) { benchmarkController.determineCanvasSize(); var configuration = document.body.className.match(/small|medium|large/); if (configuration) options[Strings.json.configuration] = configuration[0]; benchmarkRunnerClient.initialize(suites, options); var frameContainer = document.getElementById(frameContainerID); var runner = new BenchmarkRunner(suites, frameContainer, benchmarkRunnerClient); runner.runMultipleIterations(); sectionsManager.showSection("test-container"); }, startBenchmark: function() { var options = { "test-interval": 30, "display": "minimal", "tiles": "big", "controller": "ramp", "kalman-process-error": 1, "kalman-measurement-error": 4, "time-measurement": "performance" }; this._startBenchmark(Suites, options, "test-container"); }, showResults: function() { if (!this.addedKeyEvent) { document.addEventListener("keypress", this.handleKeyPress, false); this.addedKeyEvent = true; } var dashboard = benchmarkRunnerClient.results; var score = dashboard.score; var confidence = "±" + (Statistics.largestDeviationPercentage(dashboard.scoreLowerBound, score, dashboard.scoreUpperBound) * 100).toFixed(2) + "%"; sectionsManager.setSectionScore("results", score.toFixed(2), confidence); sectionsManager.populateTable("results-header", Headers.testName, dashboard); sectionsManager.populateTable("results-score", Headers.score, dashboard); sectionsManager.populateTable("results-data", Headers.details, dashboard); sectionsManager.showSection("results", true); }, handleKeyPress: function(event) { switch (event.charCode) { case 27: // esc benchmarkController.hideDebugInfo(); break; case 106: // j benchmarkController.showDebugInfo(); break; case 115: // s benchmarkController.selectResults(event.target); break; } }, hideDebugInfo: function() { var overlay = document.getElementById("overlay"); if (!overlay) return; document.body.removeChild(overlay); }, showDebugInfo: function() { if (document.getElementById("overlay")) return; var overlay = Utilities.createElement("div", { id: "overlay" }, document.body); var container = Utilities.createElement("div", {}, overlay); var header = Utilities.createElement("h3", {}, container); header.textContent = "Debug Output"; var data = Utilities.createElement("div", {}, container); data.textContent = "Please wait..."; setTimeout(function() { var output = { options: benchmarkRunnerClient.results.options, data: benchmarkRunnerClient.results.data }; data.textContent = JSON.stringify(output, function(key, value) { if (typeof value === 'number') return Utilities.toFixedNumber(value, 3); return value; }, 1); }, 0); data.onclick = function() { var selection = window.getSelection(); selection.removeAllRanges(); var range = document.createRange(); range.selectNode(data); selection.addRange(range); }; var button = Utilities.createElement("button", {}, container); button.textContent = "Done"; button.onclick = function() { benchmarkController.hideDebugInfo(); }; }, selectResults: function(target) { target.selectRange = ((target.selectRange || 0) + 1) % 3; var selection = window.getSelection(); selection.removeAllRanges(); var range = document.createRange(); switch (target.selectRange) { case 0: { range.selectNode(document.getElementById("results-score")); break; } case 1: { range.setStart(document.querySelector("#results .score"), 0); range.setEndAfter(document.querySelector("#results-score"), 0); break; } case 2: { range.selectNodeContents(document.querySelector("#results .score")); break; } } selection.addRange(range); } }; window.addEventListener("load", function() { benchmarkController.initialize(); });