diff options
Diffstat (limited to 'devtools/client/framework/test/metrics')
11 files changed, 629 insertions, 0 deletions
diff --git a/devtools/client/framework/test/metrics/browser_metrics.toml b/devtools/client/framework/test/metrics/browser_metrics.toml new file mode 100644 index 0000000000..3e95c00dfe --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics.toml @@ -0,0 +1,19 @@ +[DEFAULT] +tags = "devtools" +subsuite = "devtools" +support-files = [ + "head.js", + "!/devtools/client/shared/test/shared-head.js", + "!/devtools/client/shared/test/telemetry-test-helpers.js", +] + +# Tests counting the numbers of loaded modules have distinct manifest file to execute the test +# individually, without any other test being executed before or after, as it could impact +# the number of loaded modules. +# This manifest file is for all the _other_ tests, where such setup isn't relevant. + +["browser_metrics_pool.js"] +skip-if = [ + "debug", + "asan", +] diff --git a/devtools/client/framework/test/metrics/browser_metrics_debugger.js b/devtools/client/framework/test/metrics/browser_metrics_debugger.js new file mode 100644 index 0000000000..4a684e40fc --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_debugger.js @@ -0,0 +1,61 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This test records the number of modules loaded by DevTools, as well as the total count + * of characters in those modules, when opening the debugger. These metrics are + * retrieved by perfherder via logs. + */ + +const TEST_URL = + "data:text/html;charset=UTF-8,<div>Debugger modules load test</div>"; + +add_task(async function () { + // Disable randomly spawning processes during tests + await pushPref("dom.ipc.processPrelaunch.enabled", false); + + const toolbox = await openNewTabAndToolbox(TEST_URL, "jsdebugger"); + const toolboxBrowserLoader = toolbox.win.getBrowserLoaderForWindow(); + + // Retrieve the browser loader dedicated to the Debugger. + const panel = toolbox.getCurrentPanel(); + const debuggerLoader = panel.panelWin.getBrowserLoaderForWindow(); + + const loaders = [ + loader.loader, + toolboxBrowserLoader.loader, + debuggerLoader.loader, + ]; + + const allowedDupes = [ + "@loader/unload.js", + "@loader/options.js", + "resource://devtools/client/shared/vendor/fluent-react.js", + "resource://devtools/client/shared/vendor/react-dom.js", + "resource://devtools/client/shared/vendor/react.js", + "resource://devtools/client/shared/vendor/react-prop-types.js", + "resource://devtools/client/shared/vendor/react-dom-factories.js", + "resource://devtools/client/shared/vendor/react-redux.js", + "resource://devtools/client/shared/vendor/redux.js", + "resource://devtools/client/shared/redux/subscriber.js", + + "resource://devtools/client/shared/components/menu/MenuButton.js", + "resource://devtools/client/shared/components/menu/MenuItem.js", + "resource://devtools/client/shared/components/menu/MenuList.js", + ]; + runDuplicatedModulesTest(loaders, allowedDupes); + + runMetricsTest({ + filterString: "devtools/client/debugger", + loaders, + panelName: "debugger", + }); + + // See Bug 1637793 and Bug 1621337. + // Ideally the debugger should only resolve when the worker targets have been + // retrieved, which should be fixed by Bug 1621337 or a followup. + info("Wait for all pending requests to settle on the DevToolsClient"); + await toolbox.commands.client.waitForRequestsToSettle(); +}); diff --git a/devtools/client/framework/test/metrics/browser_metrics_debugger.toml b/devtools/client/framework/test/metrics/browser_metrics_debugger.toml new file mode 100644 index 0000000000..24d651095d --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_debugger.toml @@ -0,0 +1,18 @@ +[DEFAULT] +tags = "devtools" +subsuite = "devtools" +support-files = [ + "head.js", + "!/devtools/client/shared/test/shared-head.js", + "!/devtools/client/shared/test/telemetry-test-helpers.js", +] + +# Each metrics tests is loaded in a separate .ini file. This way the test is executed +# individually, without any other test being executed before or after. + +["browser_metrics_debugger.js"] +run-if = ["os == 'linux'"] # Results should be platform agnostic - only run on linux64-opt +skip-if = [ + "debug", + "asan", +] diff --git a/devtools/client/framework/test/metrics/browser_metrics_inspector.js b/devtools/client/framework/test/metrics/browser_metrics_inspector.js new file mode 100644 index 0000000000..284ef82372 --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_inspector.js @@ -0,0 +1,43 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This test records the number of modules loaded by DevTools, as well as the total count + * of characters in those modules, when opening the inspector. These metrics are retrieved + * by perfherder via logs. + */ + +const TEST_URL = + "data:text/html;charset=UTF-8,<div>Inspector modules load test</div>"; + +add_task(async function () { + const toolbox = await openNewTabAndToolbox(TEST_URL, "inspector"); + const toolboxBrowserLoader = toolbox.win.getBrowserLoaderForWindow(); + + // Most panels involve three loaders: + // - the global devtools loader + // - the browser loader used by the toolbox + // - a specific browser loader created for the panel + // But the inspector is a specific case, because it reuses the BrowserLoader + // of the toolbox to load its react components. This is why we only list + // two loaders here. + const loaders = [loader.loader, toolboxBrowserLoader.loader]; + + runDuplicatedModulesTest(loaders, [ + "@loader/unload.js", + "@loader/options.js", + "resource://devtools/client/shared/vendor/react.js", + "resource://devtools/client/shared/vendor/react-dom-factories.js", + "resource://devtools/client/shared/vendor/react-prop-types.js", + "resource://devtools/client/shared/vendor/redux.js", + "resource://devtools/client/shared/vendor/fluent-react.js", + ]); + + runMetricsTest({ + filterString: "devtools/client/inspector", + loaders, + panelName: "inspector", + }); +}); diff --git a/devtools/client/framework/test/metrics/browser_metrics_inspector.toml b/devtools/client/framework/test/metrics/browser_metrics_inspector.toml new file mode 100644 index 0000000000..62e2ec13ce --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_inspector.toml @@ -0,0 +1,18 @@ +[DEFAULT] +tags = "devtools" +subsuite = "devtools" +support-files = [ + "head.js", + "!/devtools/client/shared/test/shared-head.js", + "!/devtools/client/shared/test/telemetry-test-helpers.js", +] + +# Each metrics tests is loaded in a separate .ini file. This way the test is executed +# individually, without any other test being executed before or after. + +["browser_metrics_inspector.js"] +run-if = ["os == 'linux'"] # Results should be platform agnostic - only run on linux64-opt +skip-if = [ + "debug", + "asan", +] diff --git a/devtools/client/framework/test/metrics/browser_metrics_netmonitor.js b/devtools/client/framework/test/metrics/browser_metrics_netmonitor.js new file mode 100644 index 0000000000..07ae01eb95 --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_netmonitor.js @@ -0,0 +1,89 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This test records the number of modules loaded by DevTools, as well as the total count + * of characters in those modules, when opening the netmonitor. These metrics are + * retrieved by perfherder via logs. + */ + +const TEST_URL = + "data:text/html;charset=UTF-8,<div>Netmonitor modules load test</div>"; + +add_task(async function () { + const toolbox = await openNewTabAndToolbox(TEST_URL, "netmonitor"); + const toolboxBrowserLoader = toolbox.win.getBrowserLoaderForWindow(); + + // Retrieve the browser loader dedicated to the Netmonitor. + const panel = toolbox.getCurrentPanel(); + const netmonitorLoader = panel.panelWin.getBrowserLoaderForWindow(); + + const loaders = [ + loader.loader, + toolboxBrowserLoader.loader, + netmonitorLoader.loader, + ]; + + // Uncomment after Bug 1581068 is fixed, otherwise the test might fail too + // frequently. + + // const allowedDupes = [ + // "@loader/unload.js", + // "@loader/options.js", + // "resource://devtools/client/netmonitor/src/api.js", + // "resource://devtools/client/shared/vendor/redux.js", + // "resource://devtools/client/netmonitor/src/connector/index.js", + // "resource://devtools/client/netmonitor/src/create-store.js", + // "resource://devtools/client/netmonitor/src/constants.js", + // "resource://devtools/client/netmonitor/src/middleware/batching.js", + // "resource://devtools/client/netmonitor/src/middleware/prefs.js", + // "resource://devtools/client/netmonitor/src/middleware/recording.js", + // "resource://devtools/client/netmonitor/src/selectors/index.js", + // "resource://devtools/client/netmonitor/src/selectors/requests.js", + // "resource://devtools/client/shared/vendor/reselect.js", + // "resource://devtools/client/netmonitor/src/utils/filter-predicates.js", + // "resource://devtools/client/netmonitor/src/utils/filter-text-utils.js", + // "resource://devtools/client/netmonitor/src/utils/format-utils.js", + // "resource://devtools/client/netmonitor/src/utils/l10n.js", + // "resource://devtools/client/netmonitor/src/utils/sort-predicates.js", + // "resource://devtools/client/netmonitor/src/utils/request-utils.js", + // "resource://devtools/client/netmonitor/src/selectors/search.js", + // "resource://devtools/client/netmonitor/src/selectors/timing-markers.js", + // "resource://devtools/client/netmonitor/src/selectors/ui.js", + // "resource://devtools/client/netmonitor/src/selectors/messages.js", + // "resource://devtools/client/netmonitor/src/middleware/throttling.js", + // "resource://devtools/client/shared/components/throttling/actions.js", + // "resource://devtools/client/netmonitor/src/middleware/event-telemetry.js", + // "resource://devtools/client/netmonitor/src/reducers/index.js", + // "resource://devtools/client/netmonitor/src/reducers/batching.js", + // "resource://devtools/client/netmonitor/src/reducers/requests.js", + // "resource://devtools/client/netmonitor/src/reducers/search.js", + // "resource://devtools/client/netmonitor/src/reducers/sort.js", + // "resource://devtools/client/netmonitor/src/reducers/filters.js", + // "resource://devtools/client/netmonitor/src/reducers/timing-markers.js", + // "resource://devtools/client/netmonitor/src/reducers/ui.js", + // "resource://devtools/client/netmonitor/src/reducers/messages.js", + // "resource://devtools/client/shared/components/throttling/reducer.js", + // "resource://devtools/client/netmonitor/src/actions/index.js", + // "resource://devtools/client/netmonitor/src/actions/batching.js", + // "resource://devtools/client/netmonitor/src/actions/filters.js", + // "resource://devtools/client/netmonitor/src/actions/requests.js", + // "resource://devtools/client/netmonitor/src/actions/selection.js", + // "resource://devtools/client/netmonitor/src/actions/sort.js", + // "resource://devtools/client/netmonitor/src/actions/timing-markers.js", + // "resource://devtools/client/netmonitor/src/actions/ui.js", + // "resource://devtools/client/netmonitor/src/actions/messages.js", + // "resource://devtools/client/netmonitor/src/actions/search.js", + // "resource://devtools/client/netmonitor/src/workers/search/index.js", + // "resource://devtools/client/shared/worker-utils", + // ]; + // runDuplicatedModulesTest(loaders, allowedDupes); + + runMetricsTest({ + filterString: "devtools/client/netmonitor", + loaders, + panelName: "netmonitor", + }); +}); diff --git a/devtools/client/framework/test/metrics/browser_metrics_netmonitor.toml b/devtools/client/framework/test/metrics/browser_metrics_netmonitor.toml new file mode 100644 index 0000000000..2cef61b319 --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_netmonitor.toml @@ -0,0 +1,18 @@ +[DEFAULT] +tags = "devtools" +subsuite = "devtools" +support-files = [ + "head.js", + "!/devtools/client/shared/test/shared-head.js", + "!/devtools/client/shared/test/telemetry-test-helpers.js", +] + +# Each metrics tests is loaded in a separate .ini file. This way the test is executed +# individually, without any other test being executed before or after. + +["browser_metrics_netmonitor.js"] +run-if = ["os == 'linux'"] # Results should be platform agnostic - only run on linux64-opt +skip-if = [ + "debug", + "asan", +] diff --git a/devtools/client/framework/test/metrics/browser_metrics_pool.js b/devtools/client/framework/test/metrics/browser_metrics_pool.js new file mode 100644 index 0000000000..1b2231fef9 --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_pool.js @@ -0,0 +1,118 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { + DevToolsServer, +} = require("resource://devtools/server/devtools-server.js"); +const { Pool } = require("resource://devtools/shared/protocol.js"); + +// Test parameters +const ROOT_POOLS = 100; +const POOL_DEPTH = 10; +const POOLS_BY_LEVEL = 100; +// Number of Pools that will be added once the environment is set up. +const ADDITIONAL_POOLS = 5000; + +add_task(async function () { + DevToolsServer.init(); + DevToolsServer.registerAllActors(); + const conn = DevToolsServer.connectPipe()._serverConnection; + + info("Add multiple Pools to the connection"); + const pools = setupTestEnvironment(conn); + + let sumResult = 0; + + info("Test how long it takes to manage new Pools"); + let start = performance.now(); + let parentPool = pools[pools.length - 1]; + const newPools = []; + for (let i = 0; i < ADDITIONAL_POOLS; i++) { + const pool = new Pool(conn, `${parentPool.label}-${i}`); + newPools.push(pool); + parentPool.manage(pool); + } + const manageResult = performance.now() - start; + sumResult += manageResult; + + info("Test how long it takes to manage Pools that were already managed"); + start = performance.now(); + parentPool = pools[pools.length - 2]; + for (const pool of newPools) { + parentPool.manage(pool); + } + const manageAlreadyManagedResult = performance.now() - start; + sumResult += manageAlreadyManagedResult; + + info("Test how long it takes to unmanage Pools"); + start = performance.now(); + for (const pool of newPools) { + parentPool.unmanage(pool); + } + const unmanageResult = performance.now() - start; + sumResult += unmanageResult; + + info("Test how long it takes to destroy all the Pools"); + start = performance.now(); + conn.onTransportClosed(); + const destroyResult = performance.now() - start; + sumResult += destroyResult; + + const PERFHERDER_DATA = { + framework: { + name: "devtools", + }, + suites: [ + { + name: "server.pool", + value: sumResult, + subtests: [ + { + name: "server.pool.manage", + value: manageResult, + }, + { + name: "server.pool.manage-already-managed", + value: manageAlreadyManagedResult, + }, + { + name: "server.pool.unmanage", + value: unmanageResult, + }, + { + name: "server.pool.destroy", + value: destroyResult, + }, + ], + }, + ], + }; + info("PERFHERDER_DATA: " + JSON.stringify(PERFHERDER_DATA)); +}); + +// Some Pool operations might be impacted by the number of existing pools in a connection, +// so it's important to have a sizeable number of Pools in order to assert Pool performances. +function setupTestEnvironment(conn) { + const pools = []; + for (let i = 0; i < ROOT_POOLS; i++) { + const rootPool = new Pool(conn, "root-pool-" + i); + pools.push(rootPool); + let parent = rootPool; + for (let j = 0; j < POOL_DEPTH; j++) { + const intermediatePool = new Pool(conn, `pool-${i}-${j}`); + pools.push(intermediatePool); + parent.manage(intermediatePool); + + for (let k = 0; k < POOLS_BY_LEVEL; k++) { + const pool = new Pool(conn, `pool-${i}-${j}-${k}`); + pools.push(pool); + intermediatePool.manage(pool); + } + + parent = intermediatePool; + } + } + return pools; +} diff --git a/devtools/client/framework/test/metrics/browser_metrics_webconsole.js b/devtools/client/framework/test/metrics/browser_metrics_webconsole.js new file mode 100644 index 0000000000..94e24291c8 --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_webconsole.js @@ -0,0 +1,56 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This test records the number of modules loaded by DevTools, as well as the total count + * of characters in those modules, when opening the webconsole. These metrics are + * retrieved by perfherder via logs. + */ + +const TEST_URL = + "data:text/html;charset=UTF-8,<div>Webconsole modules load test</div>"; + +add_task(async function () { + const toolbox = await openNewTabAndToolbox(TEST_URL, "webconsole"); + const toolboxBrowserLoader = toolbox.win.getBrowserLoaderForWindow(); + + // Retrieve the browser loader dedicated to the WebConsole. + const panel = toolbox.getCurrentPanel(); + const webconsoleLoader = panel._frameWindow.getBrowserLoaderForWindow(); + + const loaders = [ + loader.loader, + toolboxBrowserLoader.loader, + webconsoleLoader.loader, + ]; + + const allowedDupes = [ + "@loader/unload.js", + "@loader/options.js", + "resource://devtools/client/webconsole/constants.js", + "resource://devtools/client/webconsole/utils.js", + "resource://devtools/client/webconsole/utils/messages.js", + "resource://devtools/client/webconsole/utils/l10n.js", + "resource://devtools/client/netmonitor/src/utils/request-utils.js", + "resource://devtools/client/webconsole/types.js", + "resource://devtools/client/shared/components/menu/MenuButton.js", + "resource://devtools/client/shared/components/menu/MenuItem.js", + "resource://devtools/client/shared/components/menu/MenuList.js", + "resource://devtools/client/shared/vendor/fluent-react.js", + "resource://devtools/client/shared/vendor/react.js", + "resource://devtools/client/shared/vendor/react-dom.js", + "resource://devtools/client/shared/vendor/react-prop-types.js", + "resource://devtools/client/shared/vendor/react-dom-factories.js", + "resource://devtools/client/shared/vendor/redux.js", + "resource://devtools/client/shared/redux/middleware/thunk.js", + ]; + runDuplicatedModulesTest(loaders, allowedDupes); + + runMetricsTest({ + filterString: "devtools/client/webconsole", + loaders, + panelName: "webconsole", + }); +}); diff --git a/devtools/client/framework/test/metrics/browser_metrics_webconsole.toml b/devtools/client/framework/test/metrics/browser_metrics_webconsole.toml new file mode 100644 index 0000000000..e7c8831ad0 --- /dev/null +++ b/devtools/client/framework/test/metrics/browser_metrics_webconsole.toml @@ -0,0 +1,18 @@ +[DEFAULT] +tags = "devtools" +subsuite = "devtools" +support-files = [ + "head.js", + "!/devtools/client/shared/test/shared-head.js", + "!/devtools/client/shared/test/telemetry-test-helpers.js", +] + +# Each metrics tests is loaded in a separate .ini file. This way the test is executed +# individually, without any other test being executed before or after. + +["browser_metrics_webconsole.js"] +run-if = ["os == 'linux'"] # Results should be platform agnostic - only run on linux64-opt +skip-if = [ + "debug", + "asan", +] diff --git a/devtools/client/framework/test/metrics/head.js b/devtools/client/framework/test/metrics/head.js new file mode 100644 index 0000000000..0246190b31 --- /dev/null +++ b/devtools/client/framework/test/metrics/head.js @@ -0,0 +1,171 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// shared-head.js handles imports, constants, and utility functions +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/devtools/client/shared/test/shared-head.js", + this +); + +// So that PERFHERDER data can be extracted from the logs. +SimpleTest.requestCompleteLog(); + +function getFilteredModules(filters, loaders) { + let modules = []; + for (const l of loaders) { + const loaderModulesMap = l.modules; + const loaderModulesPaths = Object.keys(loaderModulesMap); + modules = modules.concat(loaderModulesPaths); + } + return modules.filter(url => filters.some(filter => url.includes(filter))); +} + +function countCharsInModules(modules) { + return modules.reduce((sum, uri) => { + try { + return sum + require("raw!" + uri).length; + } catch (e) { + // Ignore failures + return sum; + } + }, 0); +} + +/** + * Record module loading data. + * + * @param {Object} + * - filterString {String} path to use to filter modules specific to the current panel + * - loaders {Array} Array of Loaders to check for modules + * - panelName {String} reused in identifiers for perfherder data + */ +function runMetricsTest({ filterString, loaders, panelName }) { + const allModules = getFilteredModules([""], loaders); + const panelModules = getFilteredModules([filterString], loaders); + const vendoredModules = getFilteredModules( + ["devtools/client/debugger/dist/vendors", "devtools/client/shared/vendor/"], + loaders + ); + + const allModulesCount = allModules.length; + const panelModulesCount = panelModules.length; + const vendoredModulesCount = vendoredModules.length; + + const allModulesChars = countCharsInModules(allModules); + const panelModulesChars = countCharsInModules(panelModules); + const vendoredModulesChars = countCharsInModules(vendoredModules); + + const PERFHERDER_DATA = { + framework: { + name: "devtools", + }, + suites: [ + { + name: panelName + "-metrics", + value: allModulesChars, + subtests: [ + { + name: panelName + "-modules", + value: panelModulesCount, + }, + { + name: panelName + "-chars", + value: panelModulesChars, + }, + { + name: "all-modules", + value: allModulesCount, + }, + { + name: "all-chars", + value: allModulesChars, + }, + { + name: "vendored-modules", + value: vendoredModulesCount, + }, + { + name: "vendored-chars", + value: vendoredModulesChars, + }, + ], + }, + ], + }; + info("PERFHERDER_DATA: " + JSON.stringify(PERFHERDER_DATA)); + + // Simply check that we found valid values. + ok( + allModulesCount > panelModulesCount && panelModulesCount > 0, + "Successfully recorded module count for " + panelName + ); + ok( + allModulesChars > panelModulesChars && panelModulesChars > 0, + "Successfully recorded char count for " + panelName + ); + + // Easy way to check how many vendored chars we have for a given panel. + const percentage = ((100 * vendoredModulesChars) / allModulesChars).toFixed( + 1 + ); + info(`Percentage of vendored chars for ${panelName}: ${percentage}%`); +} + +function getDuplicatedModules(loaders) { + const allModules = getFilteredModules([""], loaders); + + const uniqueModules = new Set(); + const duplicatedModules = new Set(); + for (const mod of allModules) { + if (uniqueModules.has(mod)) { + duplicatedModules.add(mod); + } + + uniqueModules.add(mod); + } + + return duplicatedModules; +} + +/** + * Check that modules are only loaded once in a given set of loaders. + * Panels might load the same module twice by mistake if they are both using + * a BrowserLoader and the regular DevTools Loader. + * + * @param {Array} loaders + * Array of Loader instances. + * @param {Array} allowedDupes + * Array of Strings which are paths to known duplicated modules. + * The test will also fail if a allowedDupesed module is not found in the + * duplicated modules. + */ +function runDuplicatedModulesTest(loaders, allowedDupes) { + const duplicatedModules = getDuplicatedModules(loaders); + + // Remove allowedDupes entries, and fail if an allowed entry is not found. + for (const mod of allowedDupes) { + const deleted = duplicatedModules.delete(mod); + if (!deleted) { + ok( + false, + "module not found in the duplicated modules: [" + + mod + + "]. The allowedDupes array should be updated to remove it." + ); + } + } + + // Prepare a log string with the paths of all duplicated modules. + let duplicatedModulesLog = ""; + for (const mod of duplicatedModules) { + duplicatedModulesLog += ` [duplicated module] ${mod}\n`; + } + + // Check that duplicatedModules Set is empty. + is( + duplicatedModules.size, + 0, + "Duplicated module load detected. List of duplicated modules:\n" + + duplicatedModulesLog + ); +} |