diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
commit | 2aa4a82499d4becd2284cdb482213d541b8804dd (patch) | |
tree | b80bf8bf13c3766139fbacc530efd0dd9d54394c /taskcluster/ci/perftest | |
parent | Initial commit. (diff) | |
download | firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip |
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/ci/perftest')
-rw-r--r-- | taskcluster/ci/perftest/android.yml | 218 | ||||
-rw-r--r-- | taskcluster/ci/perftest/kind.yml | 40 | ||||
-rw-r--r-- | taskcluster/ci/perftest/linux.yml | 210 | ||||
-rw-r--r-- | taskcluster/ci/perftest/macosx.yml | 221 | ||||
-rw-r--r-- | taskcluster/ci/perftest/windows.yml | 170 |
5 files changed, 859 insertions, 0 deletions
diff --git a/taskcluster/ci/perftest/android.yml b/taskcluster/ci/perftest/android.yml new file mode 100644 index 0000000000..e27de64a6a --- /dev/null +++ b/taskcluster/ci/perftest/android.yml @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +job-defaults: + fetches: + toolchain: + - linux64-node-10 + - linux64-geckodriver + run: + sparse-profile: perftest + checkout: true + command: >- + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +hw-g5: + worker-type: t-bitbar-gw-perf-g5 + description: Run ./mach perftest on a G5 + treeherder: + symbol: perftest-g5 + platform: android-hw-g5-7-0-arm7-api-16/opt + +hw-p2: + worker-type: t-bitbar-gw-perf-p2 + description: Run ./mach perftest on a Pixel 2 + treeherder: + symbol: perftest-p2 + platform: android-hw-p2-8-0-android-aarch64/opt + +hw-g5-view-fenix: + worker-type: t-bitbar-gw-perf-g5 + description: Run VIEW perftest on Fenix on a G5 + treeherder: + symbol: perftest(view-fenix) + platform: android-hw-g5-7-0-arm7-api-16/opt + attributes: + cron: true + batch: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --perfherder-metrics name:processLaunchToNavStart,shouldAlert:True + --android-install-apk fenix_nightlysim_multicommit_armeabi_v7a + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 25 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-p2-view-fenix: + worker-type: t-bitbar-gw-perf-p2 + description: Run VIEW perftest on Fenix on a Pixel2 + treeherder: + symbol: perftest(view-fenix) + platform: android-hw-p2-8-0-android-aarch64/opt + attributes: + cron: true + batch: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --perfherder-metrics name:processLaunchToNavStart,shouldAlert:True + --android-install-apk fenix_nightlysim_multicommit_arm64_v8a + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-perf-tuning + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 25 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-g5-view-gv: + worker-type: t-bitbar-gw-perf-g5 + description: Run VIEW perftest on GV on a G5 + treeherder: + symbol: perftest(view-gv) + platform: android-hw-g5-7-0-arm7-api-16/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --perfherder-metrics processLaunchToNavStart + --android-install-apk gve_nightly_api16 + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app geckoview + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 14 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-p2-view-gv: + worker-type: t-bitbar-gw-perf-p2 + description: Run VIEW perftest on GV on a Pixel 2 + treeherder: + symbol: perftest(view-gv) + platform: android-hw-p2-8-0-android-aarch64/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-clear-logcat + --android-capture-logcat logcat + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-perf-tuning + --perfherder-metrics processLaunchToNavStart + --android-install-apk gve_nightly_api16 + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app geckoview + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 14 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-g5-main-fenix: + worker-type: t-bitbar-gw-perf-g5 + description: Run main (home activity) perftest on Fenix on a G5 + treeherder: + symbol: perftest(main-fenix) + platform: android-hw-g5-7-0-arm7-api-16/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity .App + --android-clear-logcat + --android-capture-logcat logcat + --androidlog + --androidlog-first-timestamp ".*Start proc.*org\.mozilla\..*\..*App.*" + --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\..*" + --androidlog-subtest-name "MAIN" + --android-install-apk fenix_nightly_armeabi_v7a + --hooks testing/performance/hooks_android_main.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --browsertime-iterations 14 + testing/performance/perftest_android_main.js + +hw-p2-main-fenix: + worker-type: t-bitbar-gw-perf-p2 + description: Run main (home activity) perftest on Fenix on a Pixel 2 + treeherder: + symbol: perftest(main-fenix) + platform: android-hw-p2-8-0-android-aarch64/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity .App + --android-clear-logcat + --android-capture-logcat logcat + --android-perf-tuning + --androidlog + --androidlog-first-timestamp ".*Start proc.*org\.mozilla\..*\..*App.*" + --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\..*" + --androidlog-subtest-name "MAIN" + --android-install-apk fenix_nightly_arm64_v8a + --hooks testing/performance/hooks_android_main.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --browsertime-iterations 14 + testing/performance/perftest_android_main.js diff --git a/taskcluster/ci/perftest/kind.yml b/taskcluster/ci/perftest/kind.yml new file mode 100644 index 0000000000..dd2cee31f9 --- /dev/null +++ b/taskcluster/ci/perftest/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - build + +transforms: + - taskgraph.transforms.perftest:transforms + - taskgraph.transforms.source_test:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs-from: + - android.yml + - linux.yml + - macosx.yml + - windows.yml + +job-defaults: + run-on-projects: [] + treeherder: + kind: other + tier: 3 + worker: + taskcluster-proxy: true + max-run-time: 10800 + env: + USE_ARTIFACT: '1' + artifacts: + - type: directory + name: public/build + path: artifacts + run: + run-as-root: true + using: run-task + checkout: false diff --git a/taskcluster/ci/perftest/linux.yml b/taskcluster/ci/perftest/linux.yml new file mode 100644 index 0000000000..e8183d9fb5 --- /dev/null +++ b/taskcluster/ci/perftest/linux.yml @@ -0,0 +1,210 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-linux-talos + fetches: + toolchain: + - linux64-node-10 + - linux64-geckodriver + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.tar.bz2 + platform: linux64-shippable/opt + require-build: + linux64-shippable/opt: build-linux64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +try-browsertime: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(linux-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + --verbose + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + g-search: + - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + fb-scroll: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + g-search: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + ytw: + - [null, browsertime.waitTime=20000] + ytw-scroll: + - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 10 + --visualmetrics + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts diff --git a/taskcluster/ci/perftest/macosx.yml b/taskcluster/ci/perftest/macosx.yml new file mode 100644 index 0000000000..74aec57a3a --- /dev/null +++ b/taskcluster/ci/perftest/macosx.yml @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-osx-1014 + fetches: + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.dmg + extract: false + toolchain: + - macosx64-geckodriver + - macosx64-node + platform: macosx64-shippable/opt + require-build: + macosx64-shippable/opt: build-macosx64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on macOs + treeherder: + symbol: perftest(macos-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg + +try-browsertime: + description: Run ./mach perftest on macOs + treeherder: + symbol: perftest(macos-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test on macOS + treeherder: + symbol: perftest(macos-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg + --verbose + + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + g-search: + - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + fb-scroll: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + g-search: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + ytw: + - [null, browsertime.waitTime=20000] + ytw-scroll: + - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --browsertime-iterations 10 + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + python3 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts diff --git a/taskcluster/ci/perftest/windows.yml b/taskcluster/ci/perftest/windows.yml new file mode 100644 index 0000000000..faa718d3d4 --- /dev/null +++ b/taskcluster/ci/perftest/windows.yml @@ -0,0 +1,170 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-win10-64-hw + fetches: + build: + - artifact: target.perftests.tests.tar.gz + - artifact: target.zip + toolchain: + - win64-node-10 + - win64-geckodriver + platform: win64-shippable/opt + require-build: + win64-shippable/opt: build-win64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-browsertime: + description: Run ./mach perftest on windows + treeherder: + symbol: perftest(win-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(win-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + g-search: + - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + fb-scroll: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + g-search: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + ytw: + - [null, browsertime.waitTime=20000] + ytw-scroll: + - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --browsertime-iterations 10 + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts |