diff options
Diffstat (limited to '')
| -rw-r--r-- | taskcluster/ci/perftest/android.yml | 543 | ||||
| -rw-r--r-- | taskcluster/ci/perftest/kind.yml | 43 | ||||
| -rw-r--r-- | taskcluster/ci/perftest/linux.yml | 349 | ||||
| -rw-r--r-- | taskcluster/ci/perftest/macosx.yml | 251 | ||||
| -rw-r--r-- | taskcluster/ci/perftest/windows.yml | 200 |
5 files changed, 1386 insertions, 0 deletions
diff --git a/taskcluster/ci/perftest/android.yml b/taskcluster/ci/perftest/android.yml new file mode 100644 index 0000000000..c5b4bfe1d8 --- /dev/null +++ b/taskcluster/ci/perftest/android.yml @@ -0,0 +1,543 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +job-defaults: + fetches: + toolchain: + - linux64-node + - linux64-geckodriver + run: + sparse-profile: perftest + checkout: true + command: >- + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +hw-a51: + worker-type: t-bitbar-gw-perf-a51 + description: Run ./mach perftest on a Samsung Galaxy a51 + treeherder: + symbol: perftest-a51 + platform: android-hw-a51-11-0-arm7-shippable/opt + +# TODO: is this right? +hw-a51-aarch64: + worker-type: t-bitbar-gw-perf-a51 + description: Run ./mach perftest on a Samsung Galaxy a51 + treeherder: + symbol: perftest-a51 + platform: android-hw-a51-11-0-aarch64-shippable/opt + +hw-a51-view-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run VIEW perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(view-fenix) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: true + batch: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --perfherder-metrics name:processLaunchToNavStart,shouldAlert:True + --android-install-apk fenix_nightly_armeabi_v7a + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 25 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-a51-startup-fenix-cold-main-first-frame: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(startup-fenix-cmff) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_main_first_frame + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=fenix + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-fenix-cold-view-nav-start: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(startup-fenix-cvns) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_view_nav_start + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=fenix + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-focus-cold-main-first-frame: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Focus on a Samsung A51 + treeherder: + symbol: perftest(startup-focus-cmff) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_main_first_frame + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=focus + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-focus-cold-view-nav-start: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Focus on a Samsung A51 + treeherder: + symbol: perftest(startup-focus-cvns) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_view_nav_start + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=focus + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-geckoview-cold-main-first-frame: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Geckoview on a Samsung A51 + treeherder: + symbol: perftest(startup-geckoview-cmff) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_main_first_frame + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=geckoview_example + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-geckoview-cold-view-nav-start: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Geckoview_example on a Samsung A51 + treeherder: + symbol: perftest(startup-geckoview-cvns) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_view_nav_start + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=geckoview_example + --AndroidStartUp-release-channel=nightly + +hw-a51-aarch64-view-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run VIEW perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(view-fenix) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + batch: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --perfherder-metrics name:processLaunchToNavStart,shouldAlert:True + --android-install-apk fenix_nightly_arm64_v8a + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 25 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-a51-view-gv: + worker-type: t-bitbar-gw-perf-a51 + description: Run VIEW perftest on GV on a Samsung A51 + treeherder: + symbol: perftest(view-gv) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --perfherder-metrics processLaunchToNavStart + --android-install-apk gve_nightly_api16 + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app geckoview + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 14 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-a51-aarch64-view-gv: + worker-type: t-bitbar-gw-perf-a51 + description: Run VIEW perftest on GV on a Samsung A51 + treeherder: + symbol: perftest(view-gv) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --perfherder-metrics processLaunchToNavStart + --android-install-apk gve_nightly_aarch64 + --hooks testing/performance/hooks_android_view.py + --perfherder + --perfherder-app geckoview + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --browsertime-iterations 14 + --output $MOZ_FETCHES_DIR/../artifacts + testing/performance/perftest_android_view.js + +hw-a51-main-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run main (home activity) perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(main-fenix) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity .App + --android-clear-logcat + --android-capture-logcat logcat + --androidlog + --androidlog-first-timestamp ".*Start proc.*org\.mozilla\..*\..*App.*" + --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\..*" + --androidlog-subtest-name "MAIN" + --android-install-apk fenix_nightly_armeabi_v7a + --hooks testing/performance/hooks_android_main.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --browsertime-iterations 14 + testing/performance/perftest_android_main.js + +hw-a51-aarch64-main-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run main (home activity) perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(main-fenix) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity .App + --android-clear-logcat + --android-capture-logcat logcat + --androidlog + --androidlog-first-timestamp ".*Start proc.*org\.mozilla\..*\..*App.*" + --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\..*" + --androidlog-subtest-name "MAIN" + --android-install-apk fenix_nightly_arm64_v8a + --hooks testing/performance/hooks_android_main.py + --perfherder + --perfherder-app fenix + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --browsertime-iterations 14 + testing/performance/perftest_android_main.js + +hw-a51-perfstats-gv: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on GeckoView on a Samsung A51 + treeherder: + symbol: perftest(perfstats-gv) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk gve_nightly_api16 + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app geckoview + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-aarch64-perfstats-gv: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on GeckoView on a Samsung A51 + treeherder: + symbol: perftest(perfstats-gv) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk gve_nightly_aarch64 + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app geckoview + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-g5-perfstats-fenix: + worker-type: t-bitbar-gw-perf-g5 + description: Run PerfStats perftest on Fenix on a G5 + treeherder: + symbol: perftest(perfstats-fenix) + platform: android-hw-g5-7-0-arm7-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk fenix_nightly_armeabi_v7a + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app fenix + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-perfstats-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(perfstats-fenix) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk fenix_nightly_armeabi_v7a + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app fenix + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-aarch64-perfstats-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(perfstats-fenix) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk fenix_nightly_arm64_v8a + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app fenix + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-record-websites: + worker-type: t-bitbar-gw-perf-a51 + description: Run perftest to record mobile websites + treeherder: + symbol: perftest(rec) + platform: android-hw-a51-11-0-aarch64-shippable/opt + require-build: + android-hw-a51-11-0-aarch64-shippable/opt: build-android-aarch64-shippable/opt + attributes: + cron: false + run-on-projects: [] + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk gve_nightly_aarch64 + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --proxy + --output $MOZ_FETCHES_DIR/../artifacts + --hooks testing/performance/hooks_recording.py + testing/performance/perftest_record.js diff --git a/taskcluster/ci/perftest/kind.yml b/taskcluster/ci/perftest/kind.yml new file mode 100644 index 0000000000..0a0f8a4489 --- /dev/null +++ b/taskcluster/ci/perftest/kind.yml @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - build + +transforms: + - gecko_taskgraph.transforms.perftest:transforms + - gecko_taskgraph.transforms.source_test:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - android.yml + - linux.yml + - macosx.yml + - windows.yml + +job-defaults: + attributes: + retrigger: true + run-on-projects: [] + treeherder: + kind: other + tier: 3 + worker: + taskcluster-proxy: true + max-run-time: 10800 + env: + USE_ARTIFACT: '1' + HOSTUTILS_MANIFEST_PATH: "testing/config/tooltool-manifests/linux64/hostutils.manifest" + artifacts: + - type: directory + name: public/build + path: artifacts + run: + run-as-root: true + using: run-task + checkout: false diff --git a/taskcluster/ci/perftest/linux.yml b/taskcluster/ci/perftest/linux.yml new file mode 100644 index 0000000000..a74cc4b34c --- /dev/null +++ b/taskcluster/ci/perftest/linux.yml @@ -0,0 +1,349 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-linux-talos-1804 + fetches: + toolchain: + - linux64-node + - linux64-geckodriver + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.tar.bz2 + platform: linux1804-64-shippable/opt + require-build: + linux1804-64-shippable/opt: build-linux64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +webpagetest-firefox: + description: Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites + treeherder: + symbol: perftest(linux-webpagetest) + tier: 2 + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_firefox_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --perfherder-app=firefox + --output $MOZ_FETCHES_DIR/../artifacts + +webpagetest-chrome: + description: Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites + treeherder: + symbol: perftest-chrome(linux-webpagetest) + tier: 2 + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_chrome_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --perfherder-app=chrome + --output $MOZ_FETCHES_DIR/../artifacts + +try-browsertime: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(linux-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 10 + --visualmetrics + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(linux-perfstats) + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +record-websites: + description: Run mozperftest to record desktop websites + platform: linux1804-64/opt + require-build: + linux1804-64/opt: build-linux64/opt + treeherder: + symbol: perftest(rec) + attributes: + cron: false + run-on-projects: [] + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --flavor desktop-browser + --verbose + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --proxy + --output $MOZ_FETCHES_DIR/../artifacts + --hooks testing/performance/hooks_recording.py + testing/performance/perftest_record.js + +side-by-side: + description: Run mozperftest side-by-side + treeherder: + symbol: perftest(side-by-side) + tier: 2 + run: + using: run-task + sparse-profile: perftest + checkout: true + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + tools side-by-side + -t {test_name} + --base-platform {platform} + --base-revision {base_revision} + --new-revision {new_revision} + --base-branch {base_branch} + --new-branch {new_branch} + +mwu-change-detector: + description: Runs Mann-Whitney U-test on two revisions to detect performance changes + treeherder: + symbol: perftest-detect(mwu) + tier: 2 + run: + using: run-task + sparse-profile: perftest + checkout: true + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + tools change-detector + --task-name {task_name} + --base-revision {base_revision} + --new-revision {new_revision} + --base-branch {base_branch} + --new-branch {new_branch} diff --git a/taskcluster/ci/perftest/macosx.yml b/taskcluster/ci/perftest/macosx.yml new file mode 100644 index 0000000000..b2d8f578d2 --- /dev/null +++ b/taskcluster/ci/perftest/macosx.yml @@ -0,0 +1,251 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-osx-1015-r8 + fetches: + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.dmg + extract: false + toolchain: + - macosx64-geckodriver + - macosx64-node + platform: macosx1015-64-shippable-qr/opt + require-build: + macosx1015-64-shippable-qr/opt: build-macosx64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on macOs + treeherder: + symbol: perftest(macos-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg + +try-browsertime: + description: Run ./mach perftest on macOs + treeherder: + symbol: perftest(macos-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test on macOS + treeherder: + symbol: perftest(macos-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg + + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --browsertime-iterations 10 + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + python3 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(macos-perfstats) + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js diff --git a/taskcluster/ci/perftest/windows.yml b/taskcluster/ci/perftest/windows.yml new file mode 100644 index 0000000000..caea4026b7 --- /dev/null +++ b/taskcluster/ci/perftest/windows.yml @@ -0,0 +1,200 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-win10-64-1803-hw + fetches: + build: + - artifact: target.perftests.tests.tar.gz + - artifact: target.zip + toolchain: + - win64-node + - win64-geckodriver + platform: windows10-64-shippable/opt + require-build: + windows10-64-shippable/opt: build-win64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-browsertime: + description: Run ./mach perftest on windows + treeherder: + symbol: perftest(win-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(win-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --browsertime-iterations 10 + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(win-perfstats) + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js |