diff options
Diffstat (limited to 'taskcluster/ci/perftest/linux.yml')
| -rw-r--r-- | taskcluster/ci/perftest/linux.yml | 349 |
1 files changed, 349 insertions, 0 deletions
diff --git a/taskcluster/ci/perftest/linux.yml b/taskcluster/ci/perftest/linux.yml new file mode 100644 index 0000000000..a74cc4b34c --- /dev/null +++ b/taskcluster/ci/perftest/linux.yml @@ -0,0 +1,349 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-linux-talos-1804 + fetches: + toolchain: + - linux64-node + - linux64-geckodriver + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.tar.bz2 + platform: linux1804-64-shippable/opt + require-build: + linux1804-64-shippable/opt: build-linux64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +webpagetest-firefox: + description: Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites + treeherder: + symbol: perftest(linux-webpagetest) + tier: 2 + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_firefox_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --perfherder-app=firefox + --output $MOZ_FETCHES_DIR/../artifacts + +webpagetest-chrome: + description: Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites + treeherder: + symbol: perftest-chrome(linux-webpagetest) + tier: 2 + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_chrome_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --perfherder-app=chrome + --output $MOZ_FETCHES_DIR/../artifacts + +try-browsertime: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(linux-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 10 + --visualmetrics + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(linux-perfstats) + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +record-websites: + description: Run mozperftest to record desktop websites + platform: linux1804-64/opt + require-build: + linux1804-64/opt: build-linux64/opt + treeherder: + symbol: perftest(rec) + attributes: + cron: false + run-on-projects: [] + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + --flavor desktop-browser + --verbose + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --proxy + --output $MOZ_FETCHES_DIR/../artifacts + --hooks testing/performance/hooks_recording.py + testing/performance/perftest_record.js + +side-by-side: + description: Run mozperftest side-by-side + treeherder: + symbol: perftest(side-by-side) + tier: 2 + run: + using: run-task + sparse-profile: perftest + checkout: true + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + tools side-by-side + -t {test_name} + --base-platform {platform} + --base-revision {base_revision} + --new-revision {new_revision} + --base-branch {base_branch} + --new-branch {new_branch} + +mwu-change-detector: + description: Runs Mann-Whitney U-test on two revisions to detect performance changes + treeherder: + symbol: perftest-detect(mwu) + tier: 2 + run: + using: run-task + sparse-profile: perftest + checkout: true + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + tools change-detector + --task-name {task_name} + --base-revision {base_revision} + --new-revision {new_revision} + --base-branch {base_branch} + --new-branch {new_branch} |