diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
| commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
| tree | f435a8308119effd964b339f76abb83a57c29483 /testing/web-platform/tests/tools/ci | |
| parent | Initial commit. (diff) | |
| download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip | |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'testing/web-platform/tests/tools/ci')
58 files changed, 5957 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/ci/__init__.py b/testing/web-platform/tests/tools/ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/__init__.py diff --git a/testing/web-platform/tests/tools/ci/azure/README.md b/testing/web-platform/tests/tools/ci/azure/README.md new file mode 100644 index 0000000000..afe5021efc --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/README.md @@ -0,0 +1,2 @@ +These are step templates for Azure Pipelines, used in `.azure-pipelines.yml` +in the root of the repository. diff --git a/testing/web-platform/tests/tools/ci/azure/affected_tests.yml b/testing/web-platform/tests/tools/ci/azure/affected_tests.yml new file mode 100644 index 0000000000..566021d249 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/affected_tests.yml @@ -0,0 +1,27 @@ +parameters: + checkoutCommit: '' + affectedRange: 'HEAD^1' + artifactName: '' + +steps: +- template: checkout.yml +- ${{ if ne(parameters.checkoutCommit, '') }}: + - script: | + set -eux -o pipefail + git checkout ${{ parameters.checkoutCommit }} + displayName: 'Checkout ${{ parameters.checkoutCommit }}' +- template: install_certs.yml +- template: color_profile.yml +- template: install_safari.yml +- template: update_hosts.yml +- template: update_manifest.yml +- script: | + set -eux -o pipefail + export SYSTEM_VERSION_COMPAT=0 + ./wpt run --yes --no-pause --no-fail-on-unexpected --no-restart-on-unexpected --affected ${{ parameters.affectedRange }} --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report.json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot.txt --channel preview --kill-safari safari + displayName: 'Run tests' +- task: PublishBuildArtifacts@1 + displayName: 'Publish results' + inputs: + artifactName: '${{ parameters.artifactName }}' + condition: succeededOrFailed() diff --git a/testing/web-platform/tests/tools/ci/azure/checkout.yml b/testing/web-platform/tests/tools/ci/azure/checkout.yml new file mode 100644 index 0000000000..618c571465 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/checkout.yml @@ -0,0 +1,4 @@ +steps: +- checkout: self + fetchDepth: 50 + submodules: false diff --git a/testing/web-platform/tests/tools/ci/azure/color_profile.yml b/testing/web-platform/tests/tools/ci/azure/color_profile.yml new file mode 100644 index 0000000000..d90c6ca429 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/color_profile.yml @@ -0,0 +1,5 @@ +steps: +- script: | + ./wpt macos-color-profile + displayName: 'Set display color profile' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist b/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist new file mode 100644 index 0000000000..122080972c --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>AllowRemoteAutomation</key> + <true/> +</dict> +</plist> diff --git a/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml b/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml new file mode 100644 index 0000000000..df92ce1066 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml @@ -0,0 +1,23 @@ +# This job is used to get a run into wpt.fyi and staging.wpt.fyi, by notifying +# them with the build number and artifact to use. + +parameters: + dependsOn: '' + artifactName: '' + +jobs: +- job: ${{ parameters.dependsOn }}_hook + displayName: 'wpt.fyi hook: ${{ parameters.artifactName }}' + dependsOn: ${{ parameters.dependsOn }} + pool: + vmImage: 'ubuntu-20.04' + steps: + - checkout: none + - script: | + set -eux -o pipefail + curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://wpt.fyi/api/checks/azure/$(Build.BuildId) + displayName: 'Invoke wpt.fyi hook' + - script: | + set -eux -o pipefail + curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://staging.wpt.fyi/api/checks/azure/$(Build.BuildId) + displayName: 'Invoke staging.wpt.fyi hook' diff --git a/testing/web-platform/tests/tools/ci/azure/install_certs.yml b/testing/web-platform/tests/tools/ci/azure/install_certs.yml new file mode 100644 index 0000000000..bee5ab084e --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_certs.yml @@ -0,0 +1,11 @@ +steps: +- script: | + set -eux -o pipefail + # https://github.com/web-platform-tests/results-collection/blob/master/src/scripts/trust-root-ca.sh + # only run this on macOS < 11 + [ "11" = "`echo -e $( sw_vers -productVersion )\\\n11 | sort -V | head -n1`" ] || sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain tools/certs/cacert.pem + displayName: 'Install web-platform.test certificate (macOS)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- script: certutil –addstore -enterprise –f "Root" tools\certs\cacert.pem + displayName: 'Install web-platform.test certificate (Windows)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/testing/web-platform/tests/tools/ci/azure/install_chrome.yml b/testing/web-platform/tests/tools/ci/azure/install_chrome.yml new file mode 100644 index 0000000000..7599321be2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_chrome.yml @@ -0,0 +1,11 @@ +steps: +# The conflicting google-chrome and chromedriver casks are first uninstalled. +# The raw google-chrome-dev cask URL is used to bypass caching. +- script: | + set -eux -o pipefail + HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall --cask google-chrome || true + HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall --cask chromedriver || true + curl https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/google-chrome-dev.rb > google-chrome-dev.rb + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask google-chrome-dev.rb + displayName: 'Install Chrome Dev' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/install_edge.yml b/testing/web-platform/tests/tools/ci/azure/install_edge.yml new file mode 100644 index 0000000000..297b6e60b2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_edge.yml @@ -0,0 +1,61 @@ +parameters: + channel: dev + +# Should match https://web-platform-tests.org/running-tests/chrome.html +# Just replace chrome with edgechromium +steps: +- ${{ if eq(parameters.channel, 'stable') }}: + - powershell: | + $edgeInstallerName = 'MicrosoftEdgeSetup.exe' + # Link to Stable channel installer + Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2108834&Channel=Stable&language=en' -Destination MicrosoftEdgeSetup.exe + if (-not (Test-Path $edgeInstallerName)) { + Throw "Failed to download Edge installer to $edgeInstallerName." + } + cmd /c START /WAIT $edgeInstallerName /silent /install + $edgePath = "$env:systemdrive\Program Files (x86)\Microsoft\Edge\Application" + if (Test-Path $edgePath) { + Write-Host "##vso[task.prependpath]$edgePath" + Write-Host "Edge Stable installed at $edgePath." + (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List + } else { + Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force + Throw "Failed to install Edge at $edgePath" + } + displayName: 'Install Edge Stable' + +- ${{ if eq(parameters.channel, 'canary') }}: + - powershell: | + $edgeInstallerName = 'MicrosoftEdgeSetup.exe' + # Link to Canary channel installer + Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2084649&Channel=Canary&language=en-us' -Destination MicrosoftEdgeSetup.exe + if (-not (Test-Path $edgeInstallerName)) { + Throw "Failed to download Edge installer to $edgeInstallerName." + } + cmd /c START /WAIT $edgeInstallerName /silent /install + $edgePath = "$env:localappdata\Microsoft\Edge SxS\Application" + if (Test-Path $edgePath) { + Write-Host "##vso[task.prependpath]$edgePath" + Write-Host "Edge Canary installed at $edgePath." + (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List + } else { + Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force + Throw "Failed to install Edge Canary at $edgePath" + } + displayName: 'Install Edge Canary' +- ${{ if eq(parameters.channel, 'dev') }}: + - powershell: | + $edgeInstallerName = 'MicrosoftEdgeSetup.exe' + # Link to Dev channel installer + Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2069324&Channel=Dev&language=en-us' -Destination MicrosoftEdgeSetup.exe + cmd /c START /WAIT $edgeInstallerName /silent /install + $edgePath = "$env:systemdrive\Program Files (x86)\Microsoft\Edge Dev\Application" + if (Test-Path $edgePath) { + Write-Host "##vso[task.prependpath]$edgePath" + Write-Host "Edge Canary installed at $edgePath." + (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List + } else { + Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force + Throw "Failed to install Edge Dev at $edgePath" + } + displayName: 'Install Edge Dev' diff --git a/testing/web-platform/tests/tools/ci/azure/install_firefox.yml b/testing/web-platform/tests/tools/ci/azure/install_firefox.yml new file mode 100644 index 0000000000..73af597665 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_firefox.yml @@ -0,0 +1,9 @@ +steps: +# This is equivalent to `Homebrew/homebrew-cask-versions/firefox-nightly`, +# but the raw URL is used to bypass caching. +- script: | + set -eux -o pipefail + curl https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/firefox-nightly.rb > firefox-nightly.rb + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask firefox-nightly.rb + displayName: 'Install Firefox Nightly' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/install_fonts.yml b/testing/web-platform/tests/tools/ci/azure/install_fonts.yml new file mode 100644 index 0000000000..279c262c7e --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_fonts.yml @@ -0,0 +1,7 @@ +steps: +# Installig Ahem in /Library/Fonts instead of using --install-fonts is a +# workaround for https://github.com/web-platform-tests/wpt/issues/13803. +- script: | + set -eux -o pipefail + sudo cp fonts/Ahem.ttf /Library/Fonts + displayName: 'Install Ahem font' diff --git a/testing/web-platform/tests/tools/ci/azure/install_safari.yml b/testing/web-platform/tests/tools/ci/azure/install_safari.yml new file mode 100644 index 0000000000..1a398532da --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_safari.yml @@ -0,0 +1,29 @@ +parameters: + channel: preview + +# Should match https://web-platform-tests.org/running-tests/safari.html +steps: +- script: defaults write com.apple.WebDriver DiagnosticsEnabled 1 + displayName: 'Enable safaridriver diagnostics' + condition: eq(variables['safaridriver_diagnose'], true) +- ${{ if eq(parameters.channel, 'preview') }}: + - script: | + set -eux -o pipefail + export SYSTEM_VERSION_COMPAT=0 + ./wpt install --channel preview --download-only -d . --rename STP safari browser + sudo installer -pkg STP.pkg -target LocalSystem + # Workaround for `sudo safardriver --enable` not working on Catalina: + # https://github.com/web-platform-tests/wpt/issues/21751 + mkdir -p ~/Library/WebDriver/ + cp tools/ci/azure/com.apple.SafariTechnologyPreview.plist ~/Library/WebDriver/ + defaults write com.apple.SafariTechnologyPreview WebKitJavaScriptCanOpenWindowsAutomatically 1 + defaults write com.apple.SafariTechnologyPreview ExperimentalServerTimingEnabled 1 + displayName: 'Install Safari Technology Preview' +- ${{ if eq(parameters.channel, 'stable') }}: + - script: | + set -eux -o pipefail + export SYSTEM_VERSION_COMPAT=0 + sudo softwareupdate --install $( softwareupdate -l | grep -o '\* Label: \(Safari.*\)' | sed -e 's/* Label: //' ) + sudo safaridriver --enable + defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1 + displayName: 'Configure Safari' diff --git a/testing/web-platform/tests/tools/ci/azure/pip_install.yml b/testing/web-platform/tests/tools/ci/azure/pip_install.yml new file mode 100644 index 0000000000..18d1879e97 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/pip_install.yml @@ -0,0 +1,6 @@ +parameters: + packages: '' + +steps: +- script: pip --disable-pip-version-check install --upgrade ${{ parameters.packages }} + displayName: 'Install Python packages' diff --git a/testing/web-platform/tests/tools/ci/azure/publish_logs.yml b/testing/web-platform/tests/tools/ci/azure/publish_logs.yml new file mode 100644 index 0000000000..a49397a91a --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/publish_logs.yml @@ -0,0 +1,7 @@ +steps: +- task: PublishBuildArtifacts@1 + displayName: 'Publish safaridriver logs' + inputs: + pathtoPublish: /Users/runner/Library/Logs/com.apple.WebDriver/ + artifactName: safaridriver-logs + condition: eq(variables['safaridriver_diagnose'], true) diff --git a/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml b/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml new file mode 100644 index 0000000000..76bee7dbb8 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml @@ -0,0 +1,12 @@ +steps: +- script: | + mkdir -p /Users/runner/sysdiagnose + sudo sysdiagnose -ub -f /Users/runner/sysdiagnose -A sysdiagnose_$(System.PhaseName)-attempt-$(System.JobAttempt)-chunk-$(System.JobPositionInPhase) + displayName: 'Collect sysdiagnose' + condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Darwin')) +- task: PublishBuildArtifacts@1 + displayName: 'Publish sysdiagnose' + inputs: + pathtoPublish: /Users/runner/sysdiagnose/ + artifactName: sysdiagnose + condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/system_info.yml b/testing/web-platform/tests/tools/ci/azure/system_info.yml new file mode 100644 index 0000000000..cc37b35b85 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/system_info.yml @@ -0,0 +1,4 @@ +steps: +- script: systeminfo + displayName: 'Show system info (Windows)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml b/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml new file mode 100644 index 0000000000..3704ecc5be --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml @@ -0,0 +1,20 @@ +parameters: + directory: '' + toxenv: 'ALL' + +steps: +- template: pip_install.yml + parameters: + packages: tox + +- script: | + set -eux -o pipefail + tox -c ${{ parameters.directory }} -e ${{ parameters.toxenv }} -- --junitxml=results.xml + displayName: 'Run tests' + +- task: PublishTestResults@2 + inputs: + testResultsFiles: '${{ parameters.directory }}/results.xml' + testRunTitle: '${{ parameters.directory }}' + displayName: 'Publish results' + condition: succeededOrFailed() diff --git a/testing/web-platform/tests/tools/ci/azure/update_hosts.yml b/testing/web-platform/tests/tools/ci/azure/update_hosts.yml new file mode 100644 index 0000000000..bcb8536a5c --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/update_hosts.yml @@ -0,0 +1,12 @@ +steps: +- script: | + set -eux -o pipefail + ./wpt make-hosts-file | sudo tee -a /etc/hosts + displayName: 'Update hosts (macOS)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- powershell: | + $hostFile = "$env:systemroot\System32\drivers\etc\hosts" + Copy-Item -Path $hostFile -Destination "$hostFile.back" -Force + python wpt make-hosts-file | Out-File $env:systemroot\System32\drivers\etc\hosts -Encoding ascii -Append + displayName: 'Update hosts (Windows)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/testing/web-platform/tests/tools/ci/azure/update_manifest.yml b/testing/web-platform/tests/tools/ci/azure/update_manifest.yml new file mode 100644 index 0000000000..453ac2ac3c --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/update_manifest.yml @@ -0,0 +1,4 @@ +steps: +# `python wpt` instead of `./wpt` is to make this work on Windows: +- script: python wpt manifest + displayName: 'Update manifest' diff --git a/testing/web-platform/tests/tools/ci/ci_built_diff.sh b/testing/web-platform/tests/tools/ci/ci_built_diff.sh new file mode 100755 index 0000000000..7cf9b23db1 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/ci_built_diff.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -ex + +SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. +cd $WPT_ROOT + +main() { + # Diff PNGs based on pixel-for-pixel identity + echo -e '[diff "img"]\n textconv = identify -quiet -format "%#"' >> .git/config + echo -e '*.png diff=img' >> .git/info/attributes + + # Exclude tests that rely on font rendering + excluded=( + 'html/canvas/element/text/2d.text.draw.fill.basic.png' + 'html/canvas/element/text/2d.text.draw.fill.maxWidth.large.png' + 'html/canvas/element/text/2d.text.draw.fill.rtl.png' + 'html/canvas/element/text/2d.text.draw.stroke.basic.png' + 'html/canvas/offscreen/text/2d.text.draw.fill.basic.png' + 'html/canvas/offscreen/text/2d.text.draw.fill.maxWidth.large.png' + 'html/canvas/offscreen/text/2d.text.draw.fill.rtl.png' + 'html/canvas/offscreen/text/2d.text.draw.stroke.basic.png' + ) + + ./wpt update-built + git update-index --assume-unchanged ${excluded[*]} + git diff --exit-code +} + +main diff --git a/testing/web-platform/tests/tools/ci/ci_resources_unittest.sh b/testing/web-platform/tests/tools/ci/ci_resources_unittest.sh new file mode 100755 index 0000000000..11190fc58d --- /dev/null +++ b/testing/web-platform/tests/tools/ci/ci_resources_unittest.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex + +SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. +cd $WPT_ROOT + +main() { + cd $WPT_ROOT + pip install --user -U tox + ./wpt install firefox browser --destination $HOME + ./wpt install firefox webdriver --destination $HOME/firefox + export PATH=$HOME/firefox:$PATH + + cd $WPT_ROOT/resources/test + tox -- --binary=$HOME/browsers/nightly/firefox/firefox +} + +main diff --git a/testing/web-platform/tests/tools/ci/ci_tools_integration_test.sh b/testing/web-platform/tests/tools/ci/ci_tools_integration_test.sh new file mode 100755 index 0000000000..76f682e254 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/ci_tools_integration_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. +cd $WPT_ROOT + +main() { + git fetch --quiet --unshallow https://github.com/web-platform-tests/wpt.git +refs/heads/*:refs/remotes/origin/* + pip install --user -U tox + + # wpt commands integration tests + cd tools/wpt + tox + cd $WPT_ROOT + + # WMAS test runner integration tests + cd tools/wave + tox + cd $WPT_ROOT +} + +main diff --git a/testing/web-platform/tests/tools/ci/ci_tools_unittest.sh b/testing/web-platform/tests/tools/ci/ci_tools_unittest.sh new file mode 100755 index 0000000000..8e16ee18de --- /dev/null +++ b/testing/web-platform/tests/tools/ci/ci_tools_unittest.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -ex + +SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. +cd $WPT_ROOT + +run_applicable_tox () { + # instead of just running TOXENV (e.g., py38) + # run all environments that start with TOXENV + # (e.g., py38-firefox as well as py38) + local OLD_TOXENV="$TOXENV" + unset TOXENV + local RUN_ENVS=$(tox -l | grep "^${OLD_TOXENV}\(\-\|\$\)" | tr "\n" ",") + if [[ -n "$RUN_ENVS" ]]; then + tox -e "$RUN_ENVS" + fi + export TOXENV="$OLD_TOXENV" +} + +if ./wpt test-jobs --includes tools_unittest; then + pip install --user -U tox + cd tools + run_applicable_tox + cd $WPT_ROOT +else + echo "Skipping tools unittest" +fi + +if ./wpt test-jobs --includes wptrunner_unittest; then + cd tools/wptrunner + run_applicable_tox + cd $WPT_ROOT +else + echo "Skipping wptrunner unittest" +fi diff --git a/testing/web-platform/tests/tools/ci/ci_wptrunner_infrastructure.sh b/testing/web-platform/tests/tools/ci/ci_wptrunner_infrastructure.sh new file mode 100755 index 0000000000..6bc2a15c01 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/ci_wptrunner_infrastructure.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +REL_DIR_NAME=$(dirname "$0") +SCRIPT_DIR=$(cd "$REL_DIR_NAME" && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. +cd "$WPT_ROOT" + +run_infra_test() { + echo "### Running Infrastructure Tests for $1 ###" + ./tools/ci/taskcluster-run.py "$1" "$2" -- --log-tbpl=- --log-wptreport="../artifacts/wptreport-$1.json" --logcat-dir="../artifacts/" --metadata=infrastructure/metadata/ --include=infrastructure/ +} + +main() { + run_infra_test "chrome" "dev" + run_infra_test "firefox" "nightly" + run_infra_test "firefox_android" "nightly" +} + +main diff --git a/testing/web-platform/tests/tools/ci/commands.json b/testing/web-platform/tests/tools/ci/commands.json new file mode 100644 index 0000000000..f26baddaca --- /dev/null +++ b/testing/web-platform/tests/tools/ci/commands.json @@ -0,0 +1,82 @@ +{ + "test-jobs": { + "path": "jobs.py", + "script": "run", + "parser": "create_parser", + "help": "List test jobs that should run for a set of commits", + "virtualenv": false + }, + "make-hosts-file": { + "path": "make_hosts_file.py", + "script": "run", + "parser": "create_parser", + "help": "Output a hosts file to stdout", + "virtualenv": false + }, + "macos-color-profile": { + "path": "macos_color_profile.py", + "script": "run", + "help": "Change the macOS color profile to sRGB", + "virtualenv": true, + "requirements": [ + "requirements_macos_color_profile.txt" + ] + }, + "regen-certs": { + "path": "regen_certs.py", + "script": "run", + "parser": "get_parser", + "help": "Regenerate the WPT certificates", + "virtualenv": false + }, + "update-built": { + "path": "update_built.py", + "script": "run", + "parser": "get_parser", + "help": "Update built tests", + "virtualenv": true, + "requirements": [ + "requirements_build.txt" + ] + }, + "tc-download": { + "path": "tc/download.py", + "script": "run", + "parser": "get_parser", + "parse_known": true, + "help": "Download logs from taskcluster", + "virtualenv": true, + "requirements": [ + "requirements_tc.txt" + ] + }, + "tc-taskgraph": { + "path": "tc/taskgraph.py", + "script": "run", + "help": "Build the taskgraph", + "virtualenv": true, + "requirements": [ + "requirements_tc.txt" + ] + }, + "tc-decision": { + "path": "tc/decision.py", + "parser": "get_parser", + "script": "run", + "help": "Run the decision task", + "virtualenv": true, + "requirements": [ + "requirements_tc.txt" + ] + }, + "tc-sink-task": { + "path": "tc/sink_task.py", + "parser": "get_parser", + "script": "run", + "help": "Run the sink task", + "virtualenv": true, + "requirements": [ + "requirements_tc.txt" + ] + } +} diff --git a/testing/web-platform/tests/tools/ci/epochs_update.sh b/testing/web-platform/tests/tools/ci/epochs_update.sh new file mode 100755 index 0000000000..1c7edf15aa --- /dev/null +++ b/testing/web-platform/tests/tools/ci/epochs_update.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -eux -o pipefail + +SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. + +EPOCHS=( +epochs/three_hourly::3h +epochs/six_hourly::6h +epochs/twelve_hourly::12h +epochs/daily::1d +epochs/weekly::1w +) + +function get_epoch_branch_name () { + echo ${1} | awk -F '::' '{print $1}' +} + +function get_epoch_timeval () { + echo ${1} | awk -F '::' '{print $2}' +} + +main () { + ALL_BRANCHES_NAMES="" + for e in "${EPOCHS[@]}"; + do + EPOCH=$(get_epoch_timeval ${e}) + EPOCH_BRANCH_NAME=$(get_epoch_branch_name ${e}) + EPOCH_SHA=$(./wpt rev-list --epoch ${EPOCH}) + if [ "${EPOCH_SHA}" = "" ]; then + echo "ERROR: Empty SHA returned from ./wpt rev-list" + exit 1 + fi + git branch "${EPOCH_BRANCH_NAME}" "${EPOCH_SHA}" + + # Only set epoch tag if is not already tagged from a previous run. + if ! git tag --points-at "${EPOCH_SHA}" | grep "${EPOCH_BRANCH_NAME}"; then + EPOCH_STAMP="$(date +%Y-%m-%d_%HH)" + git tag "${EPOCH_BRANCH_NAME}/${EPOCH_STAMP}" "${EPOCH_SHA}" + fi + + ALL_BRANCHES_NAMES="${ALL_BRANCHES_NAMES} ${EPOCH_BRANCH_NAME}" + done + # This is safe because `git push` will by default fail for a non-fast-forward + # push, for example if the remote branch is ahead of the local branch. + git push --tags ${REMOTE} ${ALL_BRANCHES_NAMES} +} + +cd $WPT_ROOT + +if [ -z "$GITHUB_TOKEN" ]; then + echo "GITHUB_TOKEN must be set as an environment variable" + exit 1 +fi + +REMOTE=https://x-access-token:$GITHUB_TOKEN@github.com/web-platform-tests/wpt.git + +main diff --git a/testing/web-platform/tests/tools/ci/interfaces_update.sh b/testing/web-platform/tests/tools/ci/interfaces_update.sh new file mode 100755 index 0000000000..6bf7c7c712 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/interfaces_update.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -eux -o pipefail + +SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P) +WPT_ROOT=$SCRIPT_DIR/../.. + +main () { + # Find the latest version of the package to install. + VERSION=$(npm info @webref/idl version) + + # Install @webref/idl in a temporary directory. + TMPDIR=$(mktemp -d) + cd $TMPDIR + npm install @webref/idl@$VERSION + + # Delete interfaces/*.idl except tentative ones + cd $WPT_ROOT + find interfaces/ -name '*.idl' -not -name '*.tentative.idl' -delete + + # Handle cssom.idl with preamble first. + cat <<EOF > interfaces/cssom.idl +// GENERATED PREAMBLE - DO NOT EDIT +// CSSOMString is an implementation-defined type of either DOMString or +// USVString in CSSOM: https://drafts.csswg.org/cssom/#cssomstring-type +// For web-platform-tests, use DOMString because USVString has additional +// requirements in type conversion and could result in spurious failures for +// implementations that use DOMString. +typedef DOMString CSSOMString; + +EOF + cat $TMPDIR/node_modules/@webref/idl/cssom.idl >> interfaces/cssom.idl + rm $TMPDIR/node_modules/@webref/idl/cssom.idl + + # Move remaining *.idl from @webref/idl to interfaces/ + mv $TMPDIR/node_modules/@webref/idl/*.idl interfaces/ + + # Cleanup + rm -rf $TMPDIR + + if [ -n "$GITHUB_ENV" ]; then + echo webref_idl_version=$VERSION >> $GITHUB_ENV + fi +} + +main diff --git a/testing/web-platform/tests/tools/ci/jobs.py b/testing/web-platform/tests/tools/ci/jobs.py new file mode 100644 index 0000000000..44de9fe1ad --- /dev/null +++ b/testing/web-platform/tests/tools/ci/jobs.py @@ -0,0 +1,150 @@ +# mypy: allow-untyped-defs + +import argparse +import os +import re +from ..wpt.testfiles import branch_point, files_changed + +from tools import localpaths # noqa: F401 + +wpt_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) + +# Common exclusions between affected_tests and stability jobs. +# Files in these dirs would trigger the execution of too many tests. +EXCLUDES = [ + "!tools/", + "!docs/", + "!conformance-checkers/", + "!.*/OWNERS", + "!.*/META.yml", + "!.*/tools/", + "!.*/README", + "!css/[^/]*$" +] + +# Rules are just regex on the path, with a leading ! indicating a regex that must not +# match for the job. Paths should be kept in sync with update-built-tests.sh. +job_path_map = { + "affected_tests": [".*/.*", "!resources/(?!idlharness.js)"] + EXCLUDES, + "stability": [".*/.*", "!resources/.*"] + EXCLUDES, + "lint": [".*"], + "manifest_upload": [".*"], + "resources_unittest": ["resources/", "tools/"], + "tools_unittest": ["tools/"], + "wptrunner_unittest": ["tools/"], + "update_built": ["update-built-tests\\.sh", + "conformance-checkers/", + "css/css-ui/", + "css/css-writing-modes/", + "html/", + "infrastructure/", + "mimesniff/"], + "wpt_integration": ["tools/"], + "wptrunner_infrastructure": ["infrastructure/", + "tools/", + "resources/", + "webdriver/tests/support"], +} + + +def _path_norm(path): + """normalize a path for both case and slashes (to /)""" + path = os.path.normcase(path) + if os.path.sep != "/": + # this must be after the normcase call as that does slash normalization + path = path.replace(os.path.sep, "/") + return path + + +class Ruleset: + def __init__(self, rules): + self.include = [] + self.exclude = [] + for rule in rules: + rule = _path_norm(rule) + self.add_rule(rule) + + def add_rule(self, rule): + if rule.startswith("!"): + target = self.exclude + rule = rule[1:] + else: + target = self.include + + target.append(re.compile("^%s" % rule)) + + def __call__(self, path): + path = _path_norm(path) + for item in self.exclude: + if item.match(path): + return False + for item in self.include: + if item.match(path): + return True + return False + + def __repr__(self): + subs = tuple(",".join(item.pattern for item in target) + for target in (self.include, self.exclude)) + return "Rules<include:[%s] exclude:[%s]>" % subs + + +def get_paths(**kwargs): + if kwargs["revish"] is None: + revish = "%s..HEAD" % branch_point() + else: + revish = kwargs["revish"] + + changed, _ = files_changed(revish, ignore_rules=[]) + all_changed = {os.path.relpath(item, wpt_root) for item in set(changed)} + return all_changed + + +def get_jobs(paths, **kwargs): + if kwargs.get("all"): + return set(job_path_map.keys()) + + jobs = set() + + rules = {} + includes = kwargs.get("includes") + if includes is not None: + includes = set(includes) + for key, value in job_path_map.items(): + if includes is None or key in includes: + rules[key] = Ruleset(value) + + for path in paths: + for job in list(rules.keys()): + ruleset = rules[job] + if ruleset(path): + rules.pop(job) + jobs.add(job) + if not rules: + break + + # Default jobs should run even if there were no changes + if not paths: + for job, path_re in job_path_map.items(): + if ".*" in path_re: + jobs.add(job) + + return jobs + + +def create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("revish", default=None, help="Commits to consider. Defaults to the commits on the current branch", nargs="?") + parser.add_argument("--all", help="List all jobs unconditionally.", action="store_true") + parser.add_argument("--includes", default=None, help="Jobs to check for. Return code is 0 if all jobs are found, otherwise 1", nargs="*") + return parser + + +def run(**kwargs): + paths = get_paths(**kwargs) + jobs = get_jobs(paths, **kwargs) + if not kwargs["includes"]: + for item in sorted(jobs): + print(item) + else: + return 0 if set(kwargs["includes"]).issubset(jobs) else 1 diff --git a/testing/web-platform/tests/tools/ci/macos_color_profile.py b/testing/web-platform/tests/tools/ci/macos_color_profile.py new file mode 100644 index 0000000000..f0919d7715 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/macos_color_profile.py @@ -0,0 +1,43 @@ +from typing import Any + +from Cocoa import NSURL +from ColorSync import ( + CGDisplayCreateUUIDFromDisplayID, + ColorSyncDeviceSetCustomProfiles, + kColorSyncDeviceDefaultProfileID, + kColorSyncDisplayDeviceClass, +) +from Quartz import ( + CGGetOnlineDisplayList, + kCGErrorSuccess, +) + + +def set_all_displays(profile_url: NSURL) -> bool: + max_displays = 10 + + (err, display_ids, display_count) = CGGetOnlineDisplayList(max_displays, None, None) + if err != kCGErrorSuccess: + raise ValueError(err) + + display_uuids = [CGDisplayCreateUUIDFromDisplayID(d) for d in display_ids] + + for display_uuid in display_uuids: + profile_info = {kColorSyncDeviceDefaultProfileID: profile_url} + + success = ColorSyncDeviceSetCustomProfiles( + kColorSyncDisplayDeviceClass, + display_uuid, + profile_info, + ) + if not success: + raise Exception(f"failed to set profile on {display_uuid}") + + return True + + +def run(venv: Any, **kwargs: Any) -> None: + srgb_profile_url = NSURL.fileURLWithPath_( + "/System/Library/ColorSync/Profiles/sRGB Profile.icc" + ) + set_all_displays(srgb_profile_url) diff --git a/testing/web-platform/tests/tools/ci/make_hosts_file.py b/testing/web-platform/tests/tools/ci/make_hosts_file.py new file mode 100644 index 0000000000..f3397281b7 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/make_hosts_file.py @@ -0,0 +1,23 @@ +# mypy: allow-untyped-defs + +import argparse +import os + +from ..localpaths import repo_root + +from ..serve.serve import build_config, make_hosts_file + + +def create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("address", default="127.0.0.1", nargs="?", + help="Address that hosts should point at") + return parser + + +def run(**kwargs): + config_builder = build_config(os.path.join(repo_root, "config.json"), + ssl={"type": "none"}) + + with config_builder as config: + print(make_hosts_file(config, kwargs["address"])) diff --git a/testing/web-platform/tests/tools/ci/manifest_build.py b/testing/web-platform/tests/tools/ci/manifest_build.py new file mode 100644 index 0000000000..15b8679f00 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/manifest_build.py @@ -0,0 +1,226 @@ +# mypy: allow-untyped-defs + +import json +import logging +import os +import subprocess +import sys +import tempfile + +import requests + +from pathlib import Path + + +here = os.path.abspath(os.path.dirname(__file__)) +wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir)) + +if wpt_root not in sys.path: + sys.path.append(wpt_root) + +from tools.wpt.testfiles import get_git_cmd + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class Status: + SUCCESS = 0 + FAIL = 1 + + +def run(cmd, return_stdout=False, **kwargs): + logger.info(" ".join(cmd)) + if return_stdout: + f = subprocess.check_output + else: + f = subprocess.check_call + return f(cmd, **kwargs) + + +def create_manifest(path): + run(["./wpt", "manifest", "-p", path]) + + +def create_web_feature_manifest(path): + run(["./wpt", "web-features-manifest", "-p", path]) + + +def compress_manifest(path): + for args in [["gzip", "-k", "-f", "--best"], + ["bzip2", "-k", "-f", "--best"], + ["zstd", "-k", "-f", "--ultra", "-22", "-q"]]: + run(args + [path]) + + +def request(url, desc, method=None, data=None, json_data=None, params=None, headers=None): + github_token = os.environ.get("GITHUB_TOKEN") + default_headers = { + "Authorization": "token %s" % github_token, + "Accept": "application/vnd.github.machine-man-preview+json" + } + + _headers = default_headers + if headers is not None: + _headers.update(headers) + + kwargs = {"params": params, + "headers": _headers} + try: + logger.info("Requesting URL %s" % url) + if json_data is not None or data is not None: + if method is None: + method = requests.post + kwargs["json"] = json_data + kwargs["data"] = data + elif method is None: + method = requests.get + + resp = method(url, **kwargs) + + except Exception as e: + logger.error(f"{desc} failed:\n{e}") + return None + + try: + resp.raise_for_status() + except requests.HTTPError: + logger.error("%s failed: Got HTTP status %s. Response:" % + (desc, resp.status_code)) + logger.error(resp.text) + return None + + try: + return resp.json() + except ValueError: + logger.error("%s failed: Returned data was not JSON Response:" % desc) + logger.error(resp.text) + + +def get_pr(owner, repo, sha): + data = request("https://api.github.com/search/issues?q=type:pr+is:merged+repo:%s/%s+sha:%s" % + (owner, repo, sha), "Getting PR") + if data is None: + return None + + items = data["items"] + if len(items) == 0: + logger.error("No PR found for %s" % sha) + return None + if len(items) > 1: + logger.warning("Found multiple PRs for %s" % sha) + + pr = items[0] + + return pr["number"] + + +def get_file_upload_details(manifest_path, sha): + """ + For a given file, generate details used to upload to GitHub. + """ + path = Path(manifest_path) + stem = path.stem + extension = path.suffix + upload_filename_prefix = f"{stem}-{sha}{extension}" + upload_label_prefix = path.name + upload_desc = f"{stem.title()} upload" + return upload_filename_prefix, upload_label_prefix, upload_desc + + +def create_release(manifest_file_paths, owner, repo, sha, tag, body): + logger.info(f"Creating a release for tag='{tag}', target_commitish='{sha}'") + create_url = f"https://api.github.com/repos/{owner}/{repo}/releases" + create_data = {"tag_name": tag, + "target_commitish": sha, + "name": tag, + "body": body, + "draft": True} + create_resp = request(create_url, "Release creation", json_data=create_data) + if not create_resp: + return False + + # Upload URL contains '{?name,label}' at the end which we want to remove + upload_url = create_resp["upload_url"].split("{", 1)[0] + + upload_exts = [".gz", ".bz2", ".zst"] + for manifest_path in manifest_file_paths: + upload_filename_prefix, upload_label_prefix, upload_desc = get_file_upload_details(manifest_path, sha) + for upload_ext in upload_exts: + upload_filename = f"{upload_filename_prefix}{upload_ext}" + params = {"name": upload_filename, + "label": f"{upload_label_prefix}{upload_ext}"} + + with open(f"{manifest_path}{upload_ext}", "rb") as f: + upload_data = f.read() + + logger.info("Uploading %s bytes" % len(upload_data)) + + upload_resp = request(upload_url, upload_desc, data=upload_data, params=params, + headers={'Content-Type': 'application/octet-stream'}) + if not upload_resp: + return False + + release_id = create_resp["id"] + edit_url = f"https://api.github.com/repos/{owner}/{repo}/releases/{release_id}" + edit_data = create_data.copy() + edit_data["draft"] = False + edit_resp = request(edit_url, "Release publishing", method=requests.patch, json_data=edit_data) + if not edit_resp: + return False + + logger.info("Released %s" % edit_resp["html_url"]) + return True + + +def should_dry_run(): + with open(os.environ["GITHUB_EVENT_PATH"]) as f: + event = json.load(f) + logger.info(json.dumps(event, indent=2)) + + if "pull_request" in event: + logger.info("Dry run for PR") + return True + if event.get("ref") != "refs/heads/master": + logger.info("Dry run for ref %s" % event.get("ref")) + return True + return False + + +def main(): + dry_run = should_dry_run() + + manifest_path = os.path.join(tempfile.mkdtemp(), "MANIFEST.json") + web_features_manifest_path = os.path.join(tempfile.mkdtemp(), "WEB_FEATURES_MANIFEST.json") + + create_manifest(manifest_path) + create_web_feature_manifest(web_features_manifest_path) + + compress_manifest(manifest_path) + compress_manifest(web_features_manifest_path) + + owner, repo = os.environ["GITHUB_REPOSITORY"].split("/", 1) + + git = get_git_cmd(wpt_root) + head_rev = git("rev-parse", "HEAD").strip() + body = git("show", "--no-patch", "--format=%B", "HEAD") + + if dry_run: + return Status.SUCCESS + + pr = get_pr(owner, repo, head_rev) + if pr is None: + return Status.FAIL + tag_name = "merge_pr_%s" % pr + + manifest_paths = [manifest_path, web_features_manifest_path] + if not create_release(manifest_paths, owner, repo, head_rev, tag_name, body): + return Status.FAIL + + return Status.SUCCESS + + +if __name__ == "__main__": + code = main() # type: ignore + assert isinstance(code, int) + sys.exit(code) diff --git a/testing/web-platform/tests/tools/ci/regen_certs.py b/testing/web-platform/tests/tools/ci/regen_certs.py new file mode 100644 index 0000000000..8f3abdcad6 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/regen_certs.py @@ -0,0 +1,102 @@ +# mypy: allow-untyped-defs + +import argparse +import base64 +import logging +import subprocess +import sys + + +logger = logging.getLogger(__name__) + + +# TODO(Issue #24180): Regenerate SXG fingerprint too. +CHROME_SPKI_CERTS_CONTENT = """\ +# This file is automatically generated by 'wpt regen-certs' +# DO NOT EDIT MANUALLY. + +# tools/certs/web-platform.test.pem +WPT_FINGERPRINT = '{wpt_fingerprint}' + +# signed-exchange/resources/127.0.0.1.sxg.pem +SXG_WPT_FINGERPRINT = '0Rt4mT6SJXojEMHTnKnlJ/hBKMBcI4kteBlhR1eTTdk=' + +IGNORE_CERTIFICATE_ERRORS_SPKI_LIST = [ + WPT_FINGERPRINT, + SXG_WPT_FINGERPRINT +] +""" + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--checkend-seconds", type=int, default=5184000, + help="The number of seconds the certificates must be valid for") + parser.add_argument("--force", action="store_true", + help="Regenerate certificates even if not reaching expiry") + return parser + + +def check_cert(certificate, checkend_seconds): + """Checks whether an x509 certificate will expire within a set period. + + Returns 0 if the certificate will not expire, non-zero otherwise.""" + cmd = [ + "openssl", "x509", + "-checkend", str(checkend_seconds), + "-noout", + "-in", certificate + ] + logger.info("Running '%s'" % " ".join(cmd)) + return subprocess.call(cmd) + + +def regen_certs(): + """Regenerate the wpt openssl certificates, by delegating to wptserve.""" + cmd = [ + sys.executable, "wpt", "serve", + "--config", "tools/certs/config.json", + "--exit-after-start", + ] + logger.info("Running '%s'" % " ".join(cmd)) + subprocess.check_call(cmd) + + +def regen_chrome_spki(): + """Regenerate the SPKI fingerprints for Chrome's ignore-cert list. + + Chrome requires us to explicitly list which certificates are ignored by its + security-checking, by listing a base64 hash of the public key. This will + change every time we replace our certificates, so we store the hashes in a + file and regenerate it here. + """ + wpt_spki = calculate_spki("tools/certs/web-platform.test.pem") + with open("tools/wptrunner/wptrunner/browsers/chrome_spki_certs.py", "w") as f: + f.write(CHROME_SPKI_CERTS_CONTENT.format(wpt_fingerprint=wpt_spki)) + + +def calculate_spki(cert_path): + """Calculate the SPKI fingerprint for a given x509 certificate.""" + # We use shell=True as we control the input |cert_path|, and piping + # correctly across processes is non-trivial in Python. + cmd = (f"openssl x509 -noout -pubkey -in {cert_path} | " + + "openssl pkey -pubin -outform der | " + + "openssl dgst -sha256 -binary") + dgst_output = subprocess.check_output(cmd, shell=True) + + return base64.b64encode(dgst_output).decode('utf-8') + + +def run(**kwargs): + logging.basicConfig() + + if kwargs["force"]: + logger.info("Force regenerating WPT certificates") + checkend_seconds = kwargs["checkend_seconds"] + if (kwargs["force"] or + check_cert("tools/certs/cacert.pem", checkend_seconds) or + check_cert("tools/certs/web-platform.test.pem", checkend_seconds)): + regen_certs() + regen_chrome_spki() + else: + logger.info("Certificates are still valid for at least %s seconds, skipping regeneration" % checkend_seconds) diff --git a/testing/web-platform/tests/tools/ci/requirements_build.txt b/testing/web-platform/tests/tools/ci/requirements_build.txt new file mode 100644 index 0000000000..54f21efbd9 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/requirements_build.txt @@ -0,0 +1,5 @@ +cairocffi==1.6.1 +fonttools==4.47.2 +genshi==0.7.7 +jinja2==3.1.3 +pyyaml==6.0.1 diff --git a/testing/web-platform/tests/tools/ci/requirements_macos_color_profile.txt b/testing/web-platform/tests/tools/ci/requirements_macos_color_profile.txt new file mode 100644 index 0000000000..7505a98d9f --- /dev/null +++ b/testing/web-platform/tests/tools/ci/requirements_macos_color_profile.txt @@ -0,0 +1,4 @@ +pyobjc-core==9.2 +pyobjc-framework-Cocoa==9.2 +pyobjc-framework-ColorSync==9.2 +pyobjc-framework-Quartz==9.2 diff --git a/testing/web-platform/tests/tools/ci/requirements_tc.txt b/testing/web-platform/tests/tools/ci/requirements_tc.txt new file mode 100644 index 0000000000..8151646605 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/requirements_tc.txt @@ -0,0 +1,4 @@ +pygithub==2.2.0 +pyyaml==6.0.1 +requests==2.31.0 +taskcluster==60.3.2 diff --git a/testing/web-platform/tests/tools/ci/run_tc.py b/testing/web-platform/tests/tools/ci/run_tc.py new file mode 100755 index 0000000000..446afadac8 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/run_tc.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs + +"""Wrapper script for running jobs in Taskcluster + +This is intended for running test jobs in Taskcluster. The script +takes a two positional arguments which are the name of the test job +and the script to actually run. + +The name of the test job is used to determine whether the script should be run +for this push (this is in lieu of having a proper decision task). There are +several ways that the script can be scheduled to run + +1. The output of wpt test-jobs includes the job name +2. The job name is included in a job declaration (see below) +3. The string "all" is included in the job declaration +4. The job name is set to "all" + +A job declaration is a line appearing in the pull request body (for +pull requests) or first commit message (for pushes) of the form: + +tc-jobs: job1,job2,[...] + +In addition, there are a number of keyword arguments used to set options for the +environment in which the jobs run. Documentation for these is in the command help. + +As well as running the script, the script sets two environment variables; +GITHUB_BRANCH which is the branch that the commits will merge into (if it's a PR) +or the branch that the commits are on (if it's a push), and GITHUB_PULL_REQUEST +which is the string "false" if the event triggering this job wasn't a pull request +or the pull request number if it was. The semantics of these variables are chosen +to match the corresponding TRAVIS_* variables. + +Note: for local testing in the Docker image the script ought to still work, but +full functionality requires that the TASK_EVENT environment variable is set to +the serialization of a GitHub event payload. +""" + +import argparse +import fnmatch +import json +import os +import subprocess +import sys +import tarfile +import tempfile +import zipfile + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +from tools.wpt.utils import get_download_to_descriptor + +root = os.path.abspath( + os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) + + +def run(cmd, return_stdout=False, **kwargs): + print(" ".join(cmd)) + if return_stdout: + f = subprocess.check_output + if "encoding" not in kwargs: + kwargs["encoding"] = "utf-8" + else: + f = subprocess.check_call + return f(cmd, **kwargs) + + +def start(cmd): + print(" ".join(cmd)) + subprocess.Popen(cmd) + + +def get_parser(): + p = argparse.ArgumentParser() + p.add_argument("--oom-killer", + action="store_true", + default=False, + help="Run userspace OOM killer") + p.add_argument("--hosts", + dest="hosts_file", + action="store_true", + default=True, + help="Setup wpt entries in hosts file") + p.add_argument("--no-hosts", + dest="hosts_file", + action="store_false", + help="Don't setup wpt entries in hosts file") + p.add_argument("--browser", + action="append", + default=[], + help="Browsers that will be used in the job") + p.add_argument("--channel", + default=None, + choices=["experimental", "canary", "dev", "nightly", "beta", "stable"], + help="Chrome browser channel") + p.add_argument("--xvfb", + action="store_true", + help="Start xvfb") + p.add_argument("--install-certificates", action="store_true", default=None, + help="Install web-platform.test certificates to UA store") + p.add_argument("--no-install-certificates", action="store_false", default=None, + help="Don't install web-platform.test certificates to UA store") + p.add_argument("--no-setup-repository", action="store_false", dest="setup_repository", + help="Don't run any repository setup steps, instead use the existing worktree. " + "This is useful for local testing.") + p.add_argument("--checkout", + help="Revision to checkout before starting job") + p.add_argument("--ref", + help="Git ref for the commit that should be run") + p.add_argument("--head-rev", + help="Commit at the head of the branch when the decision task ran") + p.add_argument("--merge-rev", + help="Provisional merge commit for PR when the decision task ran") + p.add_argument("script", + help="Script to run for the job") + p.add_argument("script_args", + nargs=argparse.REMAINDER, + help="Additional arguments to pass to the script") + return p + + +def start_userspace_oom_killer(): + # Start userspace OOM killer: https://github.com/rfjakob/earlyoom + # It will report memory usage every minute and prefer to kill browsers. + start(["sudo", "earlyoom", "-p", "-r", "60", "--prefer=(chrome|firefox)", "--avoid=python"]) + + +def make_hosts_file(): + run(["sudo", "sh", "-c", "./wpt make-hosts-file >> /etc/hosts"]) + + +def checkout_revision(rev): + run(["git", "checkout", "--quiet", rev]) + + +def install_certificates(): + run(["sudo", "cp", "tools/certs/cacert.pem", + "/usr/local/share/ca-certificates/cacert.crt"]) + run(["sudo", "update-ca-certificates"]) + + +def start_dbus(): + run(["sudo", "service", "dbus", "start"]) + # Enable dbus autolaunch for Chrome + # https://source.chromium.org/chromium/chromium/src/+/main:content/app/content_main.cc;l=220;drc=0bcc023b8cdbc073aa5c48db373810db3f765c87. + os.environ["DBUS_SESSION_BUS_ADDRESS"] = "autolaunch:" + + +def install_chrome(channel): + if channel == "canary": + # Chrome for Testing Canary is installed via --install-browser + return + if channel in ("experimental", "dev"): + deb_archive = "google-chrome-unstable_current_amd64.deb" + elif channel == "beta": + deb_archive = "google-chrome-beta_current_amd64.deb" + elif channel == "stable": + deb_archive = "google-chrome-stable_current_amd64.deb" + else: + raise ValueError("Unrecognized release channel: %s" % channel) + + dest = os.path.join("/tmp", deb_archive) + deb_url = "https://dl.google.com/linux/direct/%s" % deb_archive + with open(dest, "wb") as f: + get_download_to_descriptor(f, deb_url) + + run(["sudo", "apt-get", "-qqy", "update"]) + run(["sudo", "gdebi", "-qn", "/tmp/%s" % deb_archive]) + + +def start_xvfb(): + start(["sudo", "Xvfb", os.environ["DISPLAY"], "-screen", "0", + "%sx%sx%s" % (os.environ["SCREEN_WIDTH"], + os.environ["SCREEN_HEIGHT"], + os.environ["SCREEN_DEPTH"])]) + start(["sudo", "fluxbox", "-display", os.environ["DISPLAY"]]) + + +def set_variables(event): + # Set some variables that we use to get the commits on the current branch + ref_prefix = "refs/heads/" + pull_request = "false" + branch = None + if "pull_request" in event: + pull_request = str(event["pull_request"]["number"]) + # Note that this is the branch that a PR will merge to, + # not the branch name for the PR + branch = event["pull_request"]["base"]["ref"] + elif "ref" in event: + branch = event["ref"] + if branch.startswith(ref_prefix): + branch = branch[len(ref_prefix):] + + os.environ["GITHUB_PULL_REQUEST"] = pull_request + if branch: + os.environ["GITHUB_BRANCH"] = branch + + +def task_url(task_id): + root_url = os.environ['TASKCLUSTER_ROOT_URL'] + if root_url == 'https://taskcluster.net': + queue_base = "https://queue.taskcluster.net/v1/task" + else: + queue_base = root_url + "/api/queue/v1/task" + + return "%s/%s" % (queue_base, task_id) + + +def download_artifacts(artifacts): + artifact_list_by_task = {} + for artifact in artifacts: + base_url = task_url(artifact["task"]) + if artifact["task"] not in artifact_list_by_task: + with tempfile.TemporaryFile() as f: + get_download_to_descriptor(f, base_url + "/artifacts") + f.seek(0) + artifacts_data = json.load(f) + artifact_list_by_task[artifact["task"]] = artifacts_data + + artifacts_data = artifact_list_by_task[artifact["task"]] + print("DEBUG: Got artifacts %s" % artifacts_data) + found = False + for candidate in artifacts_data["artifacts"]: + print("DEBUG: candidate: %s glob: %s" % (candidate["name"], artifact["glob"])) + if fnmatch.fnmatch(candidate["name"], artifact["glob"]): + found = True + print("INFO: Fetching aritfact %s from task %s" % (candidate["name"], artifact["task"])) + file_name = candidate["name"].rsplit("/", 1)[1] + url = base_url + "/artifacts/" + candidate["name"] + dest_path = os.path.expanduser(os.path.join("~", artifact["dest"], file_name)) + dest_dir = os.path.dirname(dest_path) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + with open(dest_path, "wb") as f: + get_download_to_descriptor(f, url) + + if artifact.get("extract"): + unpack(dest_path) + if not found: + print("WARNING: No artifact found matching %s in task %s" % (artifact["glob"], artifact["task"])) + + +def unpack(path): + dest = os.path.dirname(path) + if tarfile.is_tarfile(path): + run(["tar", "-xf", path], cwd=os.path.dirname(path)) + elif zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as archive: + archive.extractall(dest) + else: + print("ERROR: Don't know how to extract %s" % path) + raise Exception + + +def setup_environment(args): + if "TASK_ARTIFACTS" in os.environ: + artifacts = json.loads(os.environ["TASK_ARTIFACTS"]) + download_artifacts(artifacts) + + if args.hosts_file: + make_hosts_file() + + if args.install_certificates: + install_certificates() + + if "chrome" in args.browser: + assert args.channel is not None + install_chrome(args.channel) + # Chrome is using dbus for various features. + start_dbus() + + if args.xvfb: + start_xvfb() + + if args.oom_killer: + start_userspace_oom_killer() + + +def setup_repository(args): + is_pr = os.environ.get("GITHUB_PULL_REQUEST", "false") != "false" + + # Initially task_head points at the same commit as the ref we want to test. + # However that may not be the same commit as we actually want to test if + # the branch changed since the decision task ran. The branch may have + # changed because someone has pushed more commits (either to the PR + # or later commits to the branch), or because someone has pushed to the + # base branch for the PR. + # + # In that case we take a different approach depending on whether this is a + # PR or a push to a branch. + # If this is a push to a branch, and the original commit is still fetchable, + # we try to fetch that (it may not be in the case of e.g. a force push). + # If it's not fetchable then we fail the run. + # For a PR we are testing the provisional merge commit. If that's changed it + # could be that the PR branch was updated or the base branch was updated. In the + # former case we fail the run because testing an old commit is a waste of + # resources. In the latter case we assume it's OK to use the current merge + # instead of the one at the time the decision task ran. + + if args.ref: + if is_pr: + assert args.ref.endswith("/merge") + expected_head = args.merge_rev + else: + expected_head = args.head_rev + + task_head = run(["git", "rev-parse", "task_head"], return_stdout=True).strip() + + if task_head != expected_head: + if not is_pr: + try: + run(["git", "fetch", "origin", expected_head]) + run(["git", "reset", "--hard", expected_head]) + except subprocess.CalledProcessError: + print("CRITICAL: task_head points at %s, expected %s and " + "unable to fetch expected commit.\n" + "This may be because the branch was updated" % (task_head, expected_head)) + sys.exit(1) + else: + # Convert the refs/pulls/<id>/merge to refs/pulls/<id>/head + head_ref = args.ref.rsplit("/", 1)[0] + "/head" + try: + remote_head = run(["git", "ls-remote", "origin", head_ref], + return_stdout=True).split("\t")[0] + except subprocess.CalledProcessError: + print("CRITICAL: Failed to read remote ref %s" % head_ref) + sys.exit(1) + if remote_head != args.head_rev: + print("CRITICAL: task_head points at %s, expected %s. " + "This may be because the branch was updated" % (task_head, expected_head)) + sys.exit(1) + print("INFO: Merge commit changed from %s to %s due to base branch changes. " + "Running task anyway." % (expected_head, task_head)) + + if os.environ.get("GITHUB_PULL_REQUEST", "false") != "false": + parents = run(["git", "rev-parse", "task_head^@"], + return_stdout=True).strip().split() + if len(parents) == 2: + base_head = parents[0] + pr_head = parents[1] + + run(["git", "branch", "base_head", base_head]) + run(["git", "branch", "pr_head", pr_head]) + else: + print("ERROR: Pull request HEAD wasn't a 2-parent merge commit; " + "expected to test the merge of PR into the base") + commit = run(["git", "rev-parse", "task_head"], + return_stdout=True).strip() + print("HEAD: %s" % commit) + print("Parents: %s" % ", ".join(parents)) + sys.exit(1) + + branch = os.environ.get("GITHUB_BRANCH") + if branch: + # Ensure that the remote base branch exists + # TODO: move this somewhere earlier in the task + run(["git", "fetch", "--quiet", "origin", "%s:%s" % (branch, branch)]) + + checkout_rev = args.checkout if args.checkout is not None else "task_head" + checkout_revision(checkout_rev) + + refs = run(["git", "for-each-ref", "refs/heads"], return_stdout=True) + print("INFO: git refs:\n%s" % refs) + print("INFO: checked out commit:\n%s" % run(["git", "rev-parse", "HEAD"], + return_stdout=True)) + + +def fetch_event_data(): + try: + task_id = os.environ["TASK_ID"] + except KeyError: + print("WARNING: Missing TASK_ID environment variable") + # For example under local testing + return None + + with tempfile.TemporaryFile() as f: + get_download_to_descriptor(f, task_url(task_id)) + f.seek(0) + task_data = json.load(f) + event_data = task_data.get("extra", {}).get("github_event") + if event_data is not None: + return json.loads(event_data) + + +def include_job(job): + # Only for supporting pre decision-task PRs + # Special case things that unconditionally run on pushes, + # assuming a higher layer is filtering the required list of branches + if "GITHUB_PULL_REQUEST" not in os.environ: + return True + + if (os.environ["GITHUB_PULL_REQUEST"] == "false" and + job == "run-all"): + return True + + jobs_str = run([os.path.join(root, "wpt"), + "test-jobs"], return_stdout=True) + print(jobs_str) + return job in set(jobs_str.splitlines()) + + +def main(): + args = get_parser().parse_args() + + if "TASK_EVENT" in os.environ: + event = json.loads(os.environ["TASK_EVENT"]) + else: + event = fetch_event_data() + + if event: + set_variables(event) + + if args.setup_repository: + setup_repository(args) + + # Hack for backwards compatibility + if args.script in ["run-all", "lint", "update_built", "tools_unittest", + "wpt_integration", "resources_unittest", + "wptrunner_infrastructure", "stability", "affected_tests"]: + job = args.script + if not include_job(job): + return + args.script = args.script_args[0] + args.script_args = args.script_args[1:] + + # Run the job + setup_environment(args) + os.chdir(root) + cmd = [args.script] + args.script_args + print(" ".join(cmd)) + sys.exit(subprocess.call(cmd)) + + +if __name__ == "__main__": + main() # type: ignore diff --git a/testing/web-platform/tests/tools/ci/taskcluster-run.py b/testing/web-platform/tests/tools/ci/taskcluster-run.py new file mode 100755 index 0000000000..fe9538e316 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/taskcluster-run.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs + +import argparse +import gzip +import logging +import os +import shutil +import subprocess +import sys + + +def get_browser_args(product, channel, artifact_path): + if product == "firefox": + local_binary = os.path.expanduser(os.path.join("~", "build", "firefox", "firefox")) + if os.path.exists(local_binary): + return ["--binary=%s" % local_binary] + print("WARNING: Local firefox binary not found") + return ["--install-browser", "--install-webdriver"] + if product == "firefox_android": + return ["--install-browser", "--install-webdriver", "--logcat-dir", artifact_path] + if product == "servo": + return ["--install-browser", "--processes=12"] + if product == "chrome" or product == "chromium": + # Taskcluster machines do not have GPUs, so use software rendering via --enable-swiftshader. + return ["--enable-swiftshader", "--install-browser", "--install-webdriver"] + if product == "webkitgtk_minibrowser": + # Using 4 parallel jobs gives 4x speed-up even on a 1-core machine and doesn't cause extra timeouts. + # See: https://github.com/web-platform-tests/wpt/issues/38723#issuecomment-1470938179 + return ["--install-browser", "--processes=4"] + return [] + + +def find_wptreport(args): + parser = argparse.ArgumentParser() + parser.add_argument('--log-wptreport', action='store') + return parser.parse_known_args(args)[0].log_wptreport + + +def find_wptscreenshot(args): + parser = argparse.ArgumentParser() + parser.add_argument('--log-wptscreenshot', action='store') + return parser.parse_known_args(args)[0].log_wptscreenshot + + +def gzip_file(filename, delete_original=True): + with open(filename, 'rb') as f_in: + with gzip.open('%s.gz' % filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + if delete_original: + os.unlink(filename) + + +def main(product, channel, commit_range, artifact_path, wpt_args): + """Invoke the `wpt run` command according to the needs of the Taskcluster + continuous integration service.""" + + logger = logging.getLogger("tc-run") + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + logger.addHandler(handler) + + subprocess.call(['python3', './wpt', 'manifest-download']) + + if commit_range: + logger.info( + "Running tests affected in range '%s'..." % commit_range + ) + wpt_args += ['--affected', commit_range] + else: + logger.info("Running all tests") + + wpt_args += [ + "--log-mach-level=info", + "--log-mach=-", + "-y", + "--no-pause", + "--no-restart-on-unexpected", + "--install-fonts", + "--no-headless", + "--verify-log-full" + ] + wpt_args += get_browser_args(product, channel, artifact_path) + + # Hack to run servo with one process only for wdspec + if product == "servo" and "--test-type=wdspec" in wpt_args: + wpt_args = [item for item in wpt_args if not item.startswith("--processes")] + + wpt_args.append(product) + + command = ["python3", "./wpt", "run"] + wpt_args + + logger.info("Executing command: %s" % " ".join(command)) + with open(os.path.join(artifact_path, "checkrun.md"), "a") as f: + f.write("\n**WPT Command:** `%s`\n\n" % " ".join(command)) + + retcode = subprocess.call(command, env=dict(os.environ, TERM="dumb")) + if retcode != 0: + sys.exit(retcode) + + wptreport = find_wptreport(wpt_args) + if wptreport: + gzip_file(wptreport) + wptscreenshot = find_wptscreenshot(wpt_args) + if wptscreenshot: + gzip_file(wptscreenshot) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument("--commit-range", action="store", + help="""Git commit range. If specified, this will be + supplied to the `wpt tests-affected` command to + determine the list of test to execute""") + parser.add_argument("--artifact-path", action="store", + default="/home/test/artifacts/", + help="Path to store output files") + parser.add_argument("product", action="store", + help="Browser to run tests in") + parser.add_argument("channel", action="store", + help="Channel of the browser") + parser.add_argument("wpt_args", nargs="*", + help="Arguments to forward to `wpt run` command") + main(**vars(parser.parse_args())) # type: ignore diff --git a/testing/web-platform/tests/tools/ci/tc/README.md b/testing/web-platform/tests/tools/ci/tc/README.md new file mode 100644 index 0000000000..785c82cca3 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/README.md @@ -0,0 +1,243 @@ +# Taskgraph Setup + +The taskgraph is built from a YAML file. This file has two top-level +properties: `components` and `tasks`. The full list of tasks is +defined by the `tasks` object; each task is an object with a single +property representing the task with the corresponding value an object +representing the task properties. Each task requires the following +top-level properties: + +* `provisionerId`: String. Name of Taskcluster provisioner +* `schedulerId`: String. Name of Taskcluster scheduler +* `deadline`: String. Time until the task expires +* `image`: String. Name of docker image to use for task +* `maxRunTime`: Number. Maximum time in seconds for which the task can + run. +* `artifacts`: Object. List of artifacts and directories to upload; see + Taskcluster documentation. +* `command`: String. Command to run. This is automatically wrapped in a + run_tc command +* `options`: Optional Object. Options to pass into run_tc + - xvfb: Boolean. Enable Xvfb for run + - oom-killer: Boolean. Enable xvfb for run + - hosts: Boolean. Update hosts file with wpt hosts before run + - install-certificates: Boolean. Install wpt certs into OS + certificate store for run + - browser: List. List of browser names for run + - channel: String. Browser channel for run +* `trigger`: Object. Conditions on which to consider task. One or more + of following properties: + - branch: List. List of branch names on which to trigger. + - pull-request: No value. Trigger for pull request actions +* `schedule-if`: Optional Object. Conditions on which task should be + scheduled given it meets the trigger conditions. + - `run-job`: List. Job names for which this task should be considered, + matching the output from `./wpt test-jobs` +* `env`: Optional Object. Environment variables to set when running task. +* `depends-on`: Optional list. List of task names that must be complete + before the current task is scheduled. +* `description`: String. Task description. +* `name`: Optional String. Name to use for the task overriding the + property name. This is useful in combination with substitutions + described below. +* `download-artifacts`: Optional Object. An artifact to download from + a task that this task depends on. This has the following properties: + - `task` - Name of the task producing the artifact + - `glob` - A glob pattern for the filename of the artifact + - `dest` - A directory reltive to the home directory in which to place + the artifact + - `extract` - Optional. A boolean indicating whether an archive artifact + should be extracted in-place. + +## Task Expansions + +Using the above syntax it's possble to describe each task +directly. But typically in a taskgraph there are many common +properties between tasks so it's tedious and error prone to repeat +information that's common to multiple tasks. Therefore the taskgraph +format provides several mechanisms to reuse partial task definitions +across multiple tasks. + +### Components + +The other top-level property in the taskgraph format is +`components`. The value of this property is an object containing named +partial task definitions. Each task definition may contain a property called +`use` which is a list of components to use as the basis for the task +definition. The components list is evaluated in order. If a property +is not previously defined in the output it is added to the output. If +it was previously defined, the value is updated according to the type: + * Strings and numbers are replaced with a new value + * Lists are extended with the additional values + * Objects are updated recursively following the above rules +This means that types must always match between components and the +final value. + +For example +``` +components: + example-1: + list_prop: + - first + - second + object_prop: + key1: value1 + key2: base_value + example-2: + list_prop: + - third + - fourth + object_prop: + key3: + - value3-1 + +tasks: + - example-task: + use: + - example-1 + - example-2 + object_prop: + key2: value2 + key3: + - value3-2 +``` + +will evaluate to the following task: + +``` +example-task: + list_prop: + - first + - second + - third + - fourth + object_prop: + key1: value1 + key2: value2 + key3: + - value3-1 + - value3-2 +``` + +Note that components cannot currently define `use` properties of their own. + +## Substitutions + +Components and tasks can define a property `vars` that holds variables +which are later substituted into the task definition using the syntax +`${vars.property-name}`. For example: + +``` +components: + generic-component: + prop: ${vars.value} + +tasks: + - first: + use: + - generic-component + vars: + value: value1 + - second: + use: + - generic-component + vars: + value: value2 +``` + +Results in the following tasks: + +``` +first: + prop: value1 +second: + prop: value2 +``` + +## Maps + +Instead of defining a task directly, an item in the tasks property may +be an object with a single property `$map`. This object itself has two +child properties; `for` and `do`. The value of `for` is a list of +objects, and the value of `do` is either an object or a list of +objects. For each object in the `for` property, a set of tasks is +created by taking a copy of that object for each task in the `do` +property, updating the object with the properties from the +corresponding `do` object, using the same rules as for components +above, and then processing as for a normal task. `$map` rules can also +be nested. + +Note: Although `$map` shares a name with the `$map` used in json-e +(used. in `.taskcluster.yml`), the semantics are different. + +For example + +``` +components: {} +tasks: + $map: + for: + - vars: + example: value1 + - vars: + example: value2 + do: + example-${vars.example} + prop: ${vars.example} +``` + +Results in the tasks + +``` +example-value1: + prop: value1 +example-value2: + prop: value2 +``` + +Note that in combination with `$map`, variable substitutions are +applied *twice*; once after the `$map` is evaluated and once after the +`use` statements are evaluated. + +## Chunks + +A common requirements for tasks is that they are "chunked" into N +partial tasks. This is handled specially in the syntax. A top level +property `chunks` can be used to define the number of individual +chunks to create for a specific task. Each chunked task is created +with a `chunks` property set to an object containing an `id` property +containing the one-based index of the chunk an a `total` property +containing the total number of chunks. These can be substituted into +the task definition using the same syntax as for `vars` above +e.g. `${chunks.id}`. Note that because task names must be unique, it's +common to specify a `name` property on the task that will override the +property name e.g. + +``` +components: {} +tasks: + - chunked-task: + chunks:2 + command: "task-run --chunk=${chunks.id} --totalChunks=${chunks.total}" + name: task-chunk-${chunks.id} +``` + +creates tasks: + +``` +task-chunk-1: + command: "task-run --chunk=1 --totalChunks=2" +task-chunk-2: + command: "task-run --chunk=2 --totalChunks=2" +``` + +# Overall processing model + +The overall processing model for tasks is as follows: + * Evaluate maps + * Perform subsitutions + * Evaluate use statements + * Expand chunks + * Perform subsitutions + +At each point after maps are evaluated tasks must have a unique name. diff --git a/testing/web-platform/tests/tools/ci/tc/__init__.py b/testing/web-platform/tests/tools/ci/tc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/__init__.py diff --git a/testing/web-platform/tests/tools/ci/tc/decision.py b/testing/web-platform/tests/tools/ci/tc/decision.py new file mode 100644 index 0000000000..d00ba6ba19 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/decision.py @@ -0,0 +1,414 @@ +# mypy: allow-untyped-defs + +import argparse +import json +import logging +import os +import re +import subprocess +from collections import OrderedDict + +import taskcluster + +from . import taskgraph + + +here = os.path.abspath(os.path.dirname(__file__)) + + +logging.basicConfig() +logger = logging.getLogger() + + +def get_triggers(event): + # Set some variables that we use to get the commits on the current branch + ref_prefix = "refs/heads/" + is_pr = "pull_request" in event + branch = None + if not is_pr and "ref" in event: + branch = event["ref"] + if branch.startswith(ref_prefix): + branch = branch[len(ref_prefix):] + + return is_pr, branch + + +def fetch_event_data(queue): + try: + task_id = os.environ["TASK_ID"] + except KeyError: + logger.warning("Missing TASK_ID environment variable") + # For example under local testing + return None + + task_data = queue.task(task_id) + + return task_data.get("extra", {}).get("github_event") + + +def filter_triggers(event, all_tasks): + is_pr, branch = get_triggers(event) + triggered = OrderedDict() + for name, task in all_tasks.items(): + if "trigger" in task: + if is_pr and "pull-request" in task["trigger"]: + triggered[name] = task + elif branch is not None and "branch" in task["trigger"]: + for trigger_branch in task["trigger"]["branch"]: + if (trigger_branch == branch or + trigger_branch.endswith("*") and branch.startswith(trigger_branch[:-1])): + triggered[name] = task + logger.info("Triggers match tasks:\n * %s" % "\n * ".join(triggered.keys())) + return triggered + + +def get_run_jobs(event): + from tools.ci import jobs + revish = "%s..%s" % (event["pull_request"]["base"]["sha"] + if "pull_request" in event + else event["before"], + event["pull_request"]["head"]["sha"] + if "pull_request" in event + else event["after"]) + logger.info("Looking for changes in range %s" % revish) + paths = jobs.get_paths(revish=revish) + logger.info("Found changes in paths:%s" % "\n".join(paths)) + path_jobs = jobs.get_jobs(paths) + all_jobs = path_jobs | get_extra_jobs(event) + logger.info("Including jobs:\n * %s" % "\n * ".join(all_jobs)) + return all_jobs + + +def get_extra_jobs(event): + body = None + jobs = set() + if "commits" in event and event["commits"]: + body = event["commits"][0]["message"] + elif "pull_request" in event: + body = event["pull_request"]["body"] + + if not body: + return jobs + + regexp = re.compile(r"\s*tc-jobs:(.*)$") + + for line in body.splitlines(): + m = regexp.match(line) + if m: + items = m.group(1) + for item in items.split(","): + jobs.add(item.strip()) + break + return jobs + + +def filter_excluded_users(tasks, event): + # Some users' pull requests are excluded from tasks, + # such as pull requests from automated exports. + try: + submitter = event["pull_request"]["user"]["login"] + except KeyError: + # Just ignore excluded users if the + # username cannot be pulled from the event. + logger.debug("Unable to read username from event. Continuing.") + return + + excluded_tasks = [] + # A separate list of items for tasks is needed to iterate over + # because removing an item during iteration will raise an error. + for name, task in list(tasks.items()): + if submitter in task.get("exclude-users", []): + excluded_tasks.append(name) + tasks.pop(name) # removing excluded task + if excluded_tasks: + logger.info( + f"Tasks excluded for user {submitter}:\n * " + + "\n * ".join(excluded_tasks) + ) + + +def filter_schedule_if(event, tasks): + scheduled = OrderedDict() + run_jobs = None + for name, task in tasks.items(): + if "schedule-if" in task: + if "run-job" in task["schedule-if"]: + if run_jobs is None: + run_jobs = get_run_jobs(event) + if "all" in run_jobs or any(item in run_jobs for item in task["schedule-if"]["run-job"]): + scheduled[name] = task + else: + scheduled[name] = task + logger.info("Scheduling rules match tasks:\n * %s" % "\n * ".join(scheduled.keys())) + return scheduled + + +def get_fetch_rev(event): + is_pr, _ = get_triggers(event) + if is_pr: + # Try to get the actual rev so that all non-decision tasks are pinned to that + rv = ["refs/pull/%s/merge" % event["pull_request"]["number"]] + # For every PR GitHub maintains a 'head' branch with commits from the + # PR, and a 'merge' branch containing a merge commit between the base + # branch and the PR. + for ref_type in ["head", "merge"]: + ref = "refs/pull/%s/%s" % (event["pull_request"]["number"], ref_type) + sha = None + try: + output = subprocess.check_output(["git", "ls-remote", "origin", ref]) + except subprocess.CalledProcessError: + import traceback + logger.error(traceback.format_exc()) + logger.error("Failed to get commit sha1 for %s" % ref) + else: + if not output: + logger.error("Failed to get commit for %s" % ref) + else: + sha = output.decode("utf-8").split()[0] + rv.append(sha) + rv = tuple(rv) + else: + # For a branch push we have a ref and a head but no merge SHA + rv = (event["ref"], event["after"], None) + assert len(rv) == 3 + return rv + + +def build_full_command(event, task): + fetch_ref, head_sha, merge_sha = get_fetch_rev(event) + cmd_args = { + "task_name": task["name"], + "repo_url": event["repository"]["clone_url"], + "fetch_ref": fetch_ref, + "task_cmd": task["command"], + "install_str": "", + } + + options = task.get("options", {}) + options_args = [] + options_args.append("--ref=%s" % fetch_ref) + if head_sha is not None: + options_args.append("--head-rev=%s" % head_sha) + if merge_sha is not None: + options_args.append("--merge-rev=%s" % merge_sha) + if options.get("oom-killer"): + options_args.append("--oom-killer") + if options.get("xvfb"): + options_args.append("--xvfb") + if not options.get("hosts"): + options_args.append("--no-hosts") + else: + options_args.append("--hosts") + # Check out the expected SHA unless it is overridden (e.g. to base_head). + if options.get("checkout"): + options_args.append("--checkout=%s" % options["checkout"]) + for browser in options.get("browser", []): + options_args.append("--browser=%s" % browser) + if options.get("channel"): + options_args.append("--channel=%s" % options["channel"]) + if options.get("install-certificates"): + options_args.append("--install-certificates") + + cmd_args["options_str"] = " ".join(str(item) for item in options_args) + + install_packages = task.get("install") + if install_packages: + install_items = ["apt update -qqy"] + install_items.extend("apt install -qqy %s" % item + for item in install_packages) + cmd_args["install_str"] = "\n".join("sudo %s;" % item for item in install_items) + + return ["/bin/bash", + "--login", + "-xc", + """ +~/start.sh \ + %(repo_url)s \ + %(fetch_ref)s; +%(install_str)s +cd web-platform-tests; +./tools/ci/run_tc.py %(options_str)s -- %(task_cmd)s; +""" % cmd_args] + + +def get_owner(event): + if "pusher" in event: + pusher = event.get("pusher", {}).get("email", "") + if pusher and "@" in pusher: + return pusher + return "web-platform-tests@users.noreply.github.com" + + +def create_tc_task(event, task, taskgroup_id, depends_on_ids, env_extra=None): + command = build_full_command(event, task) + task_id = taskcluster.slugId() + task_data = { + "taskGroupId": taskgroup_id, + "created": taskcluster.fromNowJSON(""), + "deadline": taskcluster.fromNowJSON(task["deadline"]), + "provisionerId": task["provisionerId"], + "schedulerId": task["schedulerId"], + "workerType": task["workerType"], + "scopes": task.get("scopes", []), + "metadata": { + "name": task["name"], + "description": task.get("description", ""), + "owner": get_owner(event), + "source": event["repository"]["clone_url"] + }, + "payload": { + "artifacts": task.get("artifacts"), + "command": command, + "image": task.get("image"), + "maxRunTime": task.get("maxRunTime"), + "env": task.get("env", {}), + }, + "extra": { + "github_event": json.dumps(event) + }, + "routes": ["checks"] + } + if "extra" in task: + task_data["extra"].update(task["extra"]) + if task.get("privileged"): + if "capabilities" not in task_data["payload"]: + task_data["payload"]["capabilities"] = {} + task_data["payload"]["capabilities"]["privileged"] = True + if env_extra: + task_data["payload"]["env"].update(env_extra) + if depends_on_ids: + task_data["dependencies"] = depends_on_ids + task_data["requires"] = task.get("requires", "all-completed") + return task_id, task_data + + +def get_artifact_data(artifact, task_id_map): + task_id, data = task_id_map[artifact["task"]] + return { + "task": task_id, + "glob": artifact["glob"], + "dest": artifact["dest"], + "extract": artifact.get("extract", False) + } + + +def build_task_graph(event, all_tasks, tasks): + task_id_map = OrderedDict() + taskgroup_id = os.environ.get("TASK_ID", taskcluster.slugId()) + sink_task_depends_on = [] + + def add_task(task_name, task): + depends_on_ids = [] + if "depends-on" in task: + for depends_name in task["depends-on"]: + if depends_name not in task_id_map: + add_task(depends_name, + all_tasks[depends_name]) + depends_on_ids.append(task_id_map[depends_name][0]) + env_extra = {} + if "download-artifacts" in task: + env_extra["TASK_ARTIFACTS"] = json.dumps( + [get_artifact_data(artifact, task_id_map) + for artifact in task["download-artifacts"]]) + + task_id, task_data = create_tc_task(event, task, taskgroup_id, depends_on_ids, + env_extra=env_extra) + task_id_map[task_name] = (task_id, task_data) + + # The string conversion here is because if we use variables they are + # converted to a string, so it's easier to use a string always + if str(task.get("required", "True")) != "False" and task_name != "sink-task": + sink_task_depends_on.append(task_id) + + for task_name, task in tasks.items(): + if task_name == "sink-task": + # sink-task will be created below at the end of the ordered dict, + # so that it can depend on all other tasks. + continue + add_task(task_name, task) + + # GitHub branch protection for pull requests needs us to name explicit + # required tasks - which doesn't suffice when using a dynamic task graph. + # To work around this we declare a sink task that depends on all the other + # tasks completing, and checks if they have succeeded. We can then + # make the sink task the sole required task for pull requests. + sink_task = tasks.get("sink-task") + if sink_task: + logger.info("Scheduling sink-task") + sink_task["command"] += " {}".format(" ".join(sink_task_depends_on)) + task_id_map["sink-task"] = create_tc_task( + event, sink_task, taskgroup_id, sink_task_depends_on) + else: + logger.info("sink-task is not scheduled") + + return task_id_map + + +def create_tasks(queue, task_id_map): + for (task_id, task_data) in task_id_map.values(): + queue.createTask(task_id, task_data) + + +def get_event(queue, event_path): + if event_path is not None: + try: + with open(event_path) as f: + event_str = f.read() + except OSError: + logger.error("Missing event file at path %s" % event_path) + raise + elif "TASK_EVENT" in os.environ: + event_str = os.environ["TASK_EVENT"] + else: + event_str = fetch_event_data(queue) + if not event_str: + raise ValueError("Can't find GitHub event definition; for local testing pass --event-path") + try: + return json.loads(event_str) + except ValueError: + logger.error("Event was not valid JSON") + raise + + +def decide(event): + all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks", "test.yml")) + + triggered_tasks = filter_triggers(event, all_tasks) + scheduled_tasks = filter_schedule_if(event, triggered_tasks) + filter_excluded_users(scheduled_tasks, event) + + logger.info("UNSCHEDULED TASKS:\n %s" % "\n ".join(sorted(set(all_tasks.keys()) - + set(scheduled_tasks.keys())))) + logger.info("SCHEDULED TASKS:\n %s" % "\n ".join(sorted(scheduled_tasks.keys()))) + + task_id_map = build_task_graph(event, all_tasks, scheduled_tasks) + return task_id_map + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--event-path", + help="Path to file containing serialized GitHub event") + parser.add_argument("--dry-run", action="store_true", + help="Don't actually create the tasks, just output the tasks that " + "would be created") + parser.add_argument("--tasks-path", + help="Path to file in which to write payload for all scheduled tasks") + return parser + + +def run(venv, **kwargs): + queue = taskcluster.Queue({'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}) + event = get_event(queue, event_path=kwargs["event_path"]) + + task_id_map = decide(event) + + try: + if not kwargs["dry_run"]: + create_tasks(queue, task_id_map) + else: + print(json.dumps(task_id_map, indent=2)) + finally: + if kwargs["tasks_path"]: + with open(kwargs["tasks_path"], "w") as f: + json.dump(task_id_map, f, indent=2) diff --git a/testing/web-platform/tests/tools/ci/tc/download.py b/testing/web-platform/tests/tools/ci/tc/download.py new file mode 100644 index 0000000000..6a78935be4 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/download.py @@ -0,0 +1,111 @@ +# mypy: allow-untyped-defs + +import argparse +import os +import logging + +import requests + +import github + + +logging.basicConfig() +logger = logging.getLogger("tc-download") + +# The root URL of the Taskcluster deployment from which to download wpt reports +# (after https://bugzilla.mozilla.org/show_bug.cgi?id=1574668 lands, this will +# be https://community-tc.services.mozilla.com) +TASKCLUSTER_ROOT_URL = 'https://taskcluster.net' + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--ref", action="store", default="master", + help="Branch (in the GitHub repository) or commit to fetch logs for") + parser.add_argument("--artifact-name", action="store", default="wpt_report.json.gz", + help="Log type to fetch") + parser.add_argument("--repo-name", action="store", default="web-platform-tests/wpt", + help="GitHub repo name in the format owner/repo. " + "This must be the repo from which the Taskcluster run was scheduled " + "(for PRs this is the repo into which the PR would merge)") + parser.add_argument("--token-file", action="store", + help="File containing GitHub token") + parser.add_argument("--out-dir", action="store", default=".", + help="Path to save the logfiles") + return parser + + +def get_json(url, key=None): + resp = requests.get(url) + resp.raise_for_status() + data = resp.json() + if key: + data = data[key] + return data + + +def get(url, dest, name): + resp = requests.get(url) + resp.raise_for_status() + path = os.path.join(dest, name) + with open(path, "w") as f: + f.write(resp.content) + return path + + +def run(*args, **kwargs): + if not os.path.exists(kwargs["out_dir"]): + os.mkdir(kwargs["out_dir"]) + + if kwargs["token_file"]: + with open(kwargs["token_file"]) as f: + gh = github.Github(f.read().strip()) + else: + gh = github.Github() + + repo = gh.get_repo(kwargs["repo_name"]) + commit = repo.get_commit(kwargs["ref"]) + statuses = commit.get_statuses() + taskgroups = set() + + for status in statuses: + if not status.context.startswith("Taskcluster "): + continue + if status.state == "pending": + continue + taskgroup_id = status.target_url.rsplit("/", 1)[1] + taskgroups.add(taskgroup_id) + + if not taskgroups: + logger.error("No complete Taskcluster runs found for ref %s" % kwargs["ref"]) + return 1 + + for taskgroup in taskgroups: + if TASKCLUSTER_ROOT_URL == 'https://taskcluster.net': + # NOTE: this condition can be removed after November 9, 2019 + taskgroup_url = "https://queue.taskcluster.net/v1/task-group/%s/list" + artifacts_list_url = "https://queue.taskcluster.net/v1/task/%s/artifacts" + else: + taskgroup_url = TASKCLUSTER_ROOT_URL + "/api/queue/v1/task-group/%s/list" + artifacts_list_url = TASKCLUSTER_ROOT_URL + "/api/queue/v1/task/%s/artifacts" + tasks = get_json(taskgroup_url % taskgroup, "tasks") + for task in tasks: + task_id = task["status"]["taskId"] + url = artifacts_list_url % (task_id,) + for artifact in get_json(url, "artifacts"): + if artifact["name"].endswith(kwargs["artifact_name"]): + filename = "%s-%s-%s" % (task["task"]["metadata"]["name"], + task_id, + kwargs["artifact_name"]) + path = get("%s/%s" % (url, artifact["name"]), kwargs["out_dir"], filename) + logger.info(path) + + +def main(): + kwargs = get_parser().parse_args() + + run(None, vars(kwargs)) + + +if __name__ == "__main__": + main() # type: ignore diff --git a/testing/web-platform/tests/tools/ci/tc/github_checks_output.py b/testing/web-platform/tests/tools/ci/tc/github_checks_output.py new file mode 100644 index 0000000000..a334d39eec --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/github_checks_output.py @@ -0,0 +1,34 @@ +from typing import Optional, Text + + +class GitHubChecksOutputter: + """Provides a method to output data to be shown in the GitHub Checks UI. + + This can be useful to provide a summary of a given check (e.g. the lint) + to enable developers to quickly understand what has gone wrong. The output + supports markdown format. + + https://docs.taskcluster.net/docs/reference/integrations/github/checks#custom-text-output-in-checks + """ + def __init__(self, path: Text) -> None: + self.path = path + + def output(self, line: Text) -> None: + with open(self.path, mode="a") as f: + f.write(line) + f.write("\n") + + +__outputter = None + + +def get_gh_checks_outputter(filepath: Optional[Text]) -> Optional[GitHubChecksOutputter]: + """Return the outputter for GitHub Checks output, if enabled. + + :param filepath: The filepath to write GitHub Check output information to, + or None if not enabled. + """ + global __outputter + if filepath and __outputter is None: + __outputter = GitHubChecksOutputter(filepath) + return __outputter diff --git a/testing/web-platform/tests/tools/ci/tc/sink_task.py b/testing/web-platform/tests/tools/ci/tc/sink_task.py new file mode 100644 index 0000000000..ec3d5a47ca --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/sink_task.py @@ -0,0 +1,65 @@ +# mypy: allow-untyped-defs + +import argparse +import logging +import os + +import taskcluster + +from .github_checks_output import get_gh_checks_outputter + + +logging.basicConfig() +logger = logging.getLogger() + + +def check_task_statuses(task_ids, github_checks_outputter): + """Verifies whether a set of Taskcluster tasks completed successfully or not. + + Returns 0 if all tasks passed completed successfully, 1 otherwise.""" + + queue = taskcluster.Queue({'rootUrl': os.environ['TASKCLUSTER_ROOT_URL']}) + failed_tasks = [] + for task in task_ids: + status = queue.status(task) + state = status['status']['state'] + if state == 'failed' or state == 'exception': + logger.error(f'Task {task} failed with state "{state}"') + failed_tasks.append(status) + elif state != 'completed': + logger.error(f'Task {task} had unexpected state "{state}"') + failed_tasks.append(status) + + if failed_tasks and github_checks_outputter: + github_checks_outputter.output('Failed tasks:') + for task in failed_tasks: + # We need to make an additional call to get the task name. + task_id = task['status']['taskId'] + task_name = queue.task(task_id)['metadata']['name'] + github_checks_outputter.output('* `{}` failed with status `{}`'.format(task_name, task['status']['state'])) + else: + logger.info('All tasks completed successfully') + if github_checks_outputter: + github_checks_outputter.output('All tasks completed successfully') + return 1 if failed_tasks else 0 + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--github-checks-text-file", type=str, + help="Path to GitHub checks output file for Taskcluster runs") + parser.add_argument("tasks", nargs="+", + help="A set of Taskcluster task ids to verify the state of.") + return parser + + +def run(venv, **kwargs): + github_checks_outputter = get_gh_checks_outputter(kwargs["github_checks_text_file"]) + + if github_checks_outputter: + github_checks_outputter.output( + "This check acts as a 'sink' for all other Taskcluster-based checks. " + "A failure here means that some other check has failed, which is the " + "real blocker.\n" + ) + return check_task_statuses(kwargs['tasks'], github_checks_outputter) diff --git a/testing/web-platform/tests/tools/ci/tc/taskgraph.py b/testing/web-platform/tests/tools/ci/tc/taskgraph.py new file mode 100644 index 0000000000..54542ef0f8 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/taskgraph.py @@ -0,0 +1,175 @@ +# mypy: allow-untyped-defs + +import json +import os +import re +from collections import OrderedDict +from copy import deepcopy + +import yaml + +here = os.path.dirname(__file__) + + +def first(iterable): + # First item from a list or iterator + if not hasattr(iterable, "next"): + if hasattr(iterable, "__iter__"): + iterable = iter(iterable) + else: + raise ValueError("Object isn't iterable") + return next(iterable) + + +def load_task_file(path): + with open(path) as f: + return yaml.safe_load(f) + + +def update_recursive(data, update_data): + for key, value in update_data.items(): + if key not in data: + data[key] = value + else: + initial_value = data[key] + if isinstance(value, dict): + if not isinstance(initial_value, dict): + raise ValueError("Variable %s has inconsistent types " + "(expected object)" % key) + update_recursive(initial_value, value) + elif isinstance(value, list): + if not isinstance(initial_value, list): + raise ValueError("Variable %s has inconsistent types " + "(expected list)" % key) + initial_value.extend(value) + else: + data[key] = value + + +def resolve_use(task_data, templates): + rv = {} + if "use" in task_data: + for template_name in task_data["use"]: + update_recursive(rv, deepcopy(templates[template_name])) + update_recursive(rv, task_data) + rv.pop("use", None) + return rv + + +def resolve_name(task_data, default_name): + if "name" not in task_data: + task_data["name"] = default_name + return task_data + + +def resolve_chunks(task_data): + if "chunks" not in task_data: + return [task_data] + rv = [] + total_chunks = task_data["chunks"] + if "chunks-override" in task_data: + override = task_data["chunks-override"].get(task_data["vars"]["test-type"]) + if override is not None: + total_chunks = override + for i in range(1, total_chunks + 1): + chunk_data = deepcopy(task_data) + chunk_data["chunks"] = {"id": i, + "total": total_chunks} + rv.append(chunk_data) + return rv + + +def replace_vars(input_string, variables): + # TODO: support replacing as a non-string type? + variable_re = re.compile(r"(?<!\\)\${([^}]+)}") + + def replacer(m): + var = m.group(1).split(".") + repl = variables + for part in var: + try: + repl = repl[part] + except Exception: + # Don't substitute + return m.group(0) + return str(repl) + + return variable_re.sub(replacer, input_string) + + +def sub_variables(data, variables): + if isinstance(data, str): + return replace_vars(data, variables) + if isinstance(data, list): + return [sub_variables(item, variables) for item in data] + if isinstance(data, dict): + return {key: sub_variables(value, variables) + for key, value in data.items()} + return data + + +def substitute_variables(task): + variables = {"vars": task.get("vars", {}), + "chunks": task.get("chunks", {})} + + return sub_variables(task, variables) + + +def expand_maps(task): + name = first(task.keys()) + if name != "$map": + return [task] + + map_data = task["$map"] + if set(map_data.keys()) != {"for", "do"}: + raise ValueError("$map objects must have exactly two properties named 'for' " + "and 'do' (got %s)" % ("no properties" if not map_data.keys() + else ", ". join(map_data.keys()))) + rv = [] + for for_data in map_data["for"]: + do_items = map_data["do"] + if not isinstance(do_items, list): + do_items = expand_maps(do_items) + for do_data in do_items: + task_data = deepcopy(for_data) + if len(do_data.keys()) != 1: + raise ValueError("Each item in the 'do' list must be an object " + "with a single property") + name = first(do_data.keys()) + update_recursive(task_data, deepcopy(do_data[name])) + rv.append({name: task_data}) + return rv + + +def load_tasks(tasks_data): + map_resolved_tasks = OrderedDict() + tasks = [] + + for task in tasks_data["tasks"]: + if len(task.keys()) != 1: + raise ValueError("Each task must be an object with a single property") + for task in expand_maps(task): + if len(task.keys()) != 1: + raise ValueError("Each task must be an object with a single property") + name = first(task.keys()) + data = task[name] + new_name = sub_variables(name, {"vars": data.get("vars", {})}) + if new_name in map_resolved_tasks: + raise ValueError("Got duplicate task name %s" % new_name) + map_resolved_tasks[new_name] = substitute_variables(data) + + for task_default_name, data in map_resolved_tasks.items(): + task = resolve_use(data, tasks_data["components"]) + task = resolve_name(task, task_default_name) + tasks.extend(resolve_chunks(task)) + + tasks = [substitute_variables(task_data) for task_data in tasks] + return OrderedDict([(t["name"], t) for t in tasks]) + + +def load_tasks_from_path(path): + return load_tasks(load_task_file(path)) + + +def run(venv, **kwargs): + print(json.dumps(load_tasks_from_path(os.path.join(here, "tasks", "test.yml")), indent=2)) diff --git a/testing/web-platform/tests/tools/ci/tc/tasks/test.yml b/testing/web-platform/tests/tools/ci/tc/tasks/test.yml new file mode 100644 index 0000000000..ea9c7f9dae --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/tasks/test.yml @@ -0,0 +1,589 @@ +components: + wpt-base: + provisionerId: proj-wpt + workerType: ci + schedulerId: taskcluster-github + deadline: "24 hours" + image: webplatformtests/wpt:0.56 + maxRunTime: 7200 + artifacts: + public/results: + path: /home/test/artifacts + type: directory + extra: + github: + customCheckRun: + textArtifactName: public/results/checkrun.md + + wpt-testharness: + chunks: 16 + maxRunTime: 10800 + vars: + test-type: testharness + + wpt-reftest: + chunks: 6 + vars: + test-type: reftest + + wpt-print-reftest: + chunks: 1 + vars: + test-type: print-reftest + + wpt-wdspec: + chunks: 2 + vars: + test-type: wdspec + + wpt-crashtest: + chunks: 1 + vars: + test-type: crashtest + + run-options: + options: + xvfb: true + oom-killer: true + hosts: true + install-certificates: true + + wpt-run: + name: wpt-${vars.browser}-${vars.channel}-${vars.suite}-${chunks.id} + options: + browser: + - ${vars.browser} + channel: ${vars.channel} + command: >- + ./tools/ci/taskcluster-run.py + ${vars.browser} + ${vars.channel} + -- + --channel=${vars.channel} + --log-wptreport=../artifacts/wpt_report.json + --log-wptscreenshot=../artifacts/wpt_screenshot.txt + --no-fail-on-unexpected + --this-chunk=${chunks.id} + --total-chunks=${chunks.total} + --test-type=${vars.suite} + + trigger-master: + trigger: + branch: + - master + + trigger-push: + trigger: + branch: + - triggers/${vars.browser}_${vars.channel} + + trigger-daily: + trigger: + branch: + - epochs/daily + + trigger-weekly: + trigger: + branch: + - epochs/weekly + + trigger-pr: + trigger: + pull-request: + + browser-firefox: + depends-on: + - download-firefox-${vars.channel} + download-artifacts: + - task: download-firefox-${vars.channel} + glob: public/results/firefox-${vars.channel}.* + dest: build/ + extract: true + + browser-webkitgtk_minibrowser: {} + + browser-chrome: {} + + browser-chromium: {} + + browser-servo: {} + + browser-firefox_android: + privileged: true + scopes: + - "docker-worker:capability:privileged" + chunks-override: + testharness: 24 + + tox-python3_7: + env: + TOXENV: py37 + PY_COLORS: "0" + install: + - python3.7 + - python3.7-distutils + - python3.7-dev + - python3.7-venv + + tox-python3_11: + env: + TOXENV: py311 + PY_COLORS: "0" + install: + - python3.11 + - python3.11-distutils + - python3.11-dev + - python3.11-venv + tests-affected: + options: + browser: + - ${vars.browser} + channel: ${vars.channel} + schedule-if: + run-job: + - affected_tests + +tasks: + # The scheduling order of tasks is NOT determined by the order in which they + # are defined, but by their dependencies (depends-on). + + # Run full suites on push + - $map: + for: + - vars: + suite: testharness + - vars: + suite: reftest + - vars: + suite: wdspec + - vars: + suite: crashtest + do: + $map: + for: + - vars: + browser: firefox + channel: nightly + use: + - trigger-master + - trigger-push + - vars: + browser: firefox + channel: beta + use: + - trigger-weekly + - trigger-push + - vars: + browser: firefox + channel: stable + use: + - trigger-daily + - trigger-push + - vars: + # Chromium ToT + browser: chromium + channel: nightly + use: + - trigger-daily + - trigger-push + - vars: + browser: chrome + channel: canary + use: + - trigger-master + - trigger-push + - vars: + browser: chrome + channel: dev + use: + - trigger-master + - trigger-push + - vars: + browser: chrome + channel: beta + use: + - trigger-weekly + - trigger-push + - vars: + browser: chrome + channel: stable + use: + - trigger-daily + - trigger-push + - vars: + browser: webkitgtk_minibrowser + channel: nightly + use: + - trigger-daily + - trigger-push + - vars: + browser: webkitgtk_minibrowser + channel: stable + use: + - trigger-weekly + - trigger-push + - vars: + browser: webkitgtk_minibrowser + channel: beta + use: + - trigger-weekly + - trigger-push + - vars: + browser: servo + channel: nightly + use: + - trigger-weekly + - trigger-push + - vars: + browser: firefox_android + channel: nightly + use: + - trigger-daily + - trigger-push + do: + - ${vars.browser}-${vars.channel}-${vars.suite}: + use: + - wpt-base + - run-options + - wpt-run + - browser-${vars.browser} + - wpt-${vars.suite} + description: >- + A subset of WPT's "${vars.suite}" tests (chunk number ${chunks.id} + of ${chunks.total}), run in the ${vars.channel} release of + ${vars.browser}. + + # print-reftest are currently only supported by Chrome and Firefox. + - $map: + for: + - vars: + suite: print-reftest + do: + $map: + for: + - vars: + browser: firefox + channel: nightly + use: + - trigger-master + - trigger-push + - vars: + browser: firefox + channel: beta + use: + - trigger-weekly + - trigger-push + - vars: + browser: firefox + channel: stable + use: + - trigger-daily + - trigger-push + - vars: + # Chromium ToT + browser: chromium + channel: nightly + use: + - trigger-daily + - trigger-push + - vars: + browser: chrome + channel: canary + use: + - trigger-master + - trigger-push + - vars: + browser: chrome + channel: dev + use: + - trigger-master + - trigger-push + - vars: + browser: chrome + channel: beta + use: + - trigger-weekly + - trigger-push + - vars: + browser: chrome + channel: stable + use: + - trigger-daily + - trigger-push + do: + - ${vars.browser}-${vars.channel}-${vars.suite}: + use: + - wpt-base + - run-options + - wpt-run + - browser-${vars.browser} + - wpt-${vars.suite} + description: >- + A subset of WPT's "${vars.suite}" tests (chunk number ${chunks.id} + of ${chunks.total}), run in the ${vars.channel} release of + ${vars.browser}. + + - $map: + for: + - vars: + browser: firefox + channel: nightly + stability-exclude-users: + - moz-wptsync-bot + required: true + - vars: + browser: chrome + channel: dev + stability-exclude-users: + - chromium-wpt-export-bot + required: false + do: + - wpt-${vars.browser}-${vars.channel}-stability: + use: + - wpt-base + - run-options + - browser-${vars.browser} + - trigger-pr + - tests-affected + description: >- + Verify that all tests affected by a pull request are stable + when executed in ${vars.browser}. + command: >- + ./tools/ci/taskcluster-run.py + --commit-range base_head + ${vars.browser} + ${vars.channel} + -- + --channel=${vars.channel} + --verify + --verify-no-chaos-mode + --verify-repeat-loop=0 + --verify-repeat-restart=10 + --github-checks-text-file="/home/test/artifacts/checkrun.md" + exclude-users: ${vars.stability-exclude-users} + required: ${vars.required} + + - wpt-${vars.browser}-${vars.channel}-results: + use: + - wpt-base + - run-options + - browser-${vars.browser} + - trigger-pr + - tests-affected + description: >- + Collect results for all tests affected by a pull request in + ${vars.browser}. + command: >- + ./tools/ci/taskcluster-run.py + --commit-range base_head + ${vars.browser} + ${vars.channel} + -- + --channel=${vars.channel} + --no-fail-on-unexpected + --log-wptreport=../artifacts/wpt_report.json + --log-wptscreenshot=../artifacts/wpt_screenshot.txt + + - wpt-${vars.browser}-${vars.channel}-results-without-changes: + use: + - wpt-base + - run-options + - browser-${vars.browser} + - trigger-pr + - tests-affected + options: + checkout: base_head + description: >- + Collect results for all tests affected by a pull request in + ${vars.browser} but without the changes in the PR. + command: >- + ./tools/ci/taskcluster-run.py + --commit-range task_head + ${vars.browser} + ${vars.channel} + -- + --channel=${vars.channel} + --no-fail-on-unexpected + --log-wptreport=../artifacts/wpt_report.json + --log-wptscreenshot=../artifacts/wpt_screenshot.txt + - $map: + for: + - vars: + channel: nightly + - vars: + channel: beta + - vars: + channel: stable + do: + download-firefox-${vars.channel}: + use: + - wpt-base + command: "./wpt install --download-only --destination /home/test/artifacts/ --channel=${vars.channel} --rename=firefox-${vars.channel} firefox browser" + + - lint: + use: + - wpt-base + - trigger-master + - trigger-pr + description: >- + Lint for wpt-specific requirements + command: "./wpt lint --all --github-checks-text-file=/home/test/artifacts/checkrun.md" + + - update-built: + use: + - wpt-base + - trigger-pr + schedule-if: + run-job: + - update_built + command: "./tools/ci/ci_built_diff.sh" + + - tools/ unittests (Python 3.7): + description: >- + Unit tests for tools running under Python 3.7, excluding wptrunner + use: + - wpt-base + - trigger-pr + - tox-python3_7 + command: ./tools/ci/ci_tools_unittest.sh + env: + HYPOTHESIS_PROFILE: ci + schedule-if: + run-job: + - tools_unittest + + - tools/ unittests (Python 3.11): + description: >- + Unit tests for tools running under Python 3.11, excluding wptrunner + use: + - wpt-base + - trigger-pr + - tox-python3_11 + command: ./tools/ci/ci_tools_unittest.sh + env: + HYPOTHESIS_PROFILE: ci + schedule-if: + run-job: + - tools_unittest + + - tools/ integration tests (Python 3.7): + description: >- + Integration tests for tools running under Python 3.7 + use: + - wpt-base + - trigger-pr + - tox-python3_7 + command: ./tools/ci/ci_tools_integration_test.sh + install: + - libnss3-tools + options: + oom-killer: true + browser: + - firefox + - chrome + channel: experimental + xvfb: true + hosts: true + schedule-if: + run-job: + - wpt_integration + + - tools/ integration tests (Python 3.11): + description: >- + Integration tests for tools running under Python 3.11 + use: + - wpt-base + - trigger-pr + - tox-python3_11 + command: ./tools/ci/ci_tools_integration_test.sh + install: + - libnss3-tools + options: + oom-killer: true + browser: + - firefox + - chrome + channel: experimental + xvfb: true + hosts: true + schedule-if: + run-job: + - wpt_integration + + - resources/ tests (Python 3.7): + description: >- + Tests for testharness.js and other files in resources/ under Python 3.7 + use: + - wpt-base + - trigger-pr + - tox-python3_7 + command: ./tools/ci/ci_resources_unittest.sh + install: + - libnss3-tools + options: + browser: + - firefox + xvfb: true + hosts: true + schedule-if: + run-job: + - resources_unittest + + - resources/ tests (Python 3.11): + description: >- + Tests for testharness.js and other files in resources/ under Python 3.11 + use: + - wpt-base + - trigger-pr + - tox-python3_11 + command: ./tools/ci/ci_resources_unittest.sh + install: + - libnss3-tools + options: + browser: + - firefox + xvfb: true + hosts: true + schedule-if: + run-job: + - resources_unittest + + - infrastructure/ tests: + description: >- + Smoketests for wptrunner + vars: + channel: nightly + use: + - wpt-base + - trigger-pr + - browser-firefox + - browser-firefox_android + command: ./tools/ci/ci_wptrunner_infrastructure.sh + install: + - python3-pip + - libnss3-tools + - libappindicator1 + - fonts-liberation + options: + oom-killer: true + browser: + - firefox + - chrome + - firefox_android + channel: experimental + xvfb: true + hosts: false + schedule-if: + run-job: + - wptrunner_infrastructure + + # Note: even though sink-task does not have `depends-on`, it depends on all + # other tasks (dynamically added by tools/ci/tc/decision.py). + - sink-task: + description: >- + Sink task for all other tasks; indicates success + use: + - wpt-base + - trigger-pr + command: "./wpt tc-sink-task --github-checks-text-file=/home/test/artifacts/checkrun.md" + requires: all-resolved diff --git a/testing/web-platform/tests/tools/ci/tc/testdata/epochs_daily_push_event.json b/testing/web-platform/tests/tools/ci/tc/testdata/epochs_daily_push_event.json new file mode 100644 index 0000000000..0f74c315d2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/testdata/epochs_daily_push_event.json @@ -0,0 +1,460 @@ +{ + "ref": "refs/heads/epochs/daily", + "before": "20bb1ca5db519ee5d37ece6492868f8a6b65a2e7", + "after": "5df56b25e1cb81f81fe16c88be839f9fd538b41e", + "repository": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "name": "web-platform-tests", + "email": null, + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://github.com/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": 1330865891, + "updated_at": "2019-11-30T21:34:30Z", + "pushed_at": 1575160610, + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "https://web-platform-tests.org/", + "size": 329465, + "stargazers_count": 2543, + "watchers_count": 2543, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1838, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 1590, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1838, + "open_issues": 1590, + "watchers": 2543, + "default_branch": "master", + "stargazers": 2543, + "master_branch": "master", + "organization": "web-platform-tests" + }, + "pusher": { + "name": "github-actions[bot]", + "email": null + }, + "organization": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "url": "https://api.github.com/orgs/web-platform-tests", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks", + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "description": "" + }, + "sender": { + "login": "github-actions[bot]", + "id": 41898282, + "node_id": "MDM6Qm90NDE4OTgyODI=", + "avatar_url": "https://avatars2.githubusercontent.com/in/15368?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-actions%5Bbot%5D", + "html_url": "https://github.com/apps/github-actions", + "followers_url": "https://api.github.com/users/github-actions%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-actions%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-actions%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-actions%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-actions%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-actions%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-actions%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-actions%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-actions%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "created": false, + "deleted": false, + "forced": false, + "base_ref": "refs/heads/epochs/six_hourly", + "compare": "https://github.com/web-platform-tests/wpt/compare/20bb1ca5db51...5df56b25e1cb", + "commits": [ + { + "id": "3503c50a6452e153bde906a9c6644cb6237224fc", + "tree_id": "b735fa0ae88ebe0abd6764a1afd63aea815ac18e", + "distinct": false, + "message": "[LayoutNG] Pixel-snap column rules.\n\nBug: 829028\nChange-Id: I252901109502256f14bc68e64d4303006db50a13\nReviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1944350\nCommit-Queue: Xianzhu Wang <wangxianzhu@chromium.org>\nReviewed-by: Xianzhu Wang <wangxianzhu@chromium.org>\nCr-Commit-Position: refs/heads/master@{#720302}", + "timestamp": "2019-11-29T16:25:44-08:00", + "url": "https://github.com/web-platform-tests/wpt/commit/3503c50a6452e153bde906a9c6644cb6237224fc", + "author": { + "name": "Morten Stenshorne", + "email": "mstensho@chromium.org", + "username": "mstensho" + }, + "committer": { + "name": "Blink WPT Bot", + "email": "blink-w3c-test-autoroller@chromium.org", + "username": "chromium-wpt-export-bot" + }, + "added": [ + "css/css-multicol/equal-gap-and-rule.html" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "561b765308e6d188618f3ba73091bb598d8357ce", + "tree_id": "775ac4481c03e020819910d03019f0ec93def868", + "distinct": false, + "message": "Fix parser mXSS sanitizer bypass for <p> and <br> within foreign context\n\nPrior to this CL, the following code:\n <svg></p></svg>\nparsed to this innerHTML: <svg><p></p></svg>\n\nThis is in contrast to this code:\n <svg><p></svg>\nwhich parses to <svg></svg><p></p>\n\nThe fact that the </p> is left inside the <svg> allowed sanitizer\nbypasses as detailed in [1]. Please also see [2] for the spec\ndiscussion.\n\nWith this CL, </p> and </br> within a foreign context now cause\nthe closing of the foreign context.\n\n[1] https://research.securitum.com/dompurify-bypass-using-mxss/\n[2] https://github.com/whatwg/html/issues/5113\n\nBug: 1005713\nChange-Id: Ic07ee50de4eb1ef19b73a075bd83785c99f4f891\nReviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1940722\nReviewed-by: Kouhei Ueno <kouhei@chromium.org>\nCommit-Queue: Mason Freed <masonfreed@chromium.org>\nCr-Commit-Position: refs/heads/master@{#720315}", + "timestamp": "2019-11-30T00:22:29-08:00", + "url": "https://github.com/web-platform-tests/wpt/commit/561b765308e6d188618f3ba73091bb598d8357ce", + "author": { + "name": "Mason Freed", + "email": "masonfreed@chromium.org", + "username": "mfreed7" + }, + "committer": { + "name": "Blink WPT Bot", + "email": "blink-w3c-test-autoroller@chromium.org", + "username": "chromium-wpt-export-bot" + }, + "added": [ + "html/syntax/parsing/html_content_in_foreign_context.html" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "d31800185dab8e194294620c8ad6bf40f25bf752", + "tree_id": "c718a913e9a5197e1896f5b2ee0434f896d6725b", + "distinct": false, + "message": "[css-text-4] tests for word-boundary-expansion\n\nhttps://drafts.csswg.org/css-text-4/#word-boundary-expansion", + "timestamp": "2019-11-30T18:09:49+09:00", + "url": "https://github.com/web-platform-tests/wpt/commit/d31800185dab8e194294620c8ad6bf40f25bf752", + "author": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "committer": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "added": [ + "css/css-text/parsing/word-boundary-expansion-computed.html", + "css/css-text/parsing/word-boundary-expansion-invalid.html", + "css/css-text/parsing/word-boundary-expansion-valid.html", + "css/css-text/word-boundary/reference/word-boundary-001-ref.html", + "css/css-text/word-boundary/reference/word-boundary-002-ref.html", + "css/css-text/word-boundary/reference/word-boundary-004-ref.html", + "css/css-text/word-boundary/reference/word-boundary-007-ref.html", + "css/css-text/word-boundary/reference/word-boundary-008-ref.html", + "css/css-text/word-boundary/reference/word-boundary-009-ref.html", + "css/css-text/word-boundary/reference/word-boundary-010-ref.html", + "css/css-text/word-boundary/reference/word-boundary-011-ref.html", + "css/css-text/word-boundary/reference/word-boundary-012-ref.html", + "css/css-text/word-boundary/reference/word-boundary-013-ref.html", + "css/css-text/word-boundary/reference/word-boundary-014-ref.html", + "css/css-text/word-boundary/word-boundary-001.html", + "css/css-text/word-boundary/word-boundary-002.html", + "css/css-text/word-boundary/word-boundary-003.html", + "css/css-text/word-boundary/word-boundary-004.html", + "css/css-text/word-boundary/word-boundary-005.html", + "css/css-text/word-boundary/word-boundary-006.html", + "css/css-text/word-boundary/word-boundary-007.html", + "css/css-text/word-boundary/word-boundary-008.html", + "css/css-text/word-boundary/word-boundary-009.html", + "css/css-text/word-boundary/word-boundary-010.html", + "css/css-text/word-boundary/word-boundary-011.html", + "css/css-text/word-boundary/word-boundary-012.html", + "css/css-text/word-boundary/word-boundary-013.html", + "css/css-text/word-boundary/word-boundary-014.html", + "css/css-text/word-boundary/word-boundary-015-manual.html" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "37ddab0528d8ab49db1371188e36f68133ff5c1c", + "tree_id": "832644a697a1cdcaf93126d4f95c89f71b4f5d47", + "distinct": false, + "message": "[css-text-4] tests for word-boundary-detection\n\nhttps://drafts.csswg.org/css-text-4/#word-boundary-detection", + "timestamp": "2019-11-30T18:09:49+09:00", + "url": "https://github.com/web-platform-tests/wpt/commit/37ddab0528d8ab49db1371188e36f68133ff5c1c", + "author": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "committer": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "added": [ + "css/css-text/word-boundary/reference/word-boundary-101-ref.html", + "css/css-text/word-boundary/reference/word-boundary-102-ref.html", + "css/css-text/word-boundary/reference/word-boundary-103-ref.html", + "css/css-text/word-boundary/reference/word-boundary-104-ref.html", + "css/css-text/word-boundary/reference/word-boundary-105-ref.html", + "css/css-text/word-boundary/reference/word-boundary-106-ref.html", + "css/css-text/word-boundary/reference/word-boundary-107-ref.html", + "css/css-text/word-boundary/reference/word-boundary-108-ref.html", + "css/css-text/word-boundary/reference/word-boundary-109-a-ref.html", + "css/css-text/word-boundary/reference/word-boundary-109-b-ref.html", + "css/css-text/word-boundary/reference/word-boundary-110-a-ref.html", + "css/css-text/word-boundary/reference/word-boundary-110-b-ref.html", + "css/css-text/word-boundary/reference/word-boundary-111-ref.html", + "css/css-text/word-boundary/reference/word-boundary-112-ref.html", + "css/css-text/word-boundary/reference/word-boundary-113-ref.html", + "css/css-text/word-boundary/reference/word-boundary-114-a-ref.html", + "css/css-text/word-boundary/reference/word-boundary-114-b-ref.html", + "css/css-text/word-boundary/reference/word-boundary-115-ref.html", + "css/css-text/word-boundary/reference/word-boundary-116-ref.html", + "css/css-text/word-boundary/reference/word-boundary-117-ref.html", + "css/css-text/word-boundary/reference/word-boundary-119-ref.html", + "css/css-text/word-boundary/reference/word-boundary-120-ref.html", + "css/css-text/word-boundary/reference/word-boundary-121-ref.html", + "css/css-text/word-boundary/reference/word-boundary-122-ref.html", + "css/css-text/word-boundary/reference/word-boundary-123-ref.html", + "css/css-text/word-boundary/reference/word-boundary-124-ref.html", + "css/css-text/word-boundary/reference/word-boundary-125-ref.html", + "css/css-text/word-boundary/reference/word-boundary-126-ref.html", + "css/css-text/word-boundary/reference/word-boundary-127-ref.html", + "css/css-text/word-boundary/reference/word-boundary-128-ref.html", + "css/css-text/word-boundary/reference/word-boundary-129-ref.html", + "css/css-text/word-boundary/word-boundary-101.html", + "css/css-text/word-boundary/word-boundary-102.html", + "css/css-text/word-boundary/word-boundary-103.html", + "css/css-text/word-boundary/word-boundary-104.html", + "css/css-text/word-boundary/word-boundary-105.html", + "css/css-text/word-boundary/word-boundary-106.html", + "css/css-text/word-boundary/word-boundary-107.html", + "css/css-text/word-boundary/word-boundary-108.html", + "css/css-text/word-boundary/word-boundary-109.html", + "css/css-text/word-boundary/word-boundary-110.html", + "css/css-text/word-boundary/word-boundary-111.html", + "css/css-text/word-boundary/word-boundary-112.html", + "css/css-text/word-boundary/word-boundary-113.html", + "css/css-text/word-boundary/word-boundary-114.html", + "css/css-text/word-boundary/word-boundary-115.html", + "css/css-text/word-boundary/word-boundary-116.html", + "css/css-text/word-boundary/word-boundary-117.html", + "css/css-text/word-boundary/word-boundary-118.html", + "css/css-text/word-boundary/word-boundary-119.html", + "css/css-text/word-boundary/word-boundary-120.html", + "css/css-text/word-boundary/word-boundary-121.html", + "css/css-text/word-boundary/word-boundary-122.html", + "css/css-text/word-boundary/word-boundary-123.html", + "css/css-text/word-boundary/word-boundary-124.html", + "css/css-text/word-boundary/word-boundary-125.html", + "css/css-text/word-boundary/word-boundary-126.html", + "css/css-text/word-boundary/word-boundary-127.html", + "css/css-text/word-boundary/word-boundary-128.html", + "css/css-text/word-boundary/word-boundary-129.html" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "054edcc23aa1e0ebee50d7ddf1ce6115dd940ece", + "tree_id": "b58ca36bb7e928d440bed734e323580467dd32c7", + "distinct": false, + "message": "[css-text] Fix typo", + "timestamp": "2019-11-30T19:08:12+09:00", + "url": "https://github.com/web-platform-tests/wpt/commit/054edcc23aa1e0ebee50d7ddf1ce6115dd940ece", + "author": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "committer": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "added": [ + + ], + "removed": [ + + ], + "modified": [ + "css/css-text/parsing/word-boundary-expansion-computed.html" + ] + }, + { + "id": "d15d6d91834108a38070771025b548124d44026b", + "tree_id": "a1476d347b04acb59ed2562f0d4f845e8252e6d0", + "distinct": false, + "message": "[css-text add parsing tests for word-boundary-detection", + "timestamp": "2019-11-30T19:08:12+09:00", + "url": "https://github.com/web-platform-tests/wpt/commit/d15d6d91834108a38070771025b548124d44026b", + "author": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "committer": { + "name": "Florian Rivoal", + "email": "git@florian.rivoal.net", + "username": "frivoal" + }, + "added": [ + "css/css-text/parsing/word-boundary-detection-computed.html", + "css/css-text/parsing/word-boundary-detection-invalid.html", + "css/css-text/parsing/word-boundary-detection-valid.html" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "5df56b25e1cb81f81fe16c88be839f9fd538b41e", + "tree_id": "18da87e7701bd6218f4437b4d2d49cefe1f56af9", + "distinct": false, + "message": "Delete invalid test. (#20547)\n\nIt seems it expects `transform: rotate(1deg, 20px, 20px)` to somehow be valid.\r\n\r\nIt was introduced in cdc3032f56c86cc68121e54e169485441d9cdb1a, pointing to https://www.w3.org/TR/css-transforms-1/#svg-transform-functions, which doesn't say anything like that.\r\n\r\nDoesn't pass in any browser.", + "timestamp": "2019-11-30T13:34:24-08:00", + "url": "https://github.com/web-platform-tests/wpt/commit/5df56b25e1cb81f81fe16c88be839f9fd538b41e", + "author": { + "name": "Emilio Cobos Álvarez", + "email": "emilio@crisal.io", + "username": "emilio" + }, + "committer": { + "name": "L. David Baron", + "email": "dbaron@dbaron.org", + "username": "dbaron" + }, + "added": [ + + ], + "removed": [ + "css/css-transforms/external-styles/svg-external-styles-012.html" + ], + "modified": [ + "css/css-transforms/external-styles/support/svg-external-styles.css" + ] + } + ], + "head_commit": { + "id": "5df56b25e1cb81f81fe16c88be839f9fd538b41e", + "tree_id": "18da87e7701bd6218f4437b4d2d49cefe1f56af9", + "distinct": false, + "message": "Delete invalid test. (#20547)\n\nIt seems it expects `transform: rotate(1deg, 20px, 20px)` to somehow be valid.\r\n\r\nIt was introduced in cdc3032f56c86cc68121e54e169485441d9cdb1a, pointing to https://www.w3.org/TR/css-transforms-1/#svg-transform-functions, which doesn't say anything like that.\r\n\r\nDoesn't pass in any browser.", + "timestamp": "2019-11-30T13:34:24-08:00", + "url": "https://github.com/web-platform-tests/wpt/commit/5df56b25e1cb81f81fe16c88be839f9fd538b41e", + "author": { + "name": "Emilio Cobos Álvarez", + "email": "emilio@crisal.io", + "username": "emilio" + }, + "committer": { + "name": "L. David Baron", + "email": "dbaron@dbaron.org", + "username": "dbaron" + }, + "added": [ + + ], + "removed": [ + "css/css-transforms/external-styles/svg-external-styles-012.html" + ], + "modified": [ + "css/css-transforms/external-styles/support/svg-external-styles.css" + ] + } +} diff --git a/testing/web-platform/tests/tools/ci/tc/testdata/master_push_event.json b/testing/web-platform/tests/tools/ci/tc/testdata/master_push_event.json new file mode 100644 index 0000000000..8e79f75373 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/testdata/master_push_event.json @@ -0,0 +1,214 @@ +{ + "ref": "refs/heads/master", + "before": "a4bfa25bfc35e6dd8aabf9bc5af714bf3d70d712", + "after": "5baef702c26b8580f5a4e5e1a34ac75bb9d496ae", + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://github.com/web-platform-tests/wpt/compare/a4bfa25bfc35...5baef702c26b", + "commits": [ + { + "id": "5baef702c26b8580f5a4e5e1a34ac75bb9d496ae", + "tree_id": "045949cd04598b19f5ed1bebf2d5cbed647f3c86", + "distinct": true, + "message": "Add support for verifying taskcluster config (#15593)\n\nAdds as wpt tc-verify command that verifies that the TaskCluster\r\nconfig is a valid yaml file and computes the tasks that will run on a\r\nPR synchronize event. This can be expanded to more events and pushes\r\nin the future.", + "timestamp": "2019-03-01T14:43:07Z", + "url": "https://github.com/web-platform-tests/wpt/commit/5baef702c26b8580f5a4e5e1a34ac75bb9d496ae", + "author": { + "name": "jgraham", + "email": "james@hoppipolla.co.uk", + "username": "jgraham" + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "username": "web-flow" + }, + "added": [ + "tools/taskcluster/__init__.py", + "tools/taskcluster/commands.json", + "tools/taskcluster/testdata/pr_event.json", + "tools/taskcluster/verify.py" + ], + "removed": [ + + ], + "modified": [ + "tools/wpt/paths" + ] + } + ], + "head_commit": { + "id": "5baef702c26b8580f5a4e5e1a34ac75bb9d496ae", + "tree_id": "045949cd04598b19f5ed1bebf2d5cbed647f3c86", + "distinct": true, + "message": "Add support for verifying taskcluster config (#15593)\n\nAdds as wpt tc-verify command that verifies that the TaskCluster\r\nconfig is a valid yaml file and computes the tasks that will run on a\r\nPR synchronize event. This can be expanded to more events and pushes\r\nin the future.", + "timestamp": "2019-03-01T14:43:07Z", + "url": "https://github.com/web-platform-tests/wpt/commit/5baef702c26b8580f5a4e5e1a34ac75bb9d496ae", + "author": { + "name": "jgraham", + "email": "james@hoppipolla.co.uk", + "username": "jgraham" + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "username": "web-flow" + }, + "added": [ + "tools/taskcluster/__init__.py", + "tools/taskcluster/commands.json", + "tools/taskcluster/testdata/pr_event.json", + "tools/taskcluster/verify.py" + ], + "removed": [ + + ], + "modified": [ + "tools/wpt/paths" + ] + }, + "repository": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "name": "web-platform-tests", + "email": "", + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://github.com/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": 1330865891, + "updated_at": "2019-03-01T14:16:52Z", + "pushed_at": 1551451389, + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "http://irc.w3.org/?channels=testing", + "size": 324722, + "stargazers_count": 2060, + "watchers_count": 2060, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1605, + "mirror_url": null, + "archived": false, + "open_issues_count": 1355, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1605, + "open_issues": 1366, + "watchers": 2060, + "default_branch": "master", + "stargazers": 2060, + "master_branch": "master", + "organization": "web-platform-tests" + }, + "pusher": { + "name": "jgraham", + "email": "james@hoppipolla.co.uk" + }, + "organization": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "url": "https://api.github.com/orgs/web-platform-tests", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks", + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "description": "" + }, + "sender": { + "login": "jgraham", + "id": 294864, + "node_id": "MDQ6VXNlcjI5NDg2NA==", + "avatar_url": "https://avatars1.githubusercontent.com/u/294864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/jgraham", + "html_url": "https://github.com/jgraham", + "followers_url": "https://api.github.com/users/jgraham/followers", + "following_url": "https://api.github.com/users/jgraham/following{/other_user}", + "gists_url": "https://api.github.com/users/jgraham/gists{/gist_id}", + "starred_url": "https://api.github.com/users/jgraham/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/jgraham/subscriptions", + "organizations_url": "https://api.github.com/users/jgraham/orgs", + "repos_url": "https://api.github.com/users/jgraham/repos", + "events_url": "https://api.github.com/users/jgraham/events{/privacy}", + "received_events_url": "https://api.github.com/users/jgraham/received_events", + "type": "User", + "site_admin": false + } +} diff --git a/testing/web-platform/tests/tools/ci/tc/testdata/pr_event.json b/testing/web-platform/tests/tools/ci/tc/testdata/pr_event.json new file mode 100644 index 0000000000..5acc1a6010 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/testdata/pr_event.json @@ -0,0 +1,577 @@ +{ + "action": "synchronize", + "number": 15574, + "pull_request": { + "url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/15574", + "id": 256653065, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjU2NjUzMDY1", + "html_url": "https://github.com/web-platform-tests/wpt/pull/15574", + "diff_url": "https://github.com/web-platform-tests/wpt/pull/15574.diff", + "patch_url": "https://github.com/web-platform-tests/wpt/pull/15574.patch", + "issue_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/15574", + "number": 15574, + "state": "open", + "locked": false, + "title": "Move the lint from Travis to TaskCluster", + "user": { + "login": "jgraham", + "id": 294864, + "node_id": "MDQ6VXNlcjI5NDg2NA==", + "avatar_url": "https://avatars1.githubusercontent.com/u/294864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/jgraham", + "html_url": "https://github.com/jgraham", + "followers_url": "https://api.github.com/users/jgraham/followers", + "following_url": "https://api.github.com/users/jgraham/following{/other_user}", + "gists_url": "https://api.github.com/users/jgraham/gists{/gist_id}", + "starred_url": "https://api.github.com/users/jgraham/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/jgraham/subscriptions", + "organizations_url": "https://api.github.com/users/jgraham/orgs", + "repos_url": "https://api.github.com/users/jgraham/repos", + "events_url": "https://api.github.com/users/jgraham/events{/privacy}", + "received_events_url": "https://api.github.com/users/jgraham/received_events", + "type": "User", + "site_admin": false + }, + "body": "", + "created_at": "2019-02-27T12:03:38Z", + "updated_at": "2019-02-28T13:43:17Z", + "closed_at": null, + "merged_at": null, + "merge_commit_sha": "70a272296dad0db4f0be1133a59aa97f0a72d9ac", + "assignee": { + "login": "gsnedders", + "id": 176218, + "node_id": "MDQ6VXNlcjE3NjIxOA==", + "avatar_url": "https://avatars2.githubusercontent.com/u/176218?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/gsnedders", + "html_url": "https://github.com/gsnedders", + "followers_url": "https://api.github.com/users/gsnedders/followers", + "following_url": "https://api.github.com/users/gsnedders/following{/other_user}", + "gists_url": "https://api.github.com/users/gsnedders/gists{/gist_id}", + "starred_url": "https://api.github.com/users/gsnedders/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/gsnedders/subscriptions", + "organizations_url": "https://api.github.com/users/gsnedders/orgs", + "repos_url": "https://api.github.com/users/gsnedders/repos", + "events_url": "https://api.github.com/users/gsnedders/events{/privacy}", + "received_events_url": "https://api.github.com/users/gsnedders/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "gsnedders", + "id": 176218, + "node_id": "MDQ6VXNlcjE3NjIxOA==", + "avatar_url": "https://avatars2.githubusercontent.com/u/176218?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/gsnedders", + "html_url": "https://github.com/gsnedders", + "followers_url": "https://api.github.com/users/gsnedders/followers", + "following_url": "https://api.github.com/users/gsnedders/following{/other_user}", + "gists_url": "https://api.github.com/users/gsnedders/gists{/gist_id}", + "starred_url": "https://api.github.com/users/gsnedders/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/gsnedders/subscriptions", + "organizations_url": "https://api.github.com/users/gsnedders/orgs", + "repos_url": "https://api.github.com/users/gsnedders/repos", + "events_url": "https://api.github.com/users/gsnedders/events{/privacy}", + "received_events_url": "https://api.github.com/users/gsnedders/received_events", + "type": "User", + "site_admin": false + } + ], + "requested_reviewers": [ + { + "login": "gsnedders", + "id": 176218, + "node_id": "MDQ6VXNlcjE3NjIxOA==", + "avatar_url": "https://avatars2.githubusercontent.com/u/176218?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/gsnedders", + "html_url": "https://github.com/gsnedders", + "followers_url": "https://api.github.com/users/gsnedders/followers", + "following_url": "https://api.github.com/users/gsnedders/following{/other_user}", + "gists_url": "https://api.github.com/users/gsnedders/gists{/gist_id}", + "starred_url": "https://api.github.com/users/gsnedders/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/gsnedders/subscriptions", + "organizations_url": "https://api.github.com/users/gsnedders/orgs", + "repos_url": "https://api.github.com/users/gsnedders/repos", + "events_url": "https://api.github.com/users/gsnedders/events{/privacy}", + "received_events_url": "https://api.github.com/users/gsnedders/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "jugglinmike", + "id": 677252, + "node_id": "MDQ6VXNlcjY3NzI1Mg==", + "avatar_url": "https://avatars2.githubusercontent.com/u/677252?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/jugglinmike", + "html_url": "https://github.com/jugglinmike", + "followers_url": "https://api.github.com/users/jugglinmike/followers", + "following_url": "https://api.github.com/users/jugglinmike/following{/other_user}", + "gists_url": "https://api.github.com/users/jugglinmike/gists{/gist_id}", + "starred_url": "https://api.github.com/users/jugglinmike/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/jugglinmike/subscriptions", + "organizations_url": "https://api.github.com/users/jugglinmike/orgs", + "repos_url": "https://api.github.com/users/jugglinmike/repos", + "events_url": "https://api.github.com/users/jugglinmike/events{/privacy}", + "received_events_url": "https://api.github.com/users/jugglinmike/received_events", + "type": "User", + "site_admin": false + } + ], + "requested_teams": [ + + ], + "labels": [ + { + "id": 1012999603, + "node_id": "MDU6TGFiZWwxMDEyOTk5NjAz", + "url": "https://api.github.com/repos/web-platform-tests/wpt/labels/ci", + "name": "ci", + "color": "fef2c0", + "default": false + }, + { + "id": 45230790, + "node_id": "MDU6TGFiZWw0NTIzMDc5MA==", + "url": "https://api.github.com/repos/web-platform-tests/wpt/labels/infra", + "name": "infra", + "color": "fbca04", + "default": false + } + ], + "milestone": null, + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/15574/commits", + "review_comments_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/15574/comments", + "review_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/15574/comments", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/fef69c5d47196433234d6c37a0ff987491bd2dfc", + "head": { + "label": "web-platform-tests:taskcluster_lint", + "ref": "taskcluster_lint", + "sha": "fef69c5d47196433234d6c37a0ff987491bd2dfc", + "user": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://api.github.com/repos/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": "2012-03-04T12:58:11Z", + "updated_at": "2019-02-28T12:41:33Z", + "pushed_at": "2019-02-28T13:43:16Z", + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "http://irc.w3.org/?channels=testing", + "size": 324641, + "stargazers_count": 2058, + "watchers_count": 2058, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1604, + "mirror_url": null, + "archived": false, + "open_issues_count": 1354, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1604, + "open_issues": 1354, + "watchers": 2058, + "default_branch": "master" + } + }, + "base": { + "label": "web-platform-tests:master", + "ref": "master", + "sha": "bb657c4bd0cc4729daa27c1f3a1e1f86ef5a1dc0", + "user": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://api.github.com/repos/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": "2012-03-04T12:58:11Z", + "updated_at": "2019-02-28T12:41:33Z", + "pushed_at": "2019-02-28T13:43:16Z", + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "http://irc.w3.org/?channels=testing", + "size": 324641, + "stargazers_count": 2058, + "watchers_count": 2058, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1604, + "mirror_url": null, + "archived": false, + "open_issues_count": 1354, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1604, + "open_issues": 1354, + "watchers": 2058, + "default_branch": "master" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/15574" + }, + "html": { + "href": "https://github.com/web-platform-tests/wpt/pull/15574" + }, + "issue": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/issues/15574" + }, + "comments": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/issues/15574/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/15574/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/15574/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/statuses/fef69c5d47196433234d6c37a0ff987491bd2dfc" + } + }, + "author_association": "CONTRIBUTOR", + "draft": false, + "merged": false, + "mergeable": null, + "rebaseable": null, + "mergeable_state": "unknown", + "merged_by": null, + "comments": 2, + "review_comments": 3, + "maintainer_can_modify": false, + "commits": 2, + "additions": 55, + "deletions": 7, + "changed_files": 3 + }, + "before": "46d2f316ae10b83726dfb43150b321533bc9539f", + "after": "fef69c5d47196433234d6c37a0ff987491bd2dfc", + "repository": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://api.github.com/repos/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": "2012-03-04T12:58:11Z", + "updated_at": "2019-02-28T12:41:33Z", + "pushed_at": "2019-02-28T13:43:16Z", + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "http://irc.w3.org/?channels=testing", + "size": 324641, + "stargazers_count": 2058, + "watchers_count": 2058, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1604, + "mirror_url": null, + "archived": false, + "open_issues_count": 1354, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1604, + "open_issues": 1354, + "watchers": 2058, + "default_branch": "master" + }, + "organization": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "url": "https://api.github.com/orgs/web-platform-tests", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks", + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "description": "" + }, + "sender": { + "login": "jgraham", + "id": 294864, + "node_id": "MDQ6VXNlcjI5NDg2NA==", + "avatar_url": "https://avatars1.githubusercontent.com/u/294864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/jgraham", + "html_url": "https://github.com/jgraham", + "followers_url": "https://api.github.com/users/jgraham/followers", + "following_url": "https://api.github.com/users/jgraham/following{/other_user}", + "gists_url": "https://api.github.com/users/jgraham/gists{/gist_id}", + "starred_url": "https://api.github.com/users/jgraham/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/jgraham/subscriptions", + "organizations_url": "https://api.github.com/users/jgraham/orgs", + "repos_url": "https://api.github.com/users/jgraham/repos", + "events_url": "https://api.github.com/users/jgraham/events{/privacy}", + "received_events_url": "https://api.github.com/users/jgraham/received_events", + "type": "User", + "site_admin": false + } +} diff --git a/testing/web-platform/tests/tools/ci/tc/testdata/pr_event_tests_affected.json b/testing/web-platform/tests/tools/ci/tc/testdata/pr_event_tests_affected.json new file mode 100644 index 0000000000..792ea1bccc --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/testdata/pr_event_tests_affected.json @@ -0,0 +1,505 @@ +{ + "action": "synchronize", + "number": 20378, + "pull_request": { + "url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/20378", + "id": 344287920, + "node_id": "MDExOlB1bGxSZXF1ZXN0MzQ0Mjg3OTIw", + "html_url": "https://github.com/web-platform-tests/wpt/pull/20378", + "diff_url": "https://github.com/web-platform-tests/wpt/pull/20378.diff", + "patch_url": "https://github.com/web-platform-tests/wpt/pull/20378.patch", + "issue_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/20378", + "number": 20378, + "state": "open", + "locked": false, + "title": "Migrate more layout instability tests to WPT.", + "user": { + "login": "chromium-wpt-export-bot", + "id": 25752892, + "node_id": "MDQ6VXNlcjI1NzUyODky", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "html_url": "https://github.com/chromium-wpt-export-bot", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "type": "User", + "site_admin": false + }, + "body": "Bug: 984109\nChange-Id: Ie31c63995f63f8acbfa26d97966ffe30016edd3c\nReviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1925447\nCommit-Queue: Steve Kobes \\<skobes@chromium.org>\nReviewed-by: Nicolás Peña Moreno \\<npm@chromium.org>\nCr-Commit-Position: refs/heads/master@{#718648}\n\n", + "created_at": "2019-11-21T23:53:58Z", + "updated_at": "2019-11-25T15:54:19Z", + "closed_at": null, + "merged_at": null, + "merge_commit_sha": "36726e3e992a83e80608acf47c886fc440390691", + "assignee": null, + "assignees": [ + + ], + "requested_reviewers": [ + + ], + "requested_teams": [ + + ], + "labels": [ + { + "id": 490891502, + "node_id": "MDU6TGFiZWw0OTA4OTE1MDI=", + "url": "https://api.github.com/repos/web-platform-tests/wpt/labels/chromium-export", + "name": "chromium-export", + "color": "4788f4", + "default": false, + "description": null + }, + { + "id": 1363199651, + "node_id": "MDU6TGFiZWwxMzYzMTk5NjUx", + "url": "https://api.github.com/repos/web-platform-tests/wpt/labels/layout-instability", + "name": "layout-instability", + "color": "ededed", + "default": false, + "description": null + } + ], + "milestone": null, + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/20378/commits", + "review_comments_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/20378/comments", + "review_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/20378/comments", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/7df84e3f87f05860a2b86d0b80dc3eb06d8e7103", + "head": { + "label": "web-platform-tests:chromium-export-cl-1925447", + "ref": "chromium-export-cl-1925447", + "sha": "7df84e3f87f05860a2b86d0b80dc3eb06d8e7103", + "user": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://api.github.com/repos/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": "2012-03-04T12:58:11Z", + "updated_at": "2019-11-25T15:40:59Z", + "pushed_at": "2019-11-25T15:54:18Z", + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "https://web-platform-tests.org/", + "size": 329278, + "stargazers_count": 2536, + "watchers_count": 2536, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1837, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 1576, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1837, + "open_issues": 1576, + "watchers": 2536, + "default_branch": "master" + } + }, + "base": { + "label": "web-platform-tests:master", + "ref": "master", + "sha": "90ebe5c33c44090271759f8e9dc43d0b5bd0f8f7", + "user": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://api.github.com/repos/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": "2012-03-04T12:58:11Z", + "updated_at": "2019-11-25T15:40:59Z", + "pushed_at": "2019-11-25T15:54:18Z", + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "https://web-platform-tests.org/", + "size": 329278, + "stargazers_count": 2536, + "watchers_count": 2536, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1837, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 1576, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1837, + "open_issues": 1576, + "watchers": 2536, + "default_branch": "master" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/20378" + }, + "html": { + "href": "https://github.com/web-platform-tests/wpt/pull/20378" + }, + "issue": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/issues/20378" + }, + "comments": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/issues/20378/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/20378/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/pulls/20378/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/web-platform-tests/wpt/statuses/7df84e3f87f05860a2b86d0b80dc3eb06d8e7103" + } + }, + "author_association": "COLLABORATOR", + "draft": false, + "merged": false, + "mergeable": null, + "rebaseable": null, + "mergeable_state": "unknown", + "merged_by": null, + "comments": 0, + "review_comments": 0, + "maintainer_can_modify": false, + "commits": 1, + "additions": 431, + "deletions": 0, + "changed_files": 11 + }, + "before": "86742511fa37e1e2c1635b77431bd46958ecfb92", + "after": "7df84e3f87f05860a2b86d0b80dc3eb06d8e7103", + "repository": { + "id": 3618133, + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "name": "wpt", + "full_name": "web-platform-tests/wpt", + "private": false, + "owner": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-platform-tests", + "html_url": "https://github.com/web-platform-tests", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/web-platform-tests/wpt", + "description": "Test suites for Web platform specs — including WHATWG, W3C, and others", + "fork": false, + "url": "https://api.github.com/repos/web-platform-tests/wpt", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "created_at": "2012-03-04T12:58:11Z", + "updated_at": "2019-11-25T15:40:59Z", + "pushed_at": "2019-11-25T15:54:18Z", + "git_url": "git://github.com/web-platform-tests/wpt.git", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "svn_url": "https://github.com/web-platform-tests/wpt", + "homepage": "https://web-platform-tests.org/", + "size": 329278, + "stargazers_count": 2536, + "watchers_count": 2536, + "language": "HTML", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1837, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 1576, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 1837, + "open_issues": 1576, + "watchers": 2536, + "default_branch": "master" + }, + "organization": { + "login": "web-platform-tests", + "id": 37226233, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "url": "https://api.github.com/orgs/web-platform-tests", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks", + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "description": "" + }, + "sender": { + "login": "chromium-wpt-export-bot", + "id": 25752892, + "node_id": "MDQ6VXNlcjI1NzUyODky", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "html_url": "https://github.com/chromium-wpt-export-bot", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "type": "User", + "site_admin": false + } +} diff --git a/testing/web-platform/tests/tools/ci/tc/tests/__init__.py b/testing/web-platform/tests/tools/ci/tc/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/tests/__init__.py diff --git a/testing/web-platform/tests/tools/ci/tc/tests/test_decision.py b/testing/web-platform/tests/tools/ci/tc/tests/test_decision.py new file mode 100644 index 0000000000..f702466421 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/tests/test_decision.py @@ -0,0 +1,56 @@ +# mypy: allow-untyped-defs + +from unittest import mock + +import pytest + +from tools.ci.tc import decision + + +@pytest.mark.parametrize("run_jobs,tasks,expected", [ + ([], {"task-no-schedule-if": {}}, ["task-no-schedule-if"]), + ([], {"task-schedule-if-no-run-job": {"schedule-if": {}}}, []), + (["job"], + {"job-present": {"schedule-if": {"run-job": ["other-job", "job"]}}}, + ["job-present"]), + (["job"], {"job-missing": {"schedule-if": {"run-job": ["other-job"]}}}, []), + (["all"], {"job-all": {"schedule-if": {"run-job": ["other-job"]}}}, ["job-all"]), + (["job"], + {"job-1": {"schedule-if": {"run-job": ["job"]}}, + "job-2": {"schedule-if": {"run-job": ["other-job"]}}}, + ["job-1"]), +]) +def test_filter_schedule_if(run_jobs, tasks, expected): + with mock.patch("tools.ci.tc.decision.get_run_jobs", + return_value=run_jobs) as get_run_jobs: + assert (decision.filter_schedule_if({}, tasks) == + {name: tasks[name] for name in expected}) + get_run_jobs.call_count in (0, 1) + + +@pytest.mark.parametrize("msg,expected", [ + ("Some initial line\n\ntc-jobs:foo,bar", {"foo", "bar"}), + ("Some initial line\n\ntc-jobs:foo, bar", {"foo", "bar"}), + ("tc-jobs:foo, bar \nbaz", {"foo", "bar"}), + ("tc-jobs:all", {"all"}), + ("", set()), + ("tc-jobs:foo\ntc-jobs:bar", {"foo"})]) +@pytest.mark.parametrize("event", [ + {"commits": [{"message": "<message>"}]}, + {"pull_request": {"body": "<message>"}} +]) +def test_extra_jobs_pr(msg, expected, event): + def sub(obj): + """Copy obj, except if it's a string with the value <message> + replace it with the value of the msg argument""" + if isinstance(obj, dict): + return {key: sub(value) for (key, value) in obj.items()} + elif isinstance(obj, list): + return [sub(value) for value in obj] + elif obj == "<message>": + return msg + return obj + + event = sub(event) + + assert decision.get_extra_jobs(event) == expected diff --git a/testing/web-platform/tests/tools/ci/tc/tests/test_taskgraph.py b/testing/web-platform/tests/tools/ci/tc/tests/test_taskgraph.py new file mode 100644 index 0000000000..57528a6659 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/tests/test_taskgraph.py @@ -0,0 +1,148 @@ +# mypy: allow-untyped-defs + +import pytest +import yaml + +from tools.ci.tc import taskgraph + +@pytest.mark.parametrize("data, update_data, expected", [ + ({"a": 1}, {"b": 2}, {"a": 1, "b": 2}), + ({"a": 1}, {"a": 2}, {"a": 2}), + ({"a": [1]}, {"a": [2]}, {"a": [1, 2]}), + ({"a": {"b": 1, "c": 2}}, {"a": {"b": 2, "d": 3}}, {"a": {"b": 2, "c": 2, "d": 3}}), + ({"a": {"b": [1]}}, {"a": {"b": [2]}}, {"a": {"b": [1, 2]}}), +] +) +def test_update_recursive(data, update_data, expected): + taskgraph.update_recursive(data, update_data) + assert data == expected + + +def test_use(): + data = """ +components: + component1: + a: 1 + b: [1] + c: "c" + component2: + a: 2 + b: [2] + d: "d" +tasks: + - task1: + use: + - component1 + - component2 + b: [3] + c: "e" +""" + tasks_data = yaml.safe_load(data) + assert taskgraph.load_tasks(tasks_data) == { + "task1": { + "a": 2, + "b": [1,2,3], + "c": "e", + "d": "d", + "name": "task1" + } + } + + +def test_var(): + data = """ +components: + component1: + a: ${vars.value} +tasks: + - task1: + use: + - component1 + vars: + value: 1 +""" + tasks_data = yaml.safe_load(data) + assert taskgraph.load_tasks(tasks_data) == { + "task1": { + "a": "1", + "vars": {"value": 1}, + "name": "task1" + } + } + + +def test_map(): + data = """ +components: {} +tasks: + - $map: + for: + - vars: + a: 1 + b: [1] + - vars: + a: 2 + b: [2] + do: + - task1-${vars.a}: + a: ${vars.a} + b: [3] + - task2-${vars.a}: + a: ${vars.a} + b: [4] +""" + tasks_data = yaml.safe_load(data) + assert taskgraph.load_tasks(tasks_data) == { + "task1-1": { + "a": "1", + "b": [1, 3], + "vars": {"a": 1}, + "name": "task1-1" + }, + "task1-2": { + "a": "2", + "b": [2, 3], + "vars": {"a": 2}, + "name": "task1-2" + }, + "task2-1": { + "a": "1", + "b": [1, 4], + "vars": {"a": 1}, + "name": "task2-1" + }, + "task2-2": { + "a": "2", + "b": [2, 4], + "vars": {"a": 2}, + "name": "task2-2" + }, + + } + + +def test_chunks(): + data = """ +components: {} +tasks: + - task1: + name: task1-${chunks.id} + chunks: 2 +""" + tasks_data = yaml.safe_load(data) + assert taskgraph.load_tasks(tasks_data) == { + "task1-1": { + "name": "task1-1", + "chunks": { + "id": 1, + "total": 2 + } + }, + "task1-2": { + "name": "task1-2", + "chunks": { + "id": 2, + "total": 2 + } + } + } diff --git a/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py b/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py new file mode 100644 index 0000000000..6960a2cc47 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py @@ -0,0 +1,379 @@ +# mypy: ignore-errors + +import json +import os +from unittest import mock + +import httpx +import jsone +import pytest +import yaml +from jsonschema import validate + +from tools.ci.tc import decision + +here = os.path.dirname(__file__) +root = os.path.abspath(os.path.join(here, "..", "..", "..", "..")) + + +def data_path(filename): + return os.path.join(here, "..", "testdata", filename) + + +def test_verify_taskcluster_yml(): + """Verify that the json-e in the .taskcluster.yml is valid""" + with open(os.path.join(root, ".taskcluster.yml"), encoding="utf8") as f: + template = yaml.safe_load(f) + + events = [("pr_event.json", "github-pull-request", "Pull Request"), + ("master_push_event.json", "github-push", "Push to master")] + + for filename, tasks_for, title in events: + with open(data_path(filename), encoding="utf8") as f: + event = json.load(f) + + context = {"tasks_for": tasks_for, + "event": event, + "as_slugid": lambda x: x} + + jsone.render(template, context) + + +@pytest.mark.parametrize("event_path,expected", + [("pr_event.json", + frozenset(["lint", "wpt-chrome-dev-stability"])), + ("pr_event_tests_affected.json", frozenset(["lint"]))] + ) +def test_exclude_users(event_path, expected): + """Verify that tasks excluded by the PR submitter are properly excluded""" + tasks = { + "lint": { + "commands": "wpt example" + }, + "wpt-chrome-dev-stability": { + "commands": "wpt example", + "exclude-users": ["chromium-wpt-export-bot"] + } + } + with open(data_path(event_path), encoding="utf8") as f: + event = json.load(f) + decision.filter_excluded_users(tasks, event) + assert set(tasks) == expected + + +def test_sink_task_depends(): + """Verify that sink task only depends on required tasks""" + from tools.ci.tc.decision import decide + files_changed = {"layout-instability/clip-negative-bottom-margin.html", + "layout-instability/composited-element-movement.html"} + with open(data_path("pr_event_tests_affected.json"), encoding="utf8") as f: + event = json.load(f) + with mock.patch("tools.ci.tc.decision.get_fetch_rev", return_value=(None, None, None)): + with mock.patch("tools.wpt.testfiles.repo_files_changed", + return_value=files_changed): + # Ensure we don't exclude the Chrome jobs + event["pull_request"]["user"]["login"] = "test" + task_id_map = decide(event) + + sink_task = task_id_map["sink-task"][1] + + assert set(sink_task["dependencies"]) == ( + set(task_id for (task_name, (task_id, _)) in task_id_map.items() + if task_name not in {"sink-task", "wpt-chrome-dev-stability"})) + + +def test_verify_payload(): + """Verify that the decision task produces tasks with a valid payload""" + from tools.ci.tc.decision import decide + + r = httpx.get("https://community-tc.services.mozilla.com/schemas/queue/v1/create-task-request.json") + r.raise_for_status() + create_task_schema = r.json() + + r = httpx.get("https://community-tc.services.mozilla.com/references/schemas/docker-worker/v1/payload.json") + r.raise_for_status() + payload_schema = r.json() + + jobs = ["lint", + "manifest_upload", + "resources_unittest", + "tools_unittest", + "wpt_integration", + "wptrunner_infrastructure", + "wptrunner_unittest"] + + for filename in ["pr_event.json", "master_push_event.json"]: + with open(data_path(filename), encoding="utf8") as f: + event = json.load(f) + + with mock.patch("tools.ci.tc.decision.get_fetch_rev", return_value=(None, event["after"], None)): + with mock.patch("tools.ci.tc.decision.get_run_jobs", return_value=set(jobs)): + task_id_map = decide(event) + for name, (task_id, task_data) in task_id_map.items(): + try: + validate(instance=task_data, schema=create_task_schema) + validate(instance=task_data["payload"], schema=payload_schema) + except Exception as e: + print(f"Validation failed for task '{name}':\n{json.dumps(task_data, indent=2)}") + raise e + + +@pytest.mark.parametrize("event_path,is_pr,files_changed,expected", [ + ("master_push_event.json", False, None, + ['download-firefox-nightly', + 'wpt-firefox-nightly-testharness-1', + 'wpt-firefox-nightly-testharness-2', + 'wpt-firefox-nightly-testharness-3', + 'wpt-firefox-nightly-testharness-4', + 'wpt-firefox-nightly-testharness-5', + 'wpt-firefox-nightly-testharness-6', + 'wpt-firefox-nightly-testharness-7', + 'wpt-firefox-nightly-testharness-8', + 'wpt-firefox-nightly-testharness-9', + 'wpt-firefox-nightly-testharness-10', + 'wpt-firefox-nightly-testharness-11', + 'wpt-firefox-nightly-testharness-12', + 'wpt-firefox-nightly-testharness-13', + 'wpt-firefox-nightly-testharness-14', + 'wpt-firefox-nightly-testharness-15', + 'wpt-firefox-nightly-testharness-16', + 'wpt-chrome-canary-testharness-1', + 'wpt-chrome-canary-testharness-2', + 'wpt-chrome-canary-testharness-3', + 'wpt-chrome-canary-testharness-4', + 'wpt-chrome-canary-testharness-5', + 'wpt-chrome-canary-testharness-6', + 'wpt-chrome-canary-testharness-7', + 'wpt-chrome-canary-testharness-8', + 'wpt-chrome-canary-testharness-9', + 'wpt-chrome-canary-testharness-10', + 'wpt-chrome-canary-testharness-11', + 'wpt-chrome-canary-testharness-12', + 'wpt-chrome-canary-testharness-13', + 'wpt-chrome-canary-testharness-14', + 'wpt-chrome-canary-testharness-15', + 'wpt-chrome-canary-testharness-16', + 'wpt-chrome-dev-testharness-1', + 'wpt-chrome-dev-testharness-2', + 'wpt-chrome-dev-testharness-3', + 'wpt-chrome-dev-testharness-4', + 'wpt-chrome-dev-testharness-5', + 'wpt-chrome-dev-testharness-6', + 'wpt-chrome-dev-testharness-7', + 'wpt-chrome-dev-testharness-8', + 'wpt-chrome-dev-testharness-9', + 'wpt-chrome-dev-testharness-10', + 'wpt-chrome-dev-testharness-11', + 'wpt-chrome-dev-testharness-12', + 'wpt-chrome-dev-testharness-13', + 'wpt-chrome-dev-testharness-14', + 'wpt-chrome-dev-testharness-15', + 'wpt-chrome-dev-testharness-16', + 'wpt-firefox-nightly-reftest-1', + 'wpt-firefox-nightly-reftest-2', + 'wpt-firefox-nightly-reftest-3', + 'wpt-firefox-nightly-reftest-4', + 'wpt-firefox-nightly-reftest-5', + 'wpt-firefox-nightly-reftest-6', + 'wpt-chrome-canary-reftest-1', + 'wpt-chrome-canary-reftest-2', + 'wpt-chrome-canary-reftest-3', + 'wpt-chrome-canary-reftest-4', + 'wpt-chrome-canary-reftest-5', + 'wpt-chrome-canary-reftest-6', + 'wpt-chrome-dev-reftest-1', + 'wpt-chrome-dev-reftest-2', + 'wpt-chrome-dev-reftest-3', + 'wpt-chrome-dev-reftest-4', + 'wpt-chrome-dev-reftest-5', + 'wpt-chrome-dev-reftest-6', + 'wpt-firefox-nightly-wdspec-1', + 'wpt-firefox-nightly-wdspec-2', + 'wpt-chrome-canary-wdspec-1', + 'wpt-chrome-canary-wdspec-2', + 'wpt-chrome-dev-wdspec-1', + 'wpt-chrome-dev-wdspec-2', + 'wpt-firefox-nightly-crashtest-1', + 'wpt-chrome-canary-crashtest-1', + 'wpt-chrome-dev-crashtest-1', + 'wpt-firefox-nightly-print-reftest-1', + 'wpt-chrome-canary-print-reftest-1', + 'wpt-chrome-dev-print-reftest-1', + 'lint']), + ("pr_event.json", True, {".taskcluster.yml", ".travis.yml", "tools/ci/start.sh"}, + ['lint', + 'tools/ unittests (Python 3.7)', + 'tools/ unittests (Python 3.11)', + 'tools/ integration tests (Python 3.7)', + 'tools/ integration tests (Python 3.11)', + 'resources/ tests (Python 3.7)', + 'resources/ tests (Python 3.11)', + 'download-firefox-nightly', + 'infrastructure/ tests', + 'sink-task']), + # More tests are affected in the actual PR but it shouldn't affect the scheduled tasks + ("pr_event_tests_affected.json", True, {"layout-instability/clip-negative-bottom-margin.html", + "layout-instability/composited-element-movement.html"}, + ['download-firefox-nightly', + 'wpt-firefox-nightly-stability', + 'wpt-firefox-nightly-results', + 'wpt-firefox-nightly-results-without-changes', + 'wpt-chrome-dev-results', + 'wpt-chrome-dev-results-without-changes', + 'lint', + 'sink-task']), + ("pr_event_tests_affected.json", True, {"resources/testharness.js"}, + ['lint', + 'resources/ tests (Python 3.7)', + 'resources/ tests (Python 3.11)', + 'download-firefox-nightly', + 'infrastructure/ tests', + 'sink-task']), + ("epochs_daily_push_event.json", False, None, + ['download-firefox-stable', + 'wpt-firefox-stable-testharness-1', + 'wpt-firefox-stable-testharness-2', + 'wpt-firefox-stable-testharness-3', + 'wpt-firefox-stable-testharness-4', + 'wpt-firefox-stable-testharness-5', + 'wpt-firefox-stable-testharness-6', + 'wpt-firefox-stable-testharness-7', + 'wpt-firefox-stable-testharness-8', + 'wpt-firefox-stable-testharness-9', + 'wpt-firefox-stable-testharness-10', + 'wpt-firefox-stable-testharness-11', + 'wpt-firefox-stable-testharness-12', + 'wpt-firefox-stable-testharness-13', + 'wpt-firefox-stable-testharness-14', + 'wpt-firefox-stable-testharness-15', + 'wpt-firefox-stable-testharness-16', + 'wpt-chromium-nightly-testharness-1', + 'wpt-chromium-nightly-testharness-2', + 'wpt-chromium-nightly-testharness-3', + 'wpt-chromium-nightly-testharness-4', + 'wpt-chromium-nightly-testharness-5', + 'wpt-chromium-nightly-testharness-6', + 'wpt-chromium-nightly-testharness-7', + 'wpt-chromium-nightly-testharness-8', + 'wpt-chromium-nightly-testharness-9', + 'wpt-chromium-nightly-testharness-10', + 'wpt-chromium-nightly-testharness-11', + 'wpt-chromium-nightly-testharness-12', + 'wpt-chromium-nightly-testharness-13', + 'wpt-chromium-nightly-testharness-14', + 'wpt-chromium-nightly-testharness-15', + 'wpt-chromium-nightly-testharness-16', + 'wpt-chrome-stable-testharness-1', + 'wpt-chrome-stable-testharness-2', + 'wpt-chrome-stable-testharness-3', + 'wpt-chrome-stable-testharness-4', + 'wpt-chrome-stable-testharness-5', + 'wpt-chrome-stable-testharness-6', + 'wpt-chrome-stable-testharness-7', + 'wpt-chrome-stable-testharness-8', + 'wpt-chrome-stable-testharness-9', + 'wpt-chrome-stable-testharness-10', + 'wpt-chrome-stable-testharness-11', + 'wpt-chrome-stable-testharness-12', + 'wpt-chrome-stable-testharness-13', + 'wpt-chrome-stable-testharness-14', + 'wpt-chrome-stable-testharness-15', + 'wpt-chrome-stable-testharness-16', + 'wpt-webkitgtk_minibrowser-nightly-testharness-1', + 'wpt-webkitgtk_minibrowser-nightly-testharness-2', + 'wpt-webkitgtk_minibrowser-nightly-testharness-3', + 'wpt-webkitgtk_minibrowser-nightly-testharness-4', + 'wpt-webkitgtk_minibrowser-nightly-testharness-5', + 'wpt-webkitgtk_minibrowser-nightly-testharness-6', + 'wpt-webkitgtk_minibrowser-nightly-testharness-7', + 'wpt-webkitgtk_minibrowser-nightly-testharness-8', + 'wpt-webkitgtk_minibrowser-nightly-testharness-9', + 'wpt-webkitgtk_minibrowser-nightly-testharness-10', + 'wpt-webkitgtk_minibrowser-nightly-testharness-11', + 'wpt-webkitgtk_minibrowser-nightly-testharness-12', + 'wpt-webkitgtk_minibrowser-nightly-testharness-13', + 'wpt-webkitgtk_minibrowser-nightly-testharness-14', + 'wpt-webkitgtk_minibrowser-nightly-testharness-15', + 'wpt-webkitgtk_minibrowser-nightly-testharness-16', + 'wpt-firefox_android-nightly-testharness-1', + 'wpt-firefox_android-nightly-testharness-2', + 'wpt-firefox_android-nightly-testharness-3', + 'wpt-firefox_android-nightly-testharness-4', + 'wpt-firefox_android-nightly-testharness-5', + 'wpt-firefox_android-nightly-testharness-6', + 'wpt-firefox_android-nightly-testharness-7', + 'wpt-firefox_android-nightly-testharness-8', + 'wpt-firefox_android-nightly-testharness-9', + 'wpt-firefox_android-nightly-testharness-10', + 'wpt-firefox_android-nightly-testharness-11', + 'wpt-firefox_android-nightly-testharness-12', + 'wpt-firefox_android-nightly-testharness-13', + 'wpt-firefox_android-nightly-testharness-14', + 'wpt-firefox_android-nightly-testharness-15', + 'wpt-firefox_android-nightly-testharness-16', + 'wpt-firefox_android-nightly-testharness-17', + 'wpt-firefox_android-nightly-testharness-18', + 'wpt-firefox_android-nightly-testharness-19', + 'wpt-firefox_android-nightly-testharness-20', + 'wpt-firefox_android-nightly-testharness-21', + 'wpt-firefox_android-nightly-testharness-22', + 'wpt-firefox_android-nightly-testharness-23', + 'wpt-firefox_android-nightly-testharness-24', + 'wpt-firefox-stable-reftest-1', + 'wpt-firefox-stable-reftest-2', + 'wpt-firefox-stable-reftest-3', + 'wpt-firefox-stable-reftest-4', + 'wpt-firefox-stable-reftest-5', + 'wpt-firefox-stable-reftest-6', + 'wpt-chromium-nightly-reftest-1', + 'wpt-chromium-nightly-reftest-2', + 'wpt-chromium-nightly-reftest-3', + 'wpt-chromium-nightly-reftest-4', + 'wpt-chromium-nightly-reftest-5', + 'wpt-chromium-nightly-reftest-6', + 'wpt-chrome-stable-reftest-1', + 'wpt-chrome-stable-reftest-2', + 'wpt-chrome-stable-reftest-3', + 'wpt-chrome-stable-reftest-4', + 'wpt-chrome-stable-reftest-5', + 'wpt-chrome-stable-reftest-6', + 'wpt-webkitgtk_minibrowser-nightly-reftest-1', + 'wpt-webkitgtk_minibrowser-nightly-reftest-2', + 'wpt-webkitgtk_minibrowser-nightly-reftest-3', + 'wpt-webkitgtk_minibrowser-nightly-reftest-4', + 'wpt-webkitgtk_minibrowser-nightly-reftest-5', + 'wpt-webkitgtk_minibrowser-nightly-reftest-6', + 'wpt-firefox_android-nightly-reftest-1', + 'wpt-firefox_android-nightly-reftest-2', + 'wpt-firefox_android-nightly-reftest-3', + 'wpt-firefox_android-nightly-reftest-4', + 'wpt-firefox_android-nightly-reftest-5', + 'wpt-firefox_android-nightly-reftest-6', + 'wpt-firefox-stable-wdspec-1', + 'wpt-firefox-stable-wdspec-2', + 'wpt-chromium-nightly-wdspec-1', + 'wpt-chromium-nightly-wdspec-2', + 'wpt-chrome-stable-wdspec-1', + 'wpt-chrome-stable-wdspec-2', + 'wpt-webkitgtk_minibrowser-nightly-wdspec-1', + 'wpt-webkitgtk_minibrowser-nightly-wdspec-2', + 'wpt-firefox_android-nightly-wdspec-1', + 'wpt-firefox_android-nightly-wdspec-2', + 'wpt-firefox-stable-crashtest-1', + 'wpt-chromium-nightly-crashtest-1', + 'wpt-chrome-stable-crashtest-1', + 'wpt-webkitgtk_minibrowser-nightly-crashtest-1', + 'wpt-firefox_android-nightly-crashtest-1', + 'wpt-firefox-stable-print-reftest-1', + 'wpt-chromium-nightly-print-reftest-1', + 'wpt-chrome-stable-print-reftest-1']) +]) +def test_schedule_tasks(event_path, is_pr, files_changed, expected): + with mock.patch("tools.ci.tc.decision.get_fetch_rev", return_value=(None, None, None)): + with mock.patch("tools.wpt.testfiles.repo_files_changed", + return_value=files_changed): + with open(data_path(event_path), encoding="utf8") as event_file: + event = json.load(event_file) + scheduled = decision.decide(event) + print(list(scheduled.keys())) + assert list(scheduled.keys()) == expected diff --git a/testing/web-platform/tests/tools/ci/tests/__init__.py b/testing/web-platform/tests/tools/ci/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tests/__init__.py diff --git a/testing/web-platform/tests/tools/ci/tests/test_jobs.py b/testing/web-platform/tests/tools/ci/tests/test_jobs.py new file mode 100644 index 0000000000..421af78fef --- /dev/null +++ b/testing/web-platform/tests/tools/ci/tests/test_jobs.py @@ -0,0 +1,132 @@ +# mypy: allow-untyped-defs + +from tools.ci import jobs + +all_jobs = { + "lint", + "manifest_upload", + "resources_unittest", + "affected_tests", + "stability", + "tools_unittest", + "update_built", + "wpt_integration", + "wptrunner_infrastructure", + "wptrunner_unittest", +} + +default_jobs = {"lint", "manifest_upload"} + + +def test_all(): + assert jobs.get_jobs(["README.md"], all=True) == all_jobs + + +def test_default(): + assert jobs.get_jobs(["README.md"]) == default_jobs + + +def test_testharness(): + assert jobs.get_jobs(["resources/testharness.js"]) == default_jobs | {"resources_unittest", + "wptrunner_infrastructure"} + assert jobs.get_jobs(["resources/testharness.js"], + includes=["resources_unittest"]) == {"resources_unittest"} + assert jobs.get_jobs(["tools/wptserve/wptserve/config.py"], + includes=["resources_unittest"]) == {"resources_unittest"} + assert jobs.get_jobs(["foo/resources/testharness.js"], + includes=["resources_unittest"]) == set() + + +def test_stability(): + assert jobs.get_jobs(["dom/historical.html"], + includes=["stability"]) == {"stability"} + assert jobs.get_jobs(["tools/pytest.ini"], + includes=["stability"]) == set() + assert jobs.get_jobs(["serve"], + includes=["stability"]) == set() + assert jobs.get_jobs(["resources/testharness.js"], + includes=["stability"]) == set() + assert jobs.get_jobs(["docs/.gitignore"], + includes=["stability"]) == set() + assert jobs.get_jobs(["dom/tools/example.py"], + includes=["stability"]) == set() + assert jobs.get_jobs(["conformance-checkers/test.html"], + includes=["stability"]) == set() + assert jobs.get_jobs(["dom/README.md"], + includes=["stability"]) == set() + assert jobs.get_jobs(["css/build-css-testsuite.sh"], + includes=["stability"]) == set() + assert jobs.get_jobs(["css/CSS21/test-001.html"], + includes=["stability"]) == {"stability"} + assert jobs.get_jobs(["css/build-css-testsuite.sh", + "css/CSS21/test-001.html"], + includes=["stability"]) == {"stability"} + +def test_affected_tests(): + assert jobs.get_jobs(["dom/historical.html"], + includes=["affected_tests"]) == {"affected_tests"} + assert jobs.get_jobs(["tools/pytest.ini"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["serve"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["resources/testharness.js"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["docs/.gitignore"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["dom/tools/example.py"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["conformance-checkers/test.html"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["dom/README.md"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["css/build-css-testsuite.sh"], + includes=["affected_tests"]) == set() + assert jobs.get_jobs(["css/CSS21/test-001.html"], + includes=["affected_tests"]) == {"affected_tests"} + assert jobs.get_jobs(["css/build-css-testsuite.sh", + "css/CSS21/test-001.html"], + includes=["affected_tests"]) == {"affected_tests"} + assert jobs.get_jobs(["resources/idlharness.js"], + includes=["affected_tests"]) == {"affected_tests"} + +def test_tools_unittest(): + assert jobs.get_jobs(["tools/ci/test/test_jobs.py"], + includes=["tools_unittest"]) == {"tools_unittest"} + assert jobs.get_jobs(["dom/tools/example.py"], + includes=["tools_unittest"]) == set() + assert jobs.get_jobs(["dom/historical.html"], + includes=["tools_unittest"]) == set() + + +def test_wptrunner_unittest(): + assert jobs.get_jobs(["tools/wptrunner/wptrunner/wptrunner.py"], + includes=["wptrunner_unittest"]) == {"wptrunner_unittest"} + assert jobs.get_jobs(["tools/example.py"], + includes=["wptrunner_unittest"]) == {"wptrunner_unittest"} + + +def test_update_built(): + assert jobs.get_jobs(["html/canvas/element/foo.html"], + includes=["update_built"]) == {"update_built"} + assert jobs.get_jobs(["html/foo.html"], + includes=["update_built"]) == {"update_built"} + assert jobs.get_jobs(["html/canvas/offscreen/foo.html"], + includes=["update_built"]) == {"update_built"} + + +def test_wpt_integration(): + assert jobs.get_jobs(["tools/wpt/wpt.py"], + includes=["wpt_integration"]) == {"wpt_integration"} + assert jobs.get_jobs(["tools/wptrunner/wptrunner/wptrunner.py"], + includes=["wpt_integration"]) == {"wpt_integration"} + + +def test_wpt_infrastructure(): + assert jobs.get_jobs(["tools/hammer.html"], + includes=["wptrunner_infrastructure"]) == {"wptrunner_infrastructure"} + assert jobs.get_jobs(["infrastructure/assumptions/ahem.html"], + includes=["wptrunner_infrastructure"]) == {"wptrunner_infrastructure"} + +def test_wdspec_support(): + assert jobs.get_jobs(["webdriver/tests/support/__init__.py"], + includes=["wptrunner_infrastructure"]) == {"wptrunner_infrastructure"} diff --git a/testing/web-platform/tests/tools/ci/update_built.py b/testing/web-platform/tests/tools/ci/update_built.py new file mode 100644 index 0000000000..8e0f18589d --- /dev/null +++ b/testing/web-platform/tests/tools/ci/update_built.py @@ -0,0 +1,74 @@ +# mypy: allow-untyped-defs + +import logging +import os +import subprocess +from argparse import ArgumentParser + +logger = logging.getLogger() + +wpt_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) + +scripts = { + "canvas": ["html/canvas/tools/gentest.py"], + "conformance-checkers": ["conformance-checkers/tools/dl.py", + "conformance-checkers/tools/ins-del-datetime.py", + "conformance-checkers/tools/picture.py", + "conformance-checkers/tools/url.py"], + "css-images": ["css/css-images/tools/generate_object_view_box_tests.py"], + "css-ui": ["css/css-ui/tools/appearance-build-webkit-reftests.py"], + "css-writing-modes": ["css/css-writing-modes/tools/generators/generate.py"], + # FIXME: https://github.com/web-platform-tests/wpt/issues/32060 + # "css-text": ["css/css-text/line-breaking/tools/generate-segment-break-transformation-rules-tests.py"], + # "css-text-decor": ["css/css-text-decor/tools/generate-text-emphasis-line-height-tests.py", + # "css/css-text-decor/tools/generate-text-emphasis-position-property-tests.py", + # "css/css-text-decor/tools/generate-text-emphasis-ruby-tests.py", + # "css/css-text-decor/tools/generate-text-emphasis-style-property-tests.py"], + "fetch": ["fetch/metadata/tools/generate.py"], + "html5lib": ["html/tools/update_html5lib_tests.py"], + "infrastructure": ["infrastructure/assumptions/tools/ahem-generate-table.py"], + "mimesniff": ["mimesniff/mime-types/resources/generated-mime-types.py"], + "speculative-parsing": ["html/syntax/speculative-parsing/tools/generate.py"] +} + + +def get_parser(): + parser = ArgumentParser() + parser.add_argument("--list", action="store_true", + help="List suites that can be updated and the related script files") + parser.add_argument("--include", nargs="*", choices=scripts.keys(), default=None, + help="Suites to update (default is to update everything)") + return parser + + +def list_suites(include): + for name, script_paths in scripts.items(): + if name in include: + print(name) + for script_path in script_paths: + print(f" {script_path}") + + +def run(venv, **kwargs): + include = kwargs["include"] + if include is None: + include = list(scripts.keys()) + + if kwargs["list"]: + list_suites(include) + return 0 + + failed = False + + for target in include: + for script in scripts[target]: + script_path = script.replace("/", os.path.sep) + cmd = [os.path.join(venv.bin_path, "python3"), os.path.join(wpt_root, script_path)] + logger.info(f"Running {' '.join(cmd)}") + try: + subprocess.check_call(cmd, cwd=os.path.dirname(script_path)) + except subprocess.CalledProcessError: + logger.error(f"Update script {script} failed") + failed = True + + return 1 if failed else 0 diff --git a/testing/web-platform/tests/tools/ci/website_build.sh b/testing/web-platform/tests/tools/ci/website_build.sh new file mode 100755 index 0000000000..adfdf41ae7 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/website_build.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -ex + +neutral_status=0 +source_revision=$(git rev-parse HEAD) +# The token available in the `GITHUB_TOKEN` variable may be used to push to the +# repository, but GitHub Pages will not rebuild the website in response to such +# events. Use an access token generated for the project's machine user, +# wpt-pr-bot. +# +# https://help.github.com/en/articles/generic-jekyll-build-failures +remote_url=https://${DEPLOY_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +wpt_root=$PWD + +function json_property { + cat ${1} | \ + python -c "import json, sys; print(json.load(sys.stdin).get(\"${2}\", \"\"))" +} + +function is_pull_request { + test -n "$(json_property ${GITHUB_EVENT_PATH} pull_request)" +} + +function targets_master { + test $(json_property ${GITHUB_EVENT_PATH} ref) == 'refs/heads/master' +} + +git config --global user.email "wpt-pr-bot@users.noreply.github.com" +git config --global user.name "wpt-pr-bot" + +# Prepare the output directory so that the new build can be pushed to the +# repository as an incremental change to the prior build. +mkdir -p docs/_build +cd docs/_build +git init +git fetch --depth 1 ${remote_url} gh-pages +git checkout FETCH_HEAD +git rm -rf . + +# Build the website +unset NODE_ENV +cd ${wpt_root}/docs +npm install . +export PATH="$PWD/node_modules/.bin:$PATH" +cd ${wpt_root} + +./wpt build-docs + +cd docs/_build +# Configure DNS +echo web-platform-tests.org > CNAME +# Disable Jekyll +# https://github.blog/2009-12-29-bypassing-jekyll-on-github-pages/ +touch .nojekyll + +# Publish the website by pushing the built contents to the `gh-pages` branch +git add . + +echo This submission alters the compiled files as follows + +git diff --staged + +if is_pull_request ; then + echo Submission comes from a pull request. Exiting without publishing. + + exit ${neutral_status} +fi + +if ! targets_master ; then + echo Submission does not target the 'master' branch. Exiting without publishing. + + exit ${neutral_status} +fi + +if git diff --exit-code --quiet --staged ; then + echo No change to the website contents. Exiting without publishing. + + exit ${neutral_status} +fi + +git commit --message "Build documentation + +These files were generated from commit ${source_revision}" + +git push --force ${remote_url} HEAD:gh-pages |