diff options
Diffstat (limited to 'testing/web-platform/tests/tools/ci/azure')
19 files changed, 266 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/ci/azure/README.md b/testing/web-platform/tests/tools/ci/azure/README.md new file mode 100644 index 0000000000..afe5021efc --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/README.md @@ -0,0 +1,2 @@ +These are step templates for Azure Pipelines, used in `.azure-pipelines.yml` +in the root of the repository. diff --git a/testing/web-platform/tests/tools/ci/azure/affected_tests.yml b/testing/web-platform/tests/tools/ci/azure/affected_tests.yml new file mode 100644 index 0000000000..6d3d32acc8 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/affected_tests.yml @@ -0,0 +1,30 @@ +parameters: + checkoutCommit: '' + affectedRange: 'HEAD^1' + artifactName: '' + +steps: +- template: checkout.yml +- ${{ if ne(parameters.checkoutCommit, '') }}: + - script: | + set -eux -o pipefail + git checkout ${{ parameters.checkoutCommit }} + displayName: 'Checkout ${{ parameters.checkoutCommit }}' +- template: pip_install.yml + parameters: + packages: virtualenv +- template: install_certs.yml +- template: color_profile.yml +- template: install_safari.yml +- template: update_hosts.yml +- template: update_manifest.yml +- script: | + set -eux -o pipefail + export SYSTEM_VERSION_COMPAT=0 + ./wpt run --yes --no-pause --no-fail-on-unexpected --no-restart-on-unexpected --affected ${{ parameters.affectedRange }} --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report.json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot.txt --channel preview --kill-safari safari + displayName: 'Run tests' +- task: PublishBuildArtifacts@1 + displayName: 'Publish results' + inputs: + artifactName: '${{ parameters.artifactName }}' + condition: succeededOrFailed() diff --git a/testing/web-platform/tests/tools/ci/azure/checkout.yml b/testing/web-platform/tests/tools/ci/azure/checkout.yml new file mode 100644 index 0000000000..618c571465 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/checkout.yml @@ -0,0 +1,4 @@ +steps: +- checkout: self + fetchDepth: 50 + submodules: false diff --git a/testing/web-platform/tests/tools/ci/azure/color_profile.yml b/testing/web-platform/tests/tools/ci/azure/color_profile.yml new file mode 100644 index 0000000000..d90c6ca429 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/color_profile.yml @@ -0,0 +1,5 @@ +steps: +- script: | + ./wpt macos-color-profile + displayName: 'Set display color profile' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist b/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist new file mode 100644 index 0000000000..122080972c --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>AllowRemoteAutomation</key> + <true/> +</dict> +</plist> diff --git a/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml b/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml new file mode 100644 index 0000000000..df92ce1066 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml @@ -0,0 +1,23 @@ +# This job is used to get a run into wpt.fyi and staging.wpt.fyi, by notifying +# them with the build number and artifact to use. + +parameters: + dependsOn: '' + artifactName: '' + +jobs: +- job: ${{ parameters.dependsOn }}_hook + displayName: 'wpt.fyi hook: ${{ parameters.artifactName }}' + dependsOn: ${{ parameters.dependsOn }} + pool: + vmImage: 'ubuntu-20.04' + steps: + - checkout: none + - script: | + set -eux -o pipefail + curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://wpt.fyi/api/checks/azure/$(Build.BuildId) + displayName: 'Invoke wpt.fyi hook' + - script: | + set -eux -o pipefail + curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://staging.wpt.fyi/api/checks/azure/$(Build.BuildId) + displayName: 'Invoke staging.wpt.fyi hook' diff --git a/testing/web-platform/tests/tools/ci/azure/install_certs.yml b/testing/web-platform/tests/tools/ci/azure/install_certs.yml new file mode 100644 index 0000000000..bee5ab084e --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_certs.yml @@ -0,0 +1,11 @@ +steps: +- script: | + set -eux -o pipefail + # https://github.com/web-platform-tests/results-collection/blob/master/src/scripts/trust-root-ca.sh + # only run this on macOS < 11 + [ "11" = "`echo -e $( sw_vers -productVersion )\\\n11 | sort -V | head -n1`" ] || sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain tools/certs/cacert.pem + displayName: 'Install web-platform.test certificate (macOS)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- script: certutil –addstore -enterprise –f "Root" tools\certs\cacert.pem + displayName: 'Install web-platform.test certificate (Windows)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/testing/web-platform/tests/tools/ci/azure/install_chrome.yml b/testing/web-platform/tests/tools/ci/azure/install_chrome.yml new file mode 100644 index 0000000000..7599321be2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_chrome.yml @@ -0,0 +1,11 @@ +steps: +# The conflicting google-chrome and chromedriver casks are first uninstalled. +# The raw google-chrome-dev cask URL is used to bypass caching. +- script: | + set -eux -o pipefail + HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall --cask google-chrome || true + HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall --cask chromedriver || true + curl https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/google-chrome-dev.rb > google-chrome-dev.rb + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask google-chrome-dev.rb + displayName: 'Install Chrome Dev' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/install_edge.yml b/testing/web-platform/tests/tools/ci/azure/install_edge.yml new file mode 100644 index 0000000000..297b6e60b2 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_edge.yml @@ -0,0 +1,61 @@ +parameters: + channel: dev + +# Should match https://web-platform-tests.org/running-tests/chrome.html +# Just replace chrome with edgechromium +steps: +- ${{ if eq(parameters.channel, 'stable') }}: + - powershell: | + $edgeInstallerName = 'MicrosoftEdgeSetup.exe' + # Link to Stable channel installer + Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2108834&Channel=Stable&language=en' -Destination MicrosoftEdgeSetup.exe + if (-not (Test-Path $edgeInstallerName)) { + Throw "Failed to download Edge installer to $edgeInstallerName." + } + cmd /c START /WAIT $edgeInstallerName /silent /install + $edgePath = "$env:systemdrive\Program Files (x86)\Microsoft\Edge\Application" + if (Test-Path $edgePath) { + Write-Host "##vso[task.prependpath]$edgePath" + Write-Host "Edge Stable installed at $edgePath." + (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List + } else { + Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force + Throw "Failed to install Edge at $edgePath" + } + displayName: 'Install Edge Stable' + +- ${{ if eq(parameters.channel, 'canary') }}: + - powershell: | + $edgeInstallerName = 'MicrosoftEdgeSetup.exe' + # Link to Canary channel installer + Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2084649&Channel=Canary&language=en-us' -Destination MicrosoftEdgeSetup.exe + if (-not (Test-Path $edgeInstallerName)) { + Throw "Failed to download Edge installer to $edgeInstallerName." + } + cmd /c START /WAIT $edgeInstallerName /silent /install + $edgePath = "$env:localappdata\Microsoft\Edge SxS\Application" + if (Test-Path $edgePath) { + Write-Host "##vso[task.prependpath]$edgePath" + Write-Host "Edge Canary installed at $edgePath." + (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List + } else { + Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force + Throw "Failed to install Edge Canary at $edgePath" + } + displayName: 'Install Edge Canary' +- ${{ if eq(parameters.channel, 'dev') }}: + - powershell: | + $edgeInstallerName = 'MicrosoftEdgeSetup.exe' + # Link to Dev channel installer + Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2069324&Channel=Dev&language=en-us' -Destination MicrosoftEdgeSetup.exe + cmd /c START /WAIT $edgeInstallerName /silent /install + $edgePath = "$env:systemdrive\Program Files (x86)\Microsoft\Edge Dev\Application" + if (Test-Path $edgePath) { + Write-Host "##vso[task.prependpath]$edgePath" + Write-Host "Edge Canary installed at $edgePath." + (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List + } else { + Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force + Throw "Failed to install Edge Dev at $edgePath" + } + displayName: 'Install Edge Dev' diff --git a/testing/web-platform/tests/tools/ci/azure/install_firefox.yml b/testing/web-platform/tests/tools/ci/azure/install_firefox.yml new file mode 100644 index 0000000000..73af597665 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_firefox.yml @@ -0,0 +1,9 @@ +steps: +# This is equivalent to `Homebrew/homebrew-cask-versions/firefox-nightly`, +# but the raw URL is used to bypass caching. +- script: | + set -eux -o pipefail + curl https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/firefox-nightly.rb > firefox-nightly.rb + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask firefox-nightly.rb + displayName: 'Install Firefox Nightly' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/install_fonts.yml b/testing/web-platform/tests/tools/ci/azure/install_fonts.yml new file mode 100644 index 0000000000..279c262c7e --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_fonts.yml @@ -0,0 +1,7 @@ +steps: +# Installig Ahem in /Library/Fonts instead of using --install-fonts is a +# workaround for https://github.com/web-platform-tests/wpt/issues/13803. +- script: | + set -eux -o pipefail + sudo cp fonts/Ahem.ttf /Library/Fonts + displayName: 'Install Ahem font' diff --git a/testing/web-platform/tests/tools/ci/azure/install_safari.yml b/testing/web-platform/tests/tools/ci/azure/install_safari.yml new file mode 100644 index 0000000000..1a398532da --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/install_safari.yml @@ -0,0 +1,29 @@ +parameters: + channel: preview + +# Should match https://web-platform-tests.org/running-tests/safari.html +steps: +- script: defaults write com.apple.WebDriver DiagnosticsEnabled 1 + displayName: 'Enable safaridriver diagnostics' + condition: eq(variables['safaridriver_diagnose'], true) +- ${{ if eq(parameters.channel, 'preview') }}: + - script: | + set -eux -o pipefail + export SYSTEM_VERSION_COMPAT=0 + ./wpt install --channel preview --download-only -d . --rename STP safari browser + sudo installer -pkg STP.pkg -target LocalSystem + # Workaround for `sudo safardriver --enable` not working on Catalina: + # https://github.com/web-platform-tests/wpt/issues/21751 + mkdir -p ~/Library/WebDriver/ + cp tools/ci/azure/com.apple.SafariTechnologyPreview.plist ~/Library/WebDriver/ + defaults write com.apple.SafariTechnologyPreview WebKitJavaScriptCanOpenWindowsAutomatically 1 + defaults write com.apple.SafariTechnologyPreview ExperimentalServerTimingEnabled 1 + displayName: 'Install Safari Technology Preview' +- ${{ if eq(parameters.channel, 'stable') }}: + - script: | + set -eux -o pipefail + export SYSTEM_VERSION_COMPAT=0 + sudo softwareupdate --install $( softwareupdate -l | grep -o '\* Label: \(Safari.*\)' | sed -e 's/* Label: //' ) + sudo safaridriver --enable + defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1 + displayName: 'Configure Safari' diff --git a/testing/web-platform/tests/tools/ci/azure/pip_install.yml b/testing/web-platform/tests/tools/ci/azure/pip_install.yml new file mode 100644 index 0000000000..18d1879e97 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/pip_install.yml @@ -0,0 +1,6 @@ +parameters: + packages: '' + +steps: +- script: pip --disable-pip-version-check install --upgrade ${{ parameters.packages }} + displayName: 'Install Python packages' diff --git a/testing/web-platform/tests/tools/ci/azure/publish_logs.yml b/testing/web-platform/tests/tools/ci/azure/publish_logs.yml new file mode 100644 index 0000000000..a49397a91a --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/publish_logs.yml @@ -0,0 +1,7 @@ +steps: +- task: PublishBuildArtifacts@1 + displayName: 'Publish safaridriver logs' + inputs: + pathtoPublish: /Users/runner/Library/Logs/com.apple.WebDriver/ + artifactName: safaridriver-logs + condition: eq(variables['safaridriver_diagnose'], true) diff --git a/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml b/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml new file mode 100644 index 0000000000..97646f6914 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml @@ -0,0 +1,13 @@ +steps: +- script: | + mkdir -p /Users/runner/sysdiagnose + # No UI, and no time sensitive, generated, or archived logs + sudo sysdiagnose -ub -PGR -f /Users/runner/sysdiagnose -A sysdiagnose_$(System.JobPositionInPhase) + displayName: 'Collect sysdiagnose' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- task: PublishBuildArtifacts@1 + displayName: 'Publish sysdiagnose' + inputs: + pathtoPublish: /Users/runner/sysdiagnose/ + artifactName: sysdiagnose + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) diff --git a/testing/web-platform/tests/tools/ci/azure/system_info.yml b/testing/web-platform/tests/tools/ci/azure/system_info.yml new file mode 100644 index 0000000000..cc37b35b85 --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/system_info.yml @@ -0,0 +1,4 @@ +steps: +- script: systeminfo + displayName: 'Show system info (Windows)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml b/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml new file mode 100644 index 0000000000..3704ecc5be --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml @@ -0,0 +1,20 @@ +parameters: + directory: '' + toxenv: 'ALL' + +steps: +- template: pip_install.yml + parameters: + packages: tox + +- script: | + set -eux -o pipefail + tox -c ${{ parameters.directory }} -e ${{ parameters.toxenv }} -- --junitxml=results.xml + displayName: 'Run tests' + +- task: PublishTestResults@2 + inputs: + testResultsFiles: '${{ parameters.directory }}/results.xml' + testRunTitle: '${{ parameters.directory }}' + displayName: 'Publish results' + condition: succeededOrFailed() diff --git a/testing/web-platform/tests/tools/ci/azure/update_hosts.yml b/testing/web-platform/tests/tools/ci/azure/update_hosts.yml new file mode 100644 index 0000000000..bcb8536a5c --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/update_hosts.yml @@ -0,0 +1,12 @@ +steps: +- script: | + set -eux -o pipefail + ./wpt make-hosts-file | sudo tee -a /etc/hosts + displayName: 'Update hosts (macOS)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- powershell: | + $hostFile = "$env:systemroot\System32\drivers\etc\hosts" + Copy-Item -Path $hostFile -Destination "$hostFile.back" -Force + python wpt make-hosts-file | Out-File $env:systemroot\System32\drivers\etc\hosts -Encoding ascii -Append + displayName: 'Update hosts (Windows)' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/testing/web-platform/tests/tools/ci/azure/update_manifest.yml b/testing/web-platform/tests/tools/ci/azure/update_manifest.yml new file mode 100644 index 0000000000..453ac2ac3c --- /dev/null +++ b/testing/web-platform/tests/tools/ci/azure/update_manifest.yml @@ -0,0 +1,4 @@ +steps: +# `python wpt` instead of `./wpt` is to make this work on Windows: +- script: python wpt manifest + displayName: 'Update manifest' |