summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/tests/tools/ci/azure
diff options
context:
space:
mode:
Diffstat (limited to 'testing/web-platform/tests/tools/ci/azure')
-rw-r--r--testing/web-platform/tests/tools/ci/azure/README.md2
-rw-r--r--testing/web-platform/tests/tools/ci/azure/affected_tests.yml30
-rw-r--r--testing/web-platform/tests/tools/ci/azure/checkout.yml4
-rw-r--r--testing/web-platform/tests/tools/ci/azure/color_profile.yml5
-rw-r--r--testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist8
-rw-r--r--testing/web-platform/tests/tools/ci/azure/fyi_hook.yml23
-rw-r--r--testing/web-platform/tests/tools/ci/azure/install_certs.yml11
-rw-r--r--testing/web-platform/tests/tools/ci/azure/install_chrome.yml11
-rw-r--r--testing/web-platform/tests/tools/ci/azure/install_edge.yml61
-rw-r--r--testing/web-platform/tests/tools/ci/azure/install_firefox.yml9
-rw-r--r--testing/web-platform/tests/tools/ci/azure/install_fonts.yml7
-rw-r--r--testing/web-platform/tests/tools/ci/azure/install_safari.yml29
-rw-r--r--testing/web-platform/tests/tools/ci/azure/pip_install.yml6
-rw-r--r--testing/web-platform/tests/tools/ci/azure/publish_logs.yml7
-rw-r--r--testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml13
-rw-r--r--testing/web-platform/tests/tools/ci/azure/system_info.yml4
-rw-r--r--testing/web-platform/tests/tools/ci/azure/tox_pytest.yml20
-rw-r--r--testing/web-platform/tests/tools/ci/azure/update_hosts.yml12
-rw-r--r--testing/web-platform/tests/tools/ci/azure/update_manifest.yml4
19 files changed, 266 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/ci/azure/README.md b/testing/web-platform/tests/tools/ci/azure/README.md
new file mode 100644
index 0000000000..afe5021efc
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/README.md
@@ -0,0 +1,2 @@
+These are step templates for Azure Pipelines, used in `.azure-pipelines.yml`
+in the root of the repository.
diff --git a/testing/web-platform/tests/tools/ci/azure/affected_tests.yml b/testing/web-platform/tests/tools/ci/azure/affected_tests.yml
new file mode 100644
index 0000000000..6d3d32acc8
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/affected_tests.yml
@@ -0,0 +1,30 @@
+parameters:
+ checkoutCommit: ''
+ affectedRange: 'HEAD^1'
+ artifactName: ''
+
+steps:
+- template: checkout.yml
+- ${{ if ne(parameters.checkoutCommit, '') }}:
+ - script: |
+ set -eux -o pipefail
+ git checkout ${{ parameters.checkoutCommit }}
+ displayName: 'Checkout ${{ parameters.checkoutCommit }}'
+- template: pip_install.yml
+ parameters:
+ packages: virtualenv
+- template: install_certs.yml
+- template: color_profile.yml
+- template: install_safari.yml
+- template: update_hosts.yml
+- template: update_manifest.yml
+- script: |
+ set -eux -o pipefail
+ export SYSTEM_VERSION_COMPAT=0
+ ./wpt run --yes --no-pause --no-fail-on-unexpected --no-restart-on-unexpected --affected ${{ parameters.affectedRange }} --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report.json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot.txt --channel preview --kill-safari safari
+ displayName: 'Run tests'
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish results'
+ inputs:
+ artifactName: '${{ parameters.artifactName }}'
+ condition: succeededOrFailed()
diff --git a/testing/web-platform/tests/tools/ci/azure/checkout.yml b/testing/web-platform/tests/tools/ci/azure/checkout.yml
new file mode 100644
index 0000000000..618c571465
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/checkout.yml
@@ -0,0 +1,4 @@
+steps:
+- checkout: self
+ fetchDepth: 50
+ submodules: false
diff --git a/testing/web-platform/tests/tools/ci/azure/color_profile.yml b/testing/web-platform/tests/tools/ci/azure/color_profile.yml
new file mode 100644
index 0000000000..d90c6ca429
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/color_profile.yml
@@ -0,0 +1,5 @@
+steps:
+- script: |
+ ./wpt macos-color-profile
+ displayName: 'Set display color profile'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
diff --git a/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist b/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist
new file mode 100644
index 0000000000..122080972c
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <key>AllowRemoteAutomation</key>
+ <true/>
+</dict>
+</plist>
diff --git a/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml b/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml
new file mode 100644
index 0000000000..df92ce1066
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/fyi_hook.yml
@@ -0,0 +1,23 @@
+# This job is used to get a run into wpt.fyi and staging.wpt.fyi, by notifying
+# them with the build number and artifact to use.
+
+parameters:
+ dependsOn: ''
+ artifactName: ''
+
+jobs:
+- job: ${{ parameters.dependsOn }}_hook
+ displayName: 'wpt.fyi hook: ${{ parameters.artifactName }}'
+ dependsOn: ${{ parameters.dependsOn }}
+ pool:
+ vmImage: 'ubuntu-20.04'
+ steps:
+ - checkout: none
+ - script: |
+ set -eux -o pipefail
+ curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://wpt.fyi/api/checks/azure/$(Build.BuildId)
+ displayName: 'Invoke wpt.fyi hook'
+ - script: |
+ set -eux -o pipefail
+ curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://staging.wpt.fyi/api/checks/azure/$(Build.BuildId)
+ displayName: 'Invoke staging.wpt.fyi hook'
diff --git a/testing/web-platform/tests/tools/ci/azure/install_certs.yml b/testing/web-platform/tests/tools/ci/azure/install_certs.yml
new file mode 100644
index 0000000000..bee5ab084e
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/install_certs.yml
@@ -0,0 +1,11 @@
+steps:
+- script: |
+ set -eux -o pipefail
+ # https://github.com/web-platform-tests/results-collection/blob/master/src/scripts/trust-root-ca.sh
+ # only run this on macOS < 11
+ [ "11" = "`echo -e $( sw_vers -productVersion )\\\n11 | sort -V | head -n1`" ] || sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain tools/certs/cacert.pem
+ displayName: 'Install web-platform.test certificate (macOS)'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+- script: certutil –addstore -enterprise –f "Root" tools\certs\cacert.pem
+ displayName: 'Install web-platform.test certificate (Windows)'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
diff --git a/testing/web-platform/tests/tools/ci/azure/install_chrome.yml b/testing/web-platform/tests/tools/ci/azure/install_chrome.yml
new file mode 100644
index 0000000000..7599321be2
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/install_chrome.yml
@@ -0,0 +1,11 @@
+steps:
+# The conflicting google-chrome and chromedriver casks are first uninstalled.
+# The raw google-chrome-dev cask URL is used to bypass caching.
+- script: |
+ set -eux -o pipefail
+ HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall --cask google-chrome || true
+ HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall --cask chromedriver || true
+ curl https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/google-chrome-dev.rb > google-chrome-dev.rb
+ HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask google-chrome-dev.rb
+ displayName: 'Install Chrome Dev'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
diff --git a/testing/web-platform/tests/tools/ci/azure/install_edge.yml b/testing/web-platform/tests/tools/ci/azure/install_edge.yml
new file mode 100644
index 0000000000..297b6e60b2
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/install_edge.yml
@@ -0,0 +1,61 @@
+parameters:
+ channel: dev
+
+# Should match https://web-platform-tests.org/running-tests/chrome.html
+# Just replace chrome with edgechromium
+steps:
+- ${{ if eq(parameters.channel, 'stable') }}:
+ - powershell: |
+ $edgeInstallerName = 'MicrosoftEdgeSetup.exe'
+ # Link to Stable channel installer
+ Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2108834&Channel=Stable&language=en' -Destination MicrosoftEdgeSetup.exe
+ if (-not (Test-Path $edgeInstallerName)) {
+ Throw "Failed to download Edge installer to $edgeInstallerName."
+ }
+ cmd /c START /WAIT $edgeInstallerName /silent /install
+ $edgePath = "$env:systemdrive\Program Files (x86)\Microsoft\Edge\Application"
+ if (Test-Path $edgePath) {
+ Write-Host "##vso[task.prependpath]$edgePath"
+ Write-Host "Edge Stable installed at $edgePath."
+ (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List
+ } else {
+ Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force
+ Throw "Failed to install Edge at $edgePath"
+ }
+ displayName: 'Install Edge Stable'
+
+- ${{ if eq(parameters.channel, 'canary') }}:
+ - powershell: |
+ $edgeInstallerName = 'MicrosoftEdgeSetup.exe'
+ # Link to Canary channel installer
+ Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2084649&Channel=Canary&language=en-us' -Destination MicrosoftEdgeSetup.exe
+ if (-not (Test-Path $edgeInstallerName)) {
+ Throw "Failed to download Edge installer to $edgeInstallerName."
+ }
+ cmd /c START /WAIT $edgeInstallerName /silent /install
+ $edgePath = "$env:localappdata\Microsoft\Edge SxS\Application"
+ if (Test-Path $edgePath) {
+ Write-Host "##vso[task.prependpath]$edgePath"
+ Write-Host "Edge Canary installed at $edgePath."
+ (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List
+ } else {
+ Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force
+ Throw "Failed to install Edge Canary at $edgePath"
+ }
+ displayName: 'Install Edge Canary'
+- ${{ if eq(parameters.channel, 'dev') }}:
+ - powershell: |
+ $edgeInstallerName = 'MicrosoftEdgeSetup.exe'
+ # Link to Dev channel installer
+ Start-BitsTransfer -Source 'https://go.microsoft.com/fwlink/?linkid=2069324&Channel=Dev&language=en-us' -Destination MicrosoftEdgeSetup.exe
+ cmd /c START /WAIT $edgeInstallerName /silent /install
+ $edgePath = "$env:systemdrive\Program Files (x86)\Microsoft\Edge Dev\Application"
+ if (Test-Path $edgePath) {
+ Write-Host "##vso[task.prependpath]$edgePath"
+ Write-Host "Edge Canary installed at $edgePath."
+ (Get-Item -Path "$edgePath\msedge.exe").VersionInfo | Format-List
+ } else {
+ Copy-Item -Path "$env:temp\*edge*.log" -Destination $(Build.ArtifactStagingDirectory) -Force
+ Throw "Failed to install Edge Dev at $edgePath"
+ }
+ displayName: 'Install Edge Dev'
diff --git a/testing/web-platform/tests/tools/ci/azure/install_firefox.yml b/testing/web-platform/tests/tools/ci/azure/install_firefox.yml
new file mode 100644
index 0000000000..73af597665
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/install_firefox.yml
@@ -0,0 +1,9 @@
+steps:
+# This is equivalent to `Homebrew/homebrew-cask-versions/firefox-nightly`,
+# but the raw URL is used to bypass caching.
+- script: |
+ set -eux -o pipefail
+ curl https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/firefox-nightly.rb > firefox-nightly.rb
+ HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask firefox-nightly.rb
+ displayName: 'Install Firefox Nightly'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
diff --git a/testing/web-platform/tests/tools/ci/azure/install_fonts.yml b/testing/web-platform/tests/tools/ci/azure/install_fonts.yml
new file mode 100644
index 0000000000..279c262c7e
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/install_fonts.yml
@@ -0,0 +1,7 @@
+steps:
+# Installig Ahem in /Library/Fonts instead of using --install-fonts is a
+# workaround for https://github.com/web-platform-tests/wpt/issues/13803.
+- script: |
+ set -eux -o pipefail
+ sudo cp fonts/Ahem.ttf /Library/Fonts
+ displayName: 'Install Ahem font'
diff --git a/testing/web-platform/tests/tools/ci/azure/install_safari.yml b/testing/web-platform/tests/tools/ci/azure/install_safari.yml
new file mode 100644
index 0000000000..1a398532da
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/install_safari.yml
@@ -0,0 +1,29 @@
+parameters:
+ channel: preview
+
+# Should match https://web-platform-tests.org/running-tests/safari.html
+steps:
+- script: defaults write com.apple.WebDriver DiagnosticsEnabled 1
+ displayName: 'Enable safaridriver diagnostics'
+ condition: eq(variables['safaridriver_diagnose'], true)
+- ${{ if eq(parameters.channel, 'preview') }}:
+ - script: |
+ set -eux -o pipefail
+ export SYSTEM_VERSION_COMPAT=0
+ ./wpt install --channel preview --download-only -d . --rename STP safari browser
+ sudo installer -pkg STP.pkg -target LocalSystem
+ # Workaround for `sudo safardriver --enable` not working on Catalina:
+ # https://github.com/web-platform-tests/wpt/issues/21751
+ mkdir -p ~/Library/WebDriver/
+ cp tools/ci/azure/com.apple.SafariTechnologyPreview.plist ~/Library/WebDriver/
+ defaults write com.apple.SafariTechnologyPreview WebKitJavaScriptCanOpenWindowsAutomatically 1
+ defaults write com.apple.SafariTechnologyPreview ExperimentalServerTimingEnabled 1
+ displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'stable') }}:
+ - script: |
+ set -eux -o pipefail
+ export SYSTEM_VERSION_COMPAT=0
+ sudo softwareupdate --install $( softwareupdate -l | grep -o '\* Label: \(Safari.*\)' | sed -e 's/* Label: //' )
+ sudo safaridriver --enable
+ defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+ displayName: 'Configure Safari'
diff --git a/testing/web-platform/tests/tools/ci/azure/pip_install.yml b/testing/web-platform/tests/tools/ci/azure/pip_install.yml
new file mode 100644
index 0000000000..18d1879e97
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/pip_install.yml
@@ -0,0 +1,6 @@
+parameters:
+ packages: ''
+
+steps:
+- script: pip --disable-pip-version-check install --upgrade ${{ parameters.packages }}
+ displayName: 'Install Python packages'
diff --git a/testing/web-platform/tests/tools/ci/azure/publish_logs.yml b/testing/web-platform/tests/tools/ci/azure/publish_logs.yml
new file mode 100644
index 0000000000..a49397a91a
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/publish_logs.yml
@@ -0,0 +1,7 @@
+steps:
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish safaridriver logs'
+ inputs:
+ pathtoPublish: /Users/runner/Library/Logs/com.apple.WebDriver/
+ artifactName: safaridriver-logs
+ condition: eq(variables['safaridriver_diagnose'], true)
diff --git a/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml b/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml
new file mode 100644
index 0000000000..97646f6914
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/sysdiagnose.yml
@@ -0,0 +1,13 @@
+steps:
+- script: |
+ mkdir -p /Users/runner/sysdiagnose
+ # No UI, and no time sensitive, generated, or archived logs
+ sudo sysdiagnose -ub -PGR -f /Users/runner/sysdiagnose -A sysdiagnose_$(System.JobPositionInPhase)
+ displayName: 'Collect sysdiagnose'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish sysdiagnose'
+ inputs:
+ pathtoPublish: /Users/runner/sysdiagnose/
+ artifactName: sysdiagnose
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
diff --git a/testing/web-platform/tests/tools/ci/azure/system_info.yml b/testing/web-platform/tests/tools/ci/azure/system_info.yml
new file mode 100644
index 0000000000..cc37b35b85
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/system_info.yml
@@ -0,0 +1,4 @@
+steps:
+- script: systeminfo
+ displayName: 'Show system info (Windows)'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
diff --git a/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml b/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml
new file mode 100644
index 0000000000..3704ecc5be
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/tox_pytest.yml
@@ -0,0 +1,20 @@
+parameters:
+ directory: ''
+ toxenv: 'ALL'
+
+steps:
+- template: pip_install.yml
+ parameters:
+ packages: tox
+
+- script: |
+ set -eux -o pipefail
+ tox -c ${{ parameters.directory }} -e ${{ parameters.toxenv }} -- --junitxml=results.xml
+ displayName: 'Run tests'
+
+- task: PublishTestResults@2
+ inputs:
+ testResultsFiles: '${{ parameters.directory }}/results.xml'
+ testRunTitle: '${{ parameters.directory }}'
+ displayName: 'Publish results'
+ condition: succeededOrFailed()
diff --git a/testing/web-platform/tests/tools/ci/azure/update_hosts.yml b/testing/web-platform/tests/tools/ci/azure/update_hosts.yml
new file mode 100644
index 0000000000..bcb8536a5c
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/update_hosts.yml
@@ -0,0 +1,12 @@
+steps:
+- script: |
+ set -eux -o pipefail
+ ./wpt make-hosts-file | sudo tee -a /etc/hosts
+ displayName: 'Update hosts (macOS)'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+- powershell: |
+ $hostFile = "$env:systemroot\System32\drivers\etc\hosts"
+ Copy-Item -Path $hostFile -Destination "$hostFile.back" -Force
+ python wpt make-hosts-file | Out-File $env:systemroot\System32\drivers\etc\hosts -Encoding ascii -Append
+ displayName: 'Update hosts (Windows)'
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
diff --git a/testing/web-platform/tests/tools/ci/azure/update_manifest.yml b/testing/web-platform/tests/tools/ci/azure/update_manifest.yml
new file mode 100644
index 0000000000..453ac2ac3c
--- /dev/null
+++ b/testing/web-platform/tests/tools/ci/azure/update_manifest.yml
@@ -0,0 +1,4 @@
+steps:
+# `python wpt` instead of `./wpt` is to make this work on Windows:
+- script: python wpt manifest
+ displayName: 'Update manifest'