summaryrefslogtreecommitdiffstats
path: root/test/wpt/runner
diff options
context:
space:
mode:
Diffstat (limited to 'test/wpt/runner')
-rw-r--r--test/wpt/runner/runner.mjs356
-rw-r--r--test/wpt/runner/util.mjs172
-rw-r--r--test/wpt/runner/worker.mjs164
3 files changed, 692 insertions, 0 deletions
diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs
new file mode 100644
index 0000000..5bec326
--- /dev/null
+++ b/test/wpt/runner/runner.mjs
@@ -0,0 +1,356 @@
+import { EventEmitter, once } from 'node:events'
+import { isAbsolute, join, resolve } from 'node:path'
+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { Worker } from 'node:worker_threads'
+import { colors, handlePipes, normalizeName, parseMeta, resolveStatusPath } from './util.mjs'
+
+const basePath = fileURLToPath(join(import.meta.url, '../..'))
+const testPath = join(basePath, 'tests')
+const statusPath = join(basePath, 'status')
+
+// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
+function sanitizeUnpairedSurrogates (str) {
+ return str.replace(
+ /([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
+ function (_, low, prefix, high) {
+ let output = prefix || '' // Prefix may be undefined
+ const string = low || high // Only one of these alternates can match
+ for (let i = 0; i < string.length; i++) {
+ output += codeUnitStr(string[i])
+ }
+ return output
+ })
+}
+
+function codeUnitStr (char) {
+ return 'U+' + char.charCodeAt(0).toString(16)
+}
+
+export class WPTRunner extends EventEmitter {
+ /** @type {string} */
+ #folderName
+
+ /** @type {string} */
+ #folderPath
+
+ /** @type {string[]} */
+ #files = []
+
+ /** @type {string[]} */
+ #initScripts = []
+
+ /** @type {string} */
+ #url
+
+ /** @type {import('../../status/fetch.status.json')} */
+ #status
+
+ /** Tests that have expectedly failed mapped by file name */
+ #statusOutput = {}
+
+ #uncaughtExceptions = []
+
+ /** @type {boolean} */
+ #appendReport
+
+ /** @type {string} */
+ #reportPath
+
+ #stats = {
+ completed: 0,
+ failed: 0,
+ success: 0,
+ expectedFailures: 0,
+ skipped: 0
+ }
+
+ constructor (folder, url, { appendReport = false, reportPath } = {}) {
+ super()
+
+ this.#folderName = folder
+ this.#folderPath = join(testPath, folder)
+ this.#files.push(
+ ...WPTRunner.walk(
+ this.#folderPath,
+ (file) => file.endsWith('.any.js')
+ )
+ )
+
+ if (appendReport) {
+ if (!reportPath) {
+ throw new TypeError('reportPath must be provided when appendReport is true')
+ }
+ if (!existsSync(reportPath)) {
+ throw new TypeError('reportPath is invalid')
+ }
+ }
+
+ this.#appendReport = appendReport
+ this.#reportPath = reportPath
+
+ this.#status = JSON.parse(readFileSync(join(statusPath, `${folder}.status.json`)))
+ this.#url = url
+
+ if (this.#files.length === 0) {
+ queueMicrotask(() => {
+ this.emit('completion')
+ })
+ }
+
+ this.once('completion', () => {
+ for (const { error, test } of this.#uncaughtExceptions) {
+ console.log(colors(`Uncaught exception in "${test}":`, 'red'))
+ console.log(colors(`${error.stack}`, 'red'))
+ console.log('='.repeat(96))
+ }
+ })
+ }
+
+ static walk (dir, fn) {
+ const ini = new Set(readdirSync(dir))
+ const files = new Set()
+
+ while (ini.size !== 0) {
+ for (const d of ini) {
+ const path = resolve(dir, d)
+ ini.delete(d) // remove from set
+ const stats = statSync(path)
+
+ if (stats.isDirectory()) {
+ for (const f of readdirSync(path)) {
+ ini.add(resolve(path, f))
+ }
+ } else if (stats.isFile() && fn(d)) {
+ files.add(path)
+ }
+ }
+ }
+
+ return [...files].sort()
+ }
+
+ async run () {
+ const workerPath = fileURLToPath(join(import.meta.url, '../worker.mjs'))
+ /** @type {Set<Worker>} */
+ const activeWorkers = new Set()
+ let finishedFiles = 1
+ let total = this.#files.length
+
+ const files = this.#files.map((test) => {
+ const code = test.includes('.sub.')
+ ? handlePipes(readFileSync(test, 'utf-8'), this.#url)
+ : readFileSync(test, 'utf-8')
+ const meta = this.resolveMeta(code, test)
+
+ if (meta.variant.length) {
+ total += meta.variant.length - 1
+ }
+
+ return [test, code, meta]
+ })
+
+ console.log('='.repeat(96))
+
+ for (const [test, code, meta] of files) {
+ console.log(`Started ${test}`)
+
+ const status = resolveStatusPath(test, this.#status)
+
+ if (status.file.skip || status.topLevel.skip) {
+ this.#stats.skipped += 1
+
+ console.log(colors(`[${finishedFiles}/${total}] SKIPPED - ${test}`, 'yellow'))
+ console.log('='.repeat(96))
+
+ finishedFiles++
+ continue
+ }
+
+ const start = performance.now()
+
+ for (const variant of meta.variant.length ? meta.variant : ['']) {
+ const url = new URL(this.#url)
+ if (variant) {
+ url.search = variant
+ }
+ const worker = new Worker(workerPath, {
+ workerData: {
+ // Code to load before the test harness and tests.
+ initScripts: this.#initScripts,
+ // The test file.
+ test: code,
+ // Parsed META tag information
+ meta,
+ url: url.href,
+ path: test
+ }
+ })
+
+ let result, report
+ if (this.#appendReport) {
+ report = JSON.parse(readFileSync(this.#reportPath))
+
+ const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt')
+ fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html')
+ fileUrl.search = variant
+
+ result = {
+ test: fileUrl.href.slice(fileUrl.origin.length),
+ subtests: [],
+ status: 'OK'
+ }
+ report.results.push(result)
+ }
+
+ activeWorkers.add(worker)
+ // These values come directly from the web-platform-tests
+ const timeout = meta.timeout === 'long' ? 60_000 : 10_000
+
+ worker.on('message', (message) => {
+ if (message.type === 'result') {
+ this.handleIndividualTestCompletion(message, status, test, meta, result)
+ } else if (message.type === 'completion') {
+ this.handleTestCompletion(worker)
+ } else if (message.type === 'error') {
+ this.#uncaughtExceptions.push({ error: message.error, test })
+ this.#stats.failed += 1
+ this.#stats.success -= 1
+ }
+ })
+
+ try {
+ await once(worker, 'exit', {
+ signal: AbortSignal.timeout(timeout)
+ })
+
+ console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
+ if (variant) console.log('Variant:', variant)
+ console.log(`Test took ${(performance.now() - start).toFixed(2)}ms`)
+ console.log('='.repeat(96))
+ } catch (e) {
+ console.log(`${test} timed out after ${timeout}ms`)
+ } finally {
+ if (result?.subtests.length > 0) {
+ writeFileSync(this.#reportPath, JSON.stringify(report))
+ }
+
+ finishedFiles++
+ activeWorkers.delete(worker)
+ }
+ }
+ }
+
+ this.handleRunnerCompletion()
+ }
+
+ /**
+ * Called after a test has succeeded or failed.
+ */
+ handleIndividualTestCompletion (message, status, path, meta, wptResult) {
+ const { file, topLevel } = status
+
+ if (message.type === 'result') {
+ this.#stats.completed += 1
+
+ if (message.result.status === 1) {
+ this.#stats.failed += 1
+
+ wptResult?.subtests.push({
+ status: 'FAIL',
+ name: sanitizeUnpairedSurrogates(message.result.name),
+ message: sanitizeUnpairedSurrogates(message.result.message)
+ })
+
+ const name = normalizeName(message.result.name)
+
+ if (file.flaky?.includes(name)) {
+ this.#stats.expectedFailures += 1
+ } else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) {
+ if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) {
+ if (Array.isArray(file.fail)) {
+ this.#statusOutput[path] ??= []
+ this.#statusOutput[path].push(name)
+ }
+ }
+
+ this.#stats.expectedFailures += 1
+ } else {
+ process.exitCode = 1
+ console.error(message.result)
+ }
+ } else {
+ wptResult?.subtests.push({
+ status: 'PASS',
+ name: sanitizeUnpairedSurrogates(message.result.name)
+ })
+ this.#stats.success += 1
+ }
+ }
+ }
+
+ /**
+ * Called after all the tests in a worker are completed.
+ * @param {Worker} worker
+ */
+ handleTestCompletion (worker) {
+ worker.terminate()
+ }
+
+ /**
+ * Called after every test has completed.
+ */
+ handleRunnerCompletion () {
+ console.log(this.#statusOutput) // tests that failed
+
+ this.emit('completion')
+ const { completed, failed, success, expectedFailures, skipped } = this.#stats
+ console.log(
+ `[${this.#folderName}]: ` +
+ `Completed: ${completed}, failed: ${failed}, success: ${success}, ` +
+ `expected failures: ${expectedFailures}, ` +
+ `unexpected failures: ${failed - expectedFailures}, ` +
+ `skipped: ${skipped}`
+ )
+
+ process.exit(0)
+ }
+
+ addInitScript (code) {
+ this.#initScripts.push(code)
+ }
+
+ /**
+ * Parses META tags and resolves any script file paths.
+ * @param {string} code
+ * @param {string} path The absolute path of the test
+ */
+ resolveMeta (code, path) {
+ const meta = parseMeta(code)
+ const scripts = meta.scripts.map((filePath) => {
+ let content = ''
+
+ if (filePath === '/resources/WebIDLParser.js') {
+ // See https://github.com/web-platform-tests/wpt/pull/731
+ return readFileSync(join(testPath, '/resources/webidl2/lib/webidl2.js'), 'utf-8')
+ } else if (isAbsolute(filePath)) {
+ content = readFileSync(join(testPath, filePath), 'utf-8')
+ } else {
+ content = readFileSync(resolve(path, '..', filePath), 'utf-8')
+ }
+
+ // If the file has any built-in pipes.
+ if (filePath.includes('.sub.')) {
+ content = handlePipes(content, this.#url)
+ }
+
+ return content
+ })
+
+ return {
+ ...meta,
+ resourcePaths: meta.scripts,
+ scripts
+ }
+ }
+}
diff --git a/test/wpt/runner/util.mjs b/test/wpt/runner/util.mjs
new file mode 100644
index 0000000..ec284df
--- /dev/null
+++ b/test/wpt/runner/util.mjs
@@ -0,0 +1,172 @@
+import assert from 'node:assert'
+import { exit } from 'node:process'
+import { inspect } from 'node:util'
+import tty from 'node:tty'
+import { sep } from 'node:path'
+
+/**
+ * Parse the `Meta:` tags sometimes included in tests.
+ * These can include resources to inject, how long it should
+ * take to timeout, and which globals to expose.
+ * @example
+ * // META: timeout=long
+ * // META: global=window,worker
+ * // META: script=/common/utils.js
+ * // META: script=/common/get-host-info.sub.js
+ * // META: script=../request/request-error.js
+ * @see https://nodejs.org/api/readline.html#readline_example_read_file_stream_line_by_line
+ * @param {string} fileContents
+ */
+export function parseMeta (fileContents) {
+ const lines = fileContents.split(/\r?\n/g)
+
+ const meta = {
+ /** @type {string|null} */
+ timeout: null,
+ /** @type {string[]} */
+ global: [],
+ /** @type {string[]} */
+ scripts: [],
+ /** @type {string[]} */
+ variant: []
+ }
+
+ for (const line of lines) {
+ if (!line.startsWith('// META: ')) {
+ break
+ }
+
+ const groups = /^\/\/ META: (?<type>.*?)=(?<match>.*)$/.exec(line)?.groups
+
+ if (!groups) {
+ console.log(`Failed to parse META tag: ${line}`)
+ exit(1)
+ }
+
+ switch (groups.type) {
+ case 'variant':
+ meta[groups.type].push(groups.match)
+ break
+ case 'title':
+ case 'timeout': {
+ meta[groups.type] = groups.match
+ break
+ }
+ case 'global': {
+ // window,worker -> ['window', 'worker']
+ meta.global.push(...groups.match.split(','))
+ break
+ }
+ case 'script': {
+ // A relative or absolute file path to the resources
+ // needed for the current test.
+ meta.scripts.push(groups.match)
+ break
+ }
+ default: {
+ console.log(`Unknown META tag: ${groups.type}`)
+ exit(1)
+ }
+ }
+ }
+
+ return meta
+}
+
+/**
+ * @param {string} sub
+ */
+function parseSubBlock (sub) {
+ const subName = sub.includes('[') ? sub.slice(0, sub.indexOf('[')) : sub
+ const options = sub.matchAll(/\[(.*?)\]/gm)
+
+ return {
+ sub: subName,
+ options: [...options].map(match => match[1])
+ }
+}
+
+/**
+ * @see https://web-platform-tests.org/writing-tests/server-pipes.html?highlight=sub#built-in-pipes
+ * @param {string} code
+ * @param {string} url
+ */
+export function handlePipes (code, url) {
+ const server = new URL(url)
+
+ // "Substitutions are marked in a file using a block delimited by
+ // {{ and }}. Inside the block the following variables are available:"
+ return code.replace(/{{(.*?)}}/gm, (_, match) => {
+ const { sub } = parseSubBlock(match)
+
+ switch (sub) {
+ // "The host name of the server excluding any subdomain part."
+ // eslint-disable-next-line no-fallthrough
+ case 'host':
+ // "The domain name of a particular subdomain e.g.
+ // {{domains[www]}} for the www subdomain."
+ // eslint-disable-next-line no-fallthrough
+ case 'domains':
+ // "The domain name of a particular subdomain for a particular host.
+ // The first key may be empty (designating the “default” host) or
+ // the value alt; i.e., {{hosts[alt][]}} (designating the alternate
+ // host)."
+ // eslint-disable-next-line no-fallthrough
+ case 'hosts': {
+ return 'localhost'
+ }
+ // "The port number of servers, by protocol e.g. {{ports[http][0]}}
+ // for the first (and, depending on setup, possibly only) http server"
+ case 'ports': {
+ return server.port
+ }
+ default: {
+ throw new TypeError(`Unknown substitute "${sub}".`)
+ }
+ }
+ })
+}
+
+/**
+ * Some test names may contain characters that JSON cannot handle.
+ * @param {string} name
+ */
+export function normalizeName (name) {
+ return name.replace(/(\v)/g, (_, match) => {
+ switch (inspect(match)) {
+ case '\'\\x0B\'': return '\\x0B'
+ default: return match
+ }
+ })
+}
+
+export function colors (str, color) {
+ assert(Object.hasOwn(inspect.colors, color), `Missing color ${color}`)
+
+ if (!tty.WriteStream.prototype.hasColors()) {
+ return str
+ }
+
+ const [start, end] = inspect.colors[color]
+
+ return `\u001b[${start}m${str}\u001b[${end}m`
+}
+
+/** @param {string} path */
+export function resolveStatusPath (path, status) {
+ const paths = path
+ .slice(process.cwd().length + sep.length)
+ .split(sep)
+ .slice(3) // [test, wpt, tests, fetch, b, c.js] -> [fetch, b, c.js]
+
+ // skip the first folder name
+ for (let i = 1; i < paths.length - 1; i++) {
+ status = status[paths[i]]
+
+ if (!status) {
+ break
+ }
+ }
+
+ return { topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} }
+}
diff --git a/test/wpt/runner/worker.mjs b/test/wpt/runner/worker.mjs
new file mode 100644
index 0000000..90bfcf6
--- /dev/null
+++ b/test/wpt/runner/worker.mjs
@@ -0,0 +1,164 @@
+import buffer from 'node:buffer'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { setFlagsFromString } from 'node:v8'
+import { runInNewContext, runInThisContext } from 'node:vm'
+import { parentPort, workerData } from 'node:worker_threads'
+import {
+ fetch, File, FileReader, FormData, Headers, Request, Response, setGlobalOrigin
+} from '../../../index.js'
+import { CloseEvent } from '../../../lib/websocket/events.js'
+import { WebSocket } from '../../../lib/websocket/websocket.js'
+import { Cache } from '../../../lib/cache/cache.js'
+import { CacheStorage } from '../../../lib/cache/cachestorage.js'
+import { kConstruct } from '../../../lib/cache/symbols.js'
+
+const { initScripts, meta, test, url, path } = workerData
+
+process.on('uncaughtException', (err) => {
+ parentPort.postMessage({
+ type: 'error',
+ error: {
+ message: err.message,
+ name: err.name,
+ stack: err.stack
+ }
+ })
+})
+
+const basePath = join(process.cwd(), 'test/wpt/tests')
+const urlPath = path.slice(basePath.length)
+
+const globalPropertyDescriptors = {
+ writable: true,
+ enumerable: false,
+ configurable: true
+}
+
+Object.defineProperties(globalThis, {
+ fetch: {
+ ...globalPropertyDescriptors,
+ enumerable: true,
+ value: fetch
+ },
+ File: {
+ ...globalPropertyDescriptors,
+ value: buffer.File ?? File
+ },
+ FormData: {
+ ...globalPropertyDescriptors,
+ value: FormData
+ },
+ Headers: {
+ ...globalPropertyDescriptors,
+ value: Headers
+ },
+ Request: {
+ ...globalPropertyDescriptors,
+ value: Request
+ },
+ Response: {
+ ...globalPropertyDescriptors,
+ value: Response
+ },
+ FileReader: {
+ ...globalPropertyDescriptors,
+ value: FileReader
+ },
+ WebSocket: {
+ ...globalPropertyDescriptors,
+ value: WebSocket
+ },
+ CloseEvent: {
+ ...globalPropertyDescriptors,
+ value: CloseEvent
+ },
+ Blob: {
+ ...globalPropertyDescriptors,
+ // See https://github.com/nodejs/node/pull/45659
+ value: buffer.Blob
+ },
+ caches: {
+ ...globalPropertyDescriptors,
+ value: new CacheStorage(kConstruct)
+ },
+ Cache: {
+ ...globalPropertyDescriptors,
+ value: Cache
+ },
+ CacheStorage: {
+ ...globalPropertyDescriptors,
+ value: CacheStorage
+ }
+})
+
+// self is required by testharness
+// GLOBAL is required by self
+runInThisContext(`
+ globalThis.self = globalThis
+ globalThis.GLOBAL = {
+ isWorker () {
+ return false
+ },
+ isShadowRealm () {
+ return false
+ },
+ isWindow () {
+ return false
+ }
+ }
+ globalThis.window = globalThis
+ globalThis.location = new URL('${urlPath.replace(/\\/g, '/')}', '${url}')
+ globalThis.Window = Object.getPrototypeOf(globalThis).constructor
+`)
+
+if (meta.title) {
+ runInThisContext(`globalThis.META_TITLE = "${meta.title}"`)
+}
+
+const harness = readFileSync(join(basePath, '/resources/testharness.js'), 'utf-8')
+runInThisContext(harness)
+
+// add_*_callback comes from testharness
+// stolen from node's wpt test runner
+// eslint-disable-next-line no-undef
+add_result_callback((result) => {
+ parentPort.postMessage({
+ type: 'result',
+ result: {
+ status: result.status,
+ name: result.name,
+ message: result.message,
+ stack: result.stack
+ }
+ })
+})
+
+// eslint-disable-next-line no-undef
+add_completion_callback((_, status) => {
+ parentPort.postMessage({
+ type: 'completion',
+ status
+ })
+})
+
+setGlobalOrigin(globalThis.location)
+
+// Inject any script the user provided before
+// running the tests.
+for (const initScript of initScripts) {
+ runInThisContext(initScript)
+}
+
+// Inject any files from the META tags
+for (const script of meta.scripts) {
+ runInThisContext(script)
+}
+
+// A few tests require gc, which can't be passed to a Worker.
+// see https://github.com/nodejs/node/issues/16595#issuecomment-340288680
+setFlagsFromString('--expose-gc')
+globalThis.gc = runInNewContext('gc')
+
+// Finally, run the test.
+runInThisContext(test)
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-13 00:51:12 +0000