diff options
Diffstat (limited to '')
8 files changed, 1357 insertions, 0 deletions
diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts new file mode 100644 index 0000000000..463546c06d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts @@ -0,0 +1,278 @@ +/* eslint no-console: "off" */ + +import * as fs from 'fs'; + +import { dataCache } from '../framework/data_cache.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { prettyPrintLog } from '../internal/logging/log_message.js'; +import { Logger } from '../internal/logging/logger.js'; +import { LiveTestCaseResult } from '../internal/logging/result.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { parseExpectationsForTestQuery } from '../internal/query/query.js'; +import { Colors } from '../util/colors.js'; +import { setGPUProvider } from '../util/navigator_gpu.js'; +import { assert, unreachable } from '../util/util.js'; + +import sys from './helper/sys.js'; + +function usage(rc: number): never { + console.log(`Usage: + tools/run_${sys.type} [OPTIONS...] QUERIES... + tools/run_${sys.type} 'unittests:*' 'webgpu:buffers,*' +Options: + --colors Enable ANSI colors in output. + --coverage Emit coverage data. + --verbose Print result/log of every test as it runs. + --list Print all testcase names that match the given query and exit. + --debug Include debug messages in logging. + --print-json Print the complete result JSON in the output. + --expectations Path to expectations file. + --gpu-provider Path to node module that provides the GPU implementation. + --gpu-provider-flag Flag to set on the gpu-provider as <flag>=<value> + --unroll-const-eval-loops Unrolls loops in constant-evaluation shader execution tests + --quiet Suppress summary information in output +`); + return sys.exit(rc); +} + +// The interface that exposes creation of the GPU, and optional interface to code coverage. +interface GPUProviderModule { + // @returns a GPU with the given flags + create(flags: string[]): GPU; + // An optional interface to a CodeCoverageProvider + coverage?: CodeCoverageProvider; +} + +interface CodeCoverageProvider { + // Starts collecting code coverage + begin(): void; + // Ends collecting of code coverage, returning the coverage data. + // This data is opaque (implementation defined). + end(): string; +} + +type listModes = 'none' | 'cases' | 'unimplemented'; + +Colors.enabled = false; + +let verbose = false; +let emitCoverage = false; +let listMode: listModes = 'none'; +let debug = false; +let printJSON = false; +let quiet = false; +let loadWebGPUExpectations: Promise<unknown> | undefined = undefined; +let gpuProviderModule: GPUProviderModule | undefined = undefined; +let dataPath: string | undefined = undefined; + +const queries: string[] = []; +const gpuProviderFlags: string[] = []; +for (let i = 0; i < sys.args.length; ++i) { + const a = sys.args[i]; + if (a.startsWith('-')) { + if (a === '--colors') { + Colors.enabled = true; + } else if (a === '--coverage') { + emitCoverage = true; + } else if (a === '--verbose') { + verbose = true; + } else if (a === '--list') { + listMode = 'cases'; + } else if (a === '--list-unimplemented') { + listMode = 'unimplemented'; + } else if (a === '--debug') { + debug = true; + } else if (a === '--data') { + dataPath = sys.args[++i]; + } else if (a === '--print-json') { + printJSON = true; + } else if (a === '--expectations') { + const expectationsFile = new URL(sys.args[++i], `file://${sys.cwd()}`).pathname; + loadWebGPUExpectations = import(expectationsFile).then(m => m.expectations); + } else if (a === '--gpu-provider') { + const modulePath = sys.args[++i]; + gpuProviderModule = require(modulePath); + } else if (a === '--gpu-provider-flag') { + gpuProviderFlags.push(sys.args[++i]); + } else if (a === '--quiet') { + quiet = true; + } else if (a === '--unroll-const-eval-loops') { + globalTestConfig.unrollConstEvalLoops = true; + } else { + console.log('unrecognized flag: ', a); + usage(1); + } + } else { + queries.push(a); + } +} + +let codeCoverage: CodeCoverageProvider | undefined = undefined; + +if (gpuProviderModule) { + setGPUProvider(() => gpuProviderModule!.create(gpuProviderFlags)); + if (emitCoverage) { + codeCoverage = gpuProviderModule.coverage; + if (codeCoverage === undefined) { + console.error( + `--coverage specified, but the GPUProviderModule does not support code coverage. +Did you remember to build with code coverage instrumentation enabled?` + ); + sys.exit(1); + } + } +} + +if (dataPath !== undefined) { + dataCache.setStore({ + load: (path: string) => { + return new Promise<string>((resolve, reject) => { + fs.readFile(`${dataPath}/${path}`, 'utf8', (err, data) => { + if (err !== null) { + reject(err.message); + } else { + resolve(data); + } + }); + }); + }, + }); +} +if (verbose) { + dataCache.setDebugLogger(console.log); +} + +if (queries.length === 0) { + console.log('no queries specified'); + usage(0); +} + +(async () => { + const loader = new DefaultTestFileLoader(); + assert(queries.length === 1, 'currently, there must be exactly one query on the cmd line'); + const filterQuery = parseQuery(queries[0]); + const testcases = await loader.loadCases(filterQuery); + const expectations = parseExpectationsForTestQuery( + await (loadWebGPUExpectations ?? []), + filterQuery + ); + + Logger.globalDebugMode = debug; + const log = new Logger(); + + const failed: Array<[string, LiveTestCaseResult]> = []; + const warned: Array<[string, LiveTestCaseResult]> = []; + const skipped: Array<[string, LiveTestCaseResult]> = []; + + let total = 0; + + if (codeCoverage !== undefined) { + codeCoverage.begin(); + } + + for (const testcase of testcases) { + const name = testcase.query.toString(); + switch (listMode) { + case 'cases': + console.log(name); + continue; + case 'unimplemented': + if (testcase.isUnimplemented) { + console.log(name); + } + continue; + default: + break; + } + + const [rec, res] = log.record(name); + await testcase.run(rec, expectations); + + if (verbose) { + printResults([[name, res]]); + } + + total++; + switch (res.status) { + case 'pass': + break; + case 'fail': + failed.push([name, res]); + break; + case 'warn': + warned.push([name, res]); + break; + case 'skip': + skipped.push([name, res]); + break; + default: + unreachable('unrecognized status'); + } + } + + if (codeCoverage !== undefined) { + const coverage = codeCoverage.end(); + console.log(`Code-coverage: [[${coverage}]]`); + } + + if (listMode !== 'none') { + return; + } + + assert(total > 0, 'found no tests!'); + + // MAINTENANCE_TODO: write results out somewhere (a file?) + if (printJSON) { + console.log(log.asJSON(2)); + } + + if (!quiet) { + if (skipped.length) { + console.log(''); + console.log('** Skipped **'); + printResults(skipped); + } + if (warned.length) { + console.log(''); + console.log('** Warnings **'); + printResults(warned); + } + if (failed.length) { + console.log(''); + console.log('** Failures **'); + printResults(failed); + } + + const passed = total - warned.length - failed.length - skipped.length; + const pct = (x: number) => ((100 * x) / total).toFixed(2); + const rpt = (x: number) => { + const xs = x.toString().padStart(1 + Math.log10(total), ' '); + return `${xs} / ${total} = ${pct(x).padStart(6, ' ')}%`; + }; + console.log(''); + console.log(`** Summary ** +Passed w/o warnings = ${rpt(passed)} +Passed with warnings = ${rpt(warned.length)} +Skipped = ${rpt(skipped.length)} +Failed = ${rpt(failed.length)}`); + } + + if (failed.length || warned.length) { + sys.exit(1); + } +})().catch(ex => { + console.log(ex.stack ?? ex.toString()); + sys.exit(1); +}); + +function printResults(results: Array<[string, LiveTestCaseResult]>): void { + for (const [name, r] of results) { + console.log(`[${r.status}] ${name} (${r.timems}ms). Log:`); + if (r.logs) { + for (const l of r.logs) { + console.log(prettyPrintLog(l)); + } + } + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts new file mode 100644 index 0000000000..bec14694a3 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts @@ -0,0 +1,22 @@ +let windowURL: URL | undefined = undefined; +function getWindowURL() { + if (windowURL === undefined) { + windowURL = new URL(window.location.toString()); + } + return windowURL; +} + +export function optionEnabled( + opt: string, + searchParams: URLSearchParams = getWindowURL().searchParams +): boolean { + const val = searchParams.get(opt); + return val !== null && val !== '0'; +} + +export function optionString( + opt: string, + searchParams: URLSearchParams = getWindowURL().searchParams +): string { + return searchParams.get(opt) || ''; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts new file mode 100644 index 0000000000..d2e07ff26d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts @@ -0,0 +1,46 @@ +/* eslint no-process-exit: "off" */ +/* eslint @typescript-eslint/no-namespace: "off" */ + +function node() { + const { existsSync } = require('fs'); + + return { + type: 'node', + existsSync, + args: process.argv.slice(2), + cwd: () => process.cwd(), + exit: (code?: number | undefined) => process.exit(code), + }; +} + +declare global { + namespace Deno { + function readFileSync(path: string): Uint8Array; + const args: string[]; + const cwd: () => string; + function exit(code?: number): never; + } +} + +function deno() { + function existsSync(path: string) { + try { + Deno.readFileSync(path); + return true; + } catch (err) { + return false; + } + } + + return { + type: 'deno', + existsSync, + args: Deno.args, + cwd: Deno.cwd, + exit: Deno.exit, + }; +} + +const sys = typeof globalThis.process !== 'undefined' ? node() : deno(); + +export default sys; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts new file mode 100644 index 0000000000..9af555f36d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts @@ -0,0 +1,32 @@ +import { setBaseResourcePath } from '../../framework/resources.js'; +import { DefaultTestFileLoader } from '../../internal/file_loader.js'; +import { Logger } from '../../internal/logging/logger.js'; +import { parseQuery } from '../../internal/query/parseQuery.js'; +import { TestQueryWithExpectation } from '../../internal/query/query.js'; +import { assert } from '../../util/util.js'; + +// Should be DedicatedWorkerGlobalScope, but importing lib "webworker" conflicts with lib "dom". +/* eslint-disable-next-line @typescript-eslint/no-explicit-any */ +declare const self: any; + +const loader = new DefaultTestFileLoader(); + +setBaseResourcePath('../../../resources'); + +self.onmessage = async (ev: MessageEvent) => { + const query: string = ev.data.query; + const expectations: TestQueryWithExpectation[] = ev.data.expectations; + const debug: boolean = ev.data.debug; + + Logger.globalDebugMode = debug; + const log = new Logger(); + + const testcases = Array.from(await loader.loadCases(parseQuery(query))); + assert(testcases.length === 1, 'worker query resulted in != 1 cases'); + + const testcase = testcases[0]; + const [rec, result] = log.record(testcase.query.toString()); + await testcase.run(rec, expectations); + + self.postMessage({ query, result }); +}; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts new file mode 100644 index 0000000000..2ddc3a951b --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts @@ -0,0 +1,44 @@ +import { LogMessageWithStack } from '../../internal/logging/log_message.js'; +import { TransferredTestCaseResult, LiveTestCaseResult } from '../../internal/logging/result.js'; +import { TestCaseRecorder } from '../../internal/logging/test_case_recorder.js'; +import { TestQueryWithExpectation } from '../../internal/query/query.js'; + +export class TestWorker { + private readonly debug: boolean; + private readonly worker: Worker; + private readonly resolvers = new Map<string, (result: LiveTestCaseResult) => void>(); + + constructor(debug: boolean) { + this.debug = debug; + + const selfPath = import.meta.url; + const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); + const workerPath = selfPathDir + '/test_worker-worker.js'; + this.worker = new Worker(workerPath, { type: 'module' }); + this.worker.onmessage = ev => { + const query: string = ev.data.query; + const result: TransferredTestCaseResult = ev.data.result; + if (result.logs) { + for (const l of result.logs) { + Object.setPrototypeOf(l, LogMessageWithStack.prototype); + } + } + this.resolvers.get(query)!(result as LiveTestCaseResult); + + // MAINTENANCE_TODO(kainino0x): update the Logger with this result (or don't have a logger and + // update the entire results JSON somehow at some point). + }; + } + + async run( + rec: TestCaseRecorder, + query: string, + expectations: TestQueryWithExpectation[] = [] + ): Promise<void> { + this.worker.postMessage({ query, expectations, debug: this.debug }); + const workerResult = await new Promise<LiveTestCaseResult>(resolve => { + this.resolvers.set(query, resolve); + }); + rec.injectResult(workerResult); + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts new file mode 100644 index 0000000000..350a864a34 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts @@ -0,0 +1,227 @@ +/* eslint no-console: "off" */ + +import * as fs from 'fs'; +import * as http from 'http'; +import { AddressInfo } from 'net'; + +import { dataCache } from '../framework/data_cache.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { prettyPrintLog } from '../internal/logging/log_message.js'; +import { Logger } from '../internal/logging/logger.js'; +import { LiveTestCaseResult, Status } from '../internal/logging/result.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { TestQueryWithExpectation } from '../internal/query/query.js'; +import { TestTreeLeaf } from '../internal/tree.js'; +import { Colors } from '../util/colors.js'; +import { setGPUProvider } from '../util/navigator_gpu.js'; + +import sys from './helper/sys.js'; + +function usage(rc: number): never { + console.log(`Usage: + tools/run_${sys.type} [OPTIONS...] +Options: + --colors Enable ANSI colors in output. + --coverage Add coverage data to each result. + --data Path to the data cache directory. + --verbose Print result/log of every test as it runs. + --gpu-provider Path to node module that provides the GPU implementation. + --gpu-provider-flag Flag to set on the gpu-provider as <flag>=<value> + --unroll-const-eval-loops Unrolls loops in constant-evaluation shader execution tests + --u Flag to set on the gpu-provider as <flag>=<value> + +Provides an HTTP server used for running tests via an HTTP RPC interface +To run a test, perform an HTTP GET or POST at the URL: + http://localhost:port/run?<test-name> +To shutdown the server perform an HTTP GET or POST at the URL: + http://localhost:port/terminate +`); + return sys.exit(rc); +} + +interface RunResult { + // The result of the test + status: Status; + // Any additional messages printed + message: string; + // Code coverage data, if the server was started with `--coverage` + // This data is opaque (implementation defined). + coverageData?: string; +} + +// The interface that exposes creation of the GPU, and optional interface to code coverage. +interface GPUProviderModule { + // @returns a GPU with the given flags + create(flags: string[]): GPU; + // An optional interface to a CodeCoverageProvider + coverage?: CodeCoverageProvider; +} + +interface CodeCoverageProvider { + // Starts collecting code coverage + begin(): void; + // Ends collecting of code coverage, returning the coverage data. + // This data is opaque (implementation defined). + end(): string; +} + +if (!sys.existsSync('src/common/runtime/cmdline.ts')) { + console.log('Must be run from repository root'); + usage(1); +} + +Colors.enabled = false; + +let emitCoverage = false; +let verbose = false; +let gpuProviderModule: GPUProviderModule | undefined = undefined; +let dataPath: string | undefined = undefined; + +const gpuProviderFlags: string[] = []; +for (let i = 0; i < sys.args.length; ++i) { + const a = sys.args[i]; + if (a.startsWith('-')) { + if (a === '--colors') { + Colors.enabled = true; + } else if (a === '--coverage') { + emitCoverage = true; + } else if (a === '--data') { + dataPath = sys.args[++i]; + } else if (a === '--gpu-provider') { + const modulePath = sys.args[++i]; + gpuProviderModule = require(modulePath); + } else if (a === '--gpu-provider-flag') { + gpuProviderFlags.push(sys.args[++i]); + } else if (a === '--unroll-const-eval-loops') { + globalTestConfig.unrollConstEvalLoops = true; + } else if (a === '--help') { + usage(1); + } else if (a === '--verbose') { + verbose = true; + } else { + console.log(`unrecognised flag: ${a}`); + } + } +} + +let codeCoverage: CodeCoverageProvider | undefined = undefined; + +if (gpuProviderModule) { + setGPUProvider(() => gpuProviderModule!.create(gpuProviderFlags)); + + if (emitCoverage) { + codeCoverage = gpuProviderModule.coverage; + if (codeCoverage === undefined) { + console.error( + `--coverage specified, but the GPUProviderModule does not support code coverage. +Did you remember to build with code coverage instrumentation enabled?` + ); + sys.exit(1); + } + } +} + +if (dataPath !== undefined) { + dataCache.setStore({ + load: (path: string) => { + return new Promise<string>((resolve, reject) => { + fs.readFile(`${dataPath}/${path}`, 'utf8', (err, data) => { + if (err !== null) { + reject(err.message); + } else { + resolve(data); + } + }); + }); + }, + }); +} +if (verbose) { + dataCache.setDebugLogger(console.log); +} + +(async () => { + Logger.globalDebugMode = verbose; + const log = new Logger(); + const testcases = new Map<string, TestTreeLeaf>(); + + async function runTestcase( + testcase: TestTreeLeaf, + expectations: TestQueryWithExpectation[] = [] + ): Promise<LiveTestCaseResult> { + const name = testcase.query.toString(); + const [rec, res] = log.record(name); + await testcase.run(rec, expectations); + return res; + } + + const server = http.createServer( + async (request: http.IncomingMessage, response: http.ServerResponse) => { + if (request.url === undefined) { + response.end('invalid url'); + return; + } + + const loadCasesPrefix = '/load?'; + const runPrefix = '/run?'; + const terminatePrefix = '/terminate'; + + if (request.url.startsWith(loadCasesPrefix)) { + const query = request.url.substr(loadCasesPrefix.length); + try { + const webgpuQuery = parseQuery(query); + const loader = new DefaultTestFileLoader(); + for (const testcase of await loader.loadCases(webgpuQuery)) { + testcases.set(testcase.query.toString(), testcase); + } + response.statusCode = 200; + response.end(); + } catch (err) { + response.statusCode = 500; + response.end(`load failed with error: ${err}\n${(err as Error).stack}`); + } + } else if (request.url.startsWith(runPrefix)) { + const name = request.url.substr(runPrefix.length); + try { + const testcase = testcases.get(name); + if (testcase) { + if (codeCoverage !== undefined) { + codeCoverage.begin(); + } + const result = await runTestcase(testcase); + const coverageData = codeCoverage !== undefined ? codeCoverage.end() : undefined; + let message = ''; + if (result.logs !== undefined) { + message = result.logs.map(log => prettyPrintLog(log)).join('\n'); + } + const status = result.status; + const res: RunResult = { status, message, coverageData }; + response.statusCode = 200; + response.end(JSON.stringify(res)); + } else { + response.statusCode = 404; + response.end(`test case '${name}' not found`); + } + } catch (err) { + response.statusCode = 500; + response.end(`run failed with error: ${err}`); + } + } else if (request.url.startsWith(terminatePrefix)) { + server.close(); + sys.exit(1); + } else { + response.statusCode = 404; + response.end('unhandled url request'); + } + } + ); + + server.listen(0, () => { + const address = server.address() as AddressInfo; + console.log(`Server listening at [[${address.port}]]`); + }); +})().catch(ex => { + console.error(ex.stack ?? ex.toString()); + sys.exit(1); +}); diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts new file mode 100644 index 0000000000..0dd158fd68 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts @@ -0,0 +1,625 @@ +// Implements the standalone test runner (see also: /standalone/index.html). + +import { dataCache } from '../framework/data_cache.js'; +import { setBaseResourcePath } from '../framework/resources.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { Logger } from '../internal/logging/logger.js'; +import { LiveTestCaseResult } from '../internal/logging/result.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { TestQueryLevel } from '../internal/query/query.js'; +import { TestTreeNode, TestSubtree, TestTreeLeaf, TestTree } from '../internal/tree.js'; +import { setDefaultRequestAdapterOptions } from '../util/navigator_gpu.js'; +import { assert, ErrorWithExtra, unreachable } from '../util/util.js'; + +import { optionEnabled, optionString } from './helper/options.js'; +import { TestWorker } from './helper/test_worker.js'; + +window.onbeforeunload = () => { + // Prompt user before reloading if there are any results + return haveSomeResults ? false : undefined; +}; + +let haveSomeResults = false; + +// The possible options for the tests. +interface StandaloneOptions { + runnow: boolean; + worker: boolean; + debug: boolean; + unrollConstEvalLoops: boolean; + powerPreference: string; +} + +// Extra per option info. +interface StandaloneOptionInfo { + description: string; + parser?: (key: string) => boolean | string; + selectValueDescriptions?: { value: string; description: string }[]; +} + +// Type for info for every option. This definition means adding an option +// will generate a compile time error if not extra info is provided. +type StandaloneOptionsInfos = Record<keyof StandaloneOptions, StandaloneOptionInfo>; + +const optionsInfo: StandaloneOptionsInfos = { + runnow: { description: 'run immediately on load' }, + worker: { description: 'run in a worker' }, + debug: { description: 'show more info' }, + unrollConstEvalLoops: { description: 'unroll const eval loops in WGSL' }, + powerPreference: { + description: 'set default powerPreference for some tests', + parser: optionString, + selectValueDescriptions: [ + { value: '', description: 'default' }, + { value: 'low-power', description: 'low-power' }, + { value: 'high-performance', description: 'high-performance' }, + ], + }, +}; + +/** + * Converts camel case to snake case. + * Examples: + * fooBar -> foo_bar + * parseHTMLFile -> parse_html_file + */ +function camelCaseToSnakeCase(id: string) { + return id + .replace(/(.)([A-Z][a-z]+)/g, '$1_$2') + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .toLowerCase(); +} + +/** + * Creates a StandaloneOptions from the current URL search parameters. + */ +function getOptionsInfoFromSearchParameters( + optionsInfos: StandaloneOptionsInfos +): StandaloneOptions { + const optionValues: Record<string, boolean | string> = {}; + for (const [optionName, info] of Object.entries(optionsInfos)) { + const parser = info.parser || optionEnabled; + optionValues[optionName] = parser(camelCaseToSnakeCase(optionName)); + } + return (optionValues as unknown) as StandaloneOptions; +} + +// This is just a cast in one place. +function optionsToRecord(options: StandaloneOptions) { + return (options as unknown) as Record<string, boolean | string>; +} + +const options = getOptionsInfoFromSearchParameters(optionsInfo); +const { runnow, debug, unrollConstEvalLoops, powerPreference } = options; +globalTestConfig.unrollConstEvalLoops = unrollConstEvalLoops; + +Logger.globalDebugMode = debug; +const logger = new Logger(); + +setBaseResourcePath('../out/resources'); + +const worker = options.worker ? new TestWorker(debug) : undefined; + +const autoCloseOnPass = document.getElementById('autoCloseOnPass') as HTMLInputElement; +const resultsVis = document.getElementById('resultsVis')!; +const progressElem = document.getElementById('progress')!; +const progressTestNameElem = progressElem.querySelector('.progress-test-name')!; +const stopButtonElem = progressElem.querySelector('button')!; +let runDepth = 0; +let stopRequested = false; + +stopButtonElem.addEventListener('click', () => { + stopRequested = true; +}); + +if (powerPreference) { + setDefaultRequestAdapterOptions({ powerPreference: powerPreference as GPUPowerPreference }); +} + +dataCache.setStore({ + load: async (path: string) => { + const response = await fetch(`data/${path}`); + if (!response.ok) { + return Promise.reject(response.statusText); + } + return await response.text(); + }, +}); + +interface SubtreeResult { + pass: number; + fail: number; + warn: number; + skip: number; + total: number; + timems: number; +} + +function emptySubtreeResult() { + return { pass: 0, fail: 0, warn: 0, skip: 0, total: 0, timems: 0 }; +} + +function mergeSubtreeResults(...results: SubtreeResult[]) { + const target = emptySubtreeResult(); + for (const result of results) { + target.pass += result.pass; + target.fail += result.fail; + target.warn += result.warn; + target.skip += result.skip; + target.total += result.total; + target.timems += result.timems; + } + return target; +} + +type SetCheckedRecursively = () => void; +type GenerateSubtreeHTML = (parent: HTMLElement) => SetCheckedRecursively; +type RunSubtree = () => Promise<SubtreeResult>; + +interface VisualizedSubtree { + generateSubtreeHTML: GenerateSubtreeHTML; + runSubtree: RunSubtree; +} + +// DOM generation + +function memoize<T>(fn: () => T): () => T { + let value: T | undefined; + return () => { + if (value === undefined) { + value = fn(); + } + return value; + }; +} + +function makeTreeNodeHTML(tree: TestTreeNode, parentLevel: TestQueryLevel): VisualizedSubtree { + let subtree: VisualizedSubtree; + + if ('children' in tree) { + subtree = makeSubtreeHTML(tree, parentLevel); + } else { + subtree = makeCaseHTML(tree); + } + + const generateMyHTML = (parentElement: HTMLElement) => { + const div = $('<div>').appendTo(parentElement)[0]; + return subtree.generateSubtreeHTML(div); + }; + return { runSubtree: subtree.runSubtree, generateSubtreeHTML: generateMyHTML }; +} + +function makeCaseHTML(t: TestTreeLeaf): VisualizedSubtree { + // Becomes set once the case has been run once. + let caseResult: LiveTestCaseResult | undefined; + + // Becomes set once the DOM for this case exists. + let clearRenderedResult: (() => void) | undefined; + let updateRenderedResult: (() => void) | undefined; + + const name = t.query.toString(); + const runSubtree = async () => { + if (clearRenderedResult) clearRenderedResult(); + + const result: SubtreeResult = emptySubtreeResult(); + progressTestNameElem.textContent = name; + + haveSomeResults = true; + const [rec, res] = logger.record(name); + caseResult = res; + if (worker) { + await worker.run(rec, name); + } else { + await t.run(rec); + } + + result.total++; + result.timems += caseResult.timems; + switch (caseResult.status) { + case 'pass': + result.pass++; + break; + case 'fail': + result.fail++; + break; + case 'skip': + result.skip++; + break; + case 'warn': + result.warn++; + break; + default: + unreachable(); + } + + if (updateRenderedResult) updateRenderedResult(); + + return result; + }; + + const generateSubtreeHTML = (div: HTMLElement) => { + div.classList.add('testcase'); + + const caselogs = $('<div>').addClass('testcaselogs').hide(); + const [casehead, setChecked] = makeTreeNodeHeaderHTML(t, runSubtree, 2, checked => { + checked ? caselogs.show() : caselogs.hide(); + }); + const casetime = $('<div>').addClass('testcasetime').html('ms').appendTo(casehead); + div.appendChild(casehead); + div.appendChild(caselogs[0]); + + clearRenderedResult = () => { + div.removeAttribute('data-status'); + casetime.text('ms'); + caselogs.empty(); + }; + + updateRenderedResult = () => { + if (caseResult) { + div.setAttribute('data-status', caseResult.status); + + casetime.text(caseResult.timems.toFixed(4) + ' ms'); + + if (caseResult.logs) { + caselogs.empty(); + for (const l of caseResult.logs) { + const caselog = $('<div>').addClass('testcaselog').appendTo(caselogs); + $('<button>') + .addClass('testcaselogbtn') + .attr('alt', 'Log stack to console') + .attr('title', 'Log stack to console') + .appendTo(caselog) + .on('click', () => { + consoleLogError(l); + }); + $('<pre>').addClass('testcaselogtext').appendTo(caselog).text(l.toJSON()); + } + } + } + }; + + updateRenderedResult(); + + return setChecked; + }; + + return { runSubtree, generateSubtreeHTML }; +} + +function makeSubtreeHTML(n: TestSubtree, parentLevel: TestQueryLevel): VisualizedSubtree { + let subtreeResult: SubtreeResult = emptySubtreeResult(); + // Becomes set once the DOM for this case exists. + let clearRenderedResult: (() => void) | undefined; + let updateRenderedResult: (() => void) | undefined; + + const { runSubtree, generateSubtreeHTML } = makeSubtreeChildrenHTML( + n.children.values(), + n.query.level + ); + + const runMySubtree = async () => { + if (runDepth === 0) { + stopRequested = false; + progressElem.style.display = ''; + } + if (stopRequested) { + const result = emptySubtreeResult(); + result.skip = 1; + result.total = 1; + return result; + } + + ++runDepth; + + if (clearRenderedResult) clearRenderedResult(); + subtreeResult = await runSubtree(); + if (updateRenderedResult) updateRenderedResult(); + + --runDepth; + if (runDepth === 0) { + progressElem.style.display = 'none'; + } + + return subtreeResult; + }; + + const generateMyHTML = (div: HTMLElement) => { + const subtreeHTML = $('<div>').addClass('subtreechildren'); + const generateSubtree = memoize(() => generateSubtreeHTML(subtreeHTML[0])); + + // Hide subtree - it's not generated yet. + subtreeHTML.hide(); + const [header, setChecked] = makeTreeNodeHeaderHTML(n, runMySubtree, parentLevel, checked => { + if (checked) { + // Make sure the subtree is generated and then show it. + generateSubtree(); + subtreeHTML.show(); + } else { + subtreeHTML.hide(); + } + }); + + div.classList.add('subtree'); + div.classList.add(['', 'multifile', 'multitest', 'multicase'][n.query.level]); + div.appendChild(header); + div.appendChild(subtreeHTML[0]); + + clearRenderedResult = () => { + div.removeAttribute('data-status'); + }; + + updateRenderedResult = () => { + let status = ''; + if (subtreeResult.pass > 0) { + status += 'pass'; + } + if (subtreeResult.fail > 0) { + status += 'fail'; + } + div.setAttribute('data-status', status); + if (autoCloseOnPass.checked && status === 'pass') { + div.firstElementChild!.removeAttribute('open'); + } + }; + + updateRenderedResult(); + + return () => { + setChecked(); + const setChildrenChecked = generateSubtree(); + setChildrenChecked(); + }; + }; + + return { runSubtree: runMySubtree, generateSubtreeHTML: generateMyHTML }; +} + +function makeSubtreeChildrenHTML( + children: Iterable<TestTreeNode>, + parentLevel: TestQueryLevel +): VisualizedSubtree { + const childFns = Array.from(children, subtree => makeTreeNodeHTML(subtree, parentLevel)); + + const runMySubtree = async () => { + const results: SubtreeResult[] = []; + for (const { runSubtree } of childFns) { + results.push(await runSubtree()); + } + return mergeSubtreeResults(...results); + }; + const generateMyHTML = (div: HTMLElement) => { + const setChildrenChecked = Array.from(childFns, ({ generateSubtreeHTML }) => + generateSubtreeHTML(div) + ); + + return () => { + for (const setChildChecked of setChildrenChecked) { + setChildChecked(); + } + }; + }; + + return { runSubtree: runMySubtree, generateSubtreeHTML: generateMyHTML }; +} + +function consoleLogError(e: Error | ErrorWithExtra | undefined) { + if (e === undefined) return; + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + (globalThis as any)._stack = e; + /* eslint-disable-next-line no-console */ + console.log('_stack =', e); + if ('extra' in e && e.extra !== undefined) { + /* eslint-disable-next-line no-console */ + console.log('_stack.extra =', e.extra); + } +} + +function makeTreeNodeHeaderHTML( + n: TestTreeNode, + runSubtree: RunSubtree, + parentLevel: TestQueryLevel, + onChange: (checked: boolean) => void +): [HTMLElement, SetCheckedRecursively] { + const isLeaf = 'run' in n; + const div = $('<details>').addClass('nodeheader'); + const header = $('<summary>').appendTo(div); + + const setChecked = () => { + div.prop('open', true); // (does not fire onChange) + onChange(true); + }; + + const href = `?${worker ? 'worker&' : ''}${debug ? 'debug&' : ''}q=${n.query.toString()}`; + if (onChange) { + div.on('toggle', function (this) { + onChange((this as HTMLDetailsElement).open); + }); + + // Expand the shallower parts of the tree at load. + // Also expand completely within subtrees that are at the same query level + // (e.g. s:f:t,* and s:f:t,t,*). + if (n.query.level <= lastQueryLevelToExpand || n.query.level === parentLevel) { + setChecked(); + } + } + const runtext = isLeaf ? 'Run case' : 'Run subtree'; + $('<button>') + .addClass(isLeaf ? 'leafrun' : 'subtreerun') + .attr('alt', runtext) + .attr('title', runtext) + .on('click', () => void runSubtree()) + .appendTo(header); + $('<a>') + .addClass('nodelink') + .attr('href', href) + .attr('alt', 'Open') + .attr('title', 'Open') + .appendTo(header); + if ('testCreationStack' in n && n.testCreationStack) { + $('<button>') + .addClass('testcaselogbtn') + .attr('alt', 'Log test creation stack to console') + .attr('title', 'Log test creation stack to console') + .appendTo(header) + .on('click', () => { + consoleLogError(n.testCreationStack); + }); + } + const nodetitle = $('<div>').addClass('nodetitle').appendTo(header); + const nodecolumns = $('<span>').addClass('nodecolumns').appendTo(nodetitle); + { + $('<input>') + .attr('type', 'text') + .prop('readonly', true) + .addClass('nodequery') + .val(n.query.toString()) + .appendTo(nodecolumns); + if (n.subtreeCounts) { + $('<span>') + .attr('title', '(Nodes with TODOs) / (Total test count)') + .text(TestTree.countsToString(n)) + .appendTo(nodecolumns); + } + } + if ('description' in n && n.description) { + nodetitle.append(' '); + $('<pre>') // + .addClass('nodedescription') + .text(n.description) + .appendTo(header); + } + return [div[0], setChecked]; +} + +// Collapse s:f:t:* or s:f:t:c by default. +let lastQueryLevelToExpand: TestQueryLevel = 2; + +type ParamValue = string | undefined | null | boolean | string[]; + +/** + * Takes an array of string, ParamValue and returns an array of pairs + * of [key, value] where value is a string. Converts boolean to '0' or '1'. + */ +function keyValueToPairs([k, v]: [string, ParamValue]): [string, string][] { + const key = camelCaseToSnakeCase(k); + if (typeof v === 'boolean') { + return [[key, v ? '1' : '0']]; + } else if (Array.isArray(v)) { + return v.map(v => [key, v]); + } else { + return [[key, v!.toString()]]; + } +} + +/** + * Converts key value pairs to a search string. + * Keys will appear in order in the search string. + * Values can be undefined, null, boolean, string, or string[] + * If the value is falsy the key will not appear in the search string. + * If the value is an array the key will appear multiple times. + * + * @param params Some object with key value pairs. + * @returns a search string. + */ +function prepareParams(params: Record<string, ParamValue>): string { + const pairsArrays = Object.entries(params) + .filter(([, v]) => !!v) + .map(keyValueToPairs); + const pairs = pairsArrays.flat(); + return new URLSearchParams(pairs).toString(); +} + +void (async () => { + const loader = new DefaultTestFileLoader(); + + // MAINTENANCE_TODO: start populating page before waiting for everything to load? + const qs = new URLSearchParams(window.location.search).getAll('q'); + if (qs.length === 0) { + qs.push('webgpu:*'); + } + + // Update the URL bar to match the exact current options. + const updateURLWithCurrentOptions = () => { + const search = prepareParams(optionsToRecord(options)); + let url = `${window.location.origin}${window.location.pathname}`; + // Add in q separately to avoid escaping punctuation marks. + url += `?${search}${search ? '&' : ''}${qs.map(q => 'q=' + q).join('&')}`; + window.history.replaceState(null, '', url.toString()); + }; + updateURLWithCurrentOptions(); + + const addOptionsToPage = (options: StandaloneOptions, optionsInfos: StandaloneOptionsInfos) => { + const optionsElem = $('table#options>tbody')[0]; + const optionValues = optionsToRecord(options); + + const createCheckbox = (optionName: string) => { + return $(`<input>`) + .attr('type', 'checkbox') + .prop('checked', optionValues[optionName] as boolean) + .on('change', function () { + optionValues[optionName] = (this as HTMLInputElement).checked; + updateURLWithCurrentOptions(); + }); + }; + + const createSelect = (optionName: string, info: StandaloneOptionInfo) => { + const select = $('<select>').on('change', function () { + optionValues[optionName] = (this as HTMLInputElement).value; + updateURLWithCurrentOptions(); + }); + const currentValue = optionValues[optionName]; + for (const { value, description } of info.selectValueDescriptions!) { + $('<option>') + .text(description) + .val(value) + .prop('selected', value === currentValue) + .appendTo(select); + } + return select; + }; + + for (const [optionName, info] of Object.entries(optionsInfos)) { + const input = + typeof optionValues[optionName] === 'boolean' + ? createCheckbox(optionName) + : createSelect(optionName, info); + $('<tr>') + .append($('<td>').append(input)) + .append($('<td>').text(camelCaseToSnakeCase(optionName))) + .append($('<td>').text(info.description)) + .appendTo(optionsElem); + } + }; + addOptionsToPage(options, optionsInfo); + + assert(qs.length === 1, 'currently, there must be exactly one ?q='); + const rootQuery = parseQuery(qs[0]); + if (rootQuery.level > lastQueryLevelToExpand) { + lastQueryLevelToExpand = rootQuery.level; + } + loader.addEventListener('import', ev => { + $('#info')[0].textContent = `loading: ${ev.data.url}`; + }); + loader.addEventListener('finish', () => { + $('#info')[0].textContent = ''; + }); + const tree = await loader.loadTree(rootQuery); + + tree.dissolveSingleChildTrees(); + + const { runSubtree, generateSubtreeHTML } = makeSubtreeHTML(tree.root, 1); + const setTreeCheckedRecursively = generateSubtreeHTML(resultsVis); + + document.getElementById('expandall')!.addEventListener('click', () => { + setTreeCheckedRecursively(); + }); + + document.getElementById('copyResultsJSON')!.addEventListener('click', () => { + void navigator.clipboard.writeText(logger.asJSON(2)); + }); + + if (runnow) { + void runSubtree(); + } +})(); diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts new file mode 100644 index 0000000000..2cb9f8dbf7 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts @@ -0,0 +1,83 @@ +// Implements the wpt-embedded test runner (see also: wpt/cts.https.html). + +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { prettyPrintLog } from '../internal/logging/log_message.js'; +import { Logger } from '../internal/logging/logger.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { parseExpectationsForTestQuery, relativeQueryString } from '../internal/query/query.js'; +import { assert } from '../util/util.js'; + +import { optionEnabled } from './helper/options.js'; +import { TestWorker } from './helper/test_worker.js'; + +// testharness.js API (https://web-platform-tests.org/writing-tests/testharness-api.html) +declare interface WptTestObject { + step(f: () => void): void; + done(): void; +} +declare function setup(properties: { explicit_done?: boolean }): void; +declare function promise_test(f: (t: WptTestObject) => Promise<void>, name: string): void; +declare function done(): void; +declare function assert_unreached(description: string): void; + +declare const loadWebGPUExpectations: Promise<unknown> | undefined; +declare const shouldWebGPUCTSFailOnWarnings: Promise<boolean> | undefined; + +setup({ + // It's convenient for us to asynchronously add tests to the page. Prevent done() from being + // called implicitly when the page is finished loading. + explicit_done: true, +}); + +void (async () => { + const workerEnabled = optionEnabled('worker'); + const worker = workerEnabled ? new TestWorker(false) : undefined; + + globalTestConfig.unrollConstEvalLoops = optionEnabled('unroll_const_eval_loops'); + + const failOnWarnings = + typeof shouldWebGPUCTSFailOnWarnings !== 'undefined' && (await shouldWebGPUCTSFailOnWarnings); + + const loader = new DefaultTestFileLoader(); + const qs = new URLSearchParams(window.location.search).getAll('q'); + assert(qs.length === 1, 'currently, there must be exactly one ?q='); + const filterQuery = parseQuery(qs[0]); + const testcases = await loader.loadCases(filterQuery); + + const expectations = + typeof loadWebGPUExpectations !== 'undefined' + ? parseExpectationsForTestQuery( + await loadWebGPUExpectations, + filterQuery, + new URL(window.location.href) + ) + : []; + + const log = new Logger(); + + for (const testcase of testcases) { + const name = testcase.query.toString(); + // For brevity, display the case name "relative" to the ?q= path. + const shortName = relativeQueryString(filterQuery, testcase.query) || '(case)'; + + const wpt_fn = async () => { + const [rec, res] = log.record(name); + if (worker) { + await worker.run(rec, name, expectations); + } else { + await testcase.run(rec, expectations); + } + + // Unfortunately, it seems not possible to surface any logs for warn/skip. + if (res.status === 'fail' || (res.status === 'warn' && failOnWarnings)) { + const logs = (res.logs ?? []).map(prettyPrintLog); + assert_unreached('\n' + logs.join('\n') + '\n'); + } + }; + + promise_test(wpt_fn, shortName); + } + + done(); +})(); |