diff options
Diffstat (limited to 'dom/webgpu/tests/cts/checkout/src/common/runtime')
9 files changed, 448 insertions, 138 deletions
diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts index 44a73fb38b..2a00640f0e 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts @@ -3,6 +3,7 @@ import * as fs from 'fs'; import { dataCache } from '../framework/data_cache.js'; +import { getResourcePath, setBaseResourcePath } from '../framework/resources.js'; import { globalTestConfig } from '../framework/test_config.js'; import { DefaultTestFileLoader } from '../internal/file_loader.js'; import { prettyPrintLog } from '../internal/logging/log_message.js'; @@ -37,6 +38,12 @@ Options: return sys.exit(rc); } +if (!sys.existsSync('src/common/runtime/cmdline.ts')) { + console.log('Must be run from repository root'); + usage(1); +} +setBaseResourcePath('out-node/resources'); + // The interface that exposes creation of the GPU, and optional interface to code coverage. interface GPUProviderModule { // @returns a GPU with the given flags @@ -60,12 +67,10 @@ Colors.enabled = false; let verbose = false; let emitCoverage = false; let listMode: listModes = 'none'; -let debug = false; let printJSON = false; let quiet = false; let loadWebGPUExpectations: Promise<unknown> | undefined = undefined; let gpuProviderModule: GPUProviderModule | undefined = undefined; -let dataPath: string | undefined = undefined; const queries: string[] = []; const gpuProviderFlags: string[] = []; @@ -83,9 +88,7 @@ for (let i = 0; i < sys.args.length; ++i) { } else if (a === '--list-unimplemented') { listMode = 'unimplemented'; } else if (a === '--debug') { - debug = true; - } else if (a === '--data') { - dataPath = sys.args[++i]; + globalTestConfig.enableDebugLogs = true; } else if (a === '--print-json') { printJSON = true; } else if (a === '--expectations') { @@ -102,6 +105,10 @@ for (let i = 0; i < sys.args.length; ++i) { globalTestConfig.unrollConstEvalLoops = true; } else if (a === '--compat') { globalTestConfig.compatibility = true; + } else if (a === '--force-fallback-adapter') { + globalTestConfig.forceFallbackAdapter = true; + } else if (a === '--log-to-websocket') { + globalTestConfig.logToWebSocket = true; } else { console.log('unrecognized flag: ', a); usage(1); @@ -113,9 +120,12 @@ for (let i = 0; i < sys.args.length; ++i) { let codeCoverage: CodeCoverageProvider | undefined = undefined; -if (globalTestConfig.compatibility) { +if (globalTestConfig.compatibility || globalTestConfig.forceFallbackAdapter) { // MAINTENANCE_TODO: remove the cast once compatibilityMode is officially added - setDefaultRequestAdapterOptions({ compatibilityMode: true } as GPURequestAdapterOptions); + setDefaultRequestAdapterOptions({ + compatibilityMode: globalTestConfig.compatibility, + forceFallbackAdapter: globalTestConfig.forceFallbackAdapter, + } as GPURequestAdapterOptions); } if (gpuProviderModule) { @@ -132,21 +142,20 @@ Did you remember to build with code coverage instrumentation enabled?` } } -if (dataPath !== undefined) { - dataCache.setStore({ - load: (path: string) => { - return new Promise<Uint8Array>((resolve, reject) => { - fs.readFile(`${dataPath}/${path}`, (err, data) => { - if (err !== null) { - reject(err.message); - } else { - resolve(data); - } - }); +dataCache.setStore({ + load: (path: string) => { + return new Promise<Uint8Array>((resolve, reject) => { + fs.readFile(getResourcePath(`cache/${path}`), (err, data) => { + if (err !== null) { + reject(err.message); + } else { + resolve(data); + } }); - }, - }); -} + }); + }, +}); + if (verbose) { dataCache.setDebugLogger(console.log); } @@ -166,7 +175,6 @@ if (queries.length === 0) { filterQuery ); - Logger.globalDebugMode = debug; const log = new Logger(); const failed: Array<[string, LiveTestCaseResult]> = []; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts index 38974b803f..4a82c7d292 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts @@ -1,3 +1,5 @@ +import { unreachable } from '../../util/util.js'; + let windowURL: URL | undefined = undefined; function getWindowURL() { if (windowURL === undefined) { @@ -6,6 +8,7 @@ function getWindowURL() { return windowURL; } +/** Parse a runner option that is always boolean-typed. False if missing or '0'. */ export function optionEnabled( opt: string, searchParams: URLSearchParams = getWindowURL().searchParams @@ -14,30 +17,55 @@ export function optionEnabled( return val !== null && val !== '0'; } +/** Parse a runner option that is string-typed. If the option is missing, returns `null`. */ export function optionString( opt: string, searchParams: URLSearchParams = getWindowURL().searchParams -): string { - return searchParams.get(opt) || ''; +): string | null { + return searchParams.get(opt); +} + +/** Runtime modes for running tests in different types of workers. */ +export type WorkerMode = 'dedicated' | 'service' | 'shared'; +/** Parse a runner option for different worker modes (as in `?worker=shared`). Null if no worker. */ +export function optionWorkerMode( + opt: string, + searchParams: URLSearchParams = getWindowURL().searchParams +): WorkerMode | null { + const value = searchParams.get(opt); + if (value === null || value === '0') { + return null; + } else if (value === 'service') { + return 'service'; + } else if (value === 'shared') { + return 'shared'; + } else if (value === '' || value === '1' || value === 'dedicated') { + return 'dedicated'; + } + unreachable('invalid worker= option value'); } /** * The possible options for the tests. */ export interface CTSOptions { - worker: boolean; + worker: WorkerMode | null; debug: boolean; compatibility: boolean; + forceFallbackAdapter: boolean; unrollConstEvalLoops: boolean; - powerPreference?: GPUPowerPreference | ''; + powerPreference: GPUPowerPreference | null; + logToWebSocket: boolean; } export const kDefaultCTSOptions: CTSOptions = { - worker: false, + worker: null, debug: true, compatibility: false, + forceFallbackAdapter: false, unrollConstEvalLoops: false, - powerPreference: '', + powerPreference: null, + logToWebSocket: false, }; /** @@ -45,8 +73,8 @@ export const kDefaultCTSOptions: CTSOptions = { */ export interface OptionInfo { description: string; - parser?: (key: string, searchParams?: URLSearchParams) => boolean | string; - selectValueDescriptions?: { value: string; description: string }[]; + parser?: (key: string, searchParams?: URLSearchParams) => boolean | string | null; + selectValueDescriptions?: { value: string | null; description: string }[]; } /** @@ -59,19 +87,30 @@ export type OptionsInfos<Type> = Record<keyof Type, OptionInfo>; * Options to the CTS. */ export const kCTSOptionsInfo: OptionsInfos<CTSOptions> = { - worker: { description: 'run in a worker' }, + worker: { + description: 'run in a worker', + parser: optionWorkerMode, + selectValueDescriptions: [ + { value: null, description: 'no worker' }, + { value: 'dedicated', description: 'dedicated worker' }, + { value: 'shared', description: 'shared worker' }, + { value: 'service', description: 'service worker' }, + ], + }, debug: { description: 'show more info' }, compatibility: { description: 'run in compatibility mode' }, + forceFallbackAdapter: { description: 'pass forceFallbackAdapter: true to requestAdapter' }, unrollConstEvalLoops: { description: 'unroll const eval loops in WGSL' }, powerPreference: { description: 'set default powerPreference for some tests', parser: optionString, selectValueDescriptions: [ - { value: '', description: 'default' }, + { value: null, description: 'default' }, { value: 'low-power', description: 'low-power' }, { value: 'high-performance', description: 'high-performance' }, ], }, + logToWebSocket: { description: 'send some logs to ws://localhost:59497/' }, }; /** @@ -95,7 +134,7 @@ function getOptionsInfoFromSearchString<Type extends CTSOptions>( searchString: string ): Type { const searchParams = new URLSearchParams(searchString); - const optionValues: Record<string, boolean | string> = {}; + const optionValues: Record<string, boolean | string | null> = {}; for (const [optionName, info] of Object.entries(optionsInfos)) { const parser = info.parser || optionEnabled; optionValues[optionName] = parser(camelCaseToSnakeCase(optionName), searchParams); diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts index e8d187ea7e..ebc206c3b2 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts @@ -1,15 +1,11 @@ import { setBaseResourcePath } from '../../framework/resources.js'; -import { globalTestConfig } from '../../framework/test_config.js'; import { DefaultTestFileLoader } from '../../internal/file_loader.js'; -import { Logger } from '../../internal/logging/logger.js'; import { parseQuery } from '../../internal/query/parseQuery.js'; -import { TestQueryWithExpectation } from '../../internal/query/query.js'; -import { setDefaultRequestAdapterOptions } from '../../util/navigator_gpu.js'; import { assert } from '../../util/util.js'; -import { CTSOptions } from './options.js'; +import { setupWorkerEnvironment, WorkerTestRunRequest } from './utils_worker.js'; -// Should be DedicatedWorkerGlobalScope, but importing lib "webworker" conflicts with lib "dom". +// Should be WorkerGlobalScope, but importing lib "webworker" conflicts with lib "dom". /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ declare const self: any; @@ -17,25 +13,10 @@ const loader = new DefaultTestFileLoader(); setBaseResourcePath('../../../resources'); -self.onmessage = async (ev: MessageEvent) => { - const query: string = ev.data.query; - const expectations: TestQueryWithExpectation[] = ev.data.expectations; - const ctsOptions: CTSOptions = ev.data.ctsOptions; +async function reportTestResults(this: MessagePort | Worker, ev: MessageEvent) { + const { query, expectations, ctsOptions } = ev.data as WorkerTestRunRequest; - const { debug, unrollConstEvalLoops, powerPreference, compatibility } = ctsOptions; - globalTestConfig.unrollConstEvalLoops = unrollConstEvalLoops; - globalTestConfig.compatibility = compatibility; - - Logger.globalDebugMode = debug; - const log = new Logger(); - - if (powerPreference || compatibility) { - setDefaultRequestAdapterOptions({ - ...(powerPreference && { powerPreference }), - // MAINTENANCE_TODO: Change this to whatever the option ends up being - ...(compatibility && { compatibilityMode: true }), - }); - } + const log = setupWorkerEnvironment(ctsOptions); const testcases = Array.from(await loader.loadCases(parseQuery(query))); assert(testcases.length === 1, 'worker query resulted in != 1 cases'); @@ -44,5 +25,23 @@ self.onmessage = async (ev: MessageEvent) => { const [rec, result] = log.record(testcase.query.toString()); await testcase.run(rec, expectations); - self.postMessage({ query, result }); + this.postMessage({ + query, + result: { + ...result, + logs: result.logs?.map(l => l.toRawData()), + }, + }); +} + +self.onmessage = (ev: MessageEvent) => { + void reportTestResults.call(ev.source || self, ev); +}; + +self.onconnect = (event: MessageEvent) => { + const port = event.ports[0]; + + port.onmessage = (ev: MessageEvent) => { + void reportTestResults.call(port, ev); + }; }; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts index 9bbcab0946..f9a44bb7bc 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts @@ -2,48 +2,190 @@ import { LogMessageWithStack } from '../../internal/logging/log_message.js'; import { TransferredTestCaseResult, LiveTestCaseResult } from '../../internal/logging/result.js'; import { TestCaseRecorder } from '../../internal/logging/test_case_recorder.js'; import { TestQueryWithExpectation } from '../../internal/query/query.js'; +import { timeout } from '../../util/timeout.js'; +import { assert } from '../../util/util.js'; -import { CTSOptions, kDefaultCTSOptions } from './options.js'; +import { CTSOptions, WorkerMode, kDefaultCTSOptions } from './options.js'; +import { WorkerTestRunRequest } from './utils_worker.js'; -export class TestWorker { - private readonly ctsOptions: CTSOptions; - private readonly worker: Worker; - private readonly resolvers = new Map<string, (result: LiveTestCaseResult) => void>(); +/** Query all currently-registered service workers, and unregister them. */ +function unregisterAllServiceWorkers() { + void navigator.serviceWorker.getRegistrations().then(registrations => { + for (const registration of registrations) { + void registration.unregister(); + } + }); +} - constructor(ctsOptions?: CTSOptions) { - this.ctsOptions = { ...(ctsOptions || kDefaultCTSOptions), ...{ worker: true } }; - const selfPath = import.meta.url; - const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); - const workerPath = selfPathDir + '/test_worker-worker.js'; - this.worker = new Worker(workerPath, { type: 'module' }); - this.worker.onmessage = ev => { - const query: string = ev.data.query; - const result: TransferredTestCaseResult = ev.data.result; - if (result.logs) { - for (const l of result.logs) { - Object.setPrototypeOf(l, LogMessageWithStack.prototype); - } - } - this.resolvers.get(query)!(result as LiveTestCaseResult); +// NOTE: This code runs on startup for any runtime with worker support. Here, we use that chance to +// delete any leaked service workers, and register to clean up after ourselves at shutdown. +unregisterAllServiceWorkers(); +window.addEventListener('beforeunload', () => { + unregisterAllServiceWorkers(); +}); + +abstract class TestBaseWorker { + protected readonly ctsOptions: CTSOptions; + protected readonly resolvers = new Map<string, (result: LiveTestCaseResult) => void>(); + + constructor(worker: WorkerMode, ctsOptions?: CTSOptions) { + this.ctsOptions = { ...(ctsOptions || kDefaultCTSOptions), ...{ worker } }; + } - // MAINTENANCE_TODO(kainino0x): update the Logger with this result (or don't have a logger and - // update the entire results JSON somehow at some point). + onmessage(ev: MessageEvent) { + const query: string = ev.data.query; + const transferredResult: TransferredTestCaseResult = ev.data.result; + + const result: LiveTestCaseResult = { + status: transferredResult.status, + timems: transferredResult.timems, + logs: transferredResult.logs?.map(l => new LogMessageWithStack(l)), }; + + this.resolvers.get(query)!(result); + this.resolvers.delete(query); + + // MAINTENANCE_TODO(kainino0x): update the Logger with this result (or don't have a logger and + // update the entire results JSON somehow at some point). } - async run( - rec: TestCaseRecorder, + makeRequestAndRecordResult( + target: MessagePort | Worker | ServiceWorker, query: string, - expectations: TestQueryWithExpectation[] = [] - ): Promise<void> { - this.worker.postMessage({ + expectations: TestQueryWithExpectation[] + ): Promise<LiveTestCaseResult> { + const request: WorkerTestRunRequest = { query, expectations, ctsOptions: this.ctsOptions, - }); - const workerResult = await new Promise<LiveTestCaseResult>(resolve => { + }; + target.postMessage(request); + + return new Promise<LiveTestCaseResult>(resolve => { + assert(!this.resolvers.has(query), "can't request same query twice simultaneously"); this.resolvers.set(query, resolve); }); - rec.injectResult(workerResult); + } + + async run( + rec: TestCaseRecorder, + query: string, + expectations: TestQueryWithExpectation[] = [] + ): Promise<void> { + try { + rec.injectResult(await this.runImpl(query, expectations)); + } catch (ex) { + rec.start(); + rec.threw(ex); + rec.finish(); + } + } + + protected abstract runImpl( + query: string, + expectations: TestQueryWithExpectation[] + ): Promise<LiveTestCaseResult>; +} + +export class TestDedicatedWorker extends TestBaseWorker { + private readonly worker: Worker | Error; + + constructor(ctsOptions?: CTSOptions) { + super('dedicated', ctsOptions); + try { + if (typeof Worker === 'undefined') { + throw new Error('Dedicated Workers not available'); + } + + const selfPath = import.meta.url; + const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); + const workerPath = selfPathDir + '/test_worker-worker.js'; + this.worker = new Worker(workerPath, { type: 'module' }); + this.worker.onmessage = ev => this.onmessage(ev); + } catch (ex) { + assert(ex instanceof Error); + // Save the exception to re-throw in runImpl(). + this.worker = ex; + } + } + + override runImpl(query: string, expectations: TestQueryWithExpectation[] = []) { + if (this.worker instanceof Worker) { + return this.makeRequestAndRecordResult(this.worker, query, expectations); + } else { + throw this.worker; + } + } +} + +/** @deprecated Use TestDedicatedWorker instead. */ +export class TestWorker extends TestDedicatedWorker {} + +export class TestSharedWorker extends TestBaseWorker { + /** MessagePort to the SharedWorker, or an Error if it couldn't be initialized. */ + private readonly port: MessagePort | Error; + + constructor(ctsOptions?: CTSOptions) { + super('shared', ctsOptions); + try { + if (typeof SharedWorker === 'undefined') { + throw new Error('Shared Workers not available'); + } + + const selfPath = import.meta.url; + const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); + const workerPath = selfPathDir + '/test_worker-worker.js'; + const worker = new SharedWorker(workerPath, { type: 'module' }); + this.port = worker.port; + this.port.start(); + this.port.onmessage = ev => this.onmessage(ev); + } catch (ex) { + assert(ex instanceof Error); + // Save the exception to re-throw in runImpl(). + this.port = ex; + } + } + + override runImpl(query: string, expectations: TestQueryWithExpectation[] = []) { + if (this.port instanceof MessagePort) { + return this.makeRequestAndRecordResult(this.port, query, expectations); + } else { + throw this.port; + } + } +} + +export class TestServiceWorker extends TestBaseWorker { + constructor(ctsOptions?: CTSOptions) { + super('service', ctsOptions); + } + + override async runImpl(query: string, expectations: TestQueryWithExpectation[] = []) { + if (!('serviceWorker' in navigator)) { + throw new Error('Service Workers not available'); + } + const [suite, name] = query.split(':', 2); + const fileName = name.split(',').join('/'); + + const selfPath = import.meta.url; + const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); + // Construct the path to the worker file, then use URL to resolve the `../` components. + const serviceWorkerURL = new URL( + `${selfPathDir}/../../../${suite}/webworker/${fileName}.worker.js` + ).toString(); + + // If a registration already exists for this path, it will be ignored. + const registration = await navigator.serviceWorker.register(serviceWorkerURL, { + type: 'module', + }); + // Make sure the registration we just requested is active. (We don't worry about it being + // outdated from a previous page load, because we wipe all service workers on shutdown/startup.) + while (!registration.active || registration.active.scriptURL !== serviceWorkerURL) { + await new Promise(resolve => timeout(resolve, 0)); + } + const serviceWorker = registration.active; + + navigator.serviceWorker.onmessage = ev => this.onmessage(ev); + return this.makeRequestAndRecordResult(serviceWorker, query, expectations); } } diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/utils_worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/utils_worker.ts new file mode 100644 index 0000000000..13880635bc --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/utils_worker.ts @@ -0,0 +1,35 @@ +import { globalTestConfig } from '../../framework/test_config.js'; +import { Logger } from '../../internal/logging/logger.js'; +import { TestQueryWithExpectation } from '../../internal/query/query.js'; +import { setDefaultRequestAdapterOptions } from '../../util/navigator_gpu.js'; + +import { CTSOptions } from './options.js'; + +export interface WorkerTestRunRequest { + query: string; + expectations: TestQueryWithExpectation[]; + ctsOptions: CTSOptions; +} + +/** + * Set config environment for workers with ctsOptions and return a Logger. + */ +export function setupWorkerEnvironment(ctsOptions: CTSOptions): Logger { + const { powerPreference, compatibility } = ctsOptions; + globalTestConfig.enableDebugLogs = ctsOptions.debug; + globalTestConfig.unrollConstEvalLoops = ctsOptions.unrollConstEvalLoops; + globalTestConfig.compatibility = compatibility; + globalTestConfig.logToWebSocket = ctsOptions.logToWebSocket; + + const log = new Logger(); + + if (powerPreference || compatibility) { + setDefaultRequestAdapterOptions({ + ...(powerPreference && { powerPreference }), + // MAINTENANCE_TODO: Change this to whatever the option ends up being + ...(compatibility && { compatibilityMode: true }), + }); + } + + return log; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/wrap_for_worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/wrap_for_worker.ts new file mode 100644 index 0000000000..5f600fe89d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/wrap_for_worker.ts @@ -0,0 +1,54 @@ +import { Fixture } from '../../framework/fixture'; +import { LogMessageWithStack } from '../../internal/logging/log_message.js'; +import { comparePaths, comparePublicParamsPaths, Ordering } from '../../internal/query/compare.js'; +import { parseQuery } from '../../internal/query/parseQuery.js'; +import { TestQuerySingleCase } from '../../internal/query/query.js'; +import { TestGroup } from '../../internal/test_group.js'; +import { assert } from '../../util/util.js'; + +import { setupWorkerEnvironment, WorkerTestRunRequest } from './utils_worker.js'; + +/** + * Sets up the currently running Web Worker to wrap the TestGroup object `g`. + * `g` is the `g` exported from a `.spec.ts` file: a TestGroupBuilder<F> interface, + * which underneath is actually a TestGroup<F> object. + * + * This is used in the generated `.worker.js` files that are generated to use as service workers. + */ +export function wrapTestGroupForWorker(g: TestGroup<Fixture>) { + self.onmessage = async (ev: MessageEvent) => { + const { query, expectations, ctsOptions } = ev.data as WorkerTestRunRequest; + try { + const log = setupWorkerEnvironment(ctsOptions); + + const testQuery = parseQuery(query); + assert(testQuery instanceof TestQuerySingleCase); + let testcase = null; + for (const t of g.iterate()) { + if (comparePaths(t.testPath, testQuery.testPathParts) !== Ordering.Equal) { + continue; + } + for (const c of t.iterate(testQuery.params)) { + if (comparePublicParamsPaths(c.id.params, testQuery.params) === Ordering.Equal) { + testcase = c; + } + } + } + assert(!!testcase, 'testcase not found'); + const [rec, result] = log.record(query); + await testcase.run(rec, testQuery, expectations); + + ev.source?.postMessage({ query, result }); + } catch (thrown) { + const ex = thrown instanceof Error ? thrown : new Error(`${thrown}`); + ev.source?.postMessage({ + query, + result: { + status: 'fail', + timems: 0, + logs: [LogMessageWithStack.wrapError('INTERNAL', ex)], + }, + }); + } + }; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts index 8310784e3a..3999b285ba 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts @@ -5,6 +5,7 @@ import * as http from 'http'; import { AddressInfo } from 'net'; import { dataCache } from '../framework/data_cache.js'; +import { getResourcePath, setBaseResourcePath } from '../framework/resources.js'; import { globalTestConfig } from '../framework/test_config.js'; import { DefaultTestFileLoader } from '../internal/file_loader.js'; import { prettyPrintLog } from '../internal/logging/log_message.js'; @@ -20,21 +21,23 @@ import sys from './helper/sys.js'; function usage(rc: number): never { console.log(`Usage: - tools/run_${sys.type} [OPTIONS...] + tools/server [OPTIONS...] Options: --colors Enable ANSI colors in output. --compat Run tests in compatibility mode. --coverage Add coverage data to each result. - --data Path to the data cache directory. --verbose Print result/log of every test as it runs. + --debug Include debug messages in logging. --gpu-provider Path to node module that provides the GPU implementation. --gpu-provider-flag Flag to set on the gpu-provider as <flag>=<value> --unroll-const-eval-loops Unrolls loops in constant-evaluation shader execution tests --u Flag to set on the gpu-provider as <flag>=<value> Provides an HTTP server used for running tests via an HTTP RPC interface -To run a test, perform an HTTP GET or POST at the URL: - http://localhost:port/run?<test-name> +First, load some tree or subtree of tests: + http://localhost:port/load?unittests:basic:* +To run a single test case, perform an HTTP GET or POST at the URL: + http://localhost:port/run?unittests:basic:test,sync To shutdown the server perform an HTTP GET or POST at the URL: http://localhost:port/terminate `); @@ -46,6 +49,8 @@ interface RunResult { status: Status; // Any additional messages printed message: string; + // The time it took to execute the test + durationMS: number; // Code coverage data, if the server was started with `--coverage` // This data is opaque (implementation defined). coverageData?: string; @@ -71,13 +76,13 @@ if (!sys.existsSync('src/common/runtime/cmdline.ts')) { console.log('Must be run from repository root'); usage(1); } +setBaseResourcePath('out-node/resources'); Colors.enabled = false; let emitCoverage = false; let verbose = false; let gpuProviderModule: GPUProviderModule | undefined = undefined; -let dataPath: string | undefined = undefined; const gpuProviderFlags: string[] = []; for (let i = 0; i < sys.args.length; ++i) { @@ -89,13 +94,17 @@ for (let i = 0; i < sys.args.length; ++i) { globalTestConfig.compatibility = true; } else if (a === '--coverage') { emitCoverage = true; - } else if (a === '--data') { - dataPath = sys.args[++i]; + } else if (a === '--force-fallback-adapter') { + globalTestConfig.forceFallbackAdapter = true; + } else if (a === '--log-to-websocket') { + globalTestConfig.logToWebSocket = true; } else if (a === '--gpu-provider') { const modulePath = sys.args[++i]; gpuProviderModule = require(modulePath); } else if (a === '--gpu-provider-flag') { gpuProviderFlags.push(sys.args[++i]); + } else if (a === '--debug') { + globalTestConfig.enableDebugLogs = true; } else if (a === '--unroll-const-eval-loops') { globalTestConfig.unrollConstEvalLoops = true; } else if (a === '--help') { @@ -110,9 +119,12 @@ for (let i = 0; i < sys.args.length; ++i) { let codeCoverage: CodeCoverageProvider | undefined = undefined; -if (globalTestConfig.compatibility) { +if (globalTestConfig.compatibility || globalTestConfig.forceFallbackAdapter) { // MAINTENANCE_TODO: remove the cast once compatibilityMode is officially added - setDefaultRequestAdapterOptions({ compatibilityMode: true } as GPURequestAdapterOptions); + setDefaultRequestAdapterOptions({ + compatibilityMode: globalTestConfig.compatibility, + forceFallbackAdapter: globalTestConfig.forceFallbackAdapter, + } as GPURequestAdapterOptions); } if (gpuProviderModule) { @@ -130,28 +142,26 @@ Did you remember to build with code coverage instrumentation enabled?` } } -if (dataPath !== undefined) { - dataCache.setStore({ - load: (path: string) => { - return new Promise<Uint8Array>((resolve, reject) => { - fs.readFile(`${dataPath}/${path}`, (err, data) => { - if (err !== null) { - reject(err.message); - } else { - resolve(data); - } - }); +dataCache.setStore({ + load: (path: string) => { + return new Promise<Uint8Array>((resolve, reject) => { + fs.readFile(getResourcePath(`cache/${path}`), (err, data) => { + if (err !== null) { + reject(err.message); + } else { + resolve(data); + } }); - }, - }); -} + }); + }, +}); + if (verbose) { dataCache.setDebugLogger(console.log); } // eslint-disable-next-line @typescript-eslint/require-await (async () => { - Logger.globalDebugMode = verbose; const log = new Logger(); const testcases = new Map<string, TestTreeLeaf>(); @@ -198,14 +208,16 @@ if (verbose) { if (codeCoverage !== undefined) { codeCoverage.begin(); } + const start = performance.now(); const result = await runTestcase(testcase); + const durationMS = performance.now() - start; const coverageData = codeCoverage !== undefined ? codeCoverage.end() : undefined; let message = ''; if (result.logs !== undefined) { message = result.logs.map(log => prettyPrintLog(log)).join('\n'); } const status = result.status; - const res: RunResult = { status, message, coverageData }; + const res: RunResult = { status, message, durationMS, coverageData }; response.statusCode = 200; response.end(JSON.stringify(res)); } else { diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts index 0376f92dda..dc75e6fd01 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts @@ -2,7 +2,7 @@ /* eslint no-console: "off" */ import { dataCache } from '../framework/data_cache.js'; -import { setBaseResourcePath } from '../framework/resources.js'; +import { getResourcePath, setBaseResourcePath } from '../framework/resources.js'; import { globalTestConfig } from '../framework/test_config.js'; import { DefaultTestFileLoader } from '../internal/file_loader.js'; import { Logger } from '../internal/logging/logger.js'; @@ -21,7 +21,7 @@ import { OptionsInfos, camelCaseToSnakeCase, } from './helper/options.js'; -import { TestWorker } from './helper/test_worker.js'; +import { TestDedicatedWorker, TestSharedWorker, TestServiceWorker } from './helper/test_worker.js'; const rootQuerySpec = 'webgpu:*'; let promptBeforeReload = false; @@ -47,16 +47,26 @@ const { queries: qs, options } = parseSearchParamLikeWithOptions( kStandaloneOptionsInfos, window.location.search || rootQuerySpec ); -const { runnow, debug, unrollConstEvalLoops, powerPreference, compatibility } = options; -globalTestConfig.unrollConstEvalLoops = unrollConstEvalLoops; +const { runnow, powerPreference, compatibility, forceFallbackAdapter } = options; +globalTestConfig.enableDebugLogs = options.debug; +globalTestConfig.unrollConstEvalLoops = options.unrollConstEvalLoops; globalTestConfig.compatibility = compatibility; +globalTestConfig.logToWebSocket = options.logToWebSocket; -Logger.globalDebugMode = debug; const logger = new Logger(); setBaseResourcePath('../out/resources'); -const worker = options.worker ? new TestWorker(options) : undefined; +const testWorker = + options.worker === null + ? null + : options.worker === 'dedicated' + ? new TestDedicatedWorker(options) + : options.worker === 'shared' + ? new TestSharedWorker(options) + : options.worker === 'service' + ? new TestServiceWorker(options) + : unreachable(); const autoCloseOnPass = document.getElementById('autoCloseOnPass') as HTMLInputElement; const resultsVis = document.getElementById('resultsVis')!; @@ -70,17 +80,18 @@ stopButtonElem.addEventListener('click', () => { stopRequested = true; }); -if (powerPreference || compatibility) { +if (powerPreference || compatibility || forceFallbackAdapter) { setDefaultRequestAdapterOptions({ ...(powerPreference && { powerPreference }), // MAINTENANCE_TODO: Change this to whatever the option ends up being ...(compatibility && { compatibilityMode: true }), + ...(forceFallbackAdapter && { forceFallbackAdapter: true }), }); } dataCache.setStore({ load: async (path: string) => { - const response = await fetch(`data/${path}`); + const response = await fetch(getResourcePath(`cache/${path}`)); if (!response.ok) { return Promise.reject(response.statusText); } @@ -168,8 +179,8 @@ function makeCaseHTML(t: TestTreeLeaf): VisualizedSubtree { const [rec, res] = logger.record(name); caseResult = res; - if (worker) { - await worker.run(rec, name); + if (testWorker) { + await testWorker.run(rec, name); } else { await t.run(rec); } @@ -223,6 +234,12 @@ function makeCaseHTML(t: TestTreeLeaf): VisualizedSubtree { if (caseResult.logs) { caselogs.empty(); + // Show exceptions at the top since they are often unexpected can point out an error in the test itself vs the WebGPU implementation. + caseResult.logs + .filter(l => l.name === 'EXCEPTION') + .forEach(l => { + $('<pre>').addClass('testcaselogtext').text(l.toJSON()).appendTo(caselogs); + }); for (const l of caseResult.logs) { const caselog = $('<div>').addClass('testcaselog').appendTo(caselogs); $('<button>') @@ -500,13 +517,11 @@ function makeTreeNodeHeaderHTML( // Collapse s:f:t:* or s:f:t:c by default. let lastQueryLevelToExpand: TestQueryLevel = 2; -type ParamValue = string | undefined | null | boolean | string[]; - /** * Takes an array of string, ParamValue and returns an array of pairs * of [key, value] where value is a string. Converts boolean to '0' or '1'. */ -function keyValueToPairs([k, v]: [string, ParamValue]): [string, string][] { +function keyValueToPairs([k, v]: [string, boolean | string | null]): [string, string][] { const key = camelCaseToSnakeCase(k); if (typeof v === 'boolean') { return [[key, v ? '1' : '0']]; @@ -527,9 +542,9 @@ function keyValueToPairs([k, v]: [string, ParamValue]): [string, string][] { * @param params Some object with key value pairs. * @returns a search string. */ -function prepareParams(params: Record<string, ParamValue>): string { +function prepareParams(params: Record<string, boolean | string | null>): string { const pairsArrays = Object.entries(params) - .filter(([, v]) => !!v) + .filter(([, v]) => !(v === false || v === null || v === '0')) .map(keyValueToPairs); const pairs = pairsArrays.flat(); return new URLSearchParams(pairs).toString(); @@ -537,7 +552,7 @@ function prepareParams(params: Record<string, ParamValue>): string { // This is just a cast in one place. export function optionsToRecord(options: CTSOptions) { - return options as unknown as Record<string, boolean | string>; + return options as unknown as Record<string, boolean | string | null>; } /** @@ -597,15 +612,15 @@ void (async () => { }; const createSelect = (optionName: string, info: OptionInfo) => { - const select = $('<select>').on('change', function () { - optionValues[optionName] = (this as HTMLInputElement).value; + const select = $('<select>').on('change', function (this: HTMLSelectElement) { + optionValues[optionName] = JSON.parse(this.value); updateURLsWithCurrentOptions(); }); const currentValue = optionValues[optionName]; for (const { value, description } of info.selectValueDescriptions!) { $('<option>') .text(description) - .val(value) + .val(JSON.stringify(value)) .prop('selected', value === currentValue) .appendTo(select); } diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts index d4a4008154..79ed1b5924 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts @@ -8,8 +8,8 @@ import { parseQuery } from '../internal/query/parseQuery.js'; import { parseExpectationsForTestQuery, relativeQueryString } from '../internal/query/query.js'; import { assert } from '../util/util.js'; -import { optionEnabled } from './helper/options.js'; -import { TestWorker } from './helper/test_worker.js'; +import { optionEnabled, optionWorkerMode } from './helper/options.js'; +import { TestDedicatedWorker, TestServiceWorker, TestSharedWorker } from './helper/test_worker.js'; // testharness.js API (https://web-platform-tests.org/writing-tests/testharness-api.html) declare interface WptTestObject { @@ -31,8 +31,10 @@ setup({ }); void (async () => { - const workerEnabled = optionEnabled('worker'); - const worker = workerEnabled ? new TestWorker() : undefined; + const workerString = optionWorkerMode('worker'); + const dedicatedWorker = workerString === 'dedicated' ? new TestDedicatedWorker() : undefined; + const sharedWorker = workerString === 'shared' ? new TestSharedWorker() : undefined; + const serviceWorker = workerString === 'service' ? new TestServiceWorker() : undefined; globalTestConfig.unrollConstEvalLoops = optionEnabled('unroll_const_eval_loops'); @@ -63,8 +65,12 @@ void (async () => { const wpt_fn = async () => { const [rec, res] = log.record(name); - if (worker) { - await worker.run(rec, name, expectations); + if (dedicatedWorker) { + await dedicatedWorker.run(rec, name, expectations); + } else if (sharedWorker) { + await sharedWorker.run(rec, name, expectations); + } else if (serviceWorker) { + await serviceWorker.run(rec, name, expectations); } else { await testcase.run(rec, expectations); } |