From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- .../checkout/src/common/framework/data_cache.ts | 120 ++++ .../cts/checkout/src/common/framework/fixture.ts | 328 +++++++++++ .../src/common/framework/params_builder.ts | 337 +++++++++++ .../cts/checkout/src/common/framework/resources.ts | 110 ++++ .../checkout/src/common/framework/test_config.ts | 20 + .../checkout/src/common/framework/test_group.ts | 1 + .../checkout/src/common/internal/file_loader.ts | 95 +++ .../src/common/internal/logging/log_message.ts | 44 ++ .../checkout/src/common/internal/logging/logger.ts | 30 + .../checkout/src/common/internal/logging/result.ts | 21 + .../common/internal/logging/test_case_recorder.ts | 158 +++++ .../checkout/src/common/internal/params_utils.ts | 124 ++++ .../checkout/src/common/internal/query/compare.ts | 94 +++ .../common/internal/query/encode_selectively.ts | 23 + .../src/common/internal/query/json_param_value.ts | 83 +++ .../src/common/internal/query/parseQuery.ts | 155 +++++ .../checkout/src/common/internal/query/query.ts | 262 +++++++++ .../src/common/internal/query/separators.ts | 14 + .../src/common/internal/query/stringify_params.ts | 44 ++ .../src/common/internal/query/validQueryPart.ts | 2 + .../cts/checkout/src/common/internal/stack.ts | 82 +++ .../cts/checkout/src/common/internal/test_group.ts | 646 +++++++++++++++++++++ .../src/common/internal/test_suite_listing.ts | 15 + .../tests/cts/checkout/src/common/internal/tree.ts | 575 ++++++++++++++++++ .../tests/cts/checkout/src/common/internal/util.ts | 10 + .../cts/checkout/src/common/internal/version.ts | 1 + .../cts/checkout/src/common/runtime/cmdline.ts | 278 +++++++++ .../checkout/src/common/runtime/helper/options.ts | 22 + .../cts/checkout/src/common/runtime/helper/sys.ts | 46 ++ .../common/runtime/helper/test_worker-worker.ts | 32 + .../src/common/runtime/helper/test_worker.ts | 44 ++ .../cts/checkout/src/common/runtime/server.ts | 227 ++++++++ .../cts/checkout/src/common/runtime/standalone.ts | 625 ++++++++++++++++++++ .../tests/cts/checkout/src/common/runtime/wpt.ts | 83 +++ .../checkout/src/common/templates/cts.https.html | 32 + .../cts/checkout/src/common/tools/.eslintrc.json | 9 + .../cts/checkout/src/common/tools/checklist.ts | 138 +++++ .../tests/cts/checkout/src/common/tools/crawl.ts | 102 ++++ .../cts/checkout/src/common/tools/dev_server.ts | 189 ++++++ .../cts/checkout/src/common/tools/gen_cache.ts | 144 +++++ .../cts/checkout/src/common/tools/gen_listings.ts | 64 ++ .../checkout/src/common/tools/gen_wpt_cts_html.ts | 122 ++++ .../cts/checkout/src/common/tools/image_utils.ts | 58 ++ .../cts/checkout/src/common/tools/presubmit.ts | 19 + .../checkout/src/common/tools/run_wpt_ref_tests.ts | 446 ++++++++++++++ .../checkout/src/common/tools/setup-ts-in-node.js | 51 ++ .../tests/cts/checkout/src/common/tools/version.ts | 4 + .../checkout/src/common/util/collect_garbage.ts | 58 ++ .../tests/cts/checkout/src/common/util/colors.ts | 127 ++++ .../cts/checkout/src/common/util/data_tables.ts | 39 ++ .../cts/checkout/src/common/util/navigator_gpu.ts | 74 +++ .../cts/checkout/src/common/util/preprocessor.ts | 149 +++++ .../tests/cts/checkout/src/common/util/timeout.ts | 7 + .../tests/cts/checkout/src/common/util/types.ts | 59 ++ .../tests/cts/checkout/src/common/util/util.ts | 303 ++++++++++ .../checkout/src/common/util/wpt_reftest_wait.ts | 24 + 56 files changed, 6969 insertions(+) create mode 100644 dom/webgpu/tests/cts/checkout/src/common/framework/data_cache.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/framework/fixture.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/framework/params_builder.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/framework/resources.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/framework/test_config.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/framework/test_group.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/file_loader.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/logging/log_message.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/logging/logger.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/logging/result.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/logging/test_case_recorder.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/params_utils.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/encode_selectively.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/json_param_value.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/separators.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/stringify_params.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/query/validQueryPart.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/stack.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/util.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/internal/version.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/runtime/wpt.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/templates/cts.https.html create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/.eslintrc.json create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/checklist.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/crawl.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/dev_server.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/gen_cache.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/gen_listings.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/gen_wpt_cts_html.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/image_utils.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/presubmit.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/run_wpt_ref_tests.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/setup-ts-in-node.js create mode 100644 dom/webgpu/tests/cts/checkout/src/common/tools/version.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/collect_garbage.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/colors.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/data_tables.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/navigator_gpu.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/preprocessor.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/timeout.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/types.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/util.ts create mode 100644 dom/webgpu/tests/cts/checkout/src/common/util/wpt_reftest_wait.ts (limited to 'dom/webgpu/tests/cts/checkout/src/common') diff --git a/dom/webgpu/tests/cts/checkout/src/common/framework/data_cache.ts b/dom/webgpu/tests/cts/checkout/src/common/framework/data_cache.ts new file mode 100644 index 0000000000..6f6e80288a --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/framework/data_cache.ts @@ -0,0 +1,120 @@ +/** + * Utilities to improve the performance of the CTS, by caching data that is + * expensive to build using a two-level cache (in-memory, pre-computed file). + */ + +interface DataStore { + load(path: string): Promise; +} + +/** Logger is a basic debug logger function */ +export type Logger = (s: string) => void; + +/** DataCache is an interface to a data store used to hold cached data */ +export class DataCache { + /** setDataStore() sets the backing data store used by the data cache */ + public setStore(dataStore: DataStore) { + this.dataStore = dataStore; + } + + /** setDebugLogger() sets the verbose logger */ + public setDebugLogger(logger: Logger) { + this.debugLogger = logger; + } + + /** + * fetch() retrieves cacheable data from the data cache, first checking the + * in-memory cache, then the data store (if specified), then resorting to + * building the data and storing it in the cache. + */ + public async fetch(cacheable: Cacheable): Promise { + // First check the in-memory cache + let data = this.cache.get(cacheable.path); + if (data !== undefined) { + this.log('in-memory cache hit'); + return Promise.resolve(data as Data); + } + this.log('in-memory cache miss'); + // In in-memory cache miss. + // Next, try the data store. + if (this.dataStore !== null && !this.unavailableFiles.has(cacheable.path)) { + let serialized: string | undefined; + try { + serialized = await this.dataStore.load(cacheable.path); + this.log('loaded serialized'); + } catch (err) { + // not found in data store + this.log(`failed to load (${cacheable.path}): ${err}`); + this.unavailableFiles.add(cacheable.path); + } + if (serialized !== undefined) { + this.log(`deserializing`); + data = cacheable.deserialize(serialized); + this.cache.set(cacheable.path, data); + return data as Data; + } + } + // Not found anywhere. Build the data, and cache for future lookup. + this.log(`cache: building (${cacheable.path})`); + data = await cacheable.build(); + this.cache.set(cacheable.path, data); + return data as Data; + } + + private log(msg: string) { + if (this.debugLogger !== null) { + this.debugLogger(`DataCache: ${msg}`); + } + } + + private cache = new Map(); + private unavailableFiles = new Set(); + private dataStore: DataStore | null = null; + private debugLogger: Logger | null = null; +} + +/** The data cache */ +export const dataCache = new DataCache(); + +/** true if the current process is building the cache */ +let isBuildingDataCache = false; + +/** @returns true if the data cache is currently being built */ +export function getIsBuildingDataCache() { + return isBuildingDataCache; +} + +/** Sets whether the data cache is currently being built */ +export function setIsBuildingDataCache(value = true) { + isBuildingDataCache = value; +} + +/** + * Cacheable is the interface to something that can be stored into the + * DataCache. + * The 'npm run gen_cache' tool will look for module-scope variables of this + * interface, with the name `d`. + */ +export interface Cacheable { + /** the globally unique path for the cacheable data */ + readonly path: string; + + /** + * build() builds the cacheable data. + * This is assumed to be an expensive operation and will only happen if the + * cache does not already contain the built data. + */ + build(): Promise; + + /** + * serialize() transforms `data` to a string (usually JSON encoded) so that it + * can be stored in a text cache file. + */ + serialize(data: Data): string; + + /** + * deserialize() is the inverse of serialize(), transforming the string back + * to the Data object. + */ + deserialize(serialized: string): Data; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/framework/fixture.ts b/dom/webgpu/tests/cts/checkout/src/common/framework/fixture.ts new file mode 100644 index 0000000000..1368a3f96e --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/framework/fixture.ts @@ -0,0 +1,328 @@ +import { TestCaseRecorder } from '../internal/logging/test_case_recorder.js'; +import { JSONWithUndefined } from '../internal/params_utils.js'; +import { assert, unreachable } from '../util/util.js'; + +export class SkipTestCase extends Error {} +export class UnexpectedPassError extends Error {} + +export { TestCaseRecorder } from '../internal/logging/test_case_recorder.js'; + +/** The fully-general type for params passed to a test function invocation. */ +export type TestParams = { + readonly [k: string]: JSONWithUndefined; +}; + +type DestroyableObject = + | { destroy(): void } + | { close(): void } + | { getExtension(extensionName: 'WEBGL_lose_context'): WEBGL_lose_context }; + +export class SubcaseBatchState { + private _params: TestParams; + + constructor(params: TestParams) { + this._params = params; + } + + /** + * Returns the case parameters for this test fixture shared state. Subcase params + * are not included. + */ + get params(): TestParams { + return this._params; + } + + /** + * Runs before the `.before()` function. + * @internal MAINTENANCE_TODO: Make this not visible to test code? + */ + async init() {} + /** + * Runs between the `.before()` function and the subcases. + * @internal MAINTENANCE_TODO: Make this not visible to test code? + */ + async postInit() {} + /** + * Runs after all subcases finish. + * @internal MAINTENANCE_TODO: Make this not visible to test code? + */ + async finalize() {} +} + +/** + * A Fixture is a class used to instantiate each test sub/case at run time. + * A new instance of the Fixture is created for every single test subcase + * (i.e. every time the test function is run). + */ +export class Fixture { + private _params: unknown; + private _sharedState: S; + /** + * Interface for recording logs and test status. + * + * @internal + */ + protected rec: TestCaseRecorder; + private eventualExpectations: Array> = []; + private numOutstandingAsyncExpectations = 0; + private objectsToCleanUp: DestroyableObject[] = []; + + public static MakeSharedState(params: TestParams): SubcaseBatchState { + return new SubcaseBatchState(params); + } + + /** @internal */ + constructor(sharedState: S, rec: TestCaseRecorder, params: TestParams) { + this._sharedState = sharedState; + this.rec = rec; + this._params = params; + } + + /** + * Returns the (case+subcase) parameters for this test function invocation. + */ + get params(): unknown { + return this._params; + } + + /** + * Gets the test fixture's shared state. This object is shared between subcases + * within the same testcase. + */ + get sharedState(): S { + return this._sharedState; + } + + /** + * Override this to do additional pre-test-function work in a derived fixture. + * This has to be a member function instead of an async `createFixture` function, because + * we need to be able to ergonomically override it in subclasses. + * + * @internal MAINTENANCE_TODO: Make this not visible to test code? + */ + async init(): Promise {} + + /** + * Override this to do additional post-test-function work in a derived fixture. + * + * Called even if init was unsuccessful. + * + * @internal MAINTENANCE_TODO: Make this not visible to test code? + */ + async finalize(): Promise { + assert( + this.numOutstandingAsyncExpectations === 0, + 'there were outstanding immediateAsyncExpectations (e.g. expectUncapturedError) at the end of the test' + ); + + // Loop to exhaust the eventualExpectations in case they chain off each other. + while (this.eventualExpectations.length) { + const p = this.eventualExpectations.shift()!; + try { + await p; + } catch (ex) { + this.rec.threw(ex); + } + } + + // And clean up any objects now that they're done being used. + for (const o of this.objectsToCleanUp) { + if ('getExtension' in o) { + const WEBGL_lose_context = o.getExtension('WEBGL_lose_context'); + if (WEBGL_lose_context) WEBGL_lose_context.loseContext(); + } else if ('destroy' in o) { + o.destroy(); + } else { + o.close(); + } + } + } + + /** + * Tracks an object to be cleaned up after the test finishes. + * + * MAINTENANCE_TODO: Use this in more places. (Will be easier once .destroy() is allowed on + * invalid objects.) + */ + trackForCleanup(o: T): T { + this.objectsToCleanUp.push(o); + return o; + } + + /** Tracks an object, if it's destroyable, to be cleaned up after the test finishes. */ + tryTrackForCleanup(o: T): T { + if (typeof o === 'object' && o !== null) { + if ( + 'destroy' in o || + 'close' in o || + o instanceof WebGLRenderingContext || + o instanceof WebGL2RenderingContext + ) { + this.objectsToCleanUp.push((o as unknown) as DestroyableObject); + } + } + return o; + } + + /** Log a debug message. */ + debug(msg: string): void { + this.rec.debug(new Error(msg)); + } + + /** Throws an exception marking the subcase as skipped. */ + skip(msg: string): never { + throw new SkipTestCase(msg); + } + + /** Log a warning and increase the result status to "Warn". */ + warn(msg?: string): void { + this.rec.warn(new Error(msg)); + } + + /** Log an error and increase the result status to "ExpectFailed". */ + fail(msg?: string): void { + this.rec.expectationFailed(new Error(msg)); + } + + /** + * Wraps an async function. Tracks its status to fail if the test tries to report a test status + * before the async work has finished. + */ + protected async immediateAsyncExpectation(fn: () => Promise): Promise { + this.numOutstandingAsyncExpectations++; + const ret = await fn(); + this.numOutstandingAsyncExpectations--; + return ret; + } + + /** + * Wraps an async function, passing it an `Error` object recording the original stack trace. + * The async work will be implicitly waited upon before reporting a test status. + */ + protected eventualAsyncExpectation(fn: (niceStack: Error) => Promise): void { + const promise = fn(new Error()); + this.eventualExpectations.push(promise); + } + + private expectErrorValue(expectedError: string | true, ex: unknown, niceStack: Error): void { + if (!(ex instanceof Error)) { + niceStack.message = `THREW non-error value, of type ${typeof ex}: ${ex}`; + this.rec.expectationFailed(niceStack); + return; + } + const actualName = ex.name; + if (expectedError !== true && actualName !== expectedError) { + niceStack.message = `THREW ${actualName}, instead of ${expectedError}: ${ex}`; + this.rec.expectationFailed(niceStack); + } else { + niceStack.message = `OK: threw ${actualName}: ${ex.message}`; + this.rec.debug(niceStack); + } + } + + /** Expect that the provided promise resolves (fulfills). */ + shouldResolve(p: Promise, msg?: string): void { + this.eventualAsyncExpectation(async niceStack => { + const m = msg ? ': ' + msg : ''; + try { + await p; + niceStack.message = 'resolved as expected' + m; + } catch (ex) { + niceStack.message = `REJECTED${m}`; + if (ex instanceof Error) { + niceStack.message += '\n' + ex.message; + } + this.rec.expectationFailed(niceStack); + } + }); + } + + /** Expect that the provided promise rejects, with the provided exception name. */ + shouldReject(expectedName: string, p: Promise, msg?: string): void { + this.eventualAsyncExpectation(async niceStack => { + const m = msg ? ': ' + msg : ''; + try { + await p; + niceStack.message = 'DID NOT REJECT' + m; + this.rec.expectationFailed(niceStack); + } catch (ex) { + niceStack.message = 'rejected as expected' + m; + this.expectErrorValue(expectedName, ex, niceStack); + } + }); + } + + /** + * Expect that the provided function throws. + * If an `expectedName` is provided, expect that the throw exception has that name. + */ + shouldThrow(expectedError: string | boolean, fn: () => void, msg?: string): void { + const m = msg ? ': ' + msg : ''; + try { + fn(); + if (expectedError === false) { + this.rec.debug(new Error('did not throw, as expected' + m)); + } else { + this.rec.expectationFailed(new Error('unexpectedly did not throw' + m)); + } + } catch (ex) { + if (expectedError === false) { + this.rec.expectationFailed(new Error('threw unexpectedly' + m)); + } else { + this.expectErrorValue(expectedError, ex, new Error(m)); + } + } + } + + /** Expect that a condition is true. */ + expect(cond: boolean, msg?: string): boolean { + if (cond) { + const m = msg ? ': ' + msg : ''; + this.rec.debug(new Error('expect OK' + m)); + } else { + this.rec.expectationFailed(new Error(msg)); + } + return cond; + } + + /** + * If the argument is an `Error`, fail (or warn). If it's `undefined`, no-op. + * If the argument is an array, apply the above behavior on each of elements. + */ + expectOK( + error: Error | undefined | (Error | undefined)[], + { mode = 'fail', niceStack }: { mode?: 'fail' | 'warn'; niceStack?: Error } = {} + ): void { + const handleError = (error: Error | undefined) => { + if (error instanceof Error) { + if (niceStack) { + error.stack = niceStack.stack; + } + if (mode === 'fail') { + this.rec.expectationFailed(error); + } else if (mode === 'warn') { + this.rec.warn(error); + } else { + unreachable(); + } + } + }; + + if (Array.isArray(error)) { + for (const e of error) { + handleError(e); + } + } else { + handleError(error); + } + } + + eventualExpectOK( + error: Promise, + { mode = 'fail' }: { mode?: 'fail' | 'warn' } = {} + ) { + this.eventualAsyncExpectation(async niceStack => { + this.expectOK(await error, { mode, niceStack }); + }); + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/framework/params_builder.ts b/dom/webgpu/tests/cts/checkout/src/common/framework/params_builder.ts new file mode 100644 index 0000000000..d22444a9b6 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/framework/params_builder.ts @@ -0,0 +1,337 @@ +import { Merged, mergeParams } from '../internal/params_utils.js'; +import { stringifyPublicParams } from '../internal/query/stringify_params.js'; +import { assert, mapLazy } from '../util/util.js'; + +// ================================================================ +// "Public" ParamsBuilder API / Documentation +// ================================================================ + +/** + * Provides doc comments for the methods of CaseParamsBuilder and SubcaseParamsBuilder. + * (Also enforces rough interface match between them.) + */ +export interface ParamsBuilder { + /** + * Expands each item in `this` into zero or more items. + * Each item has its parameters expanded with those returned by the `expander`. + * + * **Note:** When only a single key is being added, use the simpler `expand` for readability. + * + * ```text + * this = [ a , b , c ] + * this.map(expander) = [ f(a) f(b) f(c) ] + * = [[a1, a2, a3] , [ b1 ] , [] ] + * merge and flatten = [ merge(a, a1), merge(a, a2), merge(a, a3), merge(b, b1) ] + * ``` + */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + expandWithParams(expander: (_: any) => any): any; + + /** + * Expands each item in `this` into zero or more items. Each item has its parameters expanded + * with one new key, `key`, and the values returned by `expander`. + */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + expand(key: string, expander: (_: any) => any): any; + + /** + * Expands each item in `this` to multiple items, one for each item in `newParams`. + * + * In other words, takes the cartesian product of [ the items in `this` ] and `newParams`. + * + * **Note:** When only a single key is being added, use the simpler `combine` for readability. + * + * ```text + * this = [ {a:1}, {b:2} ] + * newParams = [ {x:1}, {y:2} ] + * this.combineP(newParams) = [ {a:1,x:1}, {a:1,y:2}, {b:2,x:1}, {b:2,y:2} ] + * ``` + */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + combineWithParams(newParams: Iterable): any; + + /** + * Expands each item in `this` to multiple items with `{ [name]: value }` for each value. + * + * In other words, takes the cartesian product of [ the items in `this` ] + * and `[ {[name]: value} for each value in values ]` + */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + combine(key: string, newParams: Iterable): any; + + /** + * Filters `this` to only items for which `pred` returns true. + */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + filter(pred: (_: any) => boolean): any; + + /** + * Filters `this` to only items for which `pred` returns false. + */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + unless(pred: (_: any) => boolean): any; +} + +/** + * Determines the resulting parameter object type which would be generated by an object of + * the given ParamsBuilder type. + */ +export type ParamTypeOf< + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + T extends ParamsBuilder +> = T extends SubcaseParamsBuilder + ? Merged + : T extends CaseParamsBuilder + ? CaseP + : never; + +// ================================================================ +// Implementation +// ================================================================ + +/** + * Iterable over pairs of either: + * - `[case params, Iterable]` if there are subcases. + * - `[case params, undefined]` if not. + */ +export type CaseSubcaseIterable = Iterable< + readonly [CaseP, Iterable | undefined] +>; + +/** + * Base class for `CaseParamsBuilder` and `SubcaseParamsBuilder`. + */ +export abstract class ParamsBuilderBase { + protected readonly cases: () => Generator; + + constructor(cases: () => Generator) { + this.cases = cases; + } + + /** + * Hidden from test files. Use `builderIterateCasesWithSubcases` to access this. + */ + protected abstract iterateCasesWithSubcases(): CaseSubcaseIterable; +} + +/** + * Calls the (normally hidden) `iterateCasesWithSubcases()` method. + */ +export function builderIterateCasesWithSubcases(builder: ParamsBuilderBase<{}, {}>) { + interface IterableParamsBuilder { + iterateCasesWithSubcases(): CaseSubcaseIterable<{}, {}>; + } + + return ((builder as unknown) as IterableParamsBuilder).iterateCasesWithSubcases(); +} + +/** + * Builder for combinatorial test **case** parameters. + * + * CaseParamsBuilder is immutable. Each method call returns a new, immutable object, + * modifying the list of cases according to the method called. + * + * This means, for example, that the `unit` passed into `TestBuilder.params()` can be reused. + */ +export class CaseParamsBuilder + extends ParamsBuilderBase + implements Iterable, ParamsBuilder { + *iterateCasesWithSubcases(): CaseSubcaseIterable { + for (const a of this.cases()) { + yield [a, undefined]; + } + } + + [Symbol.iterator](): Iterator { + return this.cases(); + } + + /** @inheritDoc */ + expandWithParams( + expander: (_: Merged<{}, CaseP>) => Iterable + ): CaseParamsBuilder> { + const newGenerator = expanderGenerator(this.cases, expander); + return new CaseParamsBuilder(() => newGenerator({})); + } + + /** @inheritDoc */ + expand( + key: NewPKey, + expander: (_: Merged<{}, CaseP>) => Iterable + ): CaseParamsBuilder> { + return this.expandWithParams(function* (p) { + for (const value of expander(p)) { + yield { [key]: value } as { readonly [name in NewPKey]: NewPValue }; + } + }); + } + + /** @inheritDoc */ + combineWithParams( + newParams: Iterable + ): CaseParamsBuilder> { + assertNotGenerator(newParams); + const seenValues = new Set(); + for (const params of newParams) { + const paramsStr = stringifyPublicParams(params); + assert(!seenValues.has(paramsStr), `Duplicate entry in combine[WithParams]: ${paramsStr}`); + seenValues.add(paramsStr); + } + + return this.expandWithParams(() => newParams); + } + + /** @inheritDoc */ + combine( + key: NewPKey, + values: Iterable + ): CaseParamsBuilder> { + assertNotGenerator(values); + const mapped = mapLazy(values, v => ({ [key]: v } as { [name in NewPKey]: NewPValue })); + return this.combineWithParams(mapped); + } + + /** @inheritDoc */ + filter(pred: (_: Merged<{}, CaseP>) => boolean): CaseParamsBuilder { + const newGenerator = filterGenerator(this.cases, pred); + return new CaseParamsBuilder(() => newGenerator({})); + } + + /** @inheritDoc */ + unless(pred: (_: Merged<{}, CaseP>) => boolean): CaseParamsBuilder { + return this.filter(x => !pred(x)); + } + + /** + * "Finalize" the list of cases and begin defining subcases. + * Returns a new SubcaseParamsBuilder. Methods called on SubcaseParamsBuilder + * generate new subcases instead of new cases. + */ + beginSubcases(): SubcaseParamsBuilder { + return new SubcaseParamsBuilder( + () => this.cases(), + function* () { + yield {}; + } + ); + } +} + +/** + * The unit CaseParamsBuilder, representing a single case with no params: `[ {} ]`. + * + * `punit` is passed to every `.params()`/`.paramsSubcasesOnly()` call, so `kUnitCaseParamsBuilder` + * is only explicitly needed if constructing a ParamsBuilder outside of a test builder. + */ +export const kUnitCaseParamsBuilder = new CaseParamsBuilder(function* () { + yield {}; +}); + +/** + * Builder for combinatorial test _subcase_ parameters. + * + * SubcaseParamsBuilder is immutable. Each method call returns a new, immutable object, + * modifying the list of subcases according to the method called. + */ +export class SubcaseParamsBuilder + extends ParamsBuilderBase + implements ParamsBuilder { + protected readonly subcases: (_: CaseP) => Generator; + + constructor(cases: () => Generator, generator: (_: CaseP) => Generator) { + super(cases); + this.subcases = generator; + } + + *iterateCasesWithSubcases(): CaseSubcaseIterable { + for (const caseP of this.cases()) { + const subcases = Array.from(this.subcases(caseP)); + if (subcases.length) { + yield [caseP, subcases]; + } + } + } + + /** @inheritDoc */ + expandWithParams( + expander: (_: Merged) => Iterable + ): SubcaseParamsBuilder> { + return new SubcaseParamsBuilder(this.cases, expanderGenerator(this.subcases, expander)); + } + + /** @inheritDoc */ + expand( + key: NewPKey, + expander: (_: Merged) => Iterable + ): SubcaseParamsBuilder> { + return this.expandWithParams(function* (p) { + for (const value of expander(p)) { + // TypeScript doesn't know here that NewPKey is always a single literal string type. + yield { [key]: value } as { [name in NewPKey]: NewPValue }; + } + }); + } + + /** @inheritDoc */ + combineWithParams( + newParams: Iterable + ): SubcaseParamsBuilder> { + assertNotGenerator(newParams); + return this.expandWithParams(() => newParams); + } + + /** @inheritDoc */ + combine( + key: NewPKey, + values: Iterable + ): SubcaseParamsBuilder> { + assertNotGenerator(values); + return this.expand(key, () => values); + } + + /** @inheritDoc */ + filter(pred: (_: Merged) => boolean): SubcaseParamsBuilder { + return new SubcaseParamsBuilder(this.cases, filterGenerator(this.subcases, pred)); + } + + /** @inheritDoc */ + unless(pred: (_: Merged) => boolean): SubcaseParamsBuilder { + return this.filter(x => !pred(x)); + } +} + +function expanderGenerator( + baseGenerator: (_: Base) => Generator, + expander: (_: Merged) => Iterable +): (_: Base) => Generator> { + return function* (base: Base) { + for (const a of baseGenerator(base)) { + for (const b of expander(mergeParams(base, a))) { + yield mergeParams(a, b); + } + } + }; +} + +function filterGenerator( + baseGenerator: (_: Base) => Generator, + pred: (_: Merged) => boolean +): (_: Base) => Generator { + return function* (base: Base) { + for (const a of baseGenerator(base)) { + if (pred(mergeParams(base, a))) { + yield a; + } + } + }; +} + +/** Assert an object is not a Generator (a thing returned from a generator function). */ +function assertNotGenerator(x: object) { + if ('constructor' in x) { + assert( + x.constructor !== (function* () {})().constructor, + 'Argument must not be a generator, as generators are not reusable' + ); + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/framework/resources.ts b/dom/webgpu/tests/cts/checkout/src/common/framework/resources.ts new file mode 100644 index 0000000000..05451304b6 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/framework/resources.ts @@ -0,0 +1,110 @@ +/** + * Base path for resources. The default value is correct for non-worker WPT, but standalone and + * workers must access resources using a different base path, so this is overridden in + * `test_worker-worker.ts` and `standalone.ts`. + */ +let baseResourcePath = './resources'; +let crossOriginHost = ''; + +function getAbsoluteBaseResourcePath(path: string) { + // Path is already an absolute one. + if (path[0] === '/') { + return path; + } + + // Path is relative + const relparts = window.location.pathname.split('/'); + relparts.pop(); + const pathparts = path.split('/'); + + let i; + for (i = 0; i < pathparts.length; ++i) { + switch (pathparts[i]) { + case '': + break; + case '.': + break; + case '..': + relparts.pop(); + break; + default: + relparts.push(pathparts[i]); + break; + } + } + + return relparts.join('/'); +} + +function runningOnLocalHost(): boolean { + const hostname = window.location.hostname; + return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1'; +} + +/** + * Get a path to a resource in the `resources` directory relative to the current execution context + * (html file or worker .js file), for `fetch()`, ``, `(a: A, b: B): Merged { + for (const key of Object.keys(a)) { + assert(!(key in b), 'Duplicate key: ' + key); + } + return { ...a, ...b } as Merged; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts new file mode 100644 index 0000000000..e9f4b01503 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts @@ -0,0 +1,94 @@ +import { TestParams } from '../../framework/fixture.js'; +import { assert, objectEquals } from '../../util/util.js'; +import { paramKeyIsPublic } from '../params_utils.js'; + +import { TestQuery } from './query.js'; + +export const enum Ordering { + Unordered, + StrictSuperset, + Equal, + StrictSubset, +} + +/** + * Compares two queries for their ordering (which is used to build the tree). + * + * See src/unittests/query_compare.spec.ts for examples. + */ +export function compareQueries(a: TestQuery, b: TestQuery): Ordering { + if (a.suite !== b.suite) { + return Ordering.Unordered; + } + + const filePathOrdering = comparePaths(a.filePathParts, b.filePathParts); + if (filePathOrdering !== Ordering.Equal || a.isMultiFile || b.isMultiFile) { + return compareOneLevel(filePathOrdering, a.isMultiFile, b.isMultiFile); + } + assert('testPathParts' in a && 'testPathParts' in b); + + const testPathOrdering = comparePaths(a.testPathParts, b.testPathParts); + if (testPathOrdering !== Ordering.Equal || a.isMultiTest || b.isMultiTest) { + return compareOneLevel(testPathOrdering, a.isMultiTest, b.isMultiTest); + } + assert('params' in a && 'params' in b); + + const paramsPathOrdering = comparePublicParamsPaths(a.params, b.params); + if (paramsPathOrdering !== Ordering.Equal || a.isMultiCase || b.isMultiCase) { + return compareOneLevel(paramsPathOrdering, a.isMultiCase, b.isMultiCase); + } + return Ordering.Equal; +} + +/** + * Compares a single level of a query. + * + * "IsBig" means the query is big relative to the level, e.g. for test-level: + * - Anything >= `suite:a,*` is big + * - Anything <= `suite:a:*` is small + */ +function compareOneLevel(ordering: Ordering, aIsBig: boolean, bIsBig: boolean): Ordering { + assert(ordering !== Ordering.Equal || aIsBig || bIsBig); + if (ordering === Ordering.Unordered) return Ordering.Unordered; + if (aIsBig && bIsBig) return ordering; + if (!aIsBig && !bIsBig) return Ordering.Unordered; // Equal case is already handled + // Exactly one of (a, b) is big. + if (aIsBig && ordering !== Ordering.StrictSubset) return Ordering.StrictSuperset; + if (bIsBig && ordering !== Ordering.StrictSuperset) return Ordering.StrictSubset; + return Ordering.Unordered; +} + +function comparePaths(a: readonly string[], b: readonly string[]): Ordering { + const shorter = Math.min(a.length, b.length); + + for (let i = 0; i < shorter; ++i) { + if (a[i] !== b[i]) { + return Ordering.Unordered; + } + } + if (a.length === b.length) { + return Ordering.Equal; + } else if (a.length < b.length) { + return Ordering.StrictSuperset; + } else { + return Ordering.StrictSubset; + } +} + +export function comparePublicParamsPaths(a: TestParams, b: TestParams): Ordering { + const aKeys = Object.keys(a).filter(k => paramKeyIsPublic(k)); + const commonKeys = new Set(aKeys.filter(k => k in b)); + + for (const k of commonKeys) { + if (!objectEquals(a[k], b[k])) { + return Ordering.Unordered; + } + } + const bKeys = Object.keys(b).filter(k => paramKeyIsPublic(k)); + const aRemainingKeys = aKeys.length - commonKeys.size; + const bRemainingKeys = bKeys.length - commonKeys.size; + if (aRemainingKeys === 0 && bRemainingKeys === 0) return Ordering.Equal; + if (aRemainingKeys === 0) return Ordering.StrictSuperset; + if (bRemainingKeys === 0) return Ordering.StrictSubset; + return Ordering.Unordered; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/encode_selectively.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/encode_selectively.ts new file mode 100644 index 0000000000..ab1997b6e4 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/encode_selectively.ts @@ -0,0 +1,23 @@ +/** + * Encodes a stringified TestQuery so that it can be placed in a `?q=` parameter in a URL. + * + * `encodeURIComponent` encodes in accordance with `application/x-www-form-urlencoded`, + * but URLs don't actually have to be as strict as HTML form encoding + * (we interpret this purely from JavaScript). + * So we encode the component, then selectively convert some %-encoded escape codes + * back to their original form for readability/copyability. + */ +export function encodeURIComponentSelectively(s: string): string { + let ret = encodeURIComponent(s); + ret = ret.replace(/%22/g, '"'); // for JSON strings + ret = ret.replace(/%2C/g, ','); // for path separator, and JSON arrays + ret = ret.replace(/%3A/g, ':'); // for big separator + ret = ret.replace(/%3B/g, ';'); // for param separator + ret = ret.replace(/%3D/g, '='); // for params (k=v) + ret = ret.replace(/%5B/g, '['); // for JSON arrays + ret = ret.replace(/%5D/g, ']'); // for JSON arrays + ret = ret.replace(/%7B/g, '{'); // for JSON objects + ret = ret.replace(/%7D/g, '}'); // for JSON objects + ret = ret.replace(/%E2%9C%97/g, '✗'); // for jsUndefinedMagicValue + return ret; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/json_param_value.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/json_param_value.ts new file mode 100644 index 0000000000..f4be7642d3 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/json_param_value.ts @@ -0,0 +1,83 @@ +import { assert, sortObjectByKey } from '../../util/util.js'; +import { JSONWithUndefined } from '../params_utils.js'; + +// JSON can't represent various values and by default stores them as `null`. +// Instead, storing them as a magic string values in JSON. +const jsUndefinedMagicValue = '_undef_'; +const jsNaNMagicValue = '_nan_'; +const jsPositiveInfinityMagicValue = '_posinfinity_'; +const jsNegativeInfinityMagicValue = '_neginfinity_'; + +// -0 needs to be handled separately, because -0 === +0 returns true. Not +// special casing +0/0, since it behaves intuitively. Assuming that if -0 is +// being used, the differentiation from +0 is desired. +const jsNegativeZeroMagicValue = '_negzero_'; + +const toStringMagicValue = new Map([ + [undefined, jsUndefinedMagicValue], + [NaN, jsNaNMagicValue], + [Number.POSITIVE_INFINITY, jsPositiveInfinityMagicValue], + [Number.NEGATIVE_INFINITY, jsNegativeInfinityMagicValue], + // No -0 handling because it is special cased. +]); + +const fromStringMagicValue = new Map([ + [jsUndefinedMagicValue, undefined], + [jsNaNMagicValue, NaN], + [jsPositiveInfinityMagicValue, Number.POSITIVE_INFINITY], + [jsNegativeInfinityMagicValue, Number.NEGATIVE_INFINITY], + // -0 is handled in this direction because there is no comparison issue. + [jsNegativeZeroMagicValue, -0], +]); + +function stringifyFilter(k: string, v: unknown): unknown { + // Make sure no one actually uses a magic value as a parameter. + if (typeof v === 'string') { + assert( + !fromStringMagicValue.has(v), + `${v} is a magic value for stringification, so cannot be used` + ); + + assert( + v !== jsNegativeZeroMagicValue, + `${v} is a magic value for stringification, so cannot be used` + ); + } + + if (Object.is(v, -0)) { + return jsNegativeZeroMagicValue; + } + + return toStringMagicValue.has(v) ? toStringMagicValue.get(v) : v; +} + +export function stringifyParamValue(value: JSONWithUndefined): string { + return JSON.stringify(value, stringifyFilter); +} + +/** + * Like stringifyParamValue but sorts dictionaries by key, for hashing. + */ +export function stringifyParamValueUniquely(value: JSONWithUndefined): string { + return JSON.stringify(value, (k, v) => { + if (typeof v === 'object' && v !== null) { + return sortObjectByKey(v); + } + + return stringifyFilter(k, v); + }); +} + +// 'any' is part of the JSON.parse reviver interface, so cannot be avoided. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function parseParamValueReviver(k: string, v: any): any { + if (fromStringMagicValue.has(v)) { + return fromStringMagicValue.get(v); + } + + return v; +} + +export function parseParamValue(s: string): JSONWithUndefined { + return JSON.parse(s, parseParamValueReviver); +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts new file mode 100644 index 0000000000..996835b0ec --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts @@ -0,0 +1,155 @@ +import { assert } from '../../util/util.js'; +import { + TestParamsRW, + JSONWithUndefined, + badParamValueChars, + paramKeyIsPublic, +} from '../params_utils.js'; + +import { parseParamValue } from './json_param_value.js'; +import { + TestQuery, + TestQueryMultiFile, + TestQueryMultiTest, + TestQueryMultiCase, + TestQuerySingleCase, +} from './query.js'; +import { kBigSeparator, kWildcard, kPathSeparator, kParamSeparator } from './separators.js'; +import { validQueryPart } from './validQueryPart.js'; + +export function parseQuery(s: string): TestQuery { + try { + return parseQueryImpl(s); + } catch (ex) { + if (ex instanceof Error) { + ex.message += '\n on: ' + s; + } + throw ex; + } +} + +function parseQueryImpl(s: string): TestQuery { + // Undo encodeURIComponentSelectively + s = decodeURIComponent(s); + + // bigParts are: suite, file, test, params (note kBigSeparator could appear in params) + let suite: string; + let fileString: string | undefined; + let testString: string | undefined; + let paramsString: string | undefined; + { + const i1 = s.indexOf(kBigSeparator); + assert(i1 !== -1, `query string must have at least one ${kBigSeparator}`); + suite = s.substring(0, i1); + const i2 = s.indexOf(kBigSeparator, i1 + 1); + if (i2 === -1) { + fileString = s.substring(i1 + 1); + } else { + fileString = s.substring(i1 + 1, i2); + const i3 = s.indexOf(kBigSeparator, i2 + 1); + if (i3 === -1) { + testString = s.substring(i2 + 1); + } else { + testString = s.substring(i2 + 1, i3); + paramsString = s.substring(i3 + 1); + } + } + } + + const { parts: file, wildcard: filePathHasWildcard } = parseBigPart(fileString, kPathSeparator); + + if (testString === undefined) { + // Query is file-level + assert( + filePathHasWildcard, + `File-level query without wildcard ${kWildcard}. Did you want a file-level query \ +(append ${kPathSeparator}${kWildcard}) or test-level query (append ${kBigSeparator}${kWildcard})?` + ); + return new TestQueryMultiFile(suite, file); + } + assert(!filePathHasWildcard, `Wildcard ${kWildcard} must be at the end of the query string`); + + const { parts: test, wildcard: testPathHasWildcard } = parseBigPart(testString, kPathSeparator); + + if (paramsString === undefined) { + // Query is test-level + assert( + testPathHasWildcard, + `Test-level query without wildcard ${kWildcard}; did you want a test-level query \ +(append ${kPathSeparator}${kWildcard}) or case-level query (append ${kBigSeparator}${kWildcard})?` + ); + assert(file.length > 0, 'File part of test-level query was empty (::)'); + return new TestQueryMultiTest(suite, file, test); + } + + // Query is case-level + assert(!testPathHasWildcard, `Wildcard ${kWildcard} must be at the end of the query string`); + + const { parts: paramsParts, wildcard: paramsHasWildcard } = parseBigPart( + paramsString, + kParamSeparator + ); + + assert(test.length > 0, 'Test part of case-level query was empty (::)'); + + const params: TestParamsRW = {}; + for (const paramPart of paramsParts) { + const [k, v] = parseSingleParam(paramPart); + assert(validQueryPart.test(k), `param key names must match ${validQueryPart}`); + params[k] = v; + } + if (paramsHasWildcard) { + return new TestQueryMultiCase(suite, file, test, params); + } else { + return new TestQuerySingleCase(suite, file, test, params); + } +} + +// webgpu:a,b,* or webgpu:a,b,c:* +const kExampleQueries = `\ +webgpu${kBigSeparator}a${kPathSeparator}b${kPathSeparator}${kWildcard} or \ +webgpu${kBigSeparator}a${kPathSeparator}b${kPathSeparator}c${kBigSeparator}${kWildcard}`; + +function parseBigPart( + s: string, + separator: typeof kParamSeparator | typeof kPathSeparator +): { parts: string[]; wildcard: boolean } { + if (s === '') { + return { parts: [], wildcard: false }; + } + const parts = s.split(separator); + + let endsWithWildcard = false; + for (const [i, part] of parts.entries()) { + if (i === parts.length - 1) { + endsWithWildcard = part === kWildcard; + } + assert( + part.indexOf(kWildcard) === -1 || endsWithWildcard, + `Wildcard ${kWildcard} must be complete last part of a path (e.g. ${kExampleQueries})` + ); + } + if (endsWithWildcard) { + // Remove the last element of the array (which is just the wildcard). + parts.length = parts.length - 1; + } + return { parts, wildcard: endsWithWildcard }; +} + +function parseSingleParam(paramSubstring: string): [string, JSONWithUndefined] { + assert(paramSubstring !== '', 'Param in a query must not be blank (is there a trailing comma?)'); + const i = paramSubstring.indexOf('='); + assert(i !== -1, 'Param in a query must be of form key=value'); + const k = paramSubstring.substring(0, i); + assert(paramKeyIsPublic(k), 'Param in a query must not be private (start with _)'); + const v = paramSubstring.substring(i + 1); + return [k, parseSingleParamValue(v)]; +} + +function parseSingleParamValue(s: string): JSONWithUndefined { + assert( + !badParamValueChars.test(s), + `param value must not match ${badParamValueChars} - was ${s}` + ); + return parseParamValue(s); +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts new file mode 100644 index 0000000000..59e96cb538 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts @@ -0,0 +1,262 @@ +import { TestParams } from '../../framework/fixture.js'; +import { optionEnabled } from '../../runtime/helper/options.js'; +import { assert, unreachable } from '../../util/util.js'; +import { Expectation } from '../logging/result.js'; + +import { compareQueries, Ordering } from './compare.js'; +import { encodeURIComponentSelectively } from './encode_selectively.js'; +import { parseQuery } from './parseQuery.js'; +import { kBigSeparator, kPathSeparator, kWildcard } from './separators.js'; +import { stringifyPublicParams } from './stringify_params.js'; + +/** + * Represents a test query of some level. + * + * TestQuery types are immutable. + */ +export type TestQuery = + | TestQuerySingleCase + | TestQueryMultiCase + | TestQueryMultiTest + | TestQueryMultiFile; + +/** + * - 1 = MultiFile. + * - 2 = MultiTest. + * - 3 = MultiCase. + * - 4 = SingleCase. + */ +export type TestQueryLevel = 1 | 2 | 3 | 4; + +export interface TestQueryWithExpectation { + query: TestQuery; + expectation: Expectation; +} + +/** + * A multi-file test query, like `s:*` or `s:a,b,*`. + * + * Immutable (makes copies of constructor args). + */ +export class TestQueryMultiFile { + readonly level: TestQueryLevel = 1; + readonly isMultiFile: boolean = true; + readonly suite: string; + readonly filePathParts: readonly string[]; + + constructor(suite: string, file: readonly string[]) { + this.suite = suite; + this.filePathParts = [...file]; + } + + get depthInLevel() { + return this.filePathParts.length; + } + + toString(): string { + return encodeURIComponentSelectively(this.toStringHelper().join(kBigSeparator)); + } + + protected toStringHelper(): string[] { + return [this.suite, [...this.filePathParts, kWildcard].join(kPathSeparator)]; + } +} + +/** + * A multi-test test query, like `s:f:*` or `s:f:a,b,*`. + * + * Immutable (makes copies of constructor args). + */ +export class TestQueryMultiTest extends TestQueryMultiFile { + readonly level: TestQueryLevel = 2; + readonly isMultiFile: false = false; + readonly isMultiTest: boolean = true; + readonly testPathParts: readonly string[]; + + constructor(suite: string, file: readonly string[], test: readonly string[]) { + super(suite, file); + assert(file.length > 0, 'multi-test (or finer) query must have file-path'); + this.testPathParts = [...test]; + } + + get depthInLevel() { + return this.testPathParts.length; + } + + protected toStringHelper(): string[] { + return [ + this.suite, + this.filePathParts.join(kPathSeparator), + [...this.testPathParts, kWildcard].join(kPathSeparator), + ]; + } +} + +/** + * A multi-case test query, like `s:f:t:*` or `s:f:t:a,b,*`. + * + * Immutable (makes copies of constructor args), except for param values + * (which aren't normally supposed to change; they're marked readonly in TestParams). + */ +export class TestQueryMultiCase extends TestQueryMultiTest { + readonly level: TestQueryLevel = 3; + readonly isMultiTest: false = false; + readonly isMultiCase: boolean = true; + readonly params: TestParams; + + constructor(suite: string, file: readonly string[], test: readonly string[], params: TestParams) { + super(suite, file, test); + assert(test.length > 0, 'multi-case (or finer) query must have test-path'); + this.params = { ...params }; + } + + get depthInLevel() { + return Object.keys(this.params).length; + } + + protected toStringHelper(): string[] { + return [ + this.suite, + this.filePathParts.join(kPathSeparator), + this.testPathParts.join(kPathSeparator), + stringifyPublicParams(this.params, true), + ]; + } +} + +/** + * A multi-case test query, like `s:f:t:` or `s:f:t:a=1,b=1`. + * + * Immutable (makes copies of constructor args). + */ +export class TestQuerySingleCase extends TestQueryMultiCase { + readonly level: TestQueryLevel = 4; + readonly isMultiCase: false = false; + + get depthInLevel() { + return 0; + } + + protected toStringHelper(): string[] { + return [ + this.suite, + this.filePathParts.join(kPathSeparator), + this.testPathParts.join(kPathSeparator), + stringifyPublicParams(this.params), + ]; + } +} + +/** + * Parse raw expectations input into TestQueryWithExpectation[], filtering so that only + * expectations that are relevant for the provided query and wptURL. + * + * `rawExpectations` should be @type {{ query: string, expectation: Expectation }[]} + * + * The `rawExpectations` are parsed and validated that they are in the correct format. + * If `wptURL` is passed, the query string should be of the full path format such + * as `path/to/cts.https.html?worker=0&q=suite:test_path:test_name:foo=1;bar=2;*`. + * If `wptURL` is `undefined`, the query string should be only the query + * `suite:test_path:test_name:foo=1;bar=2;*`. + */ +export function parseExpectationsForTestQuery( + rawExpectations: + | unknown + | { + query: string; + expectation: Expectation; + }[], + query: TestQuery, + wptURL?: URL +) { + if (!Array.isArray(rawExpectations)) { + unreachable('Expectations should be an array'); + } + const expectations: TestQueryWithExpectation[] = []; + for (const entry of rawExpectations) { + assert(typeof entry === 'object'); + const rawExpectation = entry as { query?: string; expectation?: string }; + assert(rawExpectation.query !== undefined, 'Expectation missing query string'); + assert(rawExpectation.expectation !== undefined, 'Expectation missing expectation string'); + + let expectationQuery: TestQuery; + if (wptURL !== undefined) { + const expectationURL = new URL(`${wptURL.origin}/${entry.query}`); + if (expectationURL.pathname !== wptURL.pathname) { + continue; + } + assert( + expectationURL.pathname === wptURL.pathname, + `Invalid expectation path ${expectationURL.pathname} +Expectation should be of the form path/to/cts.https.html?worker=0&q=suite:test_path:test_name:foo=1;bar=2;... + ` + ); + + const params = expectationURL.searchParams; + if (optionEnabled('worker', params) !== optionEnabled('worker', wptURL.searchParams)) { + continue; + } + + const qs = params.getAll('q'); + assert(qs.length === 1, 'currently, there must be exactly one ?q= in the expectation string'); + expectationQuery = parseQuery(qs[0]); + } else { + expectationQuery = parseQuery(entry.query); + } + + // Strip params from multicase expectations so that an expectation of foo=2;* + // is stored if the test query is bar=3;* + const queryForFilter = + expectationQuery instanceof TestQueryMultiCase + ? new TestQueryMultiCase( + expectationQuery.suite, + expectationQuery.filePathParts, + expectationQuery.testPathParts, + {} + ) + : expectationQuery; + + if (compareQueries(query, queryForFilter) === Ordering.Unordered) { + continue; + } + + switch (entry.expectation) { + case 'pass': + case 'skip': + case 'fail': + break; + default: + unreachable(`Invalid expectation ${entry.expectation}`); + } + + expectations.push({ + query: expectationQuery, + expectation: entry.expectation, + }); + } + return expectations; +} + +/** + * For display purposes only, produces a "relative" query string from parent to child. + * Used in the wpt runtime to reduce the verbosity of logs. + */ +export function relativeQueryString(parent: TestQuery, child: TestQuery): string { + const ordering = compareQueries(parent, child); + if (ordering === Ordering.Equal) { + return ''; + } else if (ordering === Ordering.StrictSuperset) { + const parentString = parent.toString(); + assert(parentString.endsWith(kWildcard)); + const childString = child.toString(); + assert( + childString.startsWith(parentString.substring(0, parentString.length - 2)), + 'impossible?: childString does not start with parentString[:-2]' + ); + return childString.substring(parentString.length - 2); + } else { + unreachable( + `relativeQueryString arguments have invalid ordering ${ordering}:\n${parent}\n${child}` + ); + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/separators.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/separators.ts new file mode 100644 index 0000000000..0c8f6ea9a9 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/separators.ts @@ -0,0 +1,14 @@ +/** Separator between big parts: suite:file:test:case */ +export const kBigSeparator = ':'; + +/** Separator between path,to,file or path,to,test */ +export const kPathSeparator = ','; + +/** Separator between k=v;k=v */ +export const kParamSeparator = ';'; + +/** Separator between key and value in k=v */ +export const kParamKVSeparator = '='; + +/** Final wildcard, if query is not single-case */ +export const kWildcard = '*'; diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/stringify_params.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/stringify_params.ts new file mode 100644 index 0000000000..907cc0791a --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/stringify_params.ts @@ -0,0 +1,44 @@ +import { TestParams } from '../../framework/fixture.js'; +import { assert } from '../../util/util.js'; +import { JSONWithUndefined, badParamValueChars, paramKeyIsPublic } from '../params_utils.js'; + +import { stringifyParamValue, stringifyParamValueUniquely } from './json_param_value.js'; +import { kParamKVSeparator, kParamSeparator, kWildcard } from './separators.js'; + +export function stringifyPublicParams(p: TestParams, addWildcard = false): string { + const parts = Object.keys(p) + .filter(k => paramKeyIsPublic(k)) + .map(k => stringifySingleParam(k, p[k])); + + if (addWildcard) parts.push(kWildcard); + + return parts.join(kParamSeparator); +} + +/** + * An _approximately_ unique string representing a CaseParams value. + */ +export function stringifyPublicParamsUniquely(p: TestParams): string { + const keys = Object.keys(p).sort(); + return keys + .filter(k => paramKeyIsPublic(k)) + .map(k => stringifySingleParamUniquely(k, p[k])) + .join(kParamSeparator); +} + +export function stringifySingleParam(k: string, v: JSONWithUndefined) { + return `${k}${kParamKVSeparator}${stringifySingleParamValue(v)}`; +} + +function stringifySingleParamUniquely(k: string, v: JSONWithUndefined) { + return `${k}${kParamKVSeparator}${stringifyParamValueUniquely(v)}`; +} + +function stringifySingleParamValue(v: JSONWithUndefined): string { + const s = stringifyParamValue(v); + assert( + !badParamValueChars.test(s), + `JSON.stringified param value must not match ${badParamValueChars} - was ${s}` + ); + return s; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/validQueryPart.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/validQueryPart.ts new file mode 100644 index 0000000000..62184adb62 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/validQueryPart.ts @@ -0,0 +1,2 @@ +/** Applies to group parts, test parts, params keys. */ +export const validQueryPart = /^[a-zA-Z0-9_]+$/; diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/stack.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/stack.ts new file mode 100644 index 0000000000..5de54088c8 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/stack.ts @@ -0,0 +1,82 @@ +// Returns the stack trace of an Error, but without the extra boilerplate at the bottom +// (e.g. RunCaseSpecific, processTicksAndRejections, etc.), for logging. +export function extractImportantStackTrace(e: Error): string { + let stack = e.stack; + if (!stack) { + return ''; + } + const redundantMessage = 'Error: ' + e.message + '\n'; + if (stack.startsWith(redundantMessage)) { + stack = stack.substring(redundantMessage.length); + } + + const lines = stack.split('\n'); + for (let i = lines.length - 1; i >= 0; --i) { + const line = lines[i]; + if (line.indexOf('.spec.') !== -1) { + return lines.slice(0, i + 1).join('\n'); + } + } + return stack; +} + +// *** Examples *** +// +// Node fail() +// > Error: +// > at CaseRecorder.fail (/Users/kainino/src/cts/src/common/framework/logger.ts:99:30) +// > at RunCaseSpecific.exports.g.test.t [as fn] (/Users/kainino/src/cts/src/unittests/logger.spec.ts:80:7) +// x at RunCaseSpecific.run (/Users/kainino/src/cts/src/common/framework/test_group.ts:121:18) +// x at processTicksAndRejections (internal/process/task_queues.js:86:5) +// +// Node throw +// > Error: hello +// > at RunCaseSpecific.g.test.t [as fn] (/Users/kainino/src/cts/src/unittests/test_group.spec.ts:51:11) +// x at RunCaseSpecific.run (/Users/kainino/src/cts/src/common/framework/test_group.ts:121:18) +// x at processTicksAndRejections (internal/process/task_queues.js:86:5) +// +// Firefox fail() +// > fail@http://localhost:8080/out/framework/logger.js:104:30 +// > expect@http://localhost:8080/out/framework/default_fixture.js:59:16 +// > @http://localhost:8080/out/unittests/util.spec.js:35:5 +// x run@http://localhost:8080/out/framework/test_group.js:119:18 +// +// Firefox throw +// > @http://localhost:8080/out/unittests/test_group.spec.js:48:11 +// x run@http://localhost:8080/out/framework/test_group.js:119:18 +// +// Safari fail() +// > fail@http://localhost:8080/out/framework/logger.js:104:39 +// > expect@http://localhost:8080/out/framework/default_fixture.js:59:20 +// > http://localhost:8080/out/unittests/util.spec.js:35:11 +// x http://localhost:8080/out/framework/test_group.js:119:20 +// x asyncFunctionResume@[native code] +// x [native code] +// x promiseReactionJob@[native code] +// +// Safari throw +// > http://localhost:8080/out/unittests/test_group.spec.js:48:20 +// x http://localhost:8080/out/framework/test_group.js:119:20 +// x asyncFunctionResume@[native code] +// x [native code] +// x promiseReactionJob@[native code] +// +// Chrome fail() +// x Error +// x at CaseRecorder.fail (http://localhost:8080/out/framework/logger.js:104:30) +// x at DefaultFixture.expect (http://localhost:8080/out/framework/default_fixture.js:59:16) +// > at RunCaseSpecific.fn (http://localhost:8080/out/unittests/util.spec.js:35:5) +// x at RunCaseSpecific.run (http://localhost:8080/out/framework/test_group.js:119:18) +// x at async runCase (http://localhost:8080/out/runtime/standalone.js:37:17) +// x at async http://localhost:8080/out/runtime/standalone.js:102:7 +// +// Chrome throw +// x Error: hello +// > at RunCaseSpecific.fn (http://localhost:8080/out/unittests/test_group.spec.js:48:11) +// x at RunCaseSpecific.run (http://localhost:8080/out/framework/test_group.js:119:18)" +// x at async Promise.all (index 0) +// x at async TestGroupTest.run (http://localhost:8080/out/unittests/test_group_test.js:6:5) +// x at async RunCaseSpecific.fn (http://localhost:8080/out/unittests/test_group.spec.js:53:15) +// x at async RunCaseSpecific.run (http://localhost:8080/out/framework/test_group.js:119:7) +// x at async runCase (http://localhost:8080/out/runtime/standalone.js:37:17) +// x at async http://localhost:8080/out/runtime/standalone.js:102:7 diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts new file mode 100644 index 0000000000..63f017083c --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts @@ -0,0 +1,646 @@ +import { + Fixture, + SubcaseBatchState, + SkipTestCase, + TestParams, + UnexpectedPassError, +} from '../framework/fixture.js'; +import { + CaseParamsBuilder, + builderIterateCasesWithSubcases, + kUnitCaseParamsBuilder, + ParamsBuilderBase, + SubcaseParamsBuilder, +} from '../framework/params_builder.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { Expectation } from '../internal/logging/result.js'; +import { TestCaseRecorder } from '../internal/logging/test_case_recorder.js'; +import { extractPublicParams, Merged, mergeParams } from '../internal/params_utils.js'; +import { compareQueries, Ordering } from '../internal/query/compare.js'; +import { TestQuerySingleCase, TestQueryWithExpectation } from '../internal/query/query.js'; +import { kPathSeparator } from '../internal/query/separators.js'; +import { + stringifyPublicParams, + stringifyPublicParamsUniquely, +} from '../internal/query/stringify_params.js'; +import { validQueryPart } from '../internal/query/validQueryPart.js'; +import { assert, unreachable } from '../util/util.js'; + +export type RunFn = ( + rec: TestCaseRecorder, + expectations?: TestQueryWithExpectation[] +) => Promise; + +export interface TestCaseID { + readonly test: readonly string[]; + readonly params: TestParams; +} + +export interface RunCase { + readonly id: TestCaseID; + readonly isUnimplemented: boolean; + run( + rec: TestCaseRecorder, + selfQuery: TestQuerySingleCase, + expectations: TestQueryWithExpectation[] + ): Promise; +} + +// Interface for defining tests +export interface TestGroupBuilder> { + test(name: string): TestBuilderWithName; +} +export function makeTestGroup>( + fixture: FixtureClass +): TestGroupBuilder { + return new TestGroup((fixture as unknown) as FixtureClass); +} + +// Interfaces for running tests +export interface IterableTestGroup { + iterate(): Iterable; + validate(): void; +} +export interface IterableTest { + testPath: string[]; + description: string | undefined; + readonly testCreationStack: Error; + iterate(): Iterable; +} + +export function makeTestGroupForUnitTesting( + fixture: FixtureClass +): TestGroup { + return new TestGroup(fixture); +} + +export type FixtureClass< + S extends SubcaseBatchState = SubcaseBatchState, + F extends Fixture = Fixture +> = { + new (sharedState: S, log: TestCaseRecorder, params: TestParams): F; + MakeSharedState(params: TestParams): S; +}; +type TestFn = (t: F & { params: P }) => Promise | void; +type BeforeAllSubcasesFn = ( + s: S & { params: P } +) => Promise | void; + +export class TestGroup> + implements TestGroupBuilder { + private fixture: FixtureClass; + private seen: Set = new Set(); + private tests: Array> = []; + + constructor(fixture: FixtureClass) { + this.fixture = fixture; + } + + iterate(): Iterable { + return this.tests; + } + + private checkName(name: string): void { + assert( + // Shouldn't happen due to the rule above. Just makes sure that treating + // unencoded strings as encoded strings is OK. + name === decodeURIComponent(name), + `Not decodeURIComponent-idempotent: ${name} !== ${decodeURIComponent(name)}` + ); + assert(!this.seen.has(name), `Duplicate test name: ${name}`); + + this.seen.add(name); + } + + test(name: string): TestBuilderWithName { + const testCreationStack = new Error(`Test created: ${name}`); + + this.checkName(name); + + const parts = name.split(kPathSeparator); + for (const p of parts) { + assert(validQueryPart.test(p), `Invalid test name part ${p}; must match ${validQueryPart}`); + } + + const test = new TestBuilder(parts, this.fixture, testCreationStack); + this.tests.push(test); + return (test as unknown) as TestBuilderWithName; + } + + validate(): void { + for (const test of this.tests) { + test.validate(); + } + } +} + +interface TestBuilderWithName> + extends TestBuilderWithParams { + desc(description: string): this; + /** + * A noop function to associate a test with the relevant part of the specification. + * + * @param url a link to the spec where test is extracted from. + */ + specURL(url: string): this; + /** + * Parameterize the test, generating multiple cases, each possibly having subcases. + * + * The `unit` value passed to the `cases` callback is an immutable constant + * `CaseParamsBuilder<{}>` representing the "unit" builder `[ {} ]`, + * provided for convenience. The non-callback overload can be used if `unit` is not needed. + */ + params( + cases: (unit: CaseParamsBuilder<{}>) => ParamsBuilderBase + ): TestBuilderWithParams; + /** + * Parameterize the test, generating multiple cases, each possibly having subcases. + * + * Use the callback overload of this method if a "unit" builder is needed. + */ + params( + cases: ParamsBuilderBase + ): TestBuilderWithParams; + + /** + * Parameterize the test, generating multiple cases, without subcases. + */ + paramsSimple

(cases: Iterable

): TestBuilderWithParams; + + /** + * Parameterize the test, generating one case with multiple subcases. + */ + paramsSubcasesOnly

(subcases: Iterable

): TestBuilderWithParams; + /** + * Parameterize the test, generating one case with multiple subcases. + * + * The `unit` value passed to the `subcases` callback is an immutable constant + * `SubcaseParamsBuilder<{}>`, with one empty case `{}` and one empty subcase `{}`. + */ + paramsSubcasesOnly

( + subcases: (unit: SubcaseParamsBuilder<{}, {}>) => SubcaseParamsBuilder<{}, P> + ): TestBuilderWithParams; +} + +interface TestBuilderWithParams< + S extends SubcaseBatchState, + F extends Fixture, + CaseP extends {}, + SubcaseP extends {} +> { + /** + * Limit subcases to a maximum number of per testcase. + * @param b the maximum number of subcases per testcase. + * + * If the number of subcases exceeds `b`, add an internal + * numeric, incrementing `batch__` param to split subcases + * into groups of at most `b` subcases. + */ + batch(b: number): this; + /** + * Run a function on shared subcase batch state before each + * batch of subcases. + * @param fn the function to run. It is called with the test + * fixture's shared subcase batch state. + * + * Generally, this function should be careful to avoid mutating + * any state on the shared subcase batch state which could result + * in unexpected order-dependent test behavior. + */ + beforeAllSubcases(fn: BeforeAllSubcasesFn): this; + /** + * Set the test function. + * @param fn the test function. + */ + fn(fn: TestFn>): void; + /** + * Mark the test as unimplemented. + */ + unimplemented(): void; +} + +class TestBuilder { + readonly testPath: string[]; + isUnimplemented: boolean; + description: string | undefined; + readonly testCreationStack: Error; + + private readonly fixture: FixtureClass; + private testFn: TestFn | undefined; + private beforeFn: BeforeAllSubcasesFn | undefined; + private testCases?: ParamsBuilderBase<{}, {}> = undefined; + private batchSize: number = 0; + + constructor(testPath: string[], fixture: FixtureClass, testCreationStack: Error) { + this.testPath = testPath; + this.isUnimplemented = false; + this.fixture = fixture; + this.testCreationStack = testCreationStack; + } + + desc(description: string): this { + this.description = description.trim(); + return this; + } + + specURL(url: string): this { + return this; + } + + beforeAllSubcases(fn: BeforeAllSubcasesFn): this { + assert(this.beforeFn === undefined); + this.beforeFn = fn; + return this; + } + + fn(fn: TestFn): void { + // eslint-disable-next-line no-warning-comments + // MAINTENANCE_TODO: add "TODO" if there's no description? (and make sure it only ends up on + // actual tests, not on test parents in the tree, which is what happens if you do it here, not + // sure why) + assert(this.testFn === undefined); + this.testFn = fn; + } + + batch(b: number): this { + this.batchSize = b; + return this; + } + + unimplemented(): void { + assert(this.testFn === undefined); + + this.description = + (this.description ? this.description + '\n\n' : '') + 'TODO: .unimplemented()'; + this.isUnimplemented = true; + + this.testFn = () => { + throw new SkipTestCase('test unimplemented'); + }; + } + + validate(): void { + const testPathString = this.testPath.join(kPathSeparator); + assert(this.testFn !== undefined, () => { + let s = `Test is missing .fn(): ${testPathString}`; + if (this.testCreationStack.stack) { + s += `\n-> test created at:\n${this.testCreationStack.stack}`; + } + return s; + }); + + if (this.testCases === undefined) { + return; + } + + const seen = new Set(); + for (const [caseParams, subcases] of builderIterateCasesWithSubcases(this.testCases)) { + for (const subcaseParams of subcases ?? [{}]) { + const params = mergeParams(caseParams, subcaseParams); + assert(this.batchSize === 0 || !('batch__' in params)); + + // stringifyPublicParams also checks for invalid params values + const testcaseString = stringifyPublicParams(params); + + // A (hopefully) unique representation of a params value. + const testcaseStringUnique = stringifyPublicParamsUniquely(params); + assert( + !seen.has(testcaseStringUnique), + `Duplicate public test case params for test ${testPathString}: ${testcaseString}` + ); + seen.add(testcaseStringUnique); + } + } + } + + params( + cases: ((unit: CaseParamsBuilder<{}>) => ParamsBuilderBase<{}, {}>) | ParamsBuilderBase<{}, {}> + ): TestBuilder { + assert(this.testCases === undefined, 'test case is already parameterized'); + if (cases instanceof Function) { + this.testCases = cases(kUnitCaseParamsBuilder); + } else { + this.testCases = cases; + } + return this; + } + + paramsSimple(cases: Iterable<{}>): TestBuilder { + assert(this.testCases === undefined, 'test case is already parameterized'); + this.testCases = kUnitCaseParamsBuilder.combineWithParams(cases); + return this; + } + + paramsSubcasesOnly( + subcases: Iterable<{}> | ((unit: SubcaseParamsBuilder<{}, {}>) => SubcaseParamsBuilder<{}, {}>) + ): TestBuilder { + if (subcases instanceof Function) { + return this.params(subcases(kUnitCaseParamsBuilder.beginSubcases())); + } else { + return this.params(kUnitCaseParamsBuilder.beginSubcases().combineWithParams(subcases)); + } + } + + *iterate(): IterableIterator { + assert(this.testFn !== undefined, 'No test function (.fn()) for test'); + this.testCases ??= kUnitCaseParamsBuilder; + for (const [caseParams, subcases] of builderIterateCasesWithSubcases(this.testCases)) { + if (this.batchSize === 0 || subcases === undefined) { + yield new RunCaseSpecific( + this.testPath, + caseParams, + this.isUnimplemented, + subcases, + this.fixture, + this.testFn, + this.beforeFn, + this.testCreationStack + ); + } else { + const subcaseArray = Array.from(subcases); + if (subcaseArray.length <= this.batchSize) { + yield new RunCaseSpecific( + this.testPath, + caseParams, + this.isUnimplemented, + subcaseArray, + this.fixture, + this.testFn, + this.beforeFn, + this.testCreationStack + ); + } else { + for (let i = 0; i < subcaseArray.length; i = i + this.batchSize) { + yield new RunCaseSpecific( + this.testPath, + { ...caseParams, batch__: i / this.batchSize }, + this.isUnimplemented, + subcaseArray.slice(i, Math.min(subcaseArray.length, i + this.batchSize)), + this.fixture, + this.testFn, + this.beforeFn, + this.testCreationStack + ); + } + } + } + } + } +} + +class RunCaseSpecific implements RunCase { + readonly id: TestCaseID; + readonly isUnimplemented: boolean; + + private readonly params: {}; + private readonly subcases: Iterable<{}> | undefined; + private readonly fixture: FixtureClass; + private readonly fn: TestFn; + private readonly beforeFn?: BeforeAllSubcasesFn; + private readonly testCreationStack: Error; + + constructor( + testPath: string[], + params: {}, + isUnimplemented: boolean, + subcases: Iterable<{}> | undefined, + fixture: FixtureClass, + fn: TestFn, + beforeFn: BeforeAllSubcasesFn | undefined, + testCreationStack: Error + ) { + this.id = { test: testPath, params: extractPublicParams(params) }; + this.isUnimplemented = isUnimplemented; + this.params = params; + this.subcases = subcases; + this.fixture = fixture; + this.fn = fn; + this.beforeFn = beforeFn; + this.testCreationStack = testCreationStack; + } + + async runTest( + rec: TestCaseRecorder, + sharedState: SubcaseBatchState, + params: TestParams, + throwSkip: boolean, + expectedStatus: Expectation + ): Promise { + try { + rec.beginSubCase(); + if (expectedStatus === 'skip') { + throw new SkipTestCase('Skipped by expectations'); + } + + const inst = new this.fixture(sharedState, rec, params); + try { + await inst.init(); + await this.fn(inst as Fixture & { params: {} }); + } finally { + // Runs as long as constructor succeeded, even if initialization or the test failed. + await inst.finalize(); + } + } catch (ex) { + // There was an exception from constructor, init, test, or finalize. + // An error from init or test may have been a SkipTestCase. + // An error from finalize may have been an eventualAsyncExpectation failure + // or unexpected validation/OOM error from the GPUDevice. + if (throwSkip && ex instanceof SkipTestCase) { + throw ex; + } + rec.threw(ex); + } finally { + try { + rec.endSubCase(expectedStatus); + } catch (ex) { + assert(ex instanceof UnexpectedPassError); + ex.message = `Testcase passed unexpectedly.`; + ex.stack = this.testCreationStack.stack; + rec.warn(ex); + } + } + } + + async run( + rec: TestCaseRecorder, + selfQuery: TestQuerySingleCase, + expectations: TestQueryWithExpectation[] + ): Promise { + const getExpectedStatus = (selfQueryWithSubParams: TestQuerySingleCase) => { + let didSeeFail = false; + for (const exp of expectations) { + const ordering = compareQueries(exp.query, selfQueryWithSubParams); + if (ordering === Ordering.Unordered || ordering === Ordering.StrictSubset) { + continue; + } + + switch (exp.expectation) { + // Skip takes precedence. If there is any expectation indicating a skip, + // signal it immediately. + case 'skip': + return 'skip'; + case 'fail': + // Otherwise, indicate that we might expect a failure. + didSeeFail = true; + break; + default: + unreachable(); + } + } + return didSeeFail ? 'fail' : 'pass'; + }; + + const { testHeartbeatCallback, maxSubcasesInFlight } = globalTestConfig; + try { + rec.start(); + const sharedState = this.fixture.MakeSharedState(this.params); + try { + await sharedState.init(); + if (this.beforeFn) { + await this.beforeFn(sharedState); + } + await sharedState.postInit(); + testHeartbeatCallback(); + + let allPreviousSubcasesFinalizedPromise: Promise = Promise.resolve(); + if (this.subcases) { + let totalCount = 0; + let skipCount = 0; + + // If there are too many subcases in flight, starting the next subcase will register + // `resolvePromiseBlockingSubcase` and wait until `subcaseFinishedCallback` is called. + let subcasesInFlight = 0; + let resolvePromiseBlockingSubcase: (() => void) | undefined = undefined; + const subcaseFinishedCallback = () => { + subcasesInFlight -= 1; + // If there is any subcase waiting on a previous subcase to finish, + // unblock it now, and clear the resolve callback. + if (resolvePromiseBlockingSubcase) { + resolvePromiseBlockingSubcase(); + resolvePromiseBlockingSubcase = undefined; + } + }; + + for (const subParams of this.subcases) { + // Make a recorder that will defer all calls until `allPreviousSubcasesFinalizedPromise` + // resolves. Waiting on `allPreviousSubcasesFinalizedPromise` ensures that + // logs from all the previous subcases have been flushed before flushing new logs. + const subcasePrefix = 'subcase: ' + stringifyPublicParams(subParams); + const subRec = new Proxy(rec, { + get: (target, k: keyof TestCaseRecorder) => { + const prop = TestCaseRecorder.prototype[k]; + if (typeof prop === 'function') { + testHeartbeatCallback(); + return function (...args: Parameters) { + void allPreviousSubcasesFinalizedPromise.then(() => { + // Prepend the subcase name to all error messages. + for (const arg of args) { + if (arg instanceof Error) { + try { + arg.message = subcasePrefix + '\n' + arg.message; + } catch { + // If that fails (e.g. on DOMException), try to put it in the stack: + let stack = subcasePrefix; + if (arg.stack) stack += '\n' + arg.stack; + try { + arg.stack = stack; + } catch { + // If that fails too, just silence it. + } + } + } + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const rv = (prop as any).apply(target, args); + // Because this proxy executes functions in a deferred manner, + // it should never be used for functions that need to return a value. + assert(rv === undefined); + }); + }; + } + return prop; + }, + }); + + const params = mergeParams(this.params, subParams); + const subcaseQuery = new TestQuerySingleCase( + selfQuery.suite, + selfQuery.filePathParts, + selfQuery.testPathParts, + params + ); + + // Limit the maximum number of subcases in flight. + if (subcasesInFlight >= maxSubcasesInFlight) { + await new Promise(resolve => { + // There should only be one subcase waiting at a time. + assert(resolvePromiseBlockingSubcase === undefined); + resolvePromiseBlockingSubcase = resolve; + }); + } + + subcasesInFlight += 1; + // Runs async without waiting so that subsequent subcases can start. + // All finalization steps will be waited on at the end of the testcase. + const finalizePromise = this.runTest( + subRec, + sharedState, + params, + /* throwSkip */ true, + getExpectedStatus(subcaseQuery) + ) + .then(() => { + subRec.info(new Error('OK')); + }) + .catch(ex => { + if (ex instanceof SkipTestCase) { + // Convert SkipTestCase to info messages + ex.message = 'subcase skipped: ' + ex.message; + subRec.info(ex); + ++skipCount; + } else { + // Since we are catching all error inside runTest(), this should never happen + subRec.threw(ex); + } + }) + .finally(subcaseFinishedCallback); + + allPreviousSubcasesFinalizedPromise = allPreviousSubcasesFinalizedPromise.then( + () => finalizePromise + ); + ++totalCount; + } + + // Wait for all subcases to finalize and report their results. + await allPreviousSubcasesFinalizedPromise; + + if (skipCount === totalCount) { + rec.skipped(new SkipTestCase('all subcases were skipped')); + } + } else { + await this.runTest( + rec, + sharedState, + this.params, + /* throwSkip */ false, + getExpectedStatus(selfQuery) + ); + } + } finally { + testHeartbeatCallback(); + // Runs as long as the shared state constructor succeeded, even if initialization or a test failed. + await sharedState.finalize(); + testHeartbeatCallback(); + } + } catch (ex) { + // There was an exception from sharedState/fixture constructor, init, beforeFn, or test. + // An error from beforeFn may have been SkipTestCase. + // An error from finalize may have been an eventualAsyncExpectation failure + // or unexpected validation/OOM error from the GPUDevice. + rec.threw(ex); + } finally { + rec.finish(); + } + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts new file mode 100644 index 0000000000..2d2b555366 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts @@ -0,0 +1,15 @@ +// A listing of all specs within a single suite. This is the (awaited) type of +// `groups` in '{cts,unittests}/listing.ts' and `listing` in the auto-generated +// 'out/{cts,unittests}/listing.js' files (see tools/gen_listings). +export type TestSuiteListing = TestSuiteListingEntry[]; + +export type TestSuiteListingEntry = TestSuiteListingEntrySpec | TestSuiteListingEntryReadme; + +interface TestSuiteListingEntrySpec { + readonly file: string[]; +} + +interface TestSuiteListingEntryReadme { + readonly file: string[]; + readonly readme: string; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts new file mode 100644 index 0000000000..204a4f693a --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts @@ -0,0 +1,575 @@ +import { RunCase, RunFn } from '../internal/test_group.js'; +import { assert } from '../util/util.js'; + +import { TestFileLoader } from './file_loader.js'; +import { TestParamsRW } from './params_utils.js'; +import { compareQueries, Ordering } from './query/compare.js'; +import { + TestQuery, + TestQueryMultiCase, + TestQuerySingleCase, + TestQueryMultiFile, + TestQueryMultiTest, +} from './query/query.js'; +import { kBigSeparator, kWildcard, kPathSeparator, kParamSeparator } from './query/separators.js'; +import { stringifySingleParam } from './query/stringify_params.js'; +import { StacklessError } from './util.js'; + +// `loadTreeForQuery()` loads a TestTree for a given queryToLoad. +// The resulting tree is a linked-list all the way from `suite:*` to queryToLoad, +// and under queryToLoad is a tree containing every case matched by queryToLoad. +// +// `subqueriesToExpand` influences the `collapsible` flag on nodes in the resulting tree. +// A node is considered "collapsible" if none of the subqueriesToExpand is a StrictSubset +// of that node. +// +// In WebKit/Blink-style web_tests, an expectation file marks individual cts.https.html "variants +// as "Failure", "Crash", etc. By passing in the list of expectations as the subqueriesToExpand, +// we can programmatically subdivide the cts.https.html "variants" list to be able to implement +// arbitrarily-fine suppressions (instead of having to suppress entire test files, which would +// lose a lot of coverage). +// +// `iterateCollapsedNodes()` produces the list of queries for the variants list. +// +// Though somewhat complicated, this system has important benefits: +// - Avoids having to suppress entire test files, which would cause large test coverage loss. +// - Minimizes the number of page loads needed for fine-grained suppressions. +// (In the naive case, we could do one page load per test case - but the test suite would +// take impossibly long to run.) +// - Enables developers to put any number of tests in one file as appropriate, without worrying +// about expectation granularity. + +interface TestTreeNodeBase { + readonly query: T; + /** + * Readable "relative" name for display in standalone runner. + * Not always the exact relative name, because sometimes there isn't + * one (e.g. s:f:* relative to s:f,*), but something that is readable. + */ + readonly readableRelativeName: string; + subtreeCounts?: { tests: number; nodesWithTODO: number }; +} + +export interface TestSubtree extends TestTreeNodeBase { + readonly children: Map; + readonly collapsible: boolean; + description?: string; + readonly testCreationStack?: Error; +} + +export interface TestTreeLeaf extends TestTreeNodeBase { + readonly run: RunFn; + readonly isUnimplemented?: boolean; + subtreeCounts?: undefined; +} + +export type TestTreeNode = TestSubtree | TestTreeLeaf; + +/** + * When iterating through "collapsed" tree nodes, indicates how many "query levels" to traverse + * through before starting to collapse nodes. + * + * Corresponds with TestQueryLevel, but excludes 4 (SingleCase): + * - 1 = MultiFile. Expands so every file is in the collapsed tree. + * - 2 = MultiTest. Expands so every test is in the collapsed tree. + * - 3 = MultiCase. Expands so every case is in the collapsed tree (i.e. collapsing disabled). + */ +export type ExpandThroughLevel = 1 | 2 | 3; + +export class TestTree { + /** + * The `queryToLoad` that this test tree was created for. + * Test trees are always rooted at `suite:*`, but they only contain nodes that fit + * within `forQuery`. + * + * This is used for `iterateCollapsedNodes` which only starts collapsing at the next + * `TestQueryLevel` after `forQuery`. + */ + readonly forQuery: TestQuery; + readonly root: TestSubtree; + + constructor(forQuery: TestQuery, root: TestSubtree) { + this.forQuery = forQuery; + TestTree.propagateCounts(root); + this.root = root; + assert( + root.query.level === 1 && root.query.depthInLevel === 0, + 'TestTree root must be the root (suite:*)' + ); + } + + /** + * Iterate through the leaves of a version of the tree which has been pruned to exclude + * subtrees which: + * - are at a deeper `TestQueryLevel` than `this.forQuery`, and + * - were not a `Ordering.StrictSubset` of any of the `subqueriesToExpand` during tree creation. + */ + iterateCollapsedNodes({ + includeIntermediateNodes = false, + includeEmptySubtrees = false, + alwaysExpandThroughLevel, + }: { + /** Whether to include intermediate tree nodes or only collapsed-leaves. */ + includeIntermediateNodes?: boolean; + /** Whether to include collapsed-leaves with no children. */ + includeEmptySubtrees?: boolean; + /** Never collapse nodes up through this level. */ + alwaysExpandThroughLevel: ExpandThroughLevel; + }): IterableIterator> { + const expandThroughLevel = Math.max(this.forQuery.level, alwaysExpandThroughLevel); + return TestTree.iterateSubtreeNodes(this.root, { + includeIntermediateNodes, + includeEmptySubtrees, + expandThroughLevel, + }); + } + + iterateLeaves(): IterableIterator> { + return TestTree.iterateSubtreeLeaves(this.root); + } + + /** + * Dissolve nodes which have only one child, e.g.: + * a,* { a,b,* { a,b:* { ... } } } + * collapses down into: + * a,* { a,b:* { ... } } + * which is less needlessly verbose when displaying the tree in the standalone runner. + */ + dissolveSingleChildTrees(): void { + const newRoot = dissolveSingleChildTrees(this.root); + assert(newRoot === this.root); + } + + toString(): string { + return TestTree.subtreeToString('(root)', this.root, ''); + } + + static *iterateSubtreeNodes( + subtree: TestSubtree, + opts: { + includeIntermediateNodes: boolean; + includeEmptySubtrees: boolean; + expandThroughLevel: number; + } + ): IterableIterator { + if (opts.includeIntermediateNodes) { + yield subtree; + } + + for (const [, child] of subtree.children) { + if ('children' in child) { + // Is a subtree + const collapsible = child.collapsible && child.query.level > opts.expandThroughLevel; + if (child.children.size > 0 && !collapsible) { + yield* TestTree.iterateSubtreeNodes(child, opts); + } else if (child.children.size > 0 || opts.includeEmptySubtrees) { + // Don't yield empty subtrees (e.g. files with no tests) unless includeEmptySubtrees + yield child; + } + } else { + // Is a leaf + yield child; + } + } + } + + static *iterateSubtreeLeaves(subtree: TestSubtree): IterableIterator { + for (const [, child] of subtree.children) { + if ('children' in child) { + yield* TestTree.iterateSubtreeLeaves(child); + } else { + yield child; + } + } + } + + /** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */ + static propagateCounts(subtree: TestSubtree): { tests: number; nodesWithTODO: number } { + subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0 }; + for (const [, child] of subtree.children) { + if ('children' in child) { + const counts = TestTree.propagateCounts(child); + subtree.subtreeCounts.tests += counts.tests; + subtree.subtreeCounts.nodesWithTODO += counts.nodesWithTODO; + } + } + return subtree.subtreeCounts; + } + + /** Displays counts in the format `(Nodes with TODOs) / (Total test count)`. */ + static countsToString(tree: TestTreeNode): string { + if (tree.subtreeCounts) { + return `${tree.subtreeCounts.nodesWithTODO} / ${tree.subtreeCounts.tests}`; + } else { + return ''; + } + } + + static subtreeToString(name: string, tree: TestTreeNode, indent: string): string { + const collapsible = 'run' in tree ? '>' : tree.collapsible ? '+' : '-'; + let s = + indent + + `${collapsible} ${TestTree.countsToString(tree)} ${JSON.stringify(name)} => ${tree.query}`; + if ('children' in tree) { + if (tree.description !== undefined) { + s += `\n${indent} | ${JSON.stringify(tree.description)}`; + } + + for (const [name, child] of tree.children) { + s += '\n' + TestTree.subtreeToString(name, child, indent + ' '); + } + } + return s; + } +} + +// MAINTENANCE_TODO: Consider having subqueriesToExpand actually impact the depth-order of params +// in the tree. +export async function loadTreeForQuery( + loader: TestFileLoader, + queryToLoad: TestQuery, + subqueriesToExpand: TestQuery[] +): Promise { + const suite = queryToLoad.suite; + const specs = await loader.listing(suite); + + const subqueriesToExpandEntries = Array.from(subqueriesToExpand.entries()); + const seenSubqueriesToExpand: boolean[] = new Array(subqueriesToExpand.length); + seenSubqueriesToExpand.fill(false); + + const isCollapsible = (subquery: TestQuery) => + subqueriesToExpandEntries.every(([i, toExpand]) => { + const ordering = compareQueries(toExpand, subquery); + + // If toExpand == subquery, no expansion is needed (but it's still "seen"). + if (ordering === Ordering.Equal) seenSubqueriesToExpand[i] = true; + return ordering !== Ordering.StrictSubset; + }); + + // L0 = suite-level, e.g. suite:* + // L1 = file-level, e.g. suite:a,b:* + // L2 = test-level, e.g. suite:a,b:c,d:* + // L3 = case-level, e.g. suite:a,b:c,d: + let foundCase = false; + // L0 is suite:* + const subtreeL0 = makeTreeForSuite(suite, isCollapsible); + for (const entry of specs) { + if (entry.file.length === 0 && 'readme' in entry) { + // Suite-level readme. + setSubtreeDescriptionAndCountTODOs(subtreeL0, entry.readme); + continue; + } + + { + const queryL1 = new TestQueryMultiFile(suite, entry.file); + const orderingL1 = compareQueries(queryL1, queryToLoad); + if (orderingL1 === Ordering.Unordered) { + // File path is not matched by this query. + continue; + } + } + + if ('readme' in entry) { + // Entry is a README that is an ancestor or descendant of the query. + // (It's included for display in the standalone runner.) + + // readmeSubtree is suite:a,b,* + // (This is always going to dedup with a file path, if there are any test spec files under + // the directory that has the README). + const readmeSubtree: TestSubtree = addSubtreeForDirPath( + subtreeL0, + entry.file, + isCollapsible + ); + setSubtreeDescriptionAndCountTODOs(readmeSubtree, entry.readme); + continue; + } + // Entry is a spec file. + + const spec = await loader.importSpecFile(queryToLoad.suite, entry.file); + // subtreeL1 is suite:a,b:* + const subtreeL1: TestSubtree = addSubtreeForFilePath( + subtreeL0, + entry.file, + isCollapsible + ); + setSubtreeDescriptionAndCountTODOs(subtreeL1, spec.description); + + let groupHasTests = false; + for (const t of spec.g.iterate()) { + groupHasTests = true; + { + const queryL2 = new TestQueryMultiCase(suite, entry.file, t.testPath, {}); + const orderingL2 = compareQueries(queryL2, queryToLoad); + if (orderingL2 === Ordering.Unordered) { + // Test path is not matched by this query. + continue; + } + } + + // subtreeL2 is suite:a,b:c,d:* + const subtreeL2: TestSubtree = addSubtreeForTestPath( + subtreeL1, + t.testPath, + t.testCreationStack, + isCollapsible + ); + // This is 1 test. Set tests=1 then count TODOs. + subtreeL2.subtreeCounts ??= { tests: 1, nodesWithTODO: 0 }; + if (t.description) setSubtreeDescriptionAndCountTODOs(subtreeL2, t.description); + + // MAINTENANCE_TODO: If tree generation gets too slow, avoid actually iterating the cases in a + // file if there's no need to (based on the subqueriesToExpand). + for (const c of t.iterate()) { + { + const queryL3 = new TestQuerySingleCase(suite, entry.file, c.id.test, c.id.params); + const orderingL3 = compareQueries(queryL3, queryToLoad); + if (orderingL3 === Ordering.Unordered || orderingL3 === Ordering.StrictSuperset) { + // Case is not matched by this query. + continue; + } + } + + // Leaf for case is suite:a,b:c,d:x=1;y=2 + addLeafForCase(subtreeL2, c, isCollapsible); + + foundCase = true; + } + } + if (!groupHasTests && !subtreeL1.subtreeCounts) { + throw new StacklessError( + `${subtreeL1.query} has no tests - it must have "TODO" in its description` + ); + } + } + + for (const [i, sq] of subqueriesToExpandEntries) { + const subquerySeen = seenSubqueriesToExpand[i]; + if (!subquerySeen) { + throw new StacklessError( + `subqueriesToExpand entry did not match anything \ +(could be wrong, or could be redundant with a previous subquery):\n ${sq.toString()}` + ); + } + } + assert(foundCase, `Query \`${queryToLoad.toString()}\` does not match any cases`); + + return new TestTree(queryToLoad, subtreeL0); +} + +function setSubtreeDescriptionAndCountTODOs( + subtree: TestSubtree, + description: string +) { + assert(subtree.description === undefined); + subtree.description = description.trim(); + subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0 }; + if (subtree.description.indexOf('TODO') !== -1) { + subtree.subtreeCounts.nodesWithTODO++; + } +} + +function makeTreeForSuite( + suite: string, + isCollapsible: (sq: TestQuery) => boolean +): TestSubtree { + const query = new TestQueryMultiFile(suite, []); + return { + readableRelativeName: suite + kBigSeparator, + query, + children: new Map(), + collapsible: isCollapsible(query), + }; +} + +function addSubtreeForDirPath( + tree: TestSubtree, + file: string[], + isCollapsible: (sq: TestQuery) => boolean +): TestSubtree { + const subqueryFile: string[] = []; + // To start, tree is suite:* + // This loop goes from that -> suite:a,* -> suite:a,b,* + for (const part of file) { + subqueryFile.push(part); + tree = getOrInsertSubtree(part, tree, () => { + const query = new TestQueryMultiFile(tree.query.suite, subqueryFile); + return { + readableRelativeName: part + kPathSeparator + kWildcard, + query, + collapsible: isCollapsible(query), + }; + }); + } + return tree; +} + +function addSubtreeForFilePath( + tree: TestSubtree, + file: string[], + isCollapsible: (sq: TestQuery) => boolean +): TestSubtree { + // To start, tree is suite:* + // This goes from that -> suite:a,* -> suite:a,b,* + tree = addSubtreeForDirPath(tree, file, isCollapsible); + // This goes from that -> suite:a,b:* + const subtree = getOrInsertSubtree('', tree, () => { + const query = new TestQueryMultiTest(tree.query.suite, tree.query.filePathParts, []); + assert(file.length > 0, 'file path is empty'); + return { + readableRelativeName: file[file.length - 1] + kBigSeparator + kWildcard, + query, + collapsible: isCollapsible(query), + }; + }); + return subtree; +} + +function addSubtreeForTestPath( + tree: TestSubtree, + test: readonly string[], + testCreationStack: Error, + isCollapsible: (sq: TestQuery) => boolean +): TestSubtree { + const subqueryTest: string[] = []; + // To start, tree is suite:a,b:* + // This loop goes from that -> suite:a,b:c,* -> suite:a,b:c,d,* + for (const part of test) { + subqueryTest.push(part); + tree = getOrInsertSubtree(part, tree, () => { + const query = new TestQueryMultiTest( + tree.query.suite, + tree.query.filePathParts, + subqueryTest + ); + return { + readableRelativeName: part + kPathSeparator + kWildcard, + query, + collapsible: isCollapsible(query), + }; + }); + } + // This goes from that -> suite:a,b:c,d:* + return getOrInsertSubtree('', tree, () => { + const query = new TestQueryMultiCase( + tree.query.suite, + tree.query.filePathParts, + subqueryTest, + {} + ); + assert(subqueryTest.length > 0, 'subqueryTest is empty'); + return { + readableRelativeName: subqueryTest[subqueryTest.length - 1] + kBigSeparator + kWildcard, + kWildcard, + query, + testCreationStack, + collapsible: isCollapsible(query), + }; + }); +} + +function addLeafForCase( + tree: TestSubtree, + t: RunCase, + checkCollapsible: (sq: TestQuery) => boolean +): void { + const query = tree.query; + let name: string = ''; + const subqueryParams: TestParamsRW = {}; + + // To start, tree is suite:a,b:c,d:* + // This loop goes from that -> suite:a,b:c,d:x=1;* -> suite:a,b:c,d:x=1;y=2;* + for (const [k, v] of Object.entries(t.id.params)) { + name = stringifySingleParam(k, v); + subqueryParams[k] = v; + + tree = getOrInsertSubtree(name, tree, () => { + const subquery = new TestQueryMultiCase( + query.suite, + query.filePathParts, + query.testPathParts, + subqueryParams + ); + return { + readableRelativeName: name + kParamSeparator + kWildcard, + query: subquery, + collapsible: checkCollapsible(subquery), + }; + }); + } + + // This goes from that -> suite:a,b:c,d:x=1;y=2 + const subquery = new TestQuerySingleCase( + query.suite, + query.filePathParts, + query.testPathParts, + subqueryParams + ); + checkCollapsible(subquery); // mark seenSubqueriesToExpand + insertLeaf(tree, subquery, t); +} + +function getOrInsertSubtree( + key: string, + parent: TestSubtree, + createSubtree: () => Omit, 'children'> +): TestSubtree { + let v: TestSubtree; + const child = parent.children.get(key); + if (child !== undefined) { + assert('children' in child); // Make sure cached subtree is not actually a leaf + v = child as TestSubtree; + } else { + v = { ...createSubtree(), children: new Map() }; + parent.children.set(key, v); + } + return v; +} + +function insertLeaf(parent: TestSubtree, query: TestQuerySingleCase, t: RunCase) { + const leaf: TestTreeLeaf = { + readableRelativeName: readableNameForCase(query), + query, + run: (rec, expectations) => t.run(rec, query, expectations || []), + isUnimplemented: t.isUnimplemented, + }; + + // This is a leaf (e.g. s:f:t:x=1;* -> s:f:t:x=1). The key is always ''. + const key = ''; + assert(!parent.children.has(key), `Duplicate testcase: ${query}`); + parent.children.set(key, leaf); +} + +function dissolveSingleChildTrees(tree: TestTreeNode): TestTreeNode { + if ('children' in tree) { + const shouldDissolveThisTree = + tree.children.size === 1 && tree.query.depthInLevel !== 0 && tree.description === undefined; + if (shouldDissolveThisTree) { + // Loops exactly once + for (const [, child] of tree.children) { + // Recurse on child + return dissolveSingleChildTrees(child); + } + } + + for (const [k, child] of tree.children) { + // Recurse on each child + const newChild = dissolveSingleChildTrees(child); + if (newChild !== child) { + tree.children.set(k, newChild); + } + } + } + return tree; +} + +/** Generate a readable relative name for a case (used in standalone). */ +function readableNameForCase(query: TestQuerySingleCase): string { + const paramsKeys = Object.keys(query.params); + if (paramsKeys.length === 0) { + return query.testPathParts[query.testPathParts.length - 1] + kBigSeparator; + } else { + const lastKey = paramsKeys[paramsKeys.length - 1]; + return stringifySingleParam(lastKey, query.params[lastKey]); + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/util.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/util.ts new file mode 100644 index 0000000000..37a5db3568 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/util.ts @@ -0,0 +1,10 @@ +/** + * Error without a stack, which can be used to fatally exit from `tool/` scripts with a + * user-friendly message (and no confusing stack). + */ +export class StacklessError extends Error { + constructor(message: string) { + super(message); + this.stack = undefined; + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/version.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/version.ts new file mode 100644 index 0000000000..53cc97482e --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/version.ts @@ -0,0 +1 @@ +export const version = 'unknown'; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts new file mode 100644 index 0000000000..463546c06d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/cmdline.ts @@ -0,0 +1,278 @@ +/* eslint no-console: "off" */ + +import * as fs from 'fs'; + +import { dataCache } from '../framework/data_cache.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { prettyPrintLog } from '../internal/logging/log_message.js'; +import { Logger } from '../internal/logging/logger.js'; +import { LiveTestCaseResult } from '../internal/logging/result.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { parseExpectationsForTestQuery } from '../internal/query/query.js'; +import { Colors } from '../util/colors.js'; +import { setGPUProvider } from '../util/navigator_gpu.js'; +import { assert, unreachable } from '../util/util.js'; + +import sys from './helper/sys.js'; + +function usage(rc: number): never { + console.log(`Usage: + tools/run_${sys.type} [OPTIONS...] QUERIES... + tools/run_${sys.type} 'unittests:*' 'webgpu:buffers,*' +Options: + --colors Enable ANSI colors in output. + --coverage Emit coverage data. + --verbose Print result/log of every test as it runs. + --list Print all testcase names that match the given query and exit. + --debug Include debug messages in logging. + --print-json Print the complete result JSON in the output. + --expectations Path to expectations file. + --gpu-provider Path to node module that provides the GPU implementation. + --gpu-provider-flag Flag to set on the gpu-provider as = + --unroll-const-eval-loops Unrolls loops in constant-evaluation shader execution tests + --quiet Suppress summary information in output +`); + return sys.exit(rc); +} + +// The interface that exposes creation of the GPU, and optional interface to code coverage. +interface GPUProviderModule { + // @returns a GPU with the given flags + create(flags: string[]): GPU; + // An optional interface to a CodeCoverageProvider + coverage?: CodeCoverageProvider; +} + +interface CodeCoverageProvider { + // Starts collecting code coverage + begin(): void; + // Ends collecting of code coverage, returning the coverage data. + // This data is opaque (implementation defined). + end(): string; +} + +type listModes = 'none' | 'cases' | 'unimplemented'; + +Colors.enabled = false; + +let verbose = false; +let emitCoverage = false; +let listMode: listModes = 'none'; +let debug = false; +let printJSON = false; +let quiet = false; +let loadWebGPUExpectations: Promise | undefined = undefined; +let gpuProviderModule: GPUProviderModule | undefined = undefined; +let dataPath: string | undefined = undefined; + +const queries: string[] = []; +const gpuProviderFlags: string[] = []; +for (let i = 0; i < sys.args.length; ++i) { + const a = sys.args[i]; + if (a.startsWith('-')) { + if (a === '--colors') { + Colors.enabled = true; + } else if (a === '--coverage') { + emitCoverage = true; + } else if (a === '--verbose') { + verbose = true; + } else if (a === '--list') { + listMode = 'cases'; + } else if (a === '--list-unimplemented') { + listMode = 'unimplemented'; + } else if (a === '--debug') { + debug = true; + } else if (a === '--data') { + dataPath = sys.args[++i]; + } else if (a === '--print-json') { + printJSON = true; + } else if (a === '--expectations') { + const expectationsFile = new URL(sys.args[++i], `file://${sys.cwd()}`).pathname; + loadWebGPUExpectations = import(expectationsFile).then(m => m.expectations); + } else if (a === '--gpu-provider') { + const modulePath = sys.args[++i]; + gpuProviderModule = require(modulePath); + } else if (a === '--gpu-provider-flag') { + gpuProviderFlags.push(sys.args[++i]); + } else if (a === '--quiet') { + quiet = true; + } else if (a === '--unroll-const-eval-loops') { + globalTestConfig.unrollConstEvalLoops = true; + } else { + console.log('unrecognized flag: ', a); + usage(1); + } + } else { + queries.push(a); + } +} + +let codeCoverage: CodeCoverageProvider | undefined = undefined; + +if (gpuProviderModule) { + setGPUProvider(() => gpuProviderModule!.create(gpuProviderFlags)); + if (emitCoverage) { + codeCoverage = gpuProviderModule.coverage; + if (codeCoverage === undefined) { + console.error( + `--coverage specified, but the GPUProviderModule does not support code coverage. +Did you remember to build with code coverage instrumentation enabled?` + ); + sys.exit(1); + } + } +} + +if (dataPath !== undefined) { + dataCache.setStore({ + load: (path: string) => { + return new Promise((resolve, reject) => { + fs.readFile(`${dataPath}/${path}`, 'utf8', (err, data) => { + if (err !== null) { + reject(err.message); + } else { + resolve(data); + } + }); + }); + }, + }); +} +if (verbose) { + dataCache.setDebugLogger(console.log); +} + +if (queries.length === 0) { + console.log('no queries specified'); + usage(0); +} + +(async () => { + const loader = new DefaultTestFileLoader(); + assert(queries.length === 1, 'currently, there must be exactly one query on the cmd line'); + const filterQuery = parseQuery(queries[0]); + const testcases = await loader.loadCases(filterQuery); + const expectations = parseExpectationsForTestQuery( + await (loadWebGPUExpectations ?? []), + filterQuery + ); + + Logger.globalDebugMode = debug; + const log = new Logger(); + + const failed: Array<[string, LiveTestCaseResult]> = []; + const warned: Array<[string, LiveTestCaseResult]> = []; + const skipped: Array<[string, LiveTestCaseResult]> = []; + + let total = 0; + + if (codeCoverage !== undefined) { + codeCoverage.begin(); + } + + for (const testcase of testcases) { + const name = testcase.query.toString(); + switch (listMode) { + case 'cases': + console.log(name); + continue; + case 'unimplemented': + if (testcase.isUnimplemented) { + console.log(name); + } + continue; + default: + break; + } + + const [rec, res] = log.record(name); + await testcase.run(rec, expectations); + + if (verbose) { + printResults([[name, res]]); + } + + total++; + switch (res.status) { + case 'pass': + break; + case 'fail': + failed.push([name, res]); + break; + case 'warn': + warned.push([name, res]); + break; + case 'skip': + skipped.push([name, res]); + break; + default: + unreachable('unrecognized status'); + } + } + + if (codeCoverage !== undefined) { + const coverage = codeCoverage.end(); + console.log(`Code-coverage: [[${coverage}]]`); + } + + if (listMode !== 'none') { + return; + } + + assert(total > 0, 'found no tests!'); + + // MAINTENANCE_TODO: write results out somewhere (a file?) + if (printJSON) { + console.log(log.asJSON(2)); + } + + if (!quiet) { + if (skipped.length) { + console.log(''); + console.log('** Skipped **'); + printResults(skipped); + } + if (warned.length) { + console.log(''); + console.log('** Warnings **'); + printResults(warned); + } + if (failed.length) { + console.log(''); + console.log('** Failures **'); + printResults(failed); + } + + const passed = total - warned.length - failed.length - skipped.length; + const pct = (x: number) => ((100 * x) / total).toFixed(2); + const rpt = (x: number) => { + const xs = x.toString().padStart(1 + Math.log10(total), ' '); + return `${xs} / ${total} = ${pct(x).padStart(6, ' ')}%`; + }; + console.log(''); + console.log(`** Summary ** +Passed w/o warnings = ${rpt(passed)} +Passed with warnings = ${rpt(warned.length)} +Skipped = ${rpt(skipped.length)} +Failed = ${rpt(failed.length)}`); + } + + if (failed.length || warned.length) { + sys.exit(1); + } +})().catch(ex => { + console.log(ex.stack ?? ex.toString()); + sys.exit(1); +}); + +function printResults(results: Array<[string, LiveTestCaseResult]>): void { + for (const [name, r] of results) { + console.log(`[${r.status}] ${name} (${r.timems}ms). Log:`); + if (r.logs) { + for (const l of r.logs) { + console.log(prettyPrintLog(l)); + } + } + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts new file mode 100644 index 0000000000..bec14694a3 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/options.ts @@ -0,0 +1,22 @@ +let windowURL: URL | undefined = undefined; +function getWindowURL() { + if (windowURL === undefined) { + windowURL = new URL(window.location.toString()); + } + return windowURL; +} + +export function optionEnabled( + opt: string, + searchParams: URLSearchParams = getWindowURL().searchParams +): boolean { + const val = searchParams.get(opt); + return val !== null && val !== '0'; +} + +export function optionString( + opt: string, + searchParams: URLSearchParams = getWindowURL().searchParams +): string { + return searchParams.get(opt) || ''; +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts new file mode 100644 index 0000000000..d2e07ff26d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/sys.ts @@ -0,0 +1,46 @@ +/* eslint no-process-exit: "off" */ +/* eslint @typescript-eslint/no-namespace: "off" */ + +function node() { + const { existsSync } = require('fs'); + + return { + type: 'node', + existsSync, + args: process.argv.slice(2), + cwd: () => process.cwd(), + exit: (code?: number | undefined) => process.exit(code), + }; +} + +declare global { + namespace Deno { + function readFileSync(path: string): Uint8Array; + const args: string[]; + const cwd: () => string; + function exit(code?: number): never; + } +} + +function deno() { + function existsSync(path: string) { + try { + Deno.readFileSync(path); + return true; + } catch (err) { + return false; + } + } + + return { + type: 'deno', + existsSync, + args: Deno.args, + cwd: Deno.cwd, + exit: Deno.exit, + }; +} + +const sys = typeof globalThis.process !== 'undefined' ? node() : deno(); + +export default sys; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts new file mode 100644 index 0000000000..9af555f36d --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker-worker.ts @@ -0,0 +1,32 @@ +import { setBaseResourcePath } from '../../framework/resources.js'; +import { DefaultTestFileLoader } from '../../internal/file_loader.js'; +import { Logger } from '../../internal/logging/logger.js'; +import { parseQuery } from '../../internal/query/parseQuery.js'; +import { TestQueryWithExpectation } from '../../internal/query/query.js'; +import { assert } from '../../util/util.js'; + +// Should be DedicatedWorkerGlobalScope, but importing lib "webworker" conflicts with lib "dom". +/* eslint-disable-next-line @typescript-eslint/no-explicit-any */ +declare const self: any; + +const loader = new DefaultTestFileLoader(); + +setBaseResourcePath('../../../resources'); + +self.onmessage = async (ev: MessageEvent) => { + const query: string = ev.data.query; + const expectations: TestQueryWithExpectation[] = ev.data.expectations; + const debug: boolean = ev.data.debug; + + Logger.globalDebugMode = debug; + const log = new Logger(); + + const testcases = Array.from(await loader.loadCases(parseQuery(query))); + assert(testcases.length === 1, 'worker query resulted in != 1 cases'); + + const testcase = testcases[0]; + const [rec, result] = log.record(testcase.query.toString()); + await testcase.run(rec, expectations); + + self.postMessage({ query, result }); +}; diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts new file mode 100644 index 0000000000..2ddc3a951b --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/helper/test_worker.ts @@ -0,0 +1,44 @@ +import { LogMessageWithStack } from '../../internal/logging/log_message.js'; +import { TransferredTestCaseResult, LiveTestCaseResult } from '../../internal/logging/result.js'; +import { TestCaseRecorder } from '../../internal/logging/test_case_recorder.js'; +import { TestQueryWithExpectation } from '../../internal/query/query.js'; + +export class TestWorker { + private readonly debug: boolean; + private readonly worker: Worker; + private readonly resolvers = new Map void>(); + + constructor(debug: boolean) { + this.debug = debug; + + const selfPath = import.meta.url; + const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); + const workerPath = selfPathDir + '/test_worker-worker.js'; + this.worker = new Worker(workerPath, { type: 'module' }); + this.worker.onmessage = ev => { + const query: string = ev.data.query; + const result: TransferredTestCaseResult = ev.data.result; + if (result.logs) { + for (const l of result.logs) { + Object.setPrototypeOf(l, LogMessageWithStack.prototype); + } + } + this.resolvers.get(query)!(result as LiveTestCaseResult); + + // MAINTENANCE_TODO(kainino0x): update the Logger with this result (or don't have a logger and + // update the entire results JSON somehow at some point). + }; + } + + async run( + rec: TestCaseRecorder, + query: string, + expectations: TestQueryWithExpectation[] = [] + ): Promise { + this.worker.postMessage({ query, expectations, debug: this.debug }); + const workerResult = await new Promise(resolve => { + this.resolvers.set(query, resolve); + }); + rec.injectResult(workerResult); + } +} diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts new file mode 100644 index 0000000000..350a864a34 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/server.ts @@ -0,0 +1,227 @@ +/* eslint no-console: "off" */ + +import * as fs from 'fs'; +import * as http from 'http'; +import { AddressInfo } from 'net'; + +import { dataCache } from '../framework/data_cache.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { prettyPrintLog } from '../internal/logging/log_message.js'; +import { Logger } from '../internal/logging/logger.js'; +import { LiveTestCaseResult, Status } from '../internal/logging/result.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { TestQueryWithExpectation } from '../internal/query/query.js'; +import { TestTreeLeaf } from '../internal/tree.js'; +import { Colors } from '../util/colors.js'; +import { setGPUProvider } from '../util/navigator_gpu.js'; + +import sys from './helper/sys.js'; + +function usage(rc: number): never { + console.log(`Usage: + tools/run_${sys.type} [OPTIONS...] +Options: + --colors Enable ANSI colors in output. + --coverage Add coverage data to each result. + --data Path to the data cache directory. + --verbose Print result/log of every test as it runs. + --gpu-provider Path to node module that provides the GPU implementation. + --gpu-provider-flag Flag to set on the gpu-provider as = + --unroll-const-eval-loops Unrolls loops in constant-evaluation shader execution tests + --u Flag to set on the gpu-provider as = + +Provides an HTTP server used for running tests via an HTTP RPC interface +To run a test, perform an HTTP GET or POST at the URL: + http://localhost:port/run? +To shutdown the server perform an HTTP GET or POST at the URL: + http://localhost:port/terminate +`); + return sys.exit(rc); +} + +interface RunResult { + // The result of the test + status: Status; + // Any additional messages printed + message: string; + // Code coverage data, if the server was started with `--coverage` + // This data is opaque (implementation defined). + coverageData?: string; +} + +// The interface that exposes creation of the GPU, and optional interface to code coverage. +interface GPUProviderModule { + // @returns a GPU with the given flags + create(flags: string[]): GPU; + // An optional interface to a CodeCoverageProvider + coverage?: CodeCoverageProvider; +} + +interface CodeCoverageProvider { + // Starts collecting code coverage + begin(): void; + // Ends collecting of code coverage, returning the coverage data. + // This data is opaque (implementation defined). + end(): string; +} + +if (!sys.existsSync('src/common/runtime/cmdline.ts')) { + console.log('Must be run from repository root'); + usage(1); +} + +Colors.enabled = false; + +let emitCoverage = false; +let verbose = false; +let gpuProviderModule: GPUProviderModule | undefined = undefined; +let dataPath: string | undefined = undefined; + +const gpuProviderFlags: string[] = []; +for (let i = 0; i < sys.args.length; ++i) { + const a = sys.args[i]; + if (a.startsWith('-')) { + if (a === '--colors') { + Colors.enabled = true; + } else if (a === '--coverage') { + emitCoverage = true; + } else if (a === '--data') { + dataPath = sys.args[++i]; + } else if (a === '--gpu-provider') { + const modulePath = sys.args[++i]; + gpuProviderModule = require(modulePath); + } else if (a === '--gpu-provider-flag') { + gpuProviderFlags.push(sys.args[++i]); + } else if (a === '--unroll-const-eval-loops') { + globalTestConfig.unrollConstEvalLoops = true; + } else if (a === '--help') { + usage(1); + } else if (a === '--verbose') { + verbose = true; + } else { + console.log(`unrecognised flag: ${a}`); + } + } +} + +let codeCoverage: CodeCoverageProvider | undefined = undefined; + +if (gpuProviderModule) { + setGPUProvider(() => gpuProviderModule!.create(gpuProviderFlags)); + + if (emitCoverage) { + codeCoverage = gpuProviderModule.coverage; + if (codeCoverage === undefined) { + console.error( + `--coverage specified, but the GPUProviderModule does not support code coverage. +Did you remember to build with code coverage instrumentation enabled?` + ); + sys.exit(1); + } + } +} + +if (dataPath !== undefined) { + dataCache.setStore({ + load: (path: string) => { + return new Promise((resolve, reject) => { + fs.readFile(`${dataPath}/${path}`, 'utf8', (err, data) => { + if (err !== null) { + reject(err.message); + } else { + resolve(data); + } + }); + }); + }, + }); +} +if (verbose) { + dataCache.setDebugLogger(console.log); +} + +(async () => { + Logger.globalDebugMode = verbose; + const log = new Logger(); + const testcases = new Map(); + + async function runTestcase( + testcase: TestTreeLeaf, + expectations: TestQueryWithExpectation[] = [] + ): Promise { + const name = testcase.query.toString(); + const [rec, res] = log.record(name); + await testcase.run(rec, expectations); + return res; + } + + const server = http.createServer( + async (request: http.IncomingMessage, response: http.ServerResponse) => { + if (request.url === undefined) { + response.end('invalid url'); + return; + } + + const loadCasesPrefix = '/load?'; + const runPrefix = '/run?'; + const terminatePrefix = '/terminate'; + + if (request.url.startsWith(loadCasesPrefix)) { + const query = request.url.substr(loadCasesPrefix.length); + try { + const webgpuQuery = parseQuery(query); + const loader = new DefaultTestFileLoader(); + for (const testcase of await loader.loadCases(webgpuQuery)) { + testcases.set(testcase.query.toString(), testcase); + } + response.statusCode = 200; + response.end(); + } catch (err) { + response.statusCode = 500; + response.end(`load failed with error: ${err}\n${(err as Error).stack}`); + } + } else if (request.url.startsWith(runPrefix)) { + const name = request.url.substr(runPrefix.length); + try { + const testcase = testcases.get(name); + if (testcase) { + if (codeCoverage !== undefined) { + codeCoverage.begin(); + } + const result = await runTestcase(testcase); + const coverageData = codeCoverage !== undefined ? codeCoverage.end() : undefined; + let message = ''; + if (result.logs !== undefined) { + message = result.logs.map(log => prettyPrintLog(log)).join('\n'); + } + const status = result.status; + const res: RunResult = { status, message, coverageData }; + response.statusCode = 200; + response.end(JSON.stringify(res)); + } else { + response.statusCode = 404; + response.end(`test case '${name}' not found`); + } + } catch (err) { + response.statusCode = 500; + response.end(`run failed with error: ${err}`); + } + } else if (request.url.startsWith(terminatePrefix)) { + server.close(); + sys.exit(1); + } else { + response.statusCode = 404; + response.end('unhandled url request'); + } + } + ); + + server.listen(0, () => { + const address = server.address() as AddressInfo; + console.log(`Server listening at [[${address.port}]]`); + }); +})().catch(ex => { + console.error(ex.stack ?? ex.toString()); + sys.exit(1); +}); diff --git a/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts b/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts new file mode 100644 index 0000000000..0dd158fd68 --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/runtime/standalone.ts @@ -0,0 +1,625 @@ +// Implements the standalone test runner (see also: /standalone/index.html). + +import { dataCache } from '../framework/data_cache.js'; +import { setBaseResourcePath } from '../framework/resources.js'; +import { globalTestConfig } from '../framework/test_config.js'; +import { DefaultTestFileLoader } from '../internal/file_loader.js'; +import { Logger } from '../internal/logging/logger.js'; +import { LiveTestCaseResult } from '../internal/logging/result.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { TestQueryLevel } from '../internal/query/query.js'; +import { TestTreeNode, TestSubtree, TestTreeLeaf, TestTree } from '../internal/tree.js'; +import { setDefaultRequestAdapterOptions } from '../util/navigator_gpu.js'; +import { assert, ErrorWithExtra, unreachable } from '../util/util.js'; + +import { optionEnabled, optionString } from './helper/options.js'; +import { TestWorker } from './helper/test_worker.js'; + +window.onbeforeunload = () => { + // Prompt user before reloading if there are any results + return haveSomeResults ? false : undefined; +}; + +let haveSomeResults = false; + +// The possible options for the tests. +interface StandaloneOptions { + runnow: boolean; + worker: boolean; + debug: boolean; + unrollConstEvalLoops: boolean; + powerPreference: string; +} + +// Extra per option info. +interface StandaloneOptionInfo { + description: string; + parser?: (key: string) => boolean | string; + selectValueDescriptions?: { value: string; description: string }[]; +} + +// Type for info for every option. This definition means adding an option +// will generate a compile time error if not extra info is provided. +type StandaloneOptionsInfos = Record; + +const optionsInfo: StandaloneOptionsInfos = { + runnow: { description: 'run immediately on load' }, + worker: { description: 'run in a worker' }, + debug: { description: 'show more info' }, + unrollConstEvalLoops: { description: 'unroll const eval loops in WGSL' }, + powerPreference: { + description: 'set default powerPreference for some tests', + parser: optionString, + selectValueDescriptions: [ + { value: '', description: 'default' }, + { value: 'low-power', description: 'low-power' }, + { value: 'high-performance', description: 'high-performance' }, + ], + }, +}; + +/** + * Converts camel case to snake case. + * Examples: + * fooBar -> foo_bar + * parseHTMLFile -> parse_html_file + */ +function camelCaseToSnakeCase(id: string) { + return id + .replace(/(.)([A-Z][a-z]+)/g, '$1_$2') + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .toLowerCase(); +} + +/** + * Creates a StandaloneOptions from the current URL search parameters. + */ +function getOptionsInfoFromSearchParameters( + optionsInfos: StandaloneOptionsInfos +): StandaloneOptions { + const optionValues: Record = {}; + for (const [optionName, info] of Object.entries(optionsInfos)) { + const parser = info.parser || optionEnabled; + optionValues[optionName] = parser(camelCaseToSnakeCase(optionName)); + } + return (optionValues as unknown) as StandaloneOptions; +} + +// This is just a cast in one place. +function optionsToRecord(options: StandaloneOptions) { + return (options as unknown) as Record; +} + +const options = getOptionsInfoFromSearchParameters(optionsInfo); +const { runnow, debug, unrollConstEvalLoops, powerPreference } = options; +globalTestConfig.unrollConstEvalLoops = unrollConstEvalLoops; + +Logger.globalDebugMode = debug; +const logger = new Logger(); + +setBaseResourcePath('../out/resources'); + +const worker = options.worker ? new TestWorker(debug) : undefined; + +const autoCloseOnPass = document.getElementById('autoCloseOnPass') as HTMLInputElement; +const resultsVis = document.getElementById('resultsVis')!; +const progressElem = document.getElementById('progress')!; +const progressTestNameElem = progressElem.querySelector('.progress-test-name')!; +const stopButtonElem = progressElem.querySelector('button')!; +let runDepth = 0; +let stopRequested = false; + +stopButtonElem.addEventListener('click', () => { + stopRequested = true; +}); + +if (powerPreference) { + setDefaultRequestAdapterOptions({ powerPreference: powerPreference as GPUPowerPreference }); +} + +dataCache.setStore({ + load: async (path: string) => { + const response = await fetch(`data/${path}`); + if (!response.ok) { + return Promise.reject(response.statusText); + } + return await response.text(); + }, +}); + +interface SubtreeResult { + pass: number; + fail: number; + warn: number; + skip: number; + total: number; + timems: number; +} + +function emptySubtreeResult() { + return { pass: 0, fail: 0, warn: 0, skip: 0, total: 0, timems: 0 }; +} + +function mergeSubtreeResults(...results: SubtreeResult[]) { + const target = emptySubtreeResult(); + for (const result of results) { + target.pass += result.pass; + target.fail += result.fail; + target.warn += result.warn; + target.skip += result.skip; + target.total += result.total; + target.timems += result.timems; + } + return target; +} + +type SetCheckedRecursively = () => void; +type GenerateSubtreeHTML = (parent: HTMLElement) => SetCheckedRecursively; +type RunSubtree = () => Promise; + +interface VisualizedSubtree { + generateSubtreeHTML: GenerateSubtreeHTML; + runSubtree: RunSubtree; +} + +// DOM generation + +function memoize(fn: () => T): () => T { + let value: T | undefined; + return () => { + if (value === undefined) { + value = fn(); + } + return value; + }; +} + +function makeTreeNodeHTML(tree: TestTreeNode, parentLevel: TestQueryLevel): VisualizedSubtree { + let subtree: VisualizedSubtree; + + if ('children' in tree) { + subtree = makeSubtreeHTML(tree, parentLevel); + } else { + subtree = makeCaseHTML(tree); + } + + const generateMyHTML = (parentElement: HTMLElement) => { + const div = $('

').appendTo(parentElement)[0]; + return subtree.generateSubtreeHTML(div); + }; + return { runSubtree: subtree.runSubtree, generateSubtreeHTML: generateMyHTML }; +} + +function makeCaseHTML(t: TestTreeLeaf): VisualizedSubtree { + // Becomes set once the case has been run once. + let caseResult: LiveTestCaseResult | undefined; + + // Becomes set once the DOM for this case exists. + let clearRenderedResult: (() => void) | undefined; + let updateRenderedResult: (() => void) | undefined; + + const name = t.query.toString(); + const runSubtree = async () => { + if (clearRenderedResult) clearRenderedResult(); + + const result: SubtreeResult = emptySubtreeResult(); + progressTestNameElem.textContent = name; + + haveSomeResults = true; + const [rec, res] = logger.record(name); + caseResult = res; + if (worker) { + await worker.run(rec, name); + } else { + await t.run(rec); + } + + result.total++; + result.timems += caseResult.timems; + switch (caseResult.status) { + case 'pass': + result.pass++; + break; + case 'fail': + result.fail++; + break; + case 'skip': + result.skip++; + break; + case 'warn': + result.warn++; + break; + default: + unreachable(); + } + + if (updateRenderedResult) updateRenderedResult(); + + return result; + }; + + const generateSubtreeHTML = (div: HTMLElement) => { + div.classList.add('testcase'); + + const caselogs = $('
').addClass('testcaselogs').hide(); + const [casehead, setChecked] = makeTreeNodeHeaderHTML(t, runSubtree, 2, checked => { + checked ? caselogs.show() : caselogs.hide(); + }); + const casetime = $('
').addClass('testcasetime').html('ms').appendTo(casehead); + div.appendChild(casehead); + div.appendChild(caselogs[0]); + + clearRenderedResult = () => { + div.removeAttribute('data-status'); + casetime.text('ms'); + caselogs.empty(); + }; + + updateRenderedResult = () => { + if (caseResult) { + div.setAttribute('data-status', caseResult.status); + + casetime.text(caseResult.timems.toFixed(4) + ' ms'); + + if (caseResult.logs) { + caselogs.empty(); + for (const l of caseResult.logs) { + const caselog = $('
').addClass('testcaselog').appendTo(caselogs); + $('