diff options
Diffstat (limited to 'dom/webgpu/tests/cts/checkout/src/common/internal')
12 files changed, 141 insertions, 35 deletions
diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/file_loader.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/file_loader.ts index b5e1b1a446..aae4b87995 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/file_loader.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/file_loader.ts @@ -73,8 +73,9 @@ export abstract class TestFileLoader extends EventTarget { query: TestQuery, { subqueriesToExpand = [], + fullyExpandSubtrees = [], maxChunkTime = Infinity, - }: { subqueriesToExpand?: string[]; maxChunkTime?: number } = {} + }: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[]; maxChunkTime?: number } = {} ): Promise<TestTree> { const tree = await loadTreeForQuery(this, query, { subqueriesToExpand: subqueriesToExpand.map(s => { @@ -82,6 +83,7 @@ export abstract class TestFileLoader extends EventTarget { assert(q.level >= 2, () => `subqueriesToExpand entries should not be multi-file:\n ${q}`); return q; }), + fullyExpandSubtrees: fullyExpandSubtrees.map(s => parseQuery(s)), maxChunkTime, }); this.dispatchEvent(new MessageEvent<void>('finish')); diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/log_message.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/log_message.ts index ee006cdeb3..b01c08b56e 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/log_message.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/log_message.ts @@ -1,19 +1,36 @@ import { ErrorWithExtra } from '../../util/util.js'; import { extractImportantStackTrace } from '../stack.js'; +import { LogMessageRawData } from './result.js'; + export class LogMessageWithStack extends Error { readonly extra: unknown; private stackHiddenMessage: string | undefined = undefined; - constructor(name: string, ex: Error | ErrorWithExtra) { - super(ex.message); + /** + * Wrap an Error (which was created to capture the stack at that point) into a + * LogMessageWithStack (which has extra stuff for good log messages). + * + * The original `ex.name` is ignored. Inclued it in the `name` parameter if it + * needs to be preserved. + */ + static wrapError(name: string, ex: Error | ErrorWithExtra) { + return new LogMessageWithStack({ + name, + message: ex.message, + stackHiddenMessage: undefined, + stack: ex.stack, + extra: 'extra' in ex ? ex.extra : undefined, + }); + } - this.name = name; - this.stack = ex.stack; - if ('extra' in ex) { - this.extra = ex.extra; - } + constructor(o: LogMessageRawData) { + super(o.message); + this.name = o.name; + this.stackHiddenMessage = o.stackHiddenMessage; + this.stack = o.stack; + this.extra = o.extra; } /** Set a flag so the stack is not printed in toJSON(). */ @@ -21,6 +38,11 @@ export class LogMessageWithStack extends Error { this.stackHiddenMessage ??= stackHiddenMessage; } + /** + * Print the message for display. + * + * Note: This is toJSON instead of toString to make it easy to save logs using JSON.stringify. + */ toJSON(): string { let m = this.name; if (this.message) m += ': ' + this.message; @@ -33,6 +55,21 @@ export class LogMessageWithStack extends Error { } return m; } + + /** + * Flatten the message for sending over a message channel. + * + * Note `extra` may get mangled by postMessage. + */ + toRawData(): LogMessageRawData { + return { + name: this.name, + message: this.message, + stackHiddenMessage: this.stackHiddenMessage, + stack: this.stack, + extra: this.extra, + }; + } } /** diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/logger.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/logger.ts index e4526cff54..6b95f48b74 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/logger.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/logger.ts @@ -1,3 +1,4 @@ +import { globalTestConfig } from '../../framework/test_config.js'; import { version } from '../version.js'; import { LiveTestCaseResult } from './result.js'; @@ -6,8 +7,6 @@ import { TestCaseRecorder } from './test_case_recorder.js'; export type LogResults = Map<string, LiveTestCaseResult>; export class Logger { - static globalDebugMode: boolean = false; - readonly overriddenDebugMode: boolean | undefined; readonly results: LogResults = new Map(); @@ -19,7 +18,7 @@ export class Logger { const result: LiveTestCaseResult = { status: 'running', timems: -1 }; this.results.set(name, result); return [ - new TestCaseRecorder(result, this.overriddenDebugMode ?? Logger.globalDebugMode), + new TestCaseRecorder(result, this.overriddenDebugMode ?? globalTestConfig.enableDebugLogs), result, ]; } diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/result.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/result.ts index 3318e8c937..9968f3d359 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/result.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/result.ts @@ -14,8 +14,24 @@ export interface LiveTestCaseResult extends TestCaseResult { logs?: LogMessageWithStack[]; } +/** + * Raw data for a test log message. + * + * This form is sendable over a message channel, except `extra` may get mangled. + */ +export interface LogMessageRawData { + name: string; + message: string; + stackHiddenMessage: string | undefined; + stack: string | undefined; + extra: unknown; +} + +/** + * Test case results in a form sendable over a message channel. + * + * Note `extra` may get mangled by postMessage. + */ export interface TransferredTestCaseResult extends TestCaseResult { - // When transferred from a worker, a LogMessageWithStack turns into a generic Error - // (its prototype gets lost and replaced with Error). - logs?: Error[]; + logs?: LogMessageRawData[]; } diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/test_case_recorder.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/test_case_recorder.ts index f5c3252b5c..78f625269e 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/logging/test_case_recorder.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/logging/test_case_recorder.ts @@ -45,8 +45,6 @@ export class TestCaseRecorder { private logs: LogMessageWithStack[] = []; private logLinesAtCurrentSeverity = 0; private debugging = false; - /** Used to dedup log messages which have identical stacks. */ - private messagesForPreviouslySeenStacks = new Map<string, LogMessageWithStack>(); constructor(result: LiveTestCaseResult, debugging: boolean) { this.result = result; @@ -143,13 +141,15 @@ export class TestCaseRecorder { this.skipped(ex); return; } - this.logImpl(LogSeverity.ThrewException, 'EXCEPTION', ex); + // logImpl will discard the original error's ex.name. Preserve it here. + const name = ex instanceof Error ? `EXCEPTION: ${ex.name}` : 'EXCEPTION'; + this.logImpl(LogSeverity.ThrewException, name, ex); } private logImpl(level: LogSeverity, name: string, baseException: unknown): void { assert(baseException instanceof Error, 'test threw a non-Error object'); globalTestConfig.testHeartbeatCallback(); - const logMessage = new LogMessageWithStack(name, baseException); + const logMessage = LogMessageWithStack.wrapError(name, baseException); // Final case status should be the "worst" of all log entries. if (this.inSubCase) { diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts index a9419b87c1..f49833f5a2 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/compare.ts @@ -58,7 +58,10 @@ function compareOneLevel(ordering: Ordering, aIsBig: boolean, bIsBig: boolean): return Ordering.Unordered; } -function comparePaths(a: readonly string[], b: readonly string[]): Ordering { +/** + * Compare two file paths, or file-local test paths, returning an Ordering between the two. + */ +export function comparePaths(a: readonly string[], b: readonly string[]): Ordering { const shorter = Math.min(a.length, b.length); for (let i = 0; i < shorter; ++i) { diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts index 996835b0ec..0a9b355804 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/parseQuery.ts @@ -17,12 +17,49 @@ import { import { kBigSeparator, kWildcard, kPathSeparator, kParamSeparator } from './separators.js'; import { validQueryPart } from './validQueryPart.js'; -export function parseQuery(s: string): TestQuery { +/** + * converts foo/bar/src/webgpu/this/that/file.spec.ts to webgpu:this,that,file,* + */ +function convertPathToQuery(path: string) { + // removes .spec.ts and splits by directory separators. + const parts = path.substring(0, path.length - 8).split(/\/|\\/g); + // Gets parts only after the last `src`. Example: returns ['webgpu', 'foo', 'bar', 'test'] + // for ['Users', 'me', 'src', 'cts', 'src', 'webgpu', 'foo', 'bar', 'test'] + const partsAfterSrc = parts.slice(parts.lastIndexOf('src') + 1); + const suite = partsAfterSrc.shift(); + return `${suite}:${partsAfterSrc.join(',')},*`; +} + +/** + * If a query looks like a path (ends in .spec.ts and has directory separators) + * then convert try to convert it to a query. + */ +function convertPathLikeToQuery(queryOrPath: string) { + return queryOrPath.endsWith('.spec.ts') && + (queryOrPath.includes('/') || queryOrPath.includes('\\')) + ? convertPathToQuery(queryOrPath) + : queryOrPath; +} + +/** + * Convert long suite names (the part before the first colon) to the + * shortest last word + * foo.bar.moo:test,subtest,foo -> moo:test,subtest,foo + */ +function shortenSuiteName(query: string) { + const parts = query.split(':'); + // converts foo.bar.moo to moo + const suite = parts.shift()?.replace(/.*\.(\w+)$/, '$1'); + return [suite, ...parts].join(':'); +} + +export function parseQuery(queryLike: string): TestQuery { try { - return parseQueryImpl(s); + const query = shortenSuiteName(convertPathLikeToQuery(queryLike)); + return parseQueryImpl(query); } catch (ex) { if (ex instanceof Error) { - ex.message += '\n on: ' + s; + ex.message += `\n on: ${queryLike}`; } throw ex; } diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts index 7c72a62f88..676ac46d38 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/query/query.ts @@ -1,5 +1,5 @@ import { TestParams } from '../../framework/fixture.js'; -import { optionEnabled } from '../../runtime/helper/options.js'; +import { optionWorkerMode } from '../../runtime/helper/options.js'; import { assert, unreachable } from '../../util/util.js'; import { Expectation } from '../logging/result.js'; @@ -188,12 +188,12 @@ export function parseExpectationsForTestQuery( assert( expectationURL.pathname === wptURL.pathname, `Invalid expectation path ${expectationURL.pathname} -Expectation should be of the form path/to/cts.https.html?worker=0&q=suite:test_path:test_name:foo=1;bar=2;... +Expectation should be of the form path/to/cts.https.html?debug=0&q=suite:test_path:test_name:foo=1;bar=2;... ` ); const params = expectationURL.searchParams; - if (optionEnabled('worker', params) !== optionEnabled('worker', wptURL.searchParams)) { + if (optionWorkerMode('worker', params) !== optionWorkerMode('worker', wptURL.searchParams)) { continue; } diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts index 632a822ef1..e1d0cde12d 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/test_group.ts @@ -34,7 +34,7 @@ import { validQueryPart } from '../internal/query/validQueryPart.js'; import { DeepReadonly } from '../util/types.js'; import { assert, unreachable } from '../util/util.js'; -import { logToWebsocket } from './websocket_logger.js'; +import { logToWebSocket } from './websocket_logger.js'; export type RunFn = ( rec: TestCaseRecorder, @@ -294,9 +294,11 @@ class TestBuilder<S extends SubcaseBatchState, F extends Fixture> { (this.description ? this.description + '\n\n' : '') + 'TODO: .unimplemented()'; this.isUnimplemented = true; - this.testFn = () => { + // Use the beforeFn to skip the test, so we don't have to iterate the subcases. + this.beforeFn = () => { throw new SkipTestCase('test unimplemented'); }; + this.testFn = () => {}; } /** Perform various validation/"lint" chenks. */ @@ -350,7 +352,7 @@ class TestBuilder<S extends SubcaseBatchState, F extends Fixture> { const testcaseStringUnique = stringifyPublicParamsUniquely(params); assert( !seen.has(testcaseStringUnique), - `Duplicate public test case+subcase params for test ${testPathString}: ${testcaseString}` + `Duplicate public test case+subcase params for test ${testPathString}: ${testcaseString} (${caseQuery})` ); seen.add(testcaseStringUnique); } @@ -737,7 +739,7 @@ class RunCaseSpecific implements RunCase { timems: rec.result.timems, nonskippedSubcaseCount: rec.nonskippedSubcaseCount, }; - logToWebsocket(JSON.stringify(msg)); + logToWebSocket(JSON.stringify(msg)); } } } diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts index 2d2b555366..c5a0e11448 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/test_suite_listing.ts @@ -1,6 +1,6 @@ // A listing of all specs within a single suite. This is the (awaited) type of // `groups` in '{cts,unittests}/listing.ts' and `listing` in the auto-generated -// 'out/{cts,unittests}/listing.js' files (see tools/gen_listings). +// 'out/{cts,unittests}/listing.js' files (see tools/gen_listings_and_webworkers). export type TestSuiteListing = TestSuiteListingEntry[]; export type TestSuiteListingEntry = TestSuiteListingEntrySpec | TestSuiteListingEntryReadme; diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts index 594837059c..f2fad59037 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/tree.ts @@ -286,8 +286,9 @@ export async function loadTreeForQuery( queryToLoad: TestQuery, { subqueriesToExpand, + fullyExpandSubtrees = [], maxChunkTime = Infinity, - }: { subqueriesToExpand: TestQuery[]; maxChunkTime?: number } + }: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[]; maxChunkTime?: number } ): Promise<TestTree> { const suite = queryToLoad.suite; const specs = await loader.listing(suite); @@ -303,6 +304,10 @@ export async function loadTreeForQuery( // If toExpand == subquery, no expansion is needed (but it's still "seen"). if (ordering === Ordering.Equal) seenSubqueriesToExpand[i] = true; return ordering !== Ordering.StrictSubset; + }) && + fullyExpandSubtrees.every(toExpand => { + const ordering = compareQueries(toExpand, subquery); + return ordering === Ordering.Unordered; }); // L0 = suite-level, e.g. suite:* diff --git a/dom/webgpu/tests/cts/checkout/src/common/internal/websocket_logger.ts b/dom/webgpu/tests/cts/checkout/src/common/internal/websocket_logger.ts index 30246df843..373378e7c2 100644 --- a/dom/webgpu/tests/cts/checkout/src/common/internal/websocket_logger.ts +++ b/dom/webgpu/tests/cts/checkout/src/common/internal/websocket_logger.ts @@ -1,3 +1,5 @@ +import { globalTestConfig } from '../framework/test_config.js'; + /** * - 'uninitialized' means we haven't tried to connect yet * - Promise means it's pending @@ -8,12 +10,15 @@ let connection: Promise<WebSocket | 'failed'> | WebSocket | 'failed' | 'uninitia 'uninitialized'; /** - * Log a string to a websocket at `localhost:59497`. See `tools/websocket-logger`. + * If the logToWebSocket option is enabled (?log_to_web_socket=1 in browser, + * --log-to-web-socket on command line, or enable it by default in options.ts), + * log a string to a websocket at `localhost:59497`. See `tools/websocket-logger`. * - * This does nothing if a connection couldn't be established on the first call. + * This does nothing if a logToWebSocket is not enabled, or if a connection + * couldn't be established on the first call. */ -export function logToWebsocket(msg: string) { - if (connection === 'failed') { +export function logToWebSocket(msg: string) { + if (!globalTestConfig.logToWebSocket || connection === 'failed') { return; } |