diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /dom/webgpu/tests/cts/checkout/src/common/tools/merge_listing_times.ts | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/webgpu/tests/cts/checkout/src/common/tools/merge_listing_times.ts')
-rw-r--r-- | dom/webgpu/tests/cts/checkout/src/common/tools/merge_listing_times.ts | 177 |
1 files changed, 177 insertions, 0 deletions
diff --git a/dom/webgpu/tests/cts/checkout/src/common/tools/merge_listing_times.ts b/dom/webgpu/tests/cts/checkout/src/common/tools/merge_listing_times.ts new file mode 100644 index 0000000000..fb33ae20fb --- /dev/null +++ b/dom/webgpu/tests/cts/checkout/src/common/tools/merge_listing_times.ts @@ -0,0 +1,177 @@ +import * as fs from 'fs'; +import * as process from 'process'; +import * as readline from 'readline'; + +import { TestMetadataListing } from '../framework/metadata.js'; +import { parseQuery } from '../internal/query/parseQuery.js'; +import { TestQueryMultiCase, TestQuerySingleCase } from '../internal/query/query.js'; +import { CaseTimingLogLine } from '../internal/test_group.js'; +import { assert } from '../util/util.js'; + +// For information on listing_meta.json file maintenance, please read +// tools/merge_listing_times first. + +function usage(rc: number): never { + console.error(`Usage: tools/merge_listing_times [options] SUITES... -- [TIMING_LOG_FILES...] + +Options: + --help Print this message and exit. + +Reads raw case timing data for each suite in SUITES, from all TIMING_LOG_FILES +(see below), and merges it into the src/*/listing_meta.json files checked into +the repository. The timing data in the listing_meta.json files is updated with +the newly-observed timing data *if the new timing is slower*. That is, it will +only increase the values in the listing_meta.json file, and will only cause WPT +chunks to become smaller. + +If there are no TIMING_LOG_FILES, this just regenerates (reformats) the file +using the data already present. + +In more detail: + +- Reads per-case timing data in any of the SUITES, from all TIMING_LOG_FILES + (ignoring skipped cases), and averages it over the number of subcases. + In the case of cases that have run multiple times, takes the max of each. +- Compiles the average time-per-subcase for each test seen. +- For each suite seen, loads its listing_meta.json, takes the max of the old and + new data, and writes it back out. + +How to generate TIMING_LOG_FILES files: + +- Launch the 'websocket-logger' tool (see its README.md), which listens for + log messages on localhost:59497. +- Run the tests you want to capture data for, on the same system. Since + logging is done through the websocket side-channel, you can run the tests + under any runtime (standalone, WPT, etc.) as long as WebSocket support is + available (always true in browsers). +- Run \`tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-*.txt\` +`); + process.exit(rc); +} + +const kHeader = `{ + "_comment": "SEMI AUTO-GENERATED: Please read docs/adding_timing_metadata.md.", +`; +const kFooter = `\ + "_end": "" +} +`; + +const argv = process.argv; +if (argv.some(v => v.startsWith('-') && v !== '--') || argv.every(v => v !== '--')) { + usage(0); +} +const suites = []; +const timingLogFilenames = []; +let seenDashDash = false; +for (const arg of argv.slice(2)) { + if (arg === '--') { + seenDashDash = true; + continue; + } else if (arg.startsWith('-')) { + usage(0); + } + + if (seenDashDash) { + timingLogFilenames.push(arg); + } else { + suites.push(arg); + } +} +if (!seenDashDash) { + usage(0); +} + +void (async () => { + // Read the log files to find the log line for each *case* query. If a case + // ran multiple times, take the one with the largest average subcase time. + const caseTimes = new Map<string, CaseTimingLogLine>(); + for (const timingLogFilename of timingLogFilenames) { + const rl = readline.createInterface({ + input: fs.createReadStream(timingLogFilename), + crlfDelay: Infinity, + }); + + for await (const line of rl) { + const parsed: CaseTimingLogLine = JSON.parse(line); + + const prev = caseTimes.get(parsed.q); + if (prev !== undefined) { + const timePerSubcase = parsed.timems / Math.max(1, parsed.nonskippedSubcaseCount); + const prevTimePerSubcase = prev.timems / Math.max(1, prev.nonskippedSubcaseCount); + + if (timePerSubcase > prevTimePerSubcase) { + caseTimes.set(parsed.q, parsed); + } + } else { + caseTimes.set(parsed.q, parsed); + } + } + } + + // Accumulate total times per test. Map of suite -> query -> {totalTimeMS, caseCount}. + const testTimes = new Map<string, Map<string, { totalTimeMS: number; subcaseCount: number }>>(); + for (const suite of suites) { + testTimes.set(suite, new Map()); + } + for (const [caseQString, caseTime] of caseTimes) { + const caseQ = parseQuery(caseQString); + assert(caseQ instanceof TestQuerySingleCase); + const suite = caseQ.suite; + const suiteTestTimes = testTimes.get(suite); + if (suiteTestTimes === undefined) { + continue; + } + + const testQ = new TestQueryMultiCase(suite, caseQ.filePathParts, caseQ.testPathParts, {}); + const testQString = testQ.toString(); + + const prev = suiteTestTimes.get(testQString); + if (prev !== undefined) { + prev.totalTimeMS += caseTime.timems; + prev.subcaseCount += caseTime.nonskippedSubcaseCount; + } else { + suiteTestTimes.set(testQString, { + totalTimeMS: caseTime.timems, + subcaseCount: caseTime.nonskippedSubcaseCount, + }); + } + } + + for (const suite of suites) { + const currentMetadata: TestMetadataListing = JSON.parse( + fs.readFileSync(`./src/${suite}/listing_meta.json`, 'utf8') + ); + + const metadata = { ...currentMetadata }; + for (const [testQString, { totalTimeMS, subcaseCount }] of testTimes.get(suite)!) { + const avgTime = totalTimeMS / Math.max(1, subcaseCount); + if (testQString in metadata) { + metadata[testQString].subcaseMS = Math.max(metadata[testQString].subcaseMS, avgTime); + } else { + metadata[testQString] = { subcaseMS: avgTime }; + } + } + + writeListings(suite, metadata); + } +})(); + +function writeListings(suite: string, metadata: TestMetadataListing) { + const output = fs.createWriteStream(`./src/${suite}/listing_meta.json`); + try { + output.write(kHeader); + const keys = Object.keys(metadata).sort(); + for (const k of keys) { + if (k.startsWith('_')) { + // Ignore json "_comments". + continue; + } + assert(k.indexOf('"') === -1); + output.write(` "${k}": { "subcaseMS": ${metadata[k].subcaseMS.toFixed(3)} },\n`); + } + output.write(kFooter); + } finally { + output.close(); + } +} |