diff options
Diffstat (limited to 'remote/test/puppeteer/tools/mocha-runner/src')
6 files changed, 1097 insertions, 0 deletions
diff --git a/remote/test/puppeteer/tools/mocha-runner/src/interface.ts b/remote/test/puppeteer/tools/mocha-runner/src/interface.ts new file mode 100644 index 0000000000..fe0f7e18b5 --- /dev/null +++ b/remote/test/puppeteer/tools/mocha-runner/src/interface.ts @@ -0,0 +1,191 @@ +/** + * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +import Mocha from 'mocha'; +import commonInterface from 'mocha/lib/interfaces/common'; +import { + setLogCapture, + getCapturedLogs, +} from 'puppeteer-core/internal/common/Debug.js'; + +import {testIdMatchesExpectationPattern} from './utils.js'; + +type SuiteFunction = ((this: Mocha.Suite) => void) | undefined; +type ExclusiveSuiteFunction = (this: Mocha.Suite) => void; + +const skippedTests: Array<{testIdPattern: string; skip: true}> = process.env[ + 'PUPPETEER_SKIPPED_TEST_CONFIG' +] + ? JSON.parse(process.env['PUPPETEER_SKIPPED_TEST_CONFIG']) + : []; + +const deflakeRetries = Number( + process.env['PUPPETEER_DEFLAKE_RETRIES'] + ? process.env['PUPPETEER_DEFLAKE_RETRIES'] + : 100 +); +const deflakeTestPattern: string | undefined = + process.env['PUPPETEER_DEFLAKE_TESTS']; + +function shouldSkipTest(test: Mocha.Test): boolean { + // TODO: more efficient lookup. + const definition = skippedTests.find(skippedTest => { + return testIdMatchesExpectationPattern(test, skippedTest.testIdPattern); + }); + if (definition && definition.skip) { + return true; + } + return false; +} + +function shouldDeflakeTest(test: Mocha.Test): boolean { + if (deflakeTestPattern) { + // TODO: cache if we have seen it already + return testIdMatchesExpectationPattern(test, deflakeTestPattern); + } + return false; +} + +function dumpLogsIfFail(this: Mocha.Context) { + if (this.currentTest?.state === 'failed') { + console.log( + `\n"${this.currentTest.fullTitle()}" failed. Here is a debug log:` + ); + console.log(getCapturedLogs().join('\n') + '\n'); + } + setLogCapture(false); +} + +function customBDDInterface(suite: Mocha.Suite) { + const suites: [Mocha.Suite] = [suite]; + + suite.on( + Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE, + function (context, file, mocha) { + const common = commonInterface(suites, context, mocha); + + context['before'] = common.before; + context['after'] = common.after; + context['beforeEach'] = common.beforeEach; + context['afterEach'] = common.afterEach; + if (mocha.options.delay) { + context['run'] = common.runWithSuite(suite); + } + function describe(title: string, fn: SuiteFunction) { + return common.suite.create({ + title: title, + file: file, + fn: fn, + }); + } + describe.only = function (title: string, fn: ExclusiveSuiteFunction) { + return common.suite.only({ + title: title, + file: file, + fn: fn, + isOnly: true, + }); + }; + + describe.skip = function (title: string, fn: SuiteFunction) { + return common.suite.skip({ + title: title, + file: file, + fn: fn, + }); + }; + + describe.withDebugLogs = function ( + description: string, + body: (this: Mocha.Suite) => void + ): void { + context['describe']('with Debug Logs', () => { + context['beforeEach'](() => { + setLogCapture(true); + }); + context['afterEach'](dumpLogsIfFail); + context['describe'](description, body); + }); + }; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + context['describe'] = describe; + + function it(title: string, fn: Mocha.TestFunction, itOnly = false) { + const suite = suites[0]! as Mocha.Suite; + const test = new Mocha.Test(title, suite.isPending() ? undefined : fn); + test.file = file; + test.parent = suite; + + const describeOnly = Boolean( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + suite.parent?._onlySuites.find(child => { + return child === suite; + }) + ); + if (shouldDeflakeTest(test)) { + const deflakeSuit = Mocha.Suite.create(suite, 'with Debug Logs'); + test.file = file; + deflakeSuit.beforeEach(function () { + setLogCapture(true); + }); + deflakeSuit.afterEach(dumpLogsIfFail); + for (let i = 0; i < deflakeRetries; i++) { + deflakeSuit.addTest(test.clone()); + } + return test; + } else if (!(itOnly || describeOnly) && shouldSkipTest(test)) { + const test = new Mocha.Test(title); + test.file = file; + suite.addTest(test); + return test; + } else { + suite.addTest(test); + return test; + } + } + + it.only = function (title: string, fn: Mocha.TestFunction) { + return common.test.only( + mocha, + (context['it'] as unknown as typeof it)(title, fn, true) + ); + }; + + it.skip = function (title: string) { + return context['it'](title); + }; + + function wrapDeflake( + func: Function + ): (repeats: number, title: string, fn: Mocha.AsyncFunc) => void { + return (repeats: number, title: string, fn: Mocha.AsyncFunc): void => { + (context['describe'] as unknown as typeof describe).withDebugLogs( + 'with Debug Logs', + () => { + for (let i = 1; i <= repeats; i++) { + func(`${i}/${title}`, fn); + } + } + ); + }; + } + + it.deflake = wrapDeflake(it); + it.deflakeOnly = wrapDeflake(it.only); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + context.it = it; + } + ); +} + +customBDDInterface.description = 'Custom BDD'; + +module.exports = customBDDInterface; diff --git a/remote/test/puppeteer/tools/mocha-runner/src/mocha-runner.ts b/remote/test/puppeteer/tools/mocha-runner/src/mocha-runner.ts new file mode 100644 index 0000000000..1707e4cc41 --- /dev/null +++ b/remote/test/puppeteer/tools/mocha-runner/src/mocha-runner.ts @@ -0,0 +1,330 @@ +#! /usr/bin/env node + +/** + * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +import {randomUUID} from 'crypto'; +import fs from 'fs'; +import {spawn} from 'node:child_process'; +import os from 'os'; +import path from 'path'; + +import {globSync} from 'glob'; +import yargs from 'yargs'; +import {hideBin} from 'yargs/helpers'; + +import { + zPlatform, + zTestSuiteFile, + type MochaResults, + type Platform, + type TestExpectation, + type TestSuite, + type TestSuiteFile, +} from './types.js'; +import { + extendProcessEnv, + filterByParameters, + filterByPlatform, + getExpectationUpdates, + printSuggestions, + readJSON, + writeJSON, + type RecommendedExpectation, +} from './utils.js'; + +const { + _: mochaArgs, + testSuite: testSuiteId, + saveStatsTo, + cdpTests: includeCdpTests, + suggestions: provideSuggestions, + coverage: useCoverage, + minTests, + shard, + reporter, + printMemory, +} = yargs(hideBin(process.argv)) + .parserConfiguration({'unknown-options-as-args': true}) + .scriptName('@puppeteer/mocha-runner') + .option('coverage', { + boolean: true, + default: true, + }) + .option('suggestions', { + boolean: true, + default: true, + }) + .option('cdp-tests', { + boolean: true, + default: true, + }) + .option('save-stats-to', { + string: true, + requiresArg: true, + }) + .option('min-tests', { + number: true, + default: 0, + requiresArg: true, + }) + .option('test-suite', { + string: true, + requiresArg: true, + }) + .option('shard', { + string: true, + requiresArg: true, + }) + .option('reporter', { + string: true, + requiresArg: true, + }) + .option('print-memory', { + boolean: true, + default: false, + }) + .parseSync(); + +function getApplicableTestSuites( + parsedSuitesFile: TestSuiteFile, + platform: Platform +): TestSuite[] { + let applicableSuites: TestSuite[] = []; + + if (!testSuiteId) { + applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform); + } else { + const testSuite = parsedSuitesFile.testSuites.find(suite => { + return suite.id === testSuiteId; + }); + + if (!testSuite) { + console.error(`Test suite ${testSuiteId} is not defined`); + process.exit(1); + } + + if (!testSuite.platforms.includes(platform)) { + console.warn( + `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.` + ); + } + + applicableSuites = [testSuite]; + } + + return applicableSuites; +} + +async function main() { + let statsPath = saveStatsTo; + if (statsPath && statsPath.includes('INSERTID')) { + statsPath = statsPath.replace(/INSERTID/gi, randomUUID()); + } + + const platform = zPlatform.parse(os.platform()); + + const expectations = readJSON( + path.join(process.cwd(), 'test', 'TestExpectations.json') + ) as TestExpectation[]; + + const parsedSuitesFile = zTestSuiteFile.parse( + readJSON(path.join(process.cwd(), 'test', 'TestSuites.json')) + ); + + const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform); + + console.log('Planning to run the following test suites', applicableSuites); + if (statsPath) { + console.log('Test stats will be saved to', statsPath); + } + + let fail = false; + const recommendations: RecommendedExpectation[] = []; + try { + for (const suite of applicableSuites) { + const parameters = suite.parameters; + + const applicableExpectations = filterByParameters( + filterByPlatform(expectations, platform), + parameters + ).reverse(); + + // Add more logging when the GitHub Action Debugging option is set + // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + const githubActionDebugging = process.env['RUNNER_DEBUG'] + ? { + DEBUG: 'puppeteer:*', + EXTRA_LAUNCH_OPTIONS: JSON.stringify({ + dumpio: true, + extraPrefsFirefox: { + 'remote.log.level': 'Trace', + }, + }), + } + : {}; + + const env = extendProcessEnv([ + ...parameters.map(param => { + return parsedSuitesFile.parameterDefinitions[param]; + }), + { + PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify( + applicableExpectations.map(ex => { + return { + testIdPattern: ex.testIdPattern, + skip: ex.expectations.includes('SKIP'), + }; + }) + ), + }, + githubActionDebugging, + ]); + + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-test-runner-') + ); + const tmpFilename = statsPath + ? statsPath + : path.join(tmpDir, 'output.json'); + console.log('Running', JSON.stringify(parameters), tmpFilename); + const args = [ + '-u', + path.join(__dirname, 'interface.js'), + '-R', + !reporter ? path.join(__dirname, 'reporter.js') : reporter, + '-O', + `output=${tmpFilename}`, + '-n', + 'trace-warnings', + ]; + + if (printMemory) { + args.push('-n', 'expose-gc'); + } + + const specPattern = 'test/build/**/*.spec.js'; + const specs = globSync(specPattern, { + ignore: !includeCdpTests ? 'test/build/cdp/**/*.spec.js' : undefined, + }).sort((a, b) => { + return a.localeCompare(b); + }); + if (shard) { + // Shard ID is 1-based. + const [shardId, shards] = shard.split('-').map(s => { + return Number(s); + }) as [number, number]; + const argsLength = args.length; + for (let i = 0; i < specs.length; i++) { + if (i % shards === shardId - 1) { + args.push(specs[i]!); + } + } + if (argsLength === args.length) { + throw new Error('Shard did not result in any test files'); + } + console.log( + `Running shard ${shardId}-${shards}. Picked ${ + args.length - argsLength + } files out of ${specs.length}.` + ); + } else { + args.push(...specs); + } + const handle = spawn( + 'npx', + [ + ...(useCoverage + ? [ + 'c8', + '--check-coverage', + '--lines', + String(suite.expectedLineCoverage), + 'npx', + ] + : []), + 'mocha', + ...mochaArgs.map(String), + ...args, + ], + { + shell: true, + cwd: process.cwd(), + stdio: 'inherit', + env, + } + ); + await new Promise<void>((resolve, reject) => { + handle.on('error', err => { + reject(err); + }); + handle.on('close', () => { + resolve(); + }); + }); + console.log('Finished', JSON.stringify(parameters)); + try { + const results = readJSON(tmpFilename) as MochaResults; + const updates = getExpectationUpdates(results, applicableExpectations, { + platforms: [os.platform()], + parameters, + }); + const totalTests = results.stats.tests; + results.parameters = parameters; + results.platform = platform; + results.date = new Date().toISOString(); + if (updates.length > 0) { + fail = true; + recommendations.push(...updates); + results.updates = updates; + writeJSON(tmpFilename, results); + } else { + if (!shard && totalTests < minTests) { + fail = true; + console.log( + `Test run matches expectations but the number of discovered tests is too low (expected: ${minTests}, actual: ${totalTests}).` + ); + writeJSON(tmpFilename, results); + continue; + } + console.log('Test run matches expectations'); + writeJSON(tmpFilename, results); + continue; + } + } catch (err) { + fail = true; + console.error(err); + } + } + } catch (err) { + fail = true; + console.error(err); + } finally { + if (!!provideSuggestions) { + printSuggestions( + recommendations, + 'add', + 'Add the following to TestExpectations.json to ignore the error:' + ); + printSuggestions( + recommendations, + 'remove', + 'Remove the following from the TestExpectations.json to ignore the error:' + ); + printSuggestions( + recommendations, + 'update', + 'Update the following expectations in the TestExpectations.json to ignore the error:' + ); + } + process.exit(fail ? 1 : 0); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/remote/test/puppeteer/tools/mocha-runner/src/reporter.ts b/remote/test/puppeteer/tools/mocha-runner/src/reporter.ts new file mode 100644 index 0000000000..7acd5319fe --- /dev/null +++ b/remote/test/puppeteer/tools/mocha-runner/src/reporter.ts @@ -0,0 +1,16 @@ +/** + * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +import Mocha from 'mocha'; + +class SpecJSONReporter extends Mocha.reporters.Spec { + constructor(runner: Mocha.Runner, options?: Mocha.MochaOptions) { + super(runner, options); + Mocha.reporters.JSON.call(this, runner, options); + } +} + +module.exports = SpecJSONReporter; diff --git a/remote/test/puppeteer/tools/mocha-runner/src/test.ts b/remote/test/puppeteer/tools/mocha-runner/src/test.ts new file mode 100644 index 0000000000..5510966235 --- /dev/null +++ b/remote/test/puppeteer/tools/mocha-runner/src/test.ts @@ -0,0 +1,212 @@ +/** + * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; + +import type {Platform, TestExpectation, MochaTestResult} from './types.js'; +import { + filterByParameters, + getTestResultForFailure, + isWildCardPattern, + testIdMatchesExpectationPattern, + getExpectationUpdates, +} from './utils.js'; +import {getFilename, extendProcessEnv} from './utils.js'; + +describe('extendProcessEnv', () => { + it('should extend env variables for the subprocess', () => { + const env = extendProcessEnv([{TEST: 'TEST'}, {TEST2: 'TEST2'}]); + assert.equal(env['TEST'], 'TEST'); + assert.equal(env['TEST2'], 'TEST2'); + }); +}); + +describe('getFilename', () => { + it('extract filename for a path', () => { + assert.equal(getFilename('/etc/test.ts'), 'test'); + assert.equal(getFilename('/etc/test.js'), 'test'); + }); +}); + +describe('getTestResultForFailure', () => { + it('should get a test result for a mocha failure', () => { + assert.equal( + getTestResultForFailure({err: {code: 'ERR_MOCHA_TIMEOUT'}}), + 'TIMEOUT' + ); + assert.equal(getTestResultForFailure({err: {code: 'ERROR'}}), 'FAIL'); + }); +}); + +describe('filterByParameters', () => { + it('should filter a list of expectations by parameters', () => { + const expectations: TestExpectation[] = [ + { + testIdPattern: + '[oopif.spec] OOPIF "after all" hook for "should keep track of a frames OOP state"', + platforms: ['darwin'], + parameters: ['firefox', 'headless'], + expectations: ['FAIL'], + }, + ]; + assert.equal( + filterByParameters(expectations, ['firefox', 'headless']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['firefox']).length, 0); + assert.equal( + filterByParameters(expectations, ['firefox', 'headless', 'other']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['other']).length, 0); + }); +}); + +describe('isWildCardPattern', () => { + it('should detect if an expectation is a wildcard pattern', () => { + assert.equal(isWildCardPattern(''), false); + assert.equal(isWildCardPattern('a'), false); + assert.equal(isWildCardPattern('*'), true); + + assert.equal(isWildCardPattern('[queryHandler.spec]'), false); + assert.equal(isWildCardPattern('[queryHandler.spec] *'), true); + assert.equal(isWildCardPattern(' [queryHandler.spec] '), false); + + assert.equal(isWildCardPattern('[queryHandler.spec] Query'), false); + assert.equal(isWildCardPattern('[queryHandler.spec] Page *'), true); + assert.equal( + isWildCardPattern('[queryHandler.spec] Page Page.goto *'), + true + ); + }); +}); + +describe('testIdMatchesExpectationPattern', () => { + const expectations: Array<[string, boolean]> = [ + ['', false], + ['*', true], + ['* should work', true], + ['* Page.setContent *', true], + ['* should work as expected', false], + ['Page.setContent *', false], + ['[page.spec]', false], + ['[page.spec] *', true], + ['[page.spec] Page *', true], + ['[page.spec] Page Page.setContent *', true], + ['[page.spec] Page Page.setContent should work', true], + ['[page.spec] Page * should work', true], + ['[page.spec] * Page.setContent *', true], + ['[jshandle.spec] *', false], + ['[jshandle.spec] JSHandle should work', false], + ]; + + it('with MochaTest', () => { + const test = { + title: 'should work', + file: 'page.spec.ts', + fullTitle() { + return 'Page Page.setContent should work'; + }, + }; + + for (const [pattern, expected] of expectations) { + assert.equal( + testIdMatchesExpectationPattern(test, pattern), + expected, + `Expected "${pattern}" to yield "${expected}"` + ); + } + }); + + it('with MochaTestResult', () => { + const test: MochaTestResult = { + title: 'should work', + file: 'page.spec.ts', + fullTitle: 'Page Page.setContent should work', + }; + + for (const [pattern, expected] of expectations) { + assert.equal( + testIdMatchesExpectationPattern(test, pattern), + expected, + `Expected "${pattern}" to yield "${expected}"` + ); + } + }); +}); + +describe('getExpectationUpdates', () => { + it('should generate an update for expectations if a test passed with a fail expectation', () => { + const mochaResults = { + stats: {tests: 1}, + pending: [], + passes: [ + { + fullTitle: 'Page Page.setContent should work', + title: 'should work', + file: 'page.spec.ts', + }, + ], + failures: [], + }; + const expectations = [ + { + testIdPattern: '[page.spec] Page Page.setContent should work', + platforms: ['darwin'] as Platform[], + parameters: ['test'], + expectations: ['FAIL' as const], + }, + ]; + const updates = getExpectationUpdates(mochaResults, expectations, { + platforms: ['darwin'] as Platform[], + parameters: ['test'], + }); + assert.deepEqual(updates, [ + { + action: 'remove', + basedOn: { + expectations: ['FAIL'], + parameters: ['test'], + platforms: ['darwin'], + testIdPattern: '[page.spec] Page Page.setContent should work', + }, + expectation: { + expectations: ['FAIL'], + parameters: ['test'], + platforms: ['darwin'], + testIdPattern: '[page.spec] Page Page.setContent should work', + }, + }, + ]); + }); + + it('should not generate an update for successful retries', () => { + const mochaResults = { + stats: {tests: 1}, + pending: [], + passes: [ + { + fullTitle: 'Page Page.setContent should work', + title: 'should work', + file: 'page.spec.ts', + }, + ], + failures: [ + { + fullTitle: 'Page Page.setContent should work', + title: 'should work', + file: 'page.spec.ts', + err: {code: 'Timeout'}, + }, + ], + }; + const updates = getExpectationUpdates(mochaResults, [], { + platforms: ['darwin'], + parameters: ['test'], + }); + assert.deepEqual(updates, []); + }); +}); diff --git a/remote/test/puppeteer/tools/mocha-runner/src/types.ts b/remote/test/puppeteer/tools/mocha-runner/src/types.ts new file mode 100644 index 0000000000..01dc4d6be6 --- /dev/null +++ b/remote/test/puppeteer/tools/mocha-runner/src/types.ts @@ -0,0 +1,57 @@ +/** + * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +import {z} from 'zod'; + +import type {RecommendedExpectation} from './utils.js'; + +export const zPlatform = z.enum(['win32', 'linux', 'darwin']); + +export type Platform = z.infer<typeof zPlatform>; + +export const zTestSuite = z.object({ + id: z.string(), + platforms: z.array(zPlatform), + parameters: z.array(z.string()), + expectedLineCoverage: z.number(), +}); + +export type TestSuite = z.infer<typeof zTestSuite>; + +export const zTestSuiteFile = z.object({ + testSuites: z.array(zTestSuite), + parameterDefinitions: z.record(z.any()), +}); + +export type TestSuiteFile = z.infer<typeof zTestSuiteFile>; + +export type TestResult = 'PASS' | 'FAIL' | 'TIMEOUT' | 'SKIP'; + +export interface TestExpectation { + testIdPattern: string; + platforms: NodeJS.Platform[]; + parameters: string[]; + expectations: TestResult[]; +} + +export interface MochaTestResult { + fullTitle: string; + title: string; + file: string; + err?: {code: string}; +} + +export interface MochaResults { + stats: {tests: number}; + pending: MochaTestResult[]; + passes: MochaTestResult[]; + failures: MochaTestResult[]; + // Added by mocha-runner. + updates?: RecommendedExpectation[]; + parameters?: string[]; + platform?: string; + date?: string; +} diff --git a/remote/test/puppeteer/tools/mocha-runner/src/utils.ts b/remote/test/puppeteer/tools/mocha-runner/src/utils.ts new file mode 100644 index 0000000000..066c5fbe57 --- /dev/null +++ b/remote/test/puppeteer/tools/mocha-runner/src/utils.ts @@ -0,0 +1,291 @@ +/** + * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'fs'; +import path from 'path'; + +import type { + MochaTestResult, + TestExpectation, + MochaResults, + TestResult, +} from './types.js'; + +export function extendProcessEnv(envs: object[]): NodeJS.ProcessEnv { + const env = envs.reduce( + (acc: object, item: object) => { + Object.assign(acc, item); + return acc; + }, + { + ...process.env, + } + ); + + if (process.env['CI']) { + const puppeteerEnv = Object.entries(env).reduce( + (acc, [key, value]) => { + if (key.startsWith('PUPPETEER_')) { + acc[key] = value; + } + + return acc; + }, + {} as Record<string, unknown> + ); + + console.log( + 'PUPPETEER env:\n', + JSON.stringify(puppeteerEnv, null, 2), + '\n' + ); + } + + return env as NodeJS.ProcessEnv; +} + +export function getFilename(file: string): string { + return path.basename(file).replace(path.extname(file), ''); +} + +export function readJSON(path: string): unknown { + return JSON.parse(fs.readFileSync(path, 'utf-8')); +} + +export function writeJSON(path: string, json: unknown): unknown { + return fs.writeFileSync(path, JSON.stringify(json, null, 2)); +} + +export function filterByPlatform<T extends {platforms: NodeJS.Platform[]}>( + items: T[], + platform: NodeJS.Platform +): T[] { + return items.filter(item => { + return item.platforms.includes(platform); + }); +} + +export function prettyPrintJSON(json: unknown): void { + console.log(JSON.stringify(json, null, 2)); +} + +export function printSuggestions( + recommendations: RecommendedExpectation[], + action: RecommendedExpectation['action'], + message: string +): void { + const toPrint = recommendations.filter(item => { + return item.action === action; + }); + if (toPrint.length) { + console.log(message); + prettyPrintJSON( + toPrint.map(item => { + return item.expectation; + }) + ); + if (action !== 'remove') { + console.log( + 'The recommendations are based on the following applied expectations:' + ); + prettyPrintJSON( + toPrint.map(item => { + return item.basedOn; + }) + ); + } + } +} + +export function filterByParameters( + expectations: TestExpectation[], + parameters: string[] +): TestExpectation[] { + const querySet = new Set(parameters); + return expectations.filter(ex => { + return ex.parameters.every(param => { + return querySet.has(param); + }); + }); +} + +/** + * The last expectation that matches an empty string as all tests pattern + * or the name of the file or the whole name of the test the filter wins. + */ +export function findEffectiveExpectationForTest( + expectations: TestExpectation[], + result: MochaTestResult +): TestExpectation | undefined { + return expectations.find(expectation => { + return testIdMatchesExpectationPattern(result, expectation.testIdPattern); + }); +} + +export interface RecommendedExpectation { + expectation: TestExpectation; + action: 'remove' | 'add' | 'update'; + basedOn?: TestExpectation; +} + +export function isWildCardPattern(testIdPattern: string): boolean { + return testIdPattern.includes('*'); +} + +export function getExpectationUpdates( + results: MochaResults, + expectations: TestExpectation[], + context: { + platforms: NodeJS.Platform[]; + parameters: string[]; + } +): RecommendedExpectation[] { + const output = new Map<string, RecommendedExpectation>(); + + const passesByKey = results.passes.reduce((acc, pass) => { + acc.add(getTestId(pass.file, pass.fullTitle)); + return acc; + }, new Set()); + + for (const pass of results.passes) { + const expectationEntry = findEffectiveExpectationForTest( + expectations, + pass + ); + if (expectationEntry && !expectationEntry.expectations.includes('PASS')) { + if (isWildCardPattern(expectationEntry.testIdPattern)) { + addEntry({ + expectation: { + testIdPattern: getTestId(pass.file, pass.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: ['PASS'], + }, + action: 'add', + basedOn: expectationEntry, + }); + } else { + addEntry({ + expectation: expectationEntry, + action: 'remove', + basedOn: expectationEntry, + }); + } + } + } + + for (const failure of results.failures) { + // If an error occurs during a hook + // the error not have a file associated with it + if (!failure.file) { + console.error('Hook failed:', failure.err); + addEntry({ + expectation: { + testIdPattern: failure.fullTitle, + platforms: context.platforms, + parameters: context.parameters, + expectations: [], + }, + action: 'add', + }); + continue; + } + + if (passesByKey.has(getTestId(failure.file, failure.fullTitle))) { + continue; + } + + const expectationEntry = findEffectiveExpectationForTest( + expectations, + failure + ); + if (expectationEntry && !expectationEntry.expectations.includes('SKIP')) { + if ( + !expectationEntry.expectations.includes( + getTestResultForFailure(failure) + ) + ) { + // If the effective explanation is a wildcard, we recommend adding a new + // expectation instead of updating the wildcard that might affect multiple + // tests. + if (isWildCardPattern(expectationEntry.testIdPattern)) { + addEntry({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + action: 'add', + basedOn: expectationEntry, + }); + } else { + addEntry({ + expectation: { + ...expectationEntry, + expectations: [ + ...expectationEntry.expectations, + getTestResultForFailure(failure), + ], + }, + action: 'update', + basedOn: expectationEntry, + }); + } + } + } else if (!expectationEntry) { + addEntry({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + action: 'add', + }); + } + } + + function addEntry(value: RecommendedExpectation) { + const key = JSON.stringify(value); + if (!output.has(key)) { + output.set(key, value); + } + } + + return [...output.values()]; +} + +export function getTestResultForFailure( + test: Pick<MochaTestResult, 'err'> +): TestResult { + return test.err?.code === 'ERR_MOCHA_TIMEOUT' ? 'TIMEOUT' : 'FAIL'; +} + +export function getTestId(file: string, fullTitle?: string): string { + return fullTitle + ? `[${getFilename(file)}] ${fullTitle}` + : `[${getFilename(file)}]`; +} + +export function testIdMatchesExpectationPattern( + test: MochaTestResult | Pick<Mocha.Test, 'title' | 'file' | 'fullTitle'>, + pattern: string +): boolean { + const patternRegExString = pattern + // Replace `*` with non special character + .replace(/\*/g, '--STAR--') + // Escape special characters https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // Replace placeholder with greedy match + .replace(/--STAR--/g, '(.*)?'); + // Match beginning and end explicitly + const patternRegEx = new RegExp(`^${patternRegExString}$`); + const fullTitle = + typeof test.fullTitle === 'string' ? test.fullTitle : test.fullTitle(); + + return patternRegEx.test(getTestId(test.file ?? '', fullTitle)); +} |