summaryrefslogtreecommitdiffstats
path: root/remote/test/puppeteer/tools/mochaRunner
diff options
context:
space:
mode:
Diffstat (limited to 'remote/test/puppeteer/tools/mochaRunner')
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/README.md73
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/src/interface.ts130
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/src/main.ts259
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/src/reporter.ts26
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/src/test.ts134
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/src/types.ts67
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/src/utils.ts265
-rw-r--r--remote/test/puppeteer/tools/mochaRunner/tsconfig.json11
8 files changed, 965 insertions, 0 deletions
diff --git a/remote/test/puppeteer/tools/mochaRunner/README.md b/remote/test/puppeteer/tools/mochaRunner/README.md
new file mode 100644
index 0000000000..1e4398a63c
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/README.md
@@ -0,0 +1,73 @@
+# Mocha Runner
+
+Mocha Runner is a test runner on top of mocha.
+It uses `/test/TestSuites.json` and `/test/TestExpectations.json` files to run mocha tests in multiple configurations and interpret results.
+
+## Running tests for Mocha Runner itself.
+
+```bash
+npm run build && npx c8 node tools/mochaRunner/lib/test.js
+```
+
+## Running tests using Mocha Runner
+
+```bash
+npm run build && npm run test
+```
+
+By default, the runner runs all test suites applicable to the current platform.
+To pick a test suite, provide the `--test-suite` arguments. For example,
+
+```bash
+npm run build && npm run test -- --test-suite chrome-headless
+```
+
+## TestSuites.json
+
+Define test suites via the `testSuites` attribute. `parameters` can be used in the `TestExpectations.json` to disable tests
+based on parameters. The meaning for parameters is defined in `parameterDefinitions` which tell what env object corresponds
+to the given parameter.
+
+## TestExpectations.json
+
+An expectation looks like this:
+
+```json
+{
+ "testIdPattern": "[accessibility.spec]",
+ "platforms": ["darwin", "win32", "linux"],
+ "parameters": ["firefox"],
+ "expectations": ["SKIP"]
+}
+```
+
+| Field | Description | Type | Match Logic |
+| --------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ----------- |
+| `testIdPattern` | Defines the full name (or pattern) to match against test name | string | - |
+| `platforms` | Defines the platforms the expectation is for | Array<`linux` \| `win32` \|`darwin`> | `OR` |
+| `parameters` | Defines the parameters that the test has to match | Array<[ParameterDefinitions](https://github.com/puppeteer/puppeteer/blob/main/test/TestSuites.json)> | `AND` |
+| `expectations` | The list of test results that are considered to be acceptable | Array<`PASS` \| `FAIL` \| `TIMEOUT` \| `SKIP`> | `OR` |
+
+> Order of defining expectations matters. The latest expectation that is set will take president over earlier ones.
+
+> Adding `SKIP` to `expectations` will prevent the test from running, no matter if there are other expectations.
+
+### Using pattern in `testIdPattern`
+
+Sometimes we want a whole group of test to run. For that we can use a
+pattern to achieve.
+Pattern are defined with the use of `*` (using greedy method).
+
+Examples:
+| Pattern | Description | Example Pattern | Example match |
+|------------------------|---------------------------------------------------------------------------------------------|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| `*` | Match all tests | - | - |
+| `[test.spec] *` | Matches tests for the given file | `[jshandle.spec] *` | `[jshandle] JSHandle JSHandle.toString should work for primitives` |
+| `[test.spec] <text> *` | Matches tests with for a given test with a specific prefixed test (usually a describe node) | `[page.spec] Page Page.goto *` | `[page.spec] Page Page.goto should work`,<br>`[page.spec] Page Page.goto should work with anchor navigation` |
+| `[test.spec] * <text>` | Matches test with a surfix | `[navigation.spec] * should work` | `[navigation.spec] navigation Page.goto should work`,<br>`[navigation.spec] navigation Page.waitForNavigation should work` |
+
+## Updating Expectations
+
+Currently, expectations are updated manually. The test runner outputs the
+suggested changes to the expectation file if the test run does not match
+expectations.
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/interface.ts b/remote/test/puppeteer/tools/mochaRunner/src/interface.ts
new file mode 100644
index 0000000000..79329fcb0d
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/interface.ts
@@ -0,0 +1,130 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Mocha from 'mocha';
+import commonInterface from 'mocha/lib/interfaces/common';
+
+import {testIdMatchesExpectationPattern} from './utils.js';
+
+type SuiteFunction = ((this: Mocha.Suite) => void) | undefined;
+type ExclusiveSuiteFunction = (this: Mocha.Suite) => void;
+
+const skippedTests: Array<{testIdPattern: string; skip: true}> = process.env[
+ 'PUPPETEER_SKIPPED_TEST_CONFIG'
+]
+ ? JSON.parse(process.env['PUPPETEER_SKIPPED_TEST_CONFIG'])
+ : [];
+
+function shouldSkipTest(test: Mocha.Test): boolean {
+ // TODO: more efficient lookup.
+ const definition = skippedTests.find(skippedTest => {
+ return testIdMatchesExpectationPattern(test, skippedTest.testIdPattern);
+ });
+ if (definition && definition.skip) {
+ return true;
+ }
+ return false;
+}
+
+function customBDDInterface(suite: Mocha.Suite) {
+ const suites = [suite];
+
+ suite.on(
+ Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE,
+ function (context, file, mocha) {
+ const common = commonInterface(suites, context, mocha);
+
+ context['before'] = common.before;
+ context['after'] = common.after;
+ context['beforeEach'] = common.beforeEach;
+ context['afterEach'] = common.afterEach;
+ if (mocha.options.delay) {
+ context['run'] = common.runWithSuite(suite);
+ }
+ function describe(title: string, fn: SuiteFunction) {
+ return common.suite.create({
+ title: title,
+ file: file,
+ fn: fn,
+ });
+ }
+ describe.only = function (title: string, fn: ExclusiveSuiteFunction) {
+ return common.suite.only({
+ title: title,
+ file: file,
+ fn: fn,
+ isOnly: true,
+ });
+ };
+
+ describe.skip = function (title: string, fn: SuiteFunction) {
+ return common.suite.skip({
+ title: title,
+ file: file,
+ fn: fn,
+ });
+ };
+
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+ // @ts-ignore
+ context['describe'] = describe;
+
+ function it(title: string, fn: Mocha.TestFunction, itOnly = false) {
+ const suite = suites[0]!;
+ const test = new Mocha.Test(title, suite.isPending() ? undefined : fn);
+ test.file = file;
+ test.parent = suite;
+
+ const describeOnly = Boolean(
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+ // @ts-ignore
+ suite.parent?._onlySuites.find(child => {
+ return child === suite;
+ })
+ );
+
+ if (shouldSkipTest(test) && !(itOnly || describeOnly)) {
+ const test = new Mocha.Test(title);
+ test.file = file;
+ suite.addTest(test);
+ return test;
+ } else {
+ suite.addTest(test);
+ return test;
+ }
+ }
+
+ it.only = function (title: string, fn: Mocha.TestFunction) {
+ return common.test.only(
+ mocha,
+ (context['it'] as typeof it)(title, fn, true)
+ );
+ };
+
+ it.skip = function (title: string) {
+ return context['it'](title);
+ };
+
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+ // @ts-ignore
+ context.it = it;
+ }
+ );
+}
+
+customBDDInterface.description = 'Custom BDD';
+
+module.exports = customBDDInterface;
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/main.ts b/remote/test/puppeteer/tools/mochaRunner/src/main.ts
new file mode 100644
index 0000000000..d2547e721c
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/main.ts
@@ -0,0 +1,259 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {randomUUID} from 'crypto';
+import fs from 'fs';
+import {spawn, SpawnOptions} from 'node:child_process';
+import os from 'os';
+import path from 'path';
+
+import {
+ TestExpectation,
+ MochaResults,
+ zTestSuiteFile,
+ zPlatform,
+ TestSuite,
+ TestSuiteFile,
+ Platform,
+} from './types.js';
+import {
+ extendProcessEnv,
+ filterByPlatform,
+ readJSON,
+ filterByParameters,
+ getExpectationUpdates,
+ printSuggestions,
+ RecommendedExpectation,
+ writeJSON,
+} from './utils.js';
+
+function getApplicableTestSuites(
+ parsedSuitesFile: TestSuiteFile,
+ platform: Platform
+): TestSuite[] {
+ const testSuiteArgIdx = process.argv.indexOf('--test-suite');
+ let applicableSuites: TestSuite[] = [];
+
+ if (testSuiteArgIdx === -1) {
+ applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform);
+ } else {
+ const testSuiteId = process.argv[testSuiteArgIdx + 1];
+ const testSuite = parsedSuitesFile.testSuites.find(suite => {
+ return suite.id === testSuiteId;
+ });
+
+ if (!testSuite) {
+ console.error(`Test suite ${testSuiteId} is not defined`);
+ process.exit(1);
+ }
+
+ if (!testSuite.platforms.includes(platform)) {
+ console.warn(
+ `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.`
+ );
+ }
+
+ applicableSuites = [testSuite];
+ }
+
+ return applicableSuites;
+}
+
+async function main() {
+ const noCoverage = process.argv.indexOf('--no-coverage') !== -1;
+ const noSuggestions = process.argv.indexOf('--no-suggestions') !== -1;
+
+ const statsFilenameIdx = process.argv.indexOf('--save-stats-to');
+ let statsFilename = '';
+ if (statsFilenameIdx !== -1) {
+ statsFilename = process.argv[statsFilenameIdx + 1] as string;
+ if (statsFilename.includes('INSERTID')) {
+ statsFilename = statsFilename.replace(/INSERTID/gi, randomUUID());
+ }
+ }
+
+ const platform = zPlatform.parse(os.platform());
+
+ const expectations = readJSON(
+ path.join(process.cwd(), 'test', 'TestExpectations.json')
+ ) as TestExpectation[];
+
+ const parsedSuitesFile = zTestSuiteFile.parse(
+ readJSON(path.join(process.cwd(), 'test', 'TestSuites.json'))
+ );
+
+ const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform);
+
+ console.log('Planning to run the following test suites', applicableSuites);
+ if (statsFilename) {
+ console.log('Test stats will be saved to', statsFilename);
+ }
+
+ let fail = false;
+ const recommendations: RecommendedExpectation[] = [];
+ try {
+ for (const suite of applicableSuites) {
+ const parameters = suite.parameters;
+
+ const applicableExpectations = filterByParameters(
+ filterByPlatform(expectations, platform),
+ parameters
+ ).reverse();
+
+ // Add more logging when the GitHub Action Debugging option is set
+ // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+ const githubActionDebugging = process.env['RUNNER_DEBUG']
+ ? {
+ DEBUG: 'puppeteer:*',
+ EXTRA_LAUNCH_OPTIONS: JSON.stringify({
+ extraPrefsFirefox: {
+ 'remote.log.level': 'Trace',
+ },
+ }),
+ }
+ : {};
+
+ const env = extendProcessEnv([
+ ...parameters.map(param => {
+ return parsedSuitesFile.parameterDefinitions[param];
+ }),
+ {
+ PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify(
+ applicableExpectations.map(ex => {
+ return {
+ testIdPattern: ex.testIdPattern,
+ skip: ex.expectations.includes('SKIP'),
+ };
+ })
+ ),
+ },
+ githubActionDebugging,
+ ]);
+
+ const tmpDir = fs.mkdtempSync(
+ path.join(os.tmpdir(), 'puppeteer-test-runner-')
+ );
+ const tmpFilename = statsFilename
+ ? statsFilename
+ : path.join(tmpDir, 'output.json');
+ console.log('Running', JSON.stringify(parameters), tmpFilename);
+ const reporterArgumentIndex = process.argv.indexOf('--reporter');
+ const args = [
+ '-u',
+ path.join(__dirname, 'interface.js'),
+ '-R',
+ reporterArgumentIndex === -1
+ ? path.join(__dirname, 'reporter.js')
+ : process.argv[reporterArgumentIndex + 1] || '',
+ '-O',
+ 'output=' + tmpFilename,
+ ];
+ const retriesArgumentIndex = process.argv.indexOf('--retries');
+ const timeoutArgumentIndex = process.argv.indexOf('--timeout');
+ if (retriesArgumentIndex > -1) {
+ args.push('--retries', process.argv[retriesArgumentIndex + 1] || '');
+ }
+ if (timeoutArgumentIndex > -1) {
+ args.push('--timeout', process.argv[timeoutArgumentIndex + 1] || '');
+ }
+ if (process.argv.indexOf('--no-parallel')) {
+ args.push('--no-parallel');
+ }
+ if (process.argv.indexOf('--fullTrace')) {
+ args.push('--fullTrace');
+ }
+ const spawnArgs: SpawnOptions = {
+ shell: true,
+ cwd: process.cwd(),
+ stdio: 'inherit',
+ env,
+ };
+ const handle = noCoverage
+ ? spawn('npx', ['mocha', ...args], spawnArgs)
+ : spawn(
+ 'npx',
+ [
+ 'c8',
+ '--check-coverage',
+ '--lines',
+ String(suite.expectedLineCoverage),
+ 'npx mocha',
+ ...args,
+ ],
+ spawnArgs
+ );
+ await new Promise<void>((resolve, reject) => {
+ handle.on('error', err => {
+ reject(err);
+ });
+ handle.on('close', () => {
+ resolve();
+ });
+ });
+ console.log('Finished', JSON.stringify(parameters));
+ try {
+ const results = readJSON(tmpFilename) as MochaResults;
+ const updates = getExpectationUpdates(results, applicableExpectations, {
+ platforms: [os.platform()],
+ parameters,
+ });
+ results.parameters = parameters;
+ results.platform = platform;
+ results.date = new Date().toISOString();
+ if (updates.length > 0) {
+ fail = true;
+ recommendations.push(...updates);
+ results.updates = updates;
+ writeJSON(tmpFilename, results);
+ } else {
+ console.log('Test run matches expectations');
+ writeJSON(tmpFilename, results);
+ continue;
+ }
+ } catch (err) {
+ fail = true;
+ console.error(err);
+ }
+ }
+ } catch (err) {
+ fail = true;
+ console.error(err);
+ } finally {
+ if (!noSuggestions) {
+ printSuggestions(
+ recommendations,
+ 'add',
+ 'Add the following to TestExpectations.json to ignore the error:'
+ );
+ printSuggestions(
+ recommendations,
+ 'remove',
+ 'Remove the following from the TestExpectations.json to ignore the error:'
+ );
+ printSuggestions(
+ recommendations,
+ 'update',
+ 'Update the following expectations in the TestExpectations.json to ignore the error:'
+ );
+ }
+ process.exit(fail ? 1 : 0);
+ }
+}
+
+main().catch(error => {
+ console.error(error);
+ process.exit(1);
+});
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/reporter.ts b/remote/test/puppeteer/tools/mochaRunner/src/reporter.ts
new file mode 100644
index 0000000000..37ca586215
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/reporter.ts
@@ -0,0 +1,26 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Mocha from 'mocha';
+
+class SpecJSONReporter extends Mocha.reporters.Spec {
+ constructor(runner: Mocha.Runner, options?: Mocha.MochaOptions) {
+ super(runner, options);
+ Mocha.reporters.JSON.call(this, runner, options);
+ }
+}
+
+module.exports = SpecJSONReporter;
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/test.ts b/remote/test/puppeteer/tools/mochaRunner/src/test.ts
new file mode 100644
index 0000000000..1e0328499c
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/test.ts
@@ -0,0 +1,134 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import assert from 'node:assert/strict';
+import {describe, test} from 'node:test';
+
+import {TestExpectation} from './types.js';
+import {
+ filterByParameters,
+ getTestResultForFailure,
+ isWildCardPattern,
+ testIdMatchesExpectationPattern,
+} from './utils.js';
+import {getFilename, extendProcessEnv} from './utils.js';
+
+void test('extendProcessEnv', () => {
+ const env = extendProcessEnv([{TEST: 'TEST'}, {TEST2: 'TEST2'}]);
+ assert.equal(env['TEST'], 'TEST');
+ assert.equal(env['TEST2'], 'TEST2');
+});
+
+void test('getFilename', () => {
+ assert.equal(getFilename('/etc/test.ts'), 'test');
+ assert.equal(getFilename('/etc/test.js'), 'test');
+});
+
+void test('getTestResultForFailure', () => {
+ assert.equal(
+ getTestResultForFailure({err: {code: 'ERR_MOCHA_TIMEOUT'}}),
+ 'TIMEOUT'
+ );
+ assert.equal(getTestResultForFailure({err: {code: 'ERROR'}}), 'FAIL');
+});
+
+void test('filterByParameters', () => {
+ const expectations: TestExpectation[] = [
+ {
+ testIdPattern:
+ '[oopif.spec] OOPIF "after all" hook for "should keep track of a frames OOP state"',
+ platforms: ['darwin'],
+ parameters: ['firefox', 'headless'],
+ expectations: ['FAIL'],
+ },
+ ];
+ assert.equal(
+ filterByParameters(expectations, ['firefox', 'headless']).length,
+ 1
+ );
+ assert.equal(filterByParameters(expectations, ['firefox']).length, 0);
+ assert.equal(
+ filterByParameters(expectations, ['firefox', 'headless', 'other']).length,
+ 1
+ );
+ assert.equal(filterByParameters(expectations, ['other']).length, 0);
+});
+
+void test('isWildCardPattern', () => {
+ assert.equal(isWildCardPattern(''), false);
+ assert.equal(isWildCardPattern('a'), false);
+ assert.equal(isWildCardPattern('*'), true);
+
+ assert.equal(isWildCardPattern('[queryHandler.spec]'), false);
+ assert.equal(isWildCardPattern('[queryHandler.spec] *'), true);
+ assert.equal(isWildCardPattern(' [queryHandler.spec] '), false);
+
+ assert.equal(isWildCardPattern('[queryHandler.spec] Query'), false);
+ assert.equal(isWildCardPattern('[queryHandler.spec] Page *'), true);
+ assert.equal(isWildCardPattern('[queryHandler.spec] Page Page.goto *'), true);
+});
+
+describe('testIdMatchesExpectationPattern', () => {
+ const expectations: Array<[string, boolean]> = [
+ ['', false],
+ ['*', true],
+ ['* should work', true],
+ ['* Page.setContent *', true],
+ ['* should work as expected', false],
+ ['Page.setContent *', false],
+ ['[page.spec]', false],
+ ['[page.spec] *', true],
+ ['[page.spec] Page *', true],
+ ['[page.spec] Page Page.setContent *', true],
+ ['[page.spec] Page Page.setContent should work', true],
+ ['[page.spec] Page * should work', true],
+ ['[page.spec] * Page.setContent *', true],
+ ['[jshandle.spec] *', false],
+ ['[jshandle.spec] JSHandle should work', false],
+ ];
+
+ void test('with MochaTest', () => {
+ const test = {
+ title: 'should work',
+ file: 'page.spec.ts',
+ fullTitle() {
+ return 'Page Page.setContent should work';
+ },
+ } as any;
+
+ for (const [pattern, expected] of expectations) {
+ assert.equal(
+ testIdMatchesExpectationPattern(test, pattern),
+ expected,
+ `Expected "${pattern}" to yield "${expected}"`
+ );
+ }
+ });
+ void test('with MochaTestResult', () => {
+ const test = {
+ title: 'should work',
+ file: 'page.spec.ts',
+ fullTitle: 'Page Page.setContent should work',
+ } as any;
+
+ for (const [pattern, expected] of expectations) {
+ assert.equal(
+ testIdMatchesExpectationPattern(test, pattern),
+ expected,
+ `Expected "${pattern}" to yield "${expected}"`
+ );
+ }
+ });
+});
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/types.ts b/remote/test/puppeteer/tools/mochaRunner/src/types.ts
new file mode 100644
index 0000000000..8d8a08ee98
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/types.ts
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {z} from 'zod';
+
+import {RecommendedExpectation} from './utils.js';
+
+export const zPlatform = z.enum(['win32', 'linux', 'darwin']);
+
+export type Platform = z.infer<typeof zPlatform>;
+
+export const zTestSuite = z.object({
+ id: z.string(),
+ platforms: z.array(zPlatform),
+ parameters: z.array(z.string()),
+ expectedLineCoverage: z.number(),
+});
+
+export type TestSuite = z.infer<typeof zTestSuite>;
+
+export const zTestSuiteFile = z.object({
+ testSuites: z.array(zTestSuite),
+ parameterDefinitions: z.record(z.any()),
+});
+
+export type TestSuiteFile = z.infer<typeof zTestSuiteFile>;
+
+export type TestResult = 'PASS' | 'FAIL' | 'TIMEOUT' | 'SKIP';
+
+export type TestExpectation = {
+ testIdPattern: string;
+ platforms: NodeJS.Platform[];
+ parameters: string[];
+ expectations: TestResult[];
+};
+
+export type MochaTestResult = {
+ fullTitle: string;
+ title: string;
+ file: string;
+ err?: {code: string};
+};
+
+export type MochaResults = {
+ stats: unknown;
+ pending: MochaTestResult[];
+ passes: MochaTestResult[];
+ failures: MochaTestResult[];
+ // Added by mochaRunner.
+ updates?: RecommendedExpectation[];
+ parameters?: string[];
+ platform?: string;
+ date?: string;
+};
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/utils.ts b/remote/test/puppeteer/tools/mochaRunner/src/utils.ts
new file mode 100644
index 0000000000..9fdbf65583
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/utils.ts
@@ -0,0 +1,265 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import {
+ MochaTestResult,
+ TestExpectation,
+ MochaResults,
+ TestResult,
+} from './types.js';
+
+export function extendProcessEnv(envs: object[]): NodeJS.ProcessEnv {
+ return envs.reduce(
+ (acc: object, item: object) => {
+ Object.assign(acc, item);
+ return acc;
+ },
+ {
+ ...process.env,
+ }
+ ) as NodeJS.ProcessEnv;
+}
+
+export function getFilename(file: string): string {
+ return path.basename(file).replace(path.extname(file), '');
+}
+
+export function readJSON(path: string): unknown {
+ return JSON.parse(fs.readFileSync(path, 'utf-8'));
+}
+
+export function writeJSON(path: string, json: unknown): unknown {
+ return fs.writeFileSync(path, JSON.stringify(json, null, 2));
+}
+
+export function filterByPlatform<T extends {platforms: NodeJS.Platform[]}>(
+ items: T[],
+ platform: NodeJS.Platform
+): T[] {
+ return items.filter(item => {
+ return item.platforms.includes(platform);
+ });
+}
+
+export function prettyPrintJSON(json: unknown): void {
+ console.log(JSON.stringify(json, null, 2));
+}
+
+export function printSuggestions(
+ recommendations: RecommendedExpectation[],
+ action: RecommendedExpectation['action'],
+ message: string
+): void {
+ const toPrint = recommendations.filter(item => {
+ return item.action === action;
+ });
+ if (toPrint.length) {
+ console.log(message);
+ prettyPrintJSON(
+ toPrint.map(item => {
+ return item.expectation;
+ })
+ );
+ console.log(
+ 'The recommendations are based on the following applied expectaions:'
+ );
+ prettyPrintJSON(
+ toPrint.map(item => {
+ return item.basedOn;
+ })
+ );
+ }
+}
+
+export function filterByParameters(
+ expectations: TestExpectation[],
+ parameters: string[]
+): TestExpectation[] {
+ const querySet = new Set(parameters);
+ return expectations.filter(ex => {
+ return ex.parameters.every(param => {
+ return querySet.has(param);
+ });
+ });
+}
+
+/**
+ * The last expectation that matches an empty string as all tests pattern
+ * or the name of the file or the whole name of the test the filter wins.
+ */
+export function findEffectiveExpectationForTest(
+ expectations: TestExpectation[],
+ result: MochaTestResult
+): TestExpectation | undefined {
+ return expectations.find(expectation => {
+ return testIdMatchesExpectationPattern(result, expectation.testIdPattern);
+ });
+}
+
+export type RecommendedExpectation = {
+ expectation: TestExpectation;
+ action: 'remove' | 'add' | 'update';
+ basedOn?: TestExpectation;
+};
+
+export function isWildCardPattern(testIdPattern: string): boolean {
+ return testIdPattern.includes('*');
+}
+
+export function getExpectationUpdates(
+ results: MochaResults,
+ expectations: TestExpectation[],
+ context: {
+ platforms: NodeJS.Platform[];
+ parameters: string[];
+ }
+): RecommendedExpectation[] {
+ const output: Map<string, RecommendedExpectation> = new Map();
+
+ for (const pass of results.passes) {
+ // If an error occurs during a hook
+ // the error not have a file associated with it
+ if (!pass.file) {
+ continue;
+ }
+
+ const expectationEntry = findEffectiveExpectationForTest(
+ expectations,
+ pass
+ );
+ if (expectationEntry && !expectationEntry.expectations.includes('PASS')) {
+ if (isWildCardPattern(expectationEntry.testIdPattern)) {
+ addEntry({
+ expectation: {
+ testIdPattern: getTestId(pass.file, pass.fullTitle),
+ platforms: context.platforms,
+ parameters: context.parameters,
+ expectations: ['PASS'],
+ },
+ action: 'add',
+ basedOn: expectationEntry,
+ });
+ } else {
+ addEntry({
+ expectation: expectationEntry,
+ action: 'remove',
+ basedOn: expectationEntry,
+ });
+ }
+ }
+ }
+
+ for (const failure of results.failures) {
+ // If an error occurs during a hook
+ // the error not have a file associated with it
+ if (!failure.file) {
+ continue;
+ }
+
+ const expectationEntry = findEffectiveExpectationForTest(
+ expectations,
+ failure
+ );
+ if (expectationEntry && !expectationEntry.expectations.includes('SKIP')) {
+ if (
+ !expectationEntry.expectations.includes(
+ getTestResultForFailure(failure)
+ )
+ ) {
+ // If the effective explanation is a wildcard, we recommend adding a new
+ // expectation instead of updating the wildcard that might affect multiple
+ // tests.
+ if (isWildCardPattern(expectationEntry.testIdPattern)) {
+ addEntry({
+ expectation: {
+ testIdPattern: getTestId(failure.file, failure.fullTitle),
+ platforms: context.platforms,
+ parameters: context.parameters,
+ expectations: [getTestResultForFailure(failure)],
+ },
+ action: 'add',
+ basedOn: expectationEntry,
+ });
+ } else {
+ addEntry({
+ expectation: {
+ ...expectationEntry,
+ expectations: [
+ ...expectationEntry.expectations,
+ getTestResultForFailure(failure),
+ ],
+ },
+ action: 'update',
+ basedOn: expectationEntry,
+ });
+ }
+ }
+ } else if (!expectationEntry) {
+ addEntry({
+ expectation: {
+ testIdPattern: getTestId(failure.file, failure.fullTitle),
+ platforms: context.platforms,
+ parameters: context.parameters,
+ expectations: [getTestResultForFailure(failure)],
+ },
+ action: 'add',
+ });
+ }
+ }
+
+ function addEntry(value: RecommendedExpectation) {
+ const key = JSON.stringify(value);
+ if (!output.has(key)) {
+ output.set(key, value);
+ }
+ }
+
+ return [...output.values()];
+}
+
+export function getTestResultForFailure(
+ test: Pick<MochaTestResult, 'err'>
+): TestResult {
+ return test.err?.code === 'ERR_MOCHA_TIMEOUT' ? 'TIMEOUT' : 'FAIL';
+}
+
+export function getTestId(file: string, fullTitle?: string): string {
+ return fullTitle
+ ? `[${getFilename(file)}] ${fullTitle}`
+ : `[${getFilename(file)}]`;
+}
+
+export function testIdMatchesExpectationPattern(
+ test: MochaTestResult | Mocha.Test,
+ pattern: string
+): boolean {
+ const patternRegExString = pattern
+ // Replace `*` with non special character
+ .replace(/\*/g, '--STAR--')
+ // Escape special characters https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping
+ .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+ // Replace placeholder with greedy match
+ .replace(/--STAR--/g, '(.*)?');
+ // Match beginning and end explicitly
+ const patternRegEx = new RegExp(`^${patternRegExString}$`);
+ const fullTitle =
+ typeof test.fullTitle === 'string' ? test.fullTitle : test.fullTitle();
+
+ return patternRegEx.test(getTestId(test.file ?? '', fullTitle));
+}
diff --git a/remote/test/puppeteer/tools/mochaRunner/tsconfig.json b/remote/test/puppeteer/tools/mochaRunner/tsconfig.json
new file mode 100644
index 0000000000..c2576c2564
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/tsconfig.json
@@ -0,0 +1,11 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "allowJs": true,
+ "composite": true,
+ "module": "CommonJS",
+ "outDir": "lib",
+ "rootDir": "src"
+ },
+ "include": ["src"]
+}