1 files changed, 259 insertions, 0 deletions
diff --git a/remote/test/puppeteer/tools/mochaRunner/src/main.ts b/remote/test/puppeteer/tools/mochaRunner/src/main.ts
new file mode 100644
index 0000000000..d2547e721c
--- /dev/null
+++ b/remote/test/puppeteer/tools/mochaRunner/src/main.ts
@@ -0,0 +1,259 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {randomUUID} from 'crypto';
+import fs from 'fs';
+import {spawn, SpawnOptions} from 'node:child_process';
+import os from 'os';
+import path from 'path';
+
+import {
+  TestExpectation,
+  MochaResults,
+  zTestSuiteFile,
+  zPlatform,
+  TestSuite,
+  TestSuiteFile,
+  Platform,
+} from './types.js';
+import {
+  extendProcessEnv,
+  filterByPlatform,
+  readJSON,
+  filterByParameters,
+  getExpectationUpdates,
+  printSuggestions,
+  RecommendedExpectation,
+  writeJSON,
+} from './utils.js';
+
+function getApplicableTestSuites(
+  parsedSuitesFile: TestSuiteFile,
+  platform: Platform
+): TestSuite[] {
+  const testSuiteArgIdx = process.argv.indexOf('--test-suite');
+  let applicableSuites: TestSuite[] = [];
+
+  if (testSuiteArgIdx === -1) {
+    applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform);
+  } else {
+    const testSuiteId = process.argv[testSuiteArgIdx + 1];
+    const testSuite = parsedSuitesFile.testSuites.find(suite => {
+      return suite.id === testSuiteId;
+    });
+
+    if (!testSuite) {
+      console.error(`Test suite ${testSuiteId} is not defined`);
+      process.exit(1);
+    }
+
+    if (!testSuite.platforms.includes(platform)) {
+      console.warn(
+        `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.`
+      );
+    }
+
+    applicableSuites = [testSuite];
+  }
+
+  return applicableSuites;
+}
+
+async function main() {
+  const noCoverage = process.argv.indexOf('--no-coverage') !== -1;
+  const noSuggestions = process.argv.indexOf('--no-suggestions') !== -1;
+
+  const statsFilenameIdx = process.argv.indexOf('--save-stats-to');
+  let statsFilename = '';
+  if (statsFilenameIdx !== -1) {
+    statsFilename = process.argv[statsFilenameIdx + 1] as string;
+    if (statsFilename.includes('INSERTID')) {
+      statsFilename = statsFilename.replace(/INSERTID/gi, randomUUID());
+    }
+  }
+
+  const platform = zPlatform.parse(os.platform());
+
+  const expectations = readJSON(
+    path.join(process.cwd(), 'test', 'TestExpectations.json')
+  ) as TestExpectation[];
+
+  const parsedSuitesFile = zTestSuiteFile.parse(
+    readJSON(path.join(process.cwd(), 'test', 'TestSuites.json'))
+  );
+
+  const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform);
+
+  console.log('Planning to run the following test suites', applicableSuites);
+  if (statsFilename) {
+    console.log('Test stats will be saved to', statsFilename);
+  }
+
+  let fail = false;
+  const recommendations: RecommendedExpectation[] = [];
+  try {
+    for (const suite of applicableSuites) {
+      const parameters = suite.parameters;
+
+      const applicableExpectations = filterByParameters(
+        filterByPlatform(expectations, platform),
+        parameters
+      ).reverse();
+
+      // Add more logging when the GitHub Action Debugging option is set
+      // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+      const githubActionDebugging = process.env['RUNNER_DEBUG']
+        ? {
+            DEBUG: 'puppeteer:*',
+            EXTRA_LAUNCH_OPTIONS: JSON.stringify({
+              extraPrefsFirefox: {
+                'remote.log.level': 'Trace',
+              },
+            }),
+          }
+        : {};
+
+      const env = extendProcessEnv([
+        ...parameters.map(param => {
+          return parsedSuitesFile.parameterDefinitions[param];
+        }),
+        {
+          PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify(
+            applicableExpectations.map(ex => {
+              return {
+                testIdPattern: ex.testIdPattern,
+                skip: ex.expectations.includes('SKIP'),
+              };
+            })
+          ),
+        },
+        githubActionDebugging,
+      ]);
+
+      const tmpDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), 'puppeteer-test-runner-')
+      );
+      const tmpFilename = statsFilename
+        ? statsFilename
+        : path.join(tmpDir, 'output.json');
+      console.log('Running', JSON.stringify(parameters), tmpFilename);
+      const reporterArgumentIndex = process.argv.indexOf('--reporter');
+      const args = [
+        '-u',
+        path.join(__dirname, 'interface.js'),
+        '-R',
+        reporterArgumentIndex === -1
+          ? path.join(__dirname, 'reporter.js')
+          : process.argv[reporterArgumentIndex + 1] || '',
+        '-O',
+        'output=' + tmpFilename,
+      ];
+      const retriesArgumentIndex = process.argv.indexOf('--retries');
+      const timeoutArgumentIndex = process.argv.indexOf('--timeout');
+      if (retriesArgumentIndex > -1) {
+        args.push('--retries', process.argv[retriesArgumentIndex + 1] || '');
+      }
+      if (timeoutArgumentIndex > -1) {
+        args.push('--timeout', process.argv[timeoutArgumentIndex + 1] || '');
+      }
+      if (process.argv.indexOf('--no-parallel')) {
+        args.push('--no-parallel');
+      }
+      if (process.argv.indexOf('--fullTrace')) {
+        args.push('--fullTrace');
+      }
+      const spawnArgs: SpawnOptions = {
+        shell: true,
+        cwd: process.cwd(),
+        stdio: 'inherit',
+        env,
+      };
+      const handle = noCoverage
+        ? spawn('npx', ['mocha', ...args], spawnArgs)
+        : spawn(
+            'npx',
+            [
+              'c8',
+              '--check-coverage',
+              '--lines',
+              String(suite.expectedLineCoverage),
+              'npx mocha',
+              ...args,
+            ],
+            spawnArgs
+          );
+      await new Promise<void>((resolve, reject) => {
+        handle.on('error', err => {
+          reject(err);
+        });
+        handle.on('close', () => {
+          resolve();
+        });
+      });
+      console.log('Finished', JSON.stringify(parameters));
+      try {
+        const results = readJSON(tmpFilename) as MochaResults;
+        const updates = getExpectationUpdates(results, applicableExpectations, {
+          platforms: [os.platform()],
+          parameters,
+        });
+        results.parameters = parameters;
+        results.platform = platform;
+        results.date = new Date().toISOString();
+        if (updates.length > 0) {
+          fail = true;
+          recommendations.push(...updates);
+          results.updates = updates;
+          writeJSON(tmpFilename, results);
+        } else {
+          console.log('Test run matches expectations');
+          writeJSON(tmpFilename, results);
+          continue;
+        }
+      } catch (err) {
+        fail = true;
+        console.error(err);
+      }
+    }
+  } catch (err) {
+    fail = true;
+    console.error(err);
+  } finally {
+    if (!noSuggestions) {
+      printSuggestions(
+        recommendations,
+        'add',
+        'Add the following to TestExpectations.json to ignore the error:'
+      );
+      printSuggestions(
+        recommendations,
+        'remove',
+        'Remove the following from the TestExpectations.json to ignore the error:'
+      );
+      printSuggestions(
+        recommendations,
+        'update',
+        'Update the following expectations in the TestExpectations.json to ignore the error:'
+      );
+    }
+    process.exit(fail ? 1 : 0);
+  }
+}
+
+main().catch(error => {
+  console.error(error);
+  process.exit(1);
+});