summaryrefslogtreecommitdiffstats
path: root/devtools/client/bin/devtools-node-test-runner.js
diff options
context:
space:
mode:
Diffstat (limited to 'devtools/client/bin/devtools-node-test-runner.js')
-rw-r--r--devtools/client/bin/devtools-node-test-runner.js322
1 files changed, 322 insertions, 0 deletions
diff --git a/devtools/client/bin/devtools-node-test-runner.js b/devtools/client/bin/devtools-node-test-runner.js
new file mode 100644
index 0000000000..e347ee5855
--- /dev/null
+++ b/devtools/client/bin/devtools-node-test-runner.js
@@ -0,0 +1,322 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+
+/* global __dirname, process */
+
+"use strict";
+
+/**
+ * This is a test runner dedicated to run DevTools node tests continuous integration
+ * platforms. It will parse the logs to output errors compliant with treeherder tooling.
+ *
+ * See taskcluster/ci/source-test/node.yml for the definition of the task running those
+ * tests on try.
+ */
+
+const { execFileSync } = require("child_process");
+const { writeFileSync } = require("fs");
+const { chdir } = require("process");
+const path = require("path");
+const os = require("os");
+
+const REPOSITORY_ROOT = __dirname.replace("devtools/client/bin", "");
+
+// All Windows platforms report "win32", even for 64bit editions.
+const isWin = os.platform() === "win32";
+
+// On Windows, the ".cmd" suffix is mandatory to invoke yarn ; or executables in
+// general.
+const YARN_PROCESS = isWin ? "yarn.cmd" : "yarn";
+
+// Supported node test suites for DevTools
+const TEST_TYPES = {
+ JEST: "jest",
+ TYPESCRIPT: "typescript",
+};
+
+const SUITES = {
+ aboutdebugging: {
+ path: "../aboutdebugging/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ accessibility: {
+ path: "../accessibility/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ application: {
+ path: "../application/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ compatibility: {
+ path: "../inspector/compatibility/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ debugger: {
+ path: "../debugger",
+ type: TEST_TYPES.JEST,
+ },
+ framework: {
+ path: "../framework/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ netmonitor: {
+ path: "../netmonitor/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ performance: {
+ path: "../performance-new",
+ type: TEST_TYPES.TYPESCRIPT,
+ },
+ shared_components: {
+ path: "../shared/components/test/node",
+ type: TEST_TYPES.JEST,
+ },
+ webconsole: {
+ path: "../webconsole/test/node",
+ type: TEST_TYPES.JEST,
+ dependencies: ["../debugger"],
+ },
+};
+
+function execOut(...args) {
+ let out;
+ let err;
+ try {
+ out = execFileSync(...args);
+ } catch (e) {
+ out = e.stdout;
+ err = e.stderr;
+ }
+ return { out: out.toString(), err: err && err.toString() };
+}
+
+function getErrors(suite, out, err, testPath) {
+ switch (SUITES[suite].type) {
+ case TEST_TYPES.JEST:
+ return getJestErrors(out, err);
+ case TEST_TYPES.TYPESCRIPT:
+ return getTypescriptErrors(out, err, testPath);
+ default:
+ throw new Error("Unsupported suite type: " + SUITES[suite].type);
+ }
+}
+
+const JEST_ERROR_SUMMARY_REGEX = /\sā—\s/;
+
+function getJestErrors(out, err) {
+ // The string out has extra content before the JSON object starts.
+ const jestJsonOut = out.substring(out.indexOf("{"), out.lastIndexOf("}") + 1);
+ const results = JSON.parse(jestJsonOut);
+
+ /**
+ * We don't have individual information, but a multiple line string in testResult.message,
+ * which looks like
+ *
+ * ā— Simple function
+ *
+ * expect(received).toEqual(expected) // deep equality
+ *
+ * Expected: false
+ * Received: true
+ *
+ * 391 | url: "test.js",
+ * 392 | });
+ * > 393 | expect(true).toEqual(false);
+ * | ^
+ * 394 | expect(actual.code).toMatchSnapshot();
+ * 395 |
+ * 396 | const smc = await new SourceMapConsumer(actual.map.toJSON());
+ *
+ * at Object.<anonymous> (src/workers/pretty-print/tests/prettyFast.spec.js:393:18)
+ * at asyncGeneratorStep (src/workers/pretty-print/tests/prettyFast.spec.js:7:103)
+ * at _next (src/workers/pretty-print/tests/prettyFast.spec.js:9:194)
+ * at src/workers/pretty-print/tests/prettyFast.spec.js:9:364
+ * at Object.<anonymous> (src/workers/pretty-print/tests/prettyFast.spec.js:9:97)
+ *
+ */
+
+ const errors = [];
+ for (const testResult of results.testResults) {
+ if (testResult.status != "failed") {
+ continue;
+ }
+ let currentError;
+ let errorLine;
+
+ const lines = testResult.message.split("\n");
+ lines.forEach((line, i) => {
+ if (line.match(JEST_ERROR_SUMMARY_REGEX) || i == lines.length - 1) {
+ // This is the name of the test, if we were gathering information from a previous
+ // error, we add it to the errors
+ if (currentError) {
+ errors.push({
+ // The file should be relative from the repository
+ file: testResult.name.replace(REPOSITORY_ROOT, ""),
+ line: errorLine,
+ // we don't have information for the column
+ column: 0,
+ message: currentError.trim(),
+ });
+ }
+
+ // Handle the new error
+ currentError = line;
+ } else {
+ // We put any line that is not a test name in the error message as it may be
+ // valuable for the user.
+ currentError += "\n" + line;
+
+ // The actual line of the error is marked with " > XXX |"
+ const res = line.match(/> (?<line>\d+) \|/);
+ if (res) {
+ errorLine = parseInt(res.groups.line, 10);
+ }
+ }
+ });
+ }
+
+ return errors;
+}
+
+function getTypescriptErrors(out, err, testPath) {
+ console.log(out);
+ // Typescript error lines look like:
+ // popup/panel.jsm.js(103,7): error TS2531: Object is possibly 'null'.
+ // Which means:
+ // {file_path}({line},{col}): error TS{error_code}: {message}
+ const tsErrorRegex =
+ /(?<file>(\w|\/|\.)+)\((?<line>\d+),(?<column>\d+)\): (?<message>error TS\d+\:.*)/;
+ const errors = [];
+ for (const line of out.split("\n")) {
+ const res = line.match(tsErrorRegex);
+ if (!res) {
+ continue;
+ }
+ // TypeScript gives us the path from the directory the command is executed in, so we
+ // need to prepend the directory path.
+ const fileAbsPath = testPath + res.groups.file;
+ errors.push({
+ // The file should be relative from the repository.
+ file: fileAbsPath.replace(REPOSITORY_ROOT, ""),
+ line: parseInt(res.groups.line, 10),
+ column: parseInt(res.groups.column, 10),
+ message: res.groups.message.trim(),
+ });
+ }
+ return errors;
+}
+
+function runTests() {
+ console.log("[devtools-node-test-runner] Extract suite argument");
+ const suiteArg = process.argv.find(arg => arg.includes("suite="));
+ const suite = suiteArg.split("=")[1];
+ if (suite !== "all" && !SUITES[suite]) {
+ throw new Error(
+ "Invalid suite argument to devtools-node-test-runner: " + suite
+ );
+ }
+
+ console.log("[devtools-node-test-runner] Check `yarn` is available");
+ try {
+ // This will throw if yarn is unavailable
+ execFileSync(YARN_PROCESS, ["--version"]);
+ } catch (e) {
+ console.log(
+ "[devtools-node-test-runner] ERROR: `yarn` is not installed. " +
+ "See https://yarnpkg.com/docs/install/ "
+ );
+ return false;
+ }
+
+ const artifactArg = process.argv.find(arg => arg.includes("artifact="));
+ const artifactFilePath = artifactArg && artifactArg.split("=")[1];
+ const artifactErrors = {};
+
+ const failedSuites = [];
+ const suites = suite == "all" ? SUITES : { [suite]: SUITES[suite] };
+ for (const [suiteName, suiteData] of Object.entries(suites)) {
+ console.log("[devtools-node-test-runner] Running suite: " + suiteName);
+
+ if (suiteData.dependencies) {
+ console.log(
+ "[devtools-node-test-runner] Running `yarn` for dependencies"
+ );
+ for (const dep of suiteData.dependencies) {
+ const depPath = path.join(__dirname, dep);
+ chdir(depPath);
+
+ console.log("[devtools-node-test-runner] Run `yarn` in " + depPath);
+ execOut(YARN_PROCESS);
+ }
+ }
+
+ const testPath = path.join(__dirname, suiteData.path);
+ chdir(testPath);
+
+ console.log("[devtools-node-test-runner] Run `yarn` in test folder");
+ execOut(YARN_PROCESS);
+
+ console.log(`TEST START | ${suiteData.type} | ${suiteName}`);
+
+ console.log("[devtools-node-test-runner] Run `yarn test` in test folder");
+ const { out, err } = execOut(YARN_PROCESS, ["test-ci"]);
+
+ if (err) {
+ console.log("[devtools-node-test-runner] Error log");
+ console.log(err);
+ }
+
+ console.log("[devtools-node-test-runner] Parse errors from the test logs");
+ const errors = getErrors(suiteName, out, err, testPath) || [];
+ if (errors.length) {
+ failedSuites.push(suiteName);
+ }
+ for (const error of errors) {
+ if (!artifactErrors[error.file]) {
+ artifactErrors[error.file] = [];
+ }
+ artifactErrors[error.file].push({
+ path: error.file,
+ line: error.line,
+ column: error.column,
+ level: "error",
+ message: error.message,
+ analyzer: suiteName,
+ });
+
+ console.log(
+ `TEST-UNEXPECTED-FAIL | ${suiteData.type} | ${suiteName} | ${error.file}:${error.line}: ${error.message}`
+ );
+ }
+ }
+
+ if (artifactFilePath) {
+ console.log(
+ `[devtools-node-test-runner] Writing artifact to ${artifactFilePath}`
+ );
+ writeFileSync(artifactFilePath, JSON.stringify(artifactErrors, null, 2));
+ }
+
+ const success = failedSuites.length === 0;
+ if (success) {
+ console.log(
+ `[devtools-node-test-runner] Test suites [${Object.keys(suites).join(
+ ", "
+ )}] succeeded`
+ );
+ } else {
+ console.log(
+ `[devtools-node-test-runner] Test suites [${failedSuites.join(
+ ", "
+ )}] failed`
+ );
+ console.log(
+ "[devtools-node-test-runner] You can find documentation about the " +
+ "devtools node tests at https://firefox-source-docs.mozilla.org/devtools/tests/node-tests.html"
+ );
+ }
+ return success;
+}
+
+process.exitCode = runTests() ? 0 : 1;