diff options
Diffstat (limited to 'remote/test/puppeteer/tools')
25 files changed, 3590 insertions, 0 deletions
diff --git a/remote/test/puppeteer/tools/analyze_issue.mjs b/remote/test/puppeteer/tools/analyze_issue.mjs new file mode 100755 index 0000000000..74e35ca3f8 --- /dev/null +++ b/remote/test/puppeteer/tools/analyze_issue.mjs @@ -0,0 +1,280 @@ +#!/usr/bin/env node +// @ts-check + +'use strict'; + +import {writeFile, mkdir, copyFile} from 'fs/promises'; +import {dirname, join} from 'path'; +import semver from 'semver'; +import {fileURLToPath} from 'url'; +import core from '@actions/core'; +import packageJson from '../packages/puppeteer-core/package.json' assert {type: 'json'}; + +const codifyAndJoinValues = values => { + return values + .map(value => { + return `\`${value}\``; + }) + .join(' ,'); +}; +const formatMessage = value => { + return value.trim(); +}; +const removeVersionPrefix = value => { + return value.startsWith('v') ? value.slice(1) : value; +}; + +const LAST_PUPPETEER_VERSION = packageJson.version; +if (!LAST_PUPPETEER_VERSION) { + core.setFailed('No maintained version found.'); +} +const LAST_SUPPORTED_NODE_VERSION = removeVersionPrefix( + packageJson.engines.node.slice(2).trim() +); + +const SUPPORTED_OSES = ['windows', 'macos', 'linux']; +const SUPPORTED_PACKAGE_MANAGERS = ['yarn', 'npm', 'pnpm']; + +const ERROR_MESSAGES = { + unsupportedOs(value) { + return formatMessage(` +This issue has an unsupported OS: \`${value}\`. Only the following operating systems are supported: ${codifyAndJoinValues( + SUPPORTED_OSES + )}. Please verify the issue on a supported OS and update the form. +`); + }, + unsupportedPackageManager(value) { + return formatMessage(` +This issue has an unsupported package manager: \`${value}\`. Only the following package managers are supported: ${codifyAndJoinValues( + SUPPORTED_PACKAGE_MANAGERS + )}. Please verify the issue using a supported package manager and update the form. +`); + }, + invalidPackageManagerVersion(value) { + return formatMessage(` +This issue has an invalid package manager version: \`${value}\`. Versions must follow [SemVer](https://semver.org/) formatting. Please update the form with a valid version. +`); + }, + unsupportedNodeVersion(value) { + return formatMessage(` +This issue has an unsupported Node.js version: \`${value}\`. Only versions above \`v${LAST_SUPPORTED_NODE_VERSION}\` are supported. Please verify the issue on a supported version of Node.js and update the form. +`); + }, + invalidNodeVersion(value) { + return formatMessage(` +This issue has an invalid Node.js version: \`${value}\`. Versions must follow [SemVer](https://semver.org/) formatting. Please update the form with a valid version. +`); + }, + unsupportedPuppeteerVersion(value) { + return formatMessage(` +This issue has an outdated Puppeteer version: \`${value}\`. Please verify your issue on the latest \`${LAST_PUPPETEER_VERSION}\` version. Then update the form accordingly. +`); + }, + invalidPuppeteerVersion(value) { + return formatMessage(` +This issue has an invalid Puppeteer version: \`${value}\`. Versions must follow [SemVer](https://semver.org/) formatting. Please update the form with a valid version. +`); + }, +}; + +(async () => { + let input = ''; + // @ts-expect-error: `iterator` is new and experimental. + for await (const chunk of process.stdin.iterator({ + destroyOnReturn: false, + })) { + input += chunk; + } + input = JSON.parse(input).trim(); + + let mvce = ''; + let error = ''; + let configuration = ''; + let puppeteerVersion = ''; + let nodeVersion = ''; + let packageManagerVersion = ''; + let packageManager = ''; + let os = ''; + const behavior = {}; + const lines = input.split('\n'); + { + /** @type {(value: string) => void} */ + let set = () => { + return void 0; + }; + let j = 1; + let i = 1; + for (; i < lines.length; ++i) { + if (lines[i].startsWith('### Bug behavior')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + if (value.match(/\[x\] Flaky/i)) { + behavior.flaky = true; + } + if (value.match(/\[x\] pdf/i)) { + behavior.noError = true; + } + }; + } else if (lines[i].startsWith('### Minimal, reproducible example')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + mvce = value; + }; + } else if (lines[i].startsWith('### Error string')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + if (value.match(/no error/i)) { + behavior.noError = true; + } else { + error = value; + } + }; + } else if (lines[i].startsWith('### Puppeteer configuration')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + configuration = value; + }; + } else if (lines[i].startsWith('### Puppeteer version')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + puppeteerVersion = removeVersionPrefix(value); + }; + } else if (lines[i].startsWith('### Node version')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + nodeVersion = removeVersionPrefix(value); + }; + } else if (lines[i].startsWith('### Package manager version')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + packageManagerVersion = removeVersionPrefix(value); + }; + } else if (lines[i].startsWith('### Package manager')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + packageManager = value.toLowerCase(); + }; + } else if (lines[i].startsWith('### Operating system')) { + set(lines.slice(j, i).join('\n').trim()); + j = i + 1; + set = value => { + os = value.toLowerCase(); + }; + } + } + set(lines.slice(j, i).join('\n').trim()); + } + + let runsOn; + switch (os) { + case 'windows': + runsOn = 'windows-latest'; + break; + case 'macos': + runsOn = 'macos-latest'; + break; + case 'linux': + runsOn = 'ubuntu-latest'; + break; + default: + core.setOutput('errorMessage', ERROR_MESSAGES.unsupportedOs(os)); + core.setFailed(`Unsupported OS: ${os}`); + } + + if (!SUPPORTED_PACKAGE_MANAGERS.includes(packageManager)) { + core.setOutput( + 'errorMessage', + ERROR_MESSAGES.unsupportedPackageManager(packageManager) + ); + core.setFailed(`Unsupported package manager: ${packageManager}`); + } + + if (!semver.valid(nodeVersion)) { + core.setOutput( + 'errorMessage', + ERROR_MESSAGES.invalidNodeVersion(nodeVersion) + ); + core.setFailed('Invalid Node version'); + } + if (semver.lt(nodeVersion, LAST_SUPPORTED_NODE_VERSION)) { + core.setOutput( + 'errorMessage', + ERROR_MESSAGES.unsupportedNodeVersion(nodeVersion) + ); + core.setFailed(`Unsupported node version: ${nodeVersion}`); + } + + if (!semver.valid(puppeteerVersion)) { + core.setOutput( + 'errorMessage', + ERROR_MESSAGES.invalidPuppeteerVersion(puppeteerVersion) + ); + core.setFailed(`Invalid puppeteer version: ${puppeteerVersion}`); + } + if ( + !LAST_PUPPETEER_VERSION || + semver.lt(puppeteerVersion, LAST_PUPPETEER_VERSION) + ) { + core.setOutput( + 'errorMessage', + ERROR_MESSAGES.unsupportedPuppeteerVersion(puppeteerVersion) + ); + core.setFailed(`Unsupported puppeteer version: ${puppeteerVersion}`); + } + + if (!semver.valid(packageManagerVersion)) { + core.setOutput( + 'errorMessage', + ERROR_MESSAGES.invalidPackageManagerVersion(packageManagerVersion) + ); + core.setFailed(`Invalid package manager version: ${packageManagerVersion}`); + } + + core.setOutput('errorMessage', ''); + core.setOutput('runsOn', runsOn); + core.setOutput('nodeVersion', nodeVersion); + core.setOutput('packageManager', packageManager); + + await mkdir('out'); + Promise.all([ + writeFile(join('out', 'main.ts'), mvce.split('\n').slice(1, -1).join('\n')), + writeFile(join('out', 'puppeteer-error.txt'), error), + writeFile( + join('out', 'puppeteer.config.js'), + configuration.split('\n').slice(1, -1).join('\n') + ), + writeFile(join('out', 'puppeteer-behavior.json'), JSON.stringify(behavior)), + writeFile( + join('out', 'package.json'), + JSON.stringify({ + packageManager: `${packageManager}@${packageManagerVersion}`, + scripts: { + start: 'tsx main.ts', + verify: 'tsx verify_issue.ts', + }, + dependencies: { + puppeteer: puppeteerVersion, + }, + devDependencies: { + tsx: 'latest', + }, + }) + ), + copyFile( + join( + dirname(fileURLToPath(import.meta.url)), + 'assets', + 'verify_issue.ts' + ), + join('out', 'verify_issue.ts') + ), + ]); +})(); diff --git a/remote/test/puppeteer/tools/assets/verify_issue.ts b/remote/test/puppeteer/tools/assets/verify_issue.ts new file mode 100755 index 0000000000..5814eff66c --- /dev/null +++ b/remote/test/puppeteer/tools/assets/verify_issue.ts @@ -0,0 +1,68 @@ +import {spawnSync} from 'child_process'; +import {readFile, writeFile} from 'fs/promises'; + +(async () => { + const error = await readFile('puppeteer-error.txt', 'utf-8'); + const behavior = JSON.parse( + await readFile('puppeteer-behavior.json', 'utf-8') + ) as {flaky?: boolean; noError?: boolean}; + + let maxRepetitions = 1; + if (behavior.flaky) { + maxRepetitions = 100; + } + + let status: number | null = null; + let stderr = ''; + let stdout = ''; + + const preHook = async () => { + console.log('Writing output and error logs...'); + await Promise.all([ + writeFile('output.log', stdout), + writeFile('error.log', stderr), + ]); + }; + + let checkStatusWithError: () => Promise<void>; + if (behavior.noError) { + checkStatusWithError = async () => { + if (status === 0) { + await preHook(); + console.log('Script ran successfully; no error found.'); + process.exit(0); + } + }; + } else { + checkStatusWithError = async () => { + if (status !== 0) { + await preHook(); + if (stderr.toLowerCase().includes(error.toLowerCase())) { + console.log('Script failed; error found.'); + process.exit(0); + } + console.error('Script failed; unknown error found.'); + process.exit(1); + } + }; + } + + for (let i = 0; i < maxRepetitions; ++i) { + const result = spawnSync('npm', ['start'], { + shell: true, + encoding: 'utf-8', + }); + status = result.status; + stdout = result.stdout ?? ''; + stderr = result.stderr ?? ''; + await checkStatusWithError(); + } + + await preHook(); + if (behavior.noError) { + console.error('Script failed; unknown error found.'); + } else { + console.error('Script ran successfully; no error found.'); + } + process.exit(1); +})(); diff --git a/remote/test/puppeteer/tools/chmod.ts b/remote/test/puppeteer/tools/chmod.ts new file mode 100644 index 0000000000..89afe0f9e7 --- /dev/null +++ b/remote/test/puppeteer/tools/chmod.ts @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; + +/** + * Calls chmod with the mode in argv[2] on paths in argv[3...length-1]. + */ + +const mode = process.argv[2]; + +for (let i = 3; i < process.argv.length; i++) { + fs.chmodSync(process.argv[i], mode); +} diff --git a/remote/test/puppeteer/tools/cp.ts b/remote/test/puppeteer/tools/cp.ts new file mode 100644 index 0000000000..438c6ccd54 --- /dev/null +++ b/remote/test/puppeteer/tools/cp.ts @@ -0,0 +1,22 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; + +/** + * Copies single file in argv[2] to argv[3] + */ +fs.cpSync(process.argv[2], process.argv[3]); diff --git a/remote/test/puppeteer/tools/ensure-pinned-deps.ts b/remote/test/puppeteer/tools/ensure-pinned-deps.ts new file mode 100644 index 0000000000..82f53df796 --- /dev/null +++ b/remote/test/puppeteer/tools/ensure-pinned-deps.ts @@ -0,0 +1,59 @@ +/** + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {readdirSync, readFileSync} from 'fs'; +import {join} from 'path'; + +import {devDependencies} from '../package.json'; + +const LOCAL_PACKAGE_NAMES: string[] = []; + +const packagesDir = join(__dirname, '..', 'packages'); +for (const packageName of readdirSync(packagesDir)) { + const {name} = JSON.parse( + readFileSync(join(packagesDir, packageName, 'package.json'), 'utf8') + ); + LOCAL_PACKAGE_NAMES.push(name); +} + +const allDeps = {...devDependencies}; + +const invalidDeps = new Map<string, string>(); + +for (const [depKey, depValue] of Object.entries(allDeps)) { + if (LOCAL_PACKAGE_NAMES.includes(depKey)) { + continue; + } + if (/[0-9]/.test(depValue[0]!)) { + continue; + } + + invalidDeps.set(depKey, depValue); +} + +if (invalidDeps.size > 0) { + console.error('Found non-pinned dependencies in package.json:'); + console.log( + [...invalidDeps.keys()] + .map(k => { + return ` ${k}`; + }) + .join('\n') + ); + process.exit(1); +} + +process.exit(0); diff --git a/remote/test/puppeteer/tools/generate-matrix.js b/remote/test/puppeteer/tools/generate-matrix.js new file mode 100644 index 0000000000..1cb65d3491 --- /dev/null +++ b/remote/test/puppeteer/tools/generate-matrix.js @@ -0,0 +1,43 @@ +const fs = require('fs'); + +const data = JSON.parse(fs.readFileSync('./test/TestSuites.json', 'utf-8')); + +/** + * @param {string} platform + * @returns {string} + */ +function mapPlatform(platform) { + switch (platform) { + case 'linux': + return 'ubuntu-latest'; + case 'win32': + return 'windows-latest'; + case 'darwin': + return 'macos-latest'; + default: + throw new Error('Unsupported platform'); + } +} + +const result = []; +for (const suite of data.testSuites) { + for (const platform of suite.platforms) { + if (platform === 'linux' && suite.id !== 'firefox-bidi') { + for (const node of [14, 16, 18]) { + result.push(`- name: ${suite.id} + machine: ${mapPlatform(platform)} + xvfb: true + node: ${node} + suite: ${suite.id}`); + } + } else { + result.push(`- name: ${suite.id} + machine: ${mapPlatform(platform)} + xvfb: ${platform === 'linux'} + node: 18 + suite: ${suite.id}`); + } + } +} + +console.log(result.join('\n')); diff --git a/remote/test/puppeteer/tools/generate_docs.ts b/remote/test/puppeteer/tools/generate_docs.ts new file mode 100644 index 0000000000..86bc92f5f8 --- /dev/null +++ b/remote/test/puppeteer/tools/generate_docs.ts @@ -0,0 +1,152 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {copyFile, readFile, rm, writeFile} from 'fs/promises'; +import {join, resolve} from 'path'; +import {chdir} from 'process'; + +import semver from 'semver'; + +import {generateDocs} from './internal/custom_markdown_action.js'; +import {job} from './internal/job.js'; +import {spawnAndLog} from './internal/util.js'; + +chdir(resolve(join(__dirname, '..'))); + +function getOffsetAndLimit( + sectionName: string, + lines: string[] +): [offset: number, limit: number] { + const offset = + lines.findIndex(line => { + return line.includes(`<!-- ${sectionName}-start -->`); + }) + 1; + const limit = lines.slice(offset).findIndex(line => { + return line.includes(`<!-- ${sectionName}-end -->`); + }); + return [offset, limit]; +} + +function spliceIntoSection( + sectionName: string, + content: string, + sectionContent: string +): string { + const lines = content.split('\n'); + const [offset, limit] = getOffsetAndLimit(sectionName, lines); + lines.splice(offset, limit, ...sectionContent.split('\n')); + return lines.join('\n'); +} + +(async () => { + const job1 = job('', async ({inputs, outputs}) => { + await copyFile(inputs[0]!, outputs[0]!); + }) + .inputs(['README.md']) + .outputs(['docs/index.md']) + .build(); + + // Chrome Versions + const job2 = job('', async ({inputs, outputs}) => { + let content = await readFile(inputs[2]!, {encoding: 'utf8'}); + const versionModulePath = join('..', inputs[0]!); + const {versionsPerRelease} = await import(versionModulePath); + const versionsArchived = JSON.parse(await readFile(inputs[1]!, 'utf8')); + + // Generate versions + const buffer: string[] = []; + for (const [chromiumVersion, puppeteerVersion] of versionsPerRelease) { + if (puppeteerVersion === 'NEXT') { + continue; + } + if (versionsArchived.includes(puppeteerVersion.substring(1))) { + if (semver.gte(puppeteerVersion, '20.0.0')) { + buffer.push( + ` * [Chrome for Testing](https://goo.gle/chrome-for-testing) ${chromiumVersion} - [Puppeteer ${puppeteerVersion}](https://pptr.dev/${puppeteerVersion.slice( + 1 + )})` + ); + } else { + buffer.push( + ` * Chromium ${chromiumVersion} - [Puppeteer ${puppeteerVersion}](https://github.com/puppeteer/puppeteer/blob/${puppeteerVersion}/docs/api/index.md)` + ); + } + } else if (semver.lt(puppeteerVersion, '15.0.0')) { + buffer.push( + ` * Chromium ${chromiumVersion} - [Puppeteer ${puppeteerVersion}](https://github.com/puppeteer/puppeteer/blob/${puppeteerVersion}/docs/api.md)` + ); + } else if (semver.gte(puppeteerVersion, '15.3.0')) { + buffer.push( + ` * Chromium ${chromiumVersion} - [Puppeteer ${puppeteerVersion}](https://pptr.dev/${puppeteerVersion.slice( + 1 + )})` + ); + } else { + buffer.push( + ` * Chromium ${chromiumVersion} - Puppeteer ${puppeteerVersion}` + ); + } + } + content = spliceIntoSection('version', content, buffer.join('\n')); + + await writeFile(outputs[0]!, content); + }) + .inputs([ + 'versions.js', + 'website/versionsArchived.json', + 'docs/chromium-support.md', + ]) + .outputs(['docs/chromium-support.md']) + .build(); + + await Promise.all([job1, job2]); + + // Generate documentation + const puppeteerDocs = job('', async ({inputs, outputs}) => { + await rm(outputs[0]!, {recursive: true, force: true}); + generateDocs(inputs[0]!, outputs[0]!); + spawnAndLog('prettier', '--ignore-path', 'none', '--write', 'docs'); + }) + .inputs([ + 'docs/puppeteer.api.json', + 'tools/internal/custom_markdown_documenter.ts', + ]) + .outputs(['docs/api']) + .build(); + + const browsersDocs = job('', async ({inputs, outputs}) => { + await rm(outputs[0]!, {recursive: true, force: true}); + generateDocs(inputs[0]!, outputs[0]!); + spawnAndLog('prettier', '--ignore-path', 'none', '--write', 'docs'); + }) + .inputs([ + 'docs/browsers.api.json', + 'tools/internal/custom_markdown_documenter.ts', + ]) + .outputs(['docs/browsers-api']) + .build(); + + await Promise.all([puppeteerDocs, browsersDocs]); + + await job('', async ({inputs, outputs}) => { + const readme = await readFile(inputs[1]!, 'utf-8'); + const index = await readFile(inputs[0]!, 'utf-8'); + await writeFile(outputs[0]!, index.replace('# API Reference\n', readme)); + }) + .inputs(['docs/browsers-api/index.md', 'packages/browsers/README.md']) + .outputs(['docs/browsers-api/index.md']) + .build(); +})(); diff --git a/remote/test/puppeteer/tools/generate_module_package_json.ts b/remote/test/puppeteer/tools/generate_module_package_json.ts new file mode 100644 index 0000000000..75737463d5 --- /dev/null +++ b/remote/test/puppeteer/tools/generate_module_package_json.ts @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {mkdirSync, writeFileSync} from 'fs'; +import {dirname} from 'path'; + +/** + * Outputs the dummy package.json file to the path specified + * by the first argument. + */ + +mkdirSync(dirname(process.argv[2]), {recursive: true}); +writeFileSync(process.argv[2], `{"type": "module"}`); diff --git a/remote/test/puppeteer/tools/get_deprecated_version_range.js b/remote/test/puppeteer/tools/get_deprecated_version_range.js new file mode 100644 index 0000000000..c817df61bf --- /dev/null +++ b/remote/test/puppeteer/tools/get_deprecated_version_range.js @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { + versionsPerRelease, + lastMaintainedChromiumVersion, +} = require('../versions.js'); +const version = versionsPerRelease.get(lastMaintainedChromiumVersion); +if (version.toLowerCase() === 'next') { + console.error('Unexpected NEXT Puppeteer version in versions.js'); + process.exit(1); +} +console.log(`< ${version.substring(1)}`); +process.exit(0); diff --git a/remote/test/puppeteer/tools/internal/custom_markdown_action.ts b/remote/test/puppeteer/tools/internal/custom_markdown_action.ts new file mode 100644 index 0000000000..d8b3228736 --- /dev/null +++ b/remote/test/puppeteer/tools/internal/custom_markdown_action.ts @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {ApiModel} from '@microsoft/api-extractor-model'; + +import {MarkdownDocumenter} from './custom_markdown_documenter.js'; + +export const generateDocs = (jsonPath: string, outputDir: string): void => { + const apiModel = new ApiModel(); + apiModel.loadPackage(jsonPath); + + const markdownDocumenter: MarkdownDocumenter = new MarkdownDocumenter({ + apiModel: apiModel, + documenterConfig: undefined, + outputFolder: outputDir, + }); + markdownDocumenter.generateFiles(); +}; diff --git a/remote/test/puppeteer/tools/internal/custom_markdown_documenter.ts b/remote/test/puppeteer/tools/internal/custom_markdown_documenter.ts new file mode 100644 index 0000000000..366dde1d55 --- /dev/null +++ b/remote/test/puppeteer/tools/internal/custom_markdown_documenter.ts @@ -0,0 +1,1481 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the +// MIT license. See LICENSE in the project root for license information. + +// Taken from +// https://github.com/microsoft/rushstack/blob/main/apps/api-documenter/src/documenters/MarkdownDocumenter.ts +// This file has been edited to morph into Docusaurus's expected inputs. + +import * as path from 'path'; + +import {DocumenterConfig} from '@microsoft/api-documenter/lib/documenters/DocumenterConfig'; +import {CustomMarkdownEmitter} from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; +import {CustomDocNodes} from '@microsoft/api-documenter/lib/nodes/CustomDocNodeKind'; +import {DocEmphasisSpan} from '@microsoft/api-documenter/lib/nodes/DocEmphasisSpan'; +import {DocHeading} from '@microsoft/api-documenter/lib/nodes/DocHeading'; +import {DocNoteBox} from '@microsoft/api-documenter/lib/nodes/DocNoteBox'; +import {DocTable} from '@microsoft/api-documenter/lib/nodes/DocTable'; +import {DocTableCell} from '@microsoft/api-documenter/lib/nodes/DocTableCell'; +import {DocTableRow} from '@microsoft/api-documenter/lib/nodes/DocTableRow'; +import {MarkdownDocumenterAccessor} from '@microsoft/api-documenter/lib/plugin/MarkdownDocumenterAccessor'; +import { + IMarkdownDocumenterFeatureOnBeforeWritePageArgs, + MarkdownDocumenterFeatureContext, +} from '@microsoft/api-documenter/lib/plugin/MarkdownDocumenterFeature'; +import {PluginLoader} from '@microsoft/api-documenter/lib/plugin/PluginLoader'; +import {Utilities} from '@microsoft/api-documenter/lib/utils/Utilities'; +import { + ApiClass, + ApiDeclaredItem, + ApiDocumentedItem, + ApiEnum, + ApiInitializerMixin, + ApiInterface, + ApiItem, + ApiItemKind, + ApiModel, + ApiNamespace, + ApiOptionalMixin, + ApiPackage, + ApiParameterListMixin, + ApiPropertyItem, + ApiProtectedMixin, + ApiReadonlyMixin, + ApiReleaseTagMixin, + ApiReturnTypeMixin, + ApiStaticMixin, + ApiTypeAlias, + Excerpt, + ExcerptToken, + ExcerptTokenKind, + IResolveDeclarationReferenceResult, + ReleaseTag, +} from '@microsoft/api-extractor-model'; +import { + DocBlock, + DocCodeSpan, + DocComment, + DocFencedCode, + DocLinkTag, + DocNodeContainer, + DocNodeKind, + DocParagraph, + DocPlainText, + DocSection, + StandardTags, + StringBuilder, + TSDocConfiguration, +} from '@microsoft/tsdoc'; +import { + FileSystem, + NewlineKind, + PackageName, +} from '@rushstack/node-core-library'; + +export interface IMarkdownDocumenterOptions { + apiModel: ApiModel; + documenterConfig: DocumenterConfig | undefined; + outputFolder: string; +} + +/** + * Renders API documentation in the Markdown file format. + * For more info: https://en.wikipedia.org/wiki/Markdown + */ +export class MarkdownDocumenter { + private readonly _apiModel: ApiModel; + private readonly _documenterConfig: DocumenterConfig | undefined; + private readonly _tsdocConfiguration: TSDocConfiguration; + private readonly _markdownEmitter: CustomMarkdownEmitter; + private readonly _outputFolder: string; + private readonly _pluginLoader: PluginLoader; + + public constructor(options: IMarkdownDocumenterOptions) { + this._apiModel = options.apiModel; + this._documenterConfig = options.documenterConfig; + this._outputFolder = options.outputFolder; + this._tsdocConfiguration = CustomDocNodes.configuration; + this._markdownEmitter = new CustomMarkdownEmitter(this._apiModel); + + this._pluginLoader = new PluginLoader(); + } + + public generateFiles(): void { + if (this._documenterConfig) { + this._pluginLoader.load(this._documenterConfig, () => { + return new MarkdownDocumenterFeatureContext({ + apiModel: this._apiModel, + outputFolder: this._outputFolder, + documenter: new MarkdownDocumenterAccessor({ + getLinkForApiItem: (apiItem: ApiItem) => { + console.log(apiItem); + return this._getLinkFilenameForApiItem(apiItem); + }, + }), + }); + }); + } + + console.log(); + this._deleteOldOutputFiles(); + + this._writeApiItemPage(this._apiModel.members[0]!); + + if (this._pluginLoader.markdownDocumenterFeature) { + this._pluginLoader.markdownDocumenterFeature.onFinished({}); + } + } + + private _writeApiItemPage(apiItem: ApiItem): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + const output: DocSection = new DocSection({ + configuration: this._tsdocConfiguration, + }); + + const scopedName: string = apiItem.getScopedNameWithinPackage(); + + switch (apiItem.kind) { + case ApiItemKind.Class: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} class`}) + ); + break; + case ApiItemKind.Enum: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} enum`}) + ); + break; + case ApiItemKind.Interface: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} interface`}) + ); + break; + case ApiItemKind.Constructor: + case ApiItemKind.ConstructSignature: + output.appendNode(new DocHeading({configuration, title: scopedName})); + break; + case ApiItemKind.Method: + case ApiItemKind.MethodSignature: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} method`}) + ); + break; + case ApiItemKind.Function: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} function`}) + ); + break; + case ApiItemKind.Model: + output.appendNode( + new DocHeading({configuration, title: `API Reference`}) + ); + break; + case ApiItemKind.Namespace: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} namespace`}) + ); + break; + case ApiItemKind.Package: + console.log(`Writing ${apiItem.displayName} package`); + output.appendNode( + new DocHeading({ + configuration, + title: `API Reference`, + }) + ); + break; + case ApiItemKind.Property: + case ApiItemKind.PropertySignature: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} property`}) + ); + break; + case ApiItemKind.TypeAlias: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} type`}) + ); + break; + case ApiItemKind.Variable: + output.appendNode( + new DocHeading({configuration, title: `${scopedName} variable`}) + ); + break; + default: + throw new Error('Unsupported API item kind: ' + apiItem.kind); + } + + if (ApiReleaseTagMixin.isBaseClassOf(apiItem)) { + if (apiItem.releaseTag === ReleaseTag.Beta) { + this._writeBetaWarning(output); + } + } + + const decoratorBlocks: DocBlock[] = []; + + if (apiItem instanceof ApiDocumentedItem) { + const tsdocComment: DocComment | undefined = apiItem.tsdocComment; + + if (tsdocComment) { + decoratorBlocks.push( + ...tsdocComment.customBlocks.filter(block => { + return ( + block.blockTag.tagNameWithUpperCase === + StandardTags.decorator.tagNameWithUpperCase + ); + }) + ); + + if (tsdocComment.deprecatedBlock) { + output.appendNode( + new DocNoteBox({configuration: this._tsdocConfiguration}, [ + new DocParagraph({configuration: this._tsdocConfiguration}, [ + new DocPlainText({ + configuration: this._tsdocConfiguration, + text: 'Warning: This API is now obsolete. ', + }), + ]), + ...tsdocComment.deprecatedBlock.content.nodes, + ]) + ); + } + + this._appendSection(output, tsdocComment.summarySection); + } + } + + if (apiItem instanceof ApiDeclaredItem) { + if (apiItem.excerpt.text.length > 0) { + output.appendNode( + new DocHeading({configuration, title: 'Signature:', level: 4}) + ); + + let code: string; + switch (apiItem.parent?.kind) { + case ApiItemKind.Class: + code = `class ${ + apiItem.parent.displayName + } {${apiItem.getExcerptWithModifiers()}}`; + break; + case ApiItemKind.Interface: + code = `interface ${ + apiItem.parent.displayName + } {${apiItem.getExcerptWithModifiers()}}`; + break; + default: + code = apiItem.getExcerptWithModifiers(); + } + output.appendNode( + new DocFencedCode({ + configuration, + code: code, + language: 'typescript', + }) + ); + } + + this._writeHeritageTypes(output, apiItem); + } + + if (decoratorBlocks.length > 0) { + output.appendNode( + new DocHeading({configuration, title: 'Decorators:', level: 4}) + ); + for (const decoratorBlock of decoratorBlocks) { + output.appendNodes(decoratorBlock.content.nodes); + } + } + + let appendRemarks = true; + switch (apiItem.kind) { + case ApiItemKind.Class: + case ApiItemKind.Interface: + case ApiItemKind.Namespace: + case ApiItemKind.Package: + this._writeRemarksSection(output, apiItem); + appendRemarks = false; + break; + } + + switch (apiItem.kind) { + case ApiItemKind.Class: + this._writeClassTables(output, apiItem as ApiClass); + break; + case ApiItemKind.Enum: + this._writeEnumTables(output, apiItem as ApiEnum); + break; + case ApiItemKind.Interface: + this._writeInterfaceTables(output, apiItem as ApiInterface); + break; + case ApiItemKind.Constructor: + case ApiItemKind.ConstructSignature: + case ApiItemKind.Method: + case ApiItemKind.MethodSignature: + case ApiItemKind.Function: + this._writeParameterTables(output, apiItem as ApiParameterListMixin); + this._writeThrowsSection(output, apiItem); + break; + case ApiItemKind.Namespace: + this._writePackageOrNamespaceTables(output, apiItem as ApiNamespace); + break; + case ApiItemKind.Model: + this._writeModelTable(output, apiItem as ApiModel); + break; + case ApiItemKind.Package: + this._writePackageOrNamespaceTables(output, apiItem as ApiPackage); + break; + case ApiItemKind.Property: + case ApiItemKind.PropertySignature: + break; + case ApiItemKind.TypeAlias: + break; + case ApiItemKind.Variable: + break; + default: + throw new Error('Unsupported API item kind: ' + apiItem.kind); + } + + this._writeDefaultValueSection(output, apiItem); + + if (appendRemarks) { + this._writeRemarksSection(output, apiItem); + } + + const filename: string = path.join( + this._outputFolder, + this._getFilenameForApiItem(apiItem) + ); + const stringBuilder: StringBuilder = new StringBuilder(); + + this._markdownEmitter.emit(stringBuilder, output, { + contextApiItem: apiItem, + onGetFilenameForApiItem: (apiItemForFilename: ApiItem) => { + return this._getLinkFilenameForApiItem(apiItemForFilename); + }, + }); + + let pageContent: string = stringBuilder.toString(); + + if (this._pluginLoader.markdownDocumenterFeature) { + // Allow the plugin to customize the pageContent + const eventArgs: IMarkdownDocumenterFeatureOnBeforeWritePageArgs = { + apiItem: apiItem, + outputFilename: filename, + pageContent: pageContent, + }; + this._pluginLoader.markdownDocumenterFeature.onBeforeWritePage(eventArgs); + pageContent = eventArgs.pageContent; + } + + pageContent = + `---\nsidebar_label: ${this._getSidebarLabelForApiItem(apiItem)}\n---` + + pageContent; + pageContent = pageContent.replace('##', '#'); + pageContent = pageContent.replace(/<!-- -->/g, ''); + pageContent = pageContent.replace(/\\\*\\\*/g, '**'); + pageContent = pageContent.replace(/<b>|<\/b>/g, '**'); + FileSystem.writeFile(filename, pageContent, { + convertLineEndings: this._documenterConfig + ? this._documenterConfig.newlineKind + : NewlineKind.CrLf, + }); + } + + private _writeHeritageTypes( + output: DocSection, + apiItem: ApiDeclaredItem + ): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + if (apiItem instanceof ApiClass) { + if (apiItem.extendsType) { + const extendsParagraph: DocParagraph = new DocParagraph( + {configuration}, + [ + new DocEmphasisSpan({configuration, bold: true}, [ + new DocPlainText({configuration, text: 'Extends: '}), + ]), + ] + ); + this._appendExcerptWithHyperlinks( + extendsParagraph, + apiItem.extendsType.excerpt + ); + output.appendNode(extendsParagraph); + } + if (apiItem.implementsTypes.length > 0) { + const extendsParagraph: DocParagraph = new DocParagraph( + {configuration}, + [ + new DocEmphasisSpan({configuration, bold: true}, [ + new DocPlainText({configuration, text: 'Implements: '}), + ]), + ] + ); + let needsComma = false; + for (const implementsType of apiItem.implementsTypes) { + if (needsComma) { + extendsParagraph.appendNode( + new DocPlainText({configuration, text: ', '}) + ); + } + this._appendExcerptWithHyperlinks( + extendsParagraph, + implementsType.excerpt + ); + needsComma = true; + } + output.appendNode(extendsParagraph); + } + } + + if (apiItem instanceof ApiInterface) { + if (apiItem.extendsTypes.length > 0) { + const extendsParagraph: DocParagraph = new DocParagraph( + {configuration}, + [ + new DocEmphasisSpan({configuration, bold: true}, [ + new DocPlainText({configuration, text: 'Extends: '}), + ]), + ] + ); + let needsComma = false; + for (const extendsType of apiItem.extendsTypes) { + if (needsComma) { + extendsParagraph.appendNode( + new DocPlainText({configuration, text: ', '}) + ); + } + this._appendExcerptWithHyperlinks( + extendsParagraph, + extendsType.excerpt + ); + needsComma = true; + } + output.appendNode(extendsParagraph); + } + } + + if (apiItem instanceof ApiTypeAlias) { + const refs: ExcerptToken[] = apiItem.excerptTokens.filter(token => { + return ( + token.kind === ExcerptTokenKind.Reference && + token.canonicalReference && + this._apiModel.resolveDeclarationReference( + token.canonicalReference, + undefined + ).resolvedApiItem + ); + }); + if (refs.length > 0) { + const referencesParagraph: DocParagraph = new DocParagraph( + {configuration}, + [ + new DocEmphasisSpan({configuration, bold: true}, [ + new DocPlainText({configuration, text: 'References: '}), + ]), + ] + ); + let needsComma = false; + const visited: Set<string> = new Set(); + for (const ref of refs) { + if (visited.has(ref.text)) { + continue; + } + visited.add(ref.text); + + if (needsComma) { + referencesParagraph.appendNode( + new DocPlainText({configuration, text: ', '}) + ); + } + + this._appendExcerptTokenWithHyperlinks(referencesParagraph, ref); + needsComma = true; + } + output.appendNode(referencesParagraph); + } + } + } + + private _writeDefaultValueSection(output: DocSection, apiItem: ApiItem) { + if (apiItem instanceof ApiDocumentedItem) { + const block = apiItem.tsdocComment?.customBlocks.find(block => { + return ( + block.blockTag.tagNameWithUpperCase === + StandardTags.defaultValue.tagNameWithUpperCase + ); + }); + if (block) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Default value:', + level: 4, + }) + ); + this._appendSection(output, block.content); + } + } + } + + private _writeRemarksSection(output: DocSection, apiItem: ApiItem): void { + if (apiItem instanceof ApiDocumentedItem) { + const tsdocComment: DocComment | undefined = apiItem.tsdocComment; + + if (tsdocComment) { + // Write the @remarks block + if (tsdocComment.remarksBlock) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Remarks', + }) + ); + this._appendSection(output, tsdocComment.remarksBlock.content); + } + + // Write the @example blocks + const exampleBlocks: DocBlock[] = tsdocComment.customBlocks.filter( + x => { + return ( + x.blockTag.tagNameWithUpperCase === + StandardTags.example.tagNameWithUpperCase + ); + } + ); + + let exampleNumber = 1; + for (const exampleBlock of exampleBlocks) { + const heading: string = + exampleBlocks.length > 1 ? `Example ${exampleNumber}` : 'Example'; + + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: heading, + }) + ); + + this._appendSection(output, exampleBlock.content); + + ++exampleNumber; + } + } + } + } + + private _writeThrowsSection(output: DocSection, apiItem: ApiItem): void { + if (apiItem instanceof ApiDocumentedItem) { + const tsdocComment: DocComment | undefined = apiItem.tsdocComment; + + if (tsdocComment) { + // Write the @throws blocks + const throwsBlocks: DocBlock[] = tsdocComment.customBlocks.filter(x => { + return ( + x.blockTag.tagNameWithUpperCase === + StandardTags.throws.tagNameWithUpperCase + ); + }); + + if (throwsBlocks.length > 0) { + const heading = 'Exceptions'; + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: heading, + }) + ); + + for (const throwsBlock of throwsBlocks) { + this._appendSection(output, throwsBlock.content); + } + } + } + } + } + + /** + * GENERATE PAGE: MODEL + */ + private _writeModelTable(output: DocSection, apiModel: ApiModel): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const packagesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Package', 'Description'], + }); + + for (const apiMember of apiModel.members) { + const row: DocTableRow = new DocTableRow({configuration}, [ + this._createTitleCell(apiMember), + this._createDescriptionCell(apiMember), + ]); + + switch (apiMember.kind) { + case ApiItemKind.Package: + packagesTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + } + } + + if (packagesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Packages', + }) + ); + output.appendNode(packagesTable); + } + } + + /** + * GENERATE PAGE: PACKAGE or NAMESPACE + */ + private _writePackageOrNamespaceTables( + output: DocSection, + apiContainer: ApiPackage | ApiNamespace + ): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const classesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Class', 'Description'], + }); + + const enumerationsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Enumeration', 'Description'], + }); + + const functionsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Function', 'Description'], + }); + + const interfacesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Interface', 'Description'], + }); + + const namespacesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Namespace', 'Description'], + }); + + const variablesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Variable', 'Description'], + }); + + const typeAliasesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Type Alias', 'Description'], + }); + + const apiMembers: readonly ApiItem[] = + apiContainer.kind === ApiItemKind.Package + ? (apiContainer as ApiPackage).entryPoints[0]!.members + : (apiContainer as ApiNamespace).members; + + for (const apiMember of apiMembers) { + const row: DocTableRow = new DocTableRow({configuration}, [ + this._createTitleCell(apiMember), + this._createDescriptionCell(apiMember), + ]); + + switch (apiMember.kind) { + case ApiItemKind.Class: + classesTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + + case ApiItemKind.Enum: + enumerationsTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + + case ApiItemKind.Interface: + interfacesTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + + case ApiItemKind.Namespace: + namespacesTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + + case ApiItemKind.Function: + functionsTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + + case ApiItemKind.TypeAlias: + typeAliasesTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + + case ApiItemKind.Variable: + variablesTable.addRow(row); + this._writeApiItemPage(apiMember); + break; + } + } + + if (classesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Classes', + }) + ); + output.appendNode(classesTable); + } + + if (enumerationsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Enumerations', + }) + ); + output.appendNode(enumerationsTable); + } + if (functionsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Functions', + }) + ); + output.appendNode(functionsTable); + } + + if (interfacesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Interfaces', + }) + ); + output.appendNode(interfacesTable); + } + + if (namespacesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Namespaces', + }) + ); + output.appendNode(namespacesTable); + } + + if (variablesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Variables', + }) + ); + output.appendNode(variablesTable); + } + + if (typeAliasesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Type Aliases', + }) + ); + output.appendNode(typeAliasesTable); + } + } + + /** + * GENERATE PAGE: CLASS + */ + private _writeClassTables(output: DocSection, apiClass: ApiClass): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const eventsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Property', 'Modifiers', 'Type', 'Description'], + }); + + const constructorsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Constructor', 'Modifiers', 'Description'], + }); + + const propertiesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Property', 'Modifiers', 'Type', 'Description'], + }); + + const methodsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Method', 'Modifiers', 'Description'], + }); + + for (const apiMember of apiClass.members) { + switch (apiMember.kind) { + case ApiItemKind.Constructor: { + constructorsTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember), + this._createModifiersCell(apiMember), + this._createDescriptionCell(apiMember), + ]) + ); + + this._writeApiItemPage(apiMember); + break; + } + case ApiItemKind.Method: { + methodsTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember), + this._createModifiersCell(apiMember), + this._createDescriptionCell(apiMember), + ]) + ); + + this._writeApiItemPage(apiMember); + break; + } + case ApiItemKind.Property: { + if ((apiMember as ApiPropertyItem).isEventProperty) { + eventsTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember, true), + this._createModifiersCell(apiMember), + this._createPropertyTypeCell(apiMember), + this._createDescriptionCell(apiMember), + ]) + ); + } else { + propertiesTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember, true), + this._createModifiersCell(apiMember), + this._createPropertyTypeCell(apiMember), + this._createDescriptionCell(apiMember), + ]) + ); + } + break; + } + } + } + + if (eventsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Events', + }) + ); + output.appendNode(eventsTable); + } + + if (constructorsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Constructors', + }) + ); + output.appendNode(constructorsTable); + } + + if (propertiesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Properties', + }) + ); + output.appendNode(propertiesTable); + } + + if (methodsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Methods', + }) + ); + output.appendNode(methodsTable); + } + } + + /** + * GENERATE PAGE: ENUM + */ + private _writeEnumTables(output: DocSection, apiEnum: ApiEnum): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const enumMembersTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Member', 'Value', 'Description'], + }); + + for (const apiEnumMember of apiEnum.members) { + enumMembersTable.addRow( + new DocTableRow({configuration}, [ + new DocTableCell({configuration}, [ + new DocParagraph({configuration}, [ + new DocPlainText({ + configuration, + text: Utilities.getConciseSignature(apiEnumMember), + }), + ]), + ]), + this._createInitializerCell(apiEnumMember), + this._createDescriptionCell(apiEnumMember), + ]) + ); + } + + if (enumMembersTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Enumeration Members', + }) + ); + output.appendNode(enumMembersTable); + } + } + + /** + * GENERATE PAGE: INTERFACE + */ + private _writeInterfaceTables( + output: DocSection, + apiClass: ApiInterface + ): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const eventsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Property', 'Modifiers', 'Type', 'Description'], + }); + + const propertiesTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Property', 'Modifiers', 'Type', 'Description', 'Default'], + }); + + const methodsTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Method', 'Description'], + }); + + for (const apiMember of apiClass.members) { + switch (apiMember.kind) { + case ApiItemKind.ConstructSignature: + case ApiItemKind.MethodSignature: { + methodsTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember), + this._createDescriptionCell(apiMember), + ]) + ); + + this._writeApiItemPage(apiMember); + break; + } + case ApiItemKind.PropertySignature: { + if ((apiMember as ApiPropertyItem).isEventProperty) { + eventsTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember, true), + this._createModifiersCell(apiMember), + this._createPropertyTypeCell(apiMember), + this._createDescriptionCell(apiMember), + ]) + ); + } else { + propertiesTable.addRow( + new DocTableRow({configuration}, [ + this._createTitleCell(apiMember, true), + this._createModifiersCell(apiMember), + this._createPropertyTypeCell(apiMember), + this._createDescriptionCell(apiMember), + this._createDefaultCell(apiMember), + ]) + ); + } + break; + } + } + } + + if (eventsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Events', + }) + ); + output.appendNode(eventsTable); + } + + if (propertiesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Properties', + }) + ); + output.appendNode(propertiesTable); + } + + if (methodsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Methods', + }) + ); + output.appendNode(methodsTable); + } + } + + /** + * GENERATE PAGE: FUNCTION-LIKE + */ + private _writeParameterTables( + output: DocSection, + apiParameterListMixin: ApiParameterListMixin + ): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const parametersTable: DocTable = new DocTable({ + configuration, + headerTitles: ['Parameter', 'Type', 'Description'], + }); + for (const apiParameter of apiParameterListMixin.parameters) { + const parameterDescription: DocSection = new DocSection({configuration}); + + if (apiParameter.isOptional) { + parameterDescription.appendNodesInParagraph([ + new DocEmphasisSpan({configuration, italic: true}, [ + new DocPlainText({configuration, text: '(Optional)'}), + ]), + new DocPlainText({configuration, text: ' '}), + ]); + } + + if (apiParameter.tsdocParamBlock) { + this._appendAndMergeSection( + parameterDescription, + apiParameter.tsdocParamBlock.content + ); + } + + parametersTable.addRow( + new DocTableRow({configuration}, [ + new DocTableCell({configuration}, [ + new DocParagraph({configuration}, [ + new DocPlainText({configuration, text: apiParameter.name}), + ]), + ]), + new DocTableCell({configuration}, [ + this._createParagraphForTypeExcerpt( + apiParameter.parameterTypeExcerpt + ), + ]), + new DocTableCell({configuration}, parameterDescription.nodes), + ]) + ); + } + + if (parametersTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Parameters', + }) + ); + output.appendNode(parametersTable); + } + + if (ApiReturnTypeMixin.isBaseClassOf(apiParameterListMixin)) { + const returnTypeExcerpt: Excerpt = + apiParameterListMixin.returnTypeExcerpt; + output.appendNode( + new DocParagraph({configuration}, [ + new DocEmphasisSpan({configuration, bold: true}, [ + new DocPlainText({configuration, text: 'Returns:'}), + ]), + ]) + ); + + output.appendNode(this._createParagraphForTypeExcerpt(returnTypeExcerpt)); + + if (apiParameterListMixin instanceof ApiDocumentedItem) { + if ( + apiParameterListMixin.tsdocComment && + apiParameterListMixin.tsdocComment.returnsBlock + ) { + this._appendSection( + output, + apiParameterListMixin.tsdocComment.returnsBlock.content + ); + } + } + } + } + + private _createParagraphForTypeExcerpt(excerpt: Excerpt): DocParagraph { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const paragraph: DocParagraph = new DocParagraph({configuration}); + if (!excerpt.text.trim()) { + paragraph.appendNode( + new DocPlainText({configuration, text: '(not declared)'}) + ); + } else { + this._appendExcerptWithHyperlinks(paragraph, excerpt); + } + + return paragraph; + } + + private _appendExcerptWithHyperlinks( + docNodeContainer: DocNodeContainer, + excerpt: Excerpt + ): void { + for (const token of excerpt.spannedTokens) { + this._appendExcerptTokenWithHyperlinks(docNodeContainer, token); + } + } + + private _appendExcerptTokenWithHyperlinks( + docNodeContainer: DocNodeContainer, + token: ExcerptToken + ): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + // Markdown doesn't provide a standardized syntax for hyperlinks inside code + // spans, so we will render the type expression as DocPlainText. Instead of + // creating multiple DocParagraphs, we can simply discard any newlines and + // let the renderer do normal word-wrapping. + const unwrappedTokenText: string = token.text.replace(/[\r\n]+/g, ' '); + + // If it's hyperlinkable, then append a DocLinkTag + if (token.kind === ExcerptTokenKind.Reference && token.canonicalReference) { + const apiItemResult: IResolveDeclarationReferenceResult = + this._apiModel.resolveDeclarationReference( + token.canonicalReference, + undefined + ); + + if (apiItemResult.resolvedApiItem) { + docNodeContainer.appendNode( + new DocLinkTag({ + configuration, + tagName: StandardTags.link.tagName, + linkText: unwrappedTokenText, + urlDestination: this._getLinkFilenameForApiItem( + apiItemResult.resolvedApiItem + ), + }) + ); + return; + } + } + + // Otherwise append non-hyperlinked text + docNodeContainer.appendNode( + new DocPlainText({configuration, text: unwrappedTokenText}) + ); + } + + private _createTitleCell(apiItem: ApiItem, plain = false): DocTableCell { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const text: string = Utilities.getConciseSignature(apiItem); + + return new DocTableCell({configuration}, [ + new DocParagraph({configuration}, [ + plain + ? new DocPlainText({configuration, text}) + : new DocLinkTag({ + configuration, + tagName: '@link', + linkText: text, + urlDestination: this._getLinkFilenameForApiItem(apiItem), + }), + ]), + ]); + } + + /** + * This generates a DocTableCell for an ApiItem including the summary section + * and "(BETA)" annotation. + * + * @remarks + * We mostly assume that the input is an ApiDocumentedItem, but it's easier to + * perform this as a runtime check than to have each caller perform a type + * cast. + */ + private _createDescriptionCell(apiItem: ApiItem): DocTableCell { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const section: DocSection = new DocSection({configuration}); + + if (ApiReleaseTagMixin.isBaseClassOf(apiItem)) { + if (apiItem.releaseTag === ReleaseTag.Beta) { + section.appendNodesInParagraph([ + new DocEmphasisSpan({configuration, bold: true, italic: true}, [ + new DocPlainText({configuration, text: '(BETA)'}), + ]), + new DocPlainText({configuration, text: ' '}), + ]); + } + } + + if (apiItem instanceof ApiDocumentedItem) { + if (apiItem.tsdocComment !== undefined) { + this._appendAndMergeSection( + section, + apiItem.tsdocComment.summarySection + ); + } + } + + return new DocTableCell({configuration}, section.nodes); + } + + private _createDefaultCell(apiItem: ApiItem): DocTableCell { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + if (apiItem instanceof ApiDocumentedItem) { + const block = apiItem.tsdocComment?.customBlocks.find(block => { + return ( + block.blockTag.tagNameWithUpperCase === + StandardTags.defaultValue.tagNameWithUpperCase + ); + }); + if (block !== undefined) { + return new DocTableCell({configuration}, block.content.getChildNodes()); + } + } + + return new DocTableCell({configuration}, []); + } + + private _createModifiersCell(apiItem: ApiItem): DocTableCell { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const section: DocSection = new DocSection({configuration}); + + if (ApiProtectedMixin.isBaseClassOf(apiItem)) { + if (apiItem.isProtected) { + section.appendNode( + new DocParagraph({configuration}, [ + new DocCodeSpan({configuration, code: 'protected'}), + ]) + ); + } + } + + if (ApiReadonlyMixin.isBaseClassOf(apiItem)) { + if (apiItem.isReadonly) { + section.appendNode( + new DocParagraph({configuration}, [ + new DocCodeSpan({configuration, code: 'readonly'}), + ]) + ); + } + } + + if (ApiStaticMixin.isBaseClassOf(apiItem)) { + if (apiItem.isStatic) { + section.appendNode( + new DocParagraph({configuration}, [ + new DocCodeSpan({configuration, code: 'static'}), + ]) + ); + } + } + + if (ApiOptionalMixin.isBaseClassOf(apiItem)) { + if (apiItem.isOptional) { + section.appendNode( + new DocParagraph({configuration}, [ + new DocCodeSpan({configuration, code: 'optional'}), + ]) + ); + } + } + + return new DocTableCell({configuration}, section.nodes); + } + + private _createPropertyTypeCell(apiItem: ApiItem): DocTableCell { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const section: DocSection = new DocSection({configuration}); + + if (apiItem instanceof ApiPropertyItem) { + section.appendNode( + this._createParagraphForTypeExcerpt(apiItem.propertyTypeExcerpt) + ); + } + + return new DocTableCell({configuration}, section.nodes); + } + + private _createInitializerCell(apiItem: ApiItem): DocTableCell { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + const section: DocSection = new DocSection({configuration}); + + if (ApiInitializerMixin.isBaseClassOf(apiItem)) { + if (apiItem.initializerExcerpt) { + section.appendNodeInParagraph( + new DocCodeSpan({ + configuration, + code: apiItem.initializerExcerpt.text, + }) + ); + } + } + + return new DocTableCell({configuration}, section.nodes); + } + + private _writeBetaWarning(output: DocSection): void { + const configuration: TSDocConfiguration = this._tsdocConfiguration; + const betaWarning: string = + 'This API is provided as a preview for developers and may change' + + ' based on feedback that we receive. Do not use this API in a production environment.'; + output.appendNode( + new DocNoteBox({configuration}, [ + new DocParagraph({configuration}, [ + new DocPlainText({configuration, text: betaWarning}), + ]), + ]) + ); + } + + private _appendSection(output: DocSection, docSection: DocSection): void { + for (const node of docSection.nodes) { + output.appendNode(node); + } + } + + private _appendAndMergeSection( + output: DocSection, + docSection: DocSection + ): void { + let firstNode = true; + for (const node of docSection.nodes) { + if (firstNode) { + if (node.kind === DocNodeKind.Paragraph) { + output.appendNodesInParagraph(node.getChildNodes()); + firstNode = false; + continue; + } + } + firstNode = false; + + output.appendNode(node); + } + } + + private _getSidebarLabelForApiItem(apiItem: ApiItem): string { + if (apiItem.kind === ApiItemKind.Package) { + return 'API'; + } + + let baseName = ''; + for (const hierarchyItem of apiItem.getHierarchy()) { + // For overloaded methods, add a suffix such as "MyClass.myMethod_2". + let qualifiedName: string = hierarchyItem.displayName; + if (ApiParameterListMixin.isBaseClassOf(hierarchyItem)) { + if (hierarchyItem.overloadIndex > 1) { + // Subtract one for compatibility with earlier releases of API Documenter. + qualifiedName += `_${hierarchyItem.overloadIndex - 1}`; + } + } + + switch (hierarchyItem.kind) { + case ApiItemKind.Model: + case ApiItemKind.EntryPoint: + case ApiItemKind.EnumMember: + case ApiItemKind.Package: + break; + default: + baseName += qualifiedName + '.'; + } + } + return baseName.slice(0, baseName.length - 1); + } + + private _getFilenameForApiItem(apiItem: ApiItem): string { + if (apiItem.kind === ApiItemKind.Package) { + return 'index.md'; + } + + let baseName = ''; + for (const hierarchyItem of apiItem.getHierarchy()) { + // For overloaded methods, add a suffix such as "MyClass.myMethod_2". + let qualifiedName: string = Utilities.getSafeFilenameForName( + hierarchyItem.displayName + ); + if (ApiParameterListMixin.isBaseClassOf(hierarchyItem)) { + if (hierarchyItem.overloadIndex > 1) { + // Subtract one for compatibility with earlier releases of API Documenter. + // (This will get revamped when we fix GitHub issue #1308) + qualifiedName += `_${hierarchyItem.overloadIndex - 1}`; + } + } + + switch (hierarchyItem.kind) { + case ApiItemKind.Model: + case ApiItemKind.EntryPoint: + case ApiItemKind.EnumMember: + break; + case ApiItemKind.Package: + baseName = Utilities.getSafeFilenameForName( + PackageName.getUnscopedName(hierarchyItem.displayName) + ); + break; + default: + baseName += '.' + qualifiedName; + } + } + return baseName + '.md'; + } + + private _getLinkFilenameForApiItem(apiItem: ApiItem): string { + return './' + this._getFilenameForApiItem(apiItem); + } + + private _deleteOldOutputFiles(): void { + console.log('Deleting old output from ' + this._outputFolder); + FileSystem.ensureEmptyFolder(this._outputFolder); + } +} diff --git a/remote/test/puppeteer/tools/internal/job.ts b/remote/test/puppeteer/tools/internal/job.ts new file mode 100644 index 0000000000..ef6ea10237 --- /dev/null +++ b/remote/test/puppeteer/tools/internal/job.ts @@ -0,0 +1,153 @@ +import {createHash} from 'crypto'; +import {existsSync, Stats} from 'fs'; +import {mkdir, readFile, stat, writeFile} from 'fs/promises'; +import {tmpdir} from 'os'; +import {dirname, join} from 'path'; + +import glob from 'glob'; + +interface JobContext { + name: string; + inputs: string[]; + outputs: string[]; +} + +class JobBuilder { + #inputs: string[] = []; + #outputs: string[] = []; + #callback: (ctx: JobContext) => Promise<void>; + #name: string; + #value = ''; + #force = false; + + constructor(name: string, callback: (ctx: JobContext) => Promise<void>) { + this.#name = name; + this.#callback = callback; + } + + get jobHash(): string { + return createHash('sha256').update(this.#name).digest('hex'); + } + + force() { + this.#force = true; + return this; + } + + value(value: string) { + this.#value = value; + return this; + } + + inputs(inputs: string[]): JobBuilder { + this.#inputs = inputs.flatMap(value => { + if (glob.hasMagic(value)) { + return glob.sync(value); + } + return value; + }); + return this; + } + + outputs(outputs: string[]): JobBuilder { + if (!this.#name) { + this.#name = outputs.join(' and '); + } + + this.#outputs = outputs; + return this; + } + + async build(): Promise<void> { + console.log(`Running job ${this.#name}...`); + // For debugging. + if (this.#force) { + return this.#run(); + } + // In case we deleted an output file on purpose. + if (!this.getOutputStats()) { + return this.#run(); + } + // Run if the job has a value, but it changes. + if (this.#value) { + if (!(await this.isValueDifferent())) { + return; + } + return this.#run(); + } + // Always run when there is no output. + if (!this.#outputs.length) { + return this.#run(); + } + // Make-like comparator. + if (!(await this.areInputsNewer())) { + return; + } + return this.#run(); + } + + async isValueDifferent(): Promise<boolean> { + const file = join(tmpdir(), `puppeteer/${this.jobHash}.txt`); + await mkdir(dirname(file), {recursive: true}); + if (!existsSync(file)) { + await writeFile(file, this.#value); + return true; + } + return this.#value !== (await readFile(file, 'utf8')); + } + + #outputStats?: Stats[]; + async getOutputStats(): Promise<Stats[] | undefined> { + if (this.#outputStats) { + return this.#outputStats; + } + try { + this.#outputStats = await Promise.all( + this.#outputs.map(output => { + return stat(output); + }) + ); + } catch {} + return this.#outputStats; + } + + async areInputsNewer(): Promise<boolean> { + const inputStats = await Promise.all( + this.#inputs.map(input => { + return stat(input); + }) + ); + const outputStats = await this.getOutputStats(); + if ( + outputStats && + outputStats.reduce(reduceMinTime, Infinity) > + inputStats.reduce(reduceMaxTime, 0) + ) { + return false; + } + return true; + } + + #run(): Promise<void> { + return this.#callback({ + name: this.#name, + inputs: this.#inputs, + outputs: this.#outputs, + }); + } +} + +export const job = ( + name: string, + callback: (ctx: JobContext) => Promise<void> +): JobBuilder => { + return new JobBuilder(name, callback); +}; + +const reduceMaxTime = (time: number, stat: Stats) => { + return time < stat.mtimeMs ? stat.mtimeMs : time; +}; + +const reduceMinTime = (time: number, stat: Stats) => { + return time > stat.mtimeMs ? stat.mtimeMs : time; +}; diff --git a/remote/test/puppeteer/tools/internal/util.ts b/remote/test/puppeteer/tools/internal/util.ts new file mode 100644 index 0000000000..4ebbe8b86b --- /dev/null +++ b/remote/test/puppeteer/tools/internal/util.ts @@ -0,0 +1,14 @@ +import {spawnSync} from 'child_process'; + +export const spawnAndLog = (...args: string[]): void => { + const {stdout, stderr} = spawnSync(args[0]!, args.slice(1), { + encoding: 'utf-8', + shell: true, + }); + if (stdout) { + console.log(stdout); + } + if (stderr) { + console.error(stderr); + } +}; diff --git a/remote/test/puppeteer/tools/mochaRunner/README.md b/remote/test/puppeteer/tools/mochaRunner/README.md new file mode 100644 index 0000000000..1e4398a63c --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/README.md @@ -0,0 +1,73 @@ +# Mocha Runner + +Mocha Runner is a test runner on top of mocha. +It uses `/test/TestSuites.json` and `/test/TestExpectations.json` files to run mocha tests in multiple configurations and interpret results. + +## Running tests for Mocha Runner itself. + +```bash +npm run build && npx c8 node tools/mochaRunner/lib/test.js +``` + +## Running tests using Mocha Runner + +```bash +npm run build && npm run test +``` + +By default, the runner runs all test suites applicable to the current platform. +To pick a test suite, provide the `--test-suite` arguments. For example, + +```bash +npm run build && npm run test -- --test-suite chrome-headless +``` + +## TestSuites.json + +Define test suites via the `testSuites` attribute. `parameters` can be used in the `TestExpectations.json` to disable tests +based on parameters. The meaning for parameters is defined in `parameterDefinitions` which tell what env object corresponds +to the given parameter. + +## TestExpectations.json + +An expectation looks like this: + +```json +{ + "testIdPattern": "[accessibility.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] +} +``` + +| Field | Description | Type | Match Logic | +| --------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ----------- | +| `testIdPattern` | Defines the full name (or pattern) to match against test name | string | - | +| `platforms` | Defines the platforms the expectation is for | Array<`linux` \| `win32` \|`darwin`> | `OR` | +| `parameters` | Defines the parameters that the test has to match | Array<[ParameterDefinitions](https://github.com/puppeteer/puppeteer/blob/main/test/TestSuites.json)> | `AND` | +| `expectations` | The list of test results that are considered to be acceptable | Array<`PASS` \| `FAIL` \| `TIMEOUT` \| `SKIP`> | `OR` | + +> Order of defining expectations matters. The latest expectation that is set will take president over earlier ones. + +> Adding `SKIP` to `expectations` will prevent the test from running, no matter if there are other expectations. + +### Using pattern in `testIdPattern` + +Sometimes we want a whole group of test to run. For that we can use a +pattern to achieve. +Pattern are defined with the use of `*` (using greedy method). + +Examples: +| Pattern | Description | Example Pattern | Example match | +|------------------------|---------------------------------------------------------------------------------------------|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------| +| `*` | Match all tests | - | - | +| `[test.spec] *` | Matches tests for the given file | `[jshandle.spec] *` | `[jshandle] JSHandle JSHandle.toString should work for primitives` | +| `[test.spec] <text> *` | Matches tests with for a given test with a specific prefixed test (usually a describe node) | `[page.spec] Page Page.goto *` | `[page.spec] Page Page.goto should work`,<br>`[page.spec] Page Page.goto should work with anchor navigation` | +| `[test.spec] * <text>` | Matches test with a surfix | `[navigation.spec] * should work` | `[navigation.spec] navigation Page.goto should work`,<br>`[navigation.spec] navigation Page.waitForNavigation should work` | + +## Updating Expectations + +Currently, expectations are updated manually. The test runner outputs the +suggested changes to the expectation file if the test run does not match +expectations. diff --git a/remote/test/puppeteer/tools/mochaRunner/src/interface.ts b/remote/test/puppeteer/tools/mochaRunner/src/interface.ts new file mode 100644 index 0000000000..79329fcb0d --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/src/interface.ts @@ -0,0 +1,130 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Mocha from 'mocha'; +import commonInterface from 'mocha/lib/interfaces/common'; + +import {testIdMatchesExpectationPattern} from './utils.js'; + +type SuiteFunction = ((this: Mocha.Suite) => void) | undefined; +type ExclusiveSuiteFunction = (this: Mocha.Suite) => void; + +const skippedTests: Array<{testIdPattern: string; skip: true}> = process.env[ + 'PUPPETEER_SKIPPED_TEST_CONFIG' +] + ? JSON.parse(process.env['PUPPETEER_SKIPPED_TEST_CONFIG']) + : []; + +function shouldSkipTest(test: Mocha.Test): boolean { + // TODO: more efficient lookup. + const definition = skippedTests.find(skippedTest => { + return testIdMatchesExpectationPattern(test, skippedTest.testIdPattern); + }); + if (definition && definition.skip) { + return true; + } + return false; +} + +function customBDDInterface(suite: Mocha.Suite) { + const suites = [suite]; + + suite.on( + Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE, + function (context, file, mocha) { + const common = commonInterface(suites, context, mocha); + + context['before'] = common.before; + context['after'] = common.after; + context['beforeEach'] = common.beforeEach; + context['afterEach'] = common.afterEach; + if (mocha.options.delay) { + context['run'] = common.runWithSuite(suite); + } + function describe(title: string, fn: SuiteFunction) { + return common.suite.create({ + title: title, + file: file, + fn: fn, + }); + } + describe.only = function (title: string, fn: ExclusiveSuiteFunction) { + return common.suite.only({ + title: title, + file: file, + fn: fn, + isOnly: true, + }); + }; + + describe.skip = function (title: string, fn: SuiteFunction) { + return common.suite.skip({ + title: title, + file: file, + fn: fn, + }); + }; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + context['describe'] = describe; + + function it(title: string, fn: Mocha.TestFunction, itOnly = false) { + const suite = suites[0]!; + const test = new Mocha.Test(title, suite.isPending() ? undefined : fn); + test.file = file; + test.parent = suite; + + const describeOnly = Boolean( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + suite.parent?._onlySuites.find(child => { + return child === suite; + }) + ); + + if (shouldSkipTest(test) && !(itOnly || describeOnly)) { + const test = new Mocha.Test(title); + test.file = file; + suite.addTest(test); + return test; + } else { + suite.addTest(test); + return test; + } + } + + it.only = function (title: string, fn: Mocha.TestFunction) { + return common.test.only( + mocha, + (context['it'] as typeof it)(title, fn, true) + ); + }; + + it.skip = function (title: string) { + return context['it'](title); + }; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + context.it = it; + } + ); +} + +customBDDInterface.description = 'Custom BDD'; + +module.exports = customBDDInterface; diff --git a/remote/test/puppeteer/tools/mochaRunner/src/main.ts b/remote/test/puppeteer/tools/mochaRunner/src/main.ts new file mode 100644 index 0000000000..d2547e721c --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/src/main.ts @@ -0,0 +1,259 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {randomUUID} from 'crypto'; +import fs from 'fs'; +import {spawn, SpawnOptions} from 'node:child_process'; +import os from 'os'; +import path from 'path'; + +import { + TestExpectation, + MochaResults, + zTestSuiteFile, + zPlatform, + TestSuite, + TestSuiteFile, + Platform, +} from './types.js'; +import { + extendProcessEnv, + filterByPlatform, + readJSON, + filterByParameters, + getExpectationUpdates, + printSuggestions, + RecommendedExpectation, + writeJSON, +} from './utils.js'; + +function getApplicableTestSuites( + parsedSuitesFile: TestSuiteFile, + platform: Platform +): TestSuite[] { + const testSuiteArgIdx = process.argv.indexOf('--test-suite'); + let applicableSuites: TestSuite[] = []; + + if (testSuiteArgIdx === -1) { + applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform); + } else { + const testSuiteId = process.argv[testSuiteArgIdx + 1]; + const testSuite = parsedSuitesFile.testSuites.find(suite => { + return suite.id === testSuiteId; + }); + + if (!testSuite) { + console.error(`Test suite ${testSuiteId} is not defined`); + process.exit(1); + } + + if (!testSuite.platforms.includes(platform)) { + console.warn( + `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.` + ); + } + + applicableSuites = [testSuite]; + } + + return applicableSuites; +} + +async function main() { + const noCoverage = process.argv.indexOf('--no-coverage') !== -1; + const noSuggestions = process.argv.indexOf('--no-suggestions') !== -1; + + const statsFilenameIdx = process.argv.indexOf('--save-stats-to'); + let statsFilename = ''; + if (statsFilenameIdx !== -1) { + statsFilename = process.argv[statsFilenameIdx + 1] as string; + if (statsFilename.includes('INSERTID')) { + statsFilename = statsFilename.replace(/INSERTID/gi, randomUUID()); + } + } + + const platform = zPlatform.parse(os.platform()); + + const expectations = readJSON( + path.join(process.cwd(), 'test', 'TestExpectations.json') + ) as TestExpectation[]; + + const parsedSuitesFile = zTestSuiteFile.parse( + readJSON(path.join(process.cwd(), 'test', 'TestSuites.json')) + ); + + const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform); + + console.log('Planning to run the following test suites', applicableSuites); + if (statsFilename) { + console.log('Test stats will be saved to', statsFilename); + } + + let fail = false; + const recommendations: RecommendedExpectation[] = []; + try { + for (const suite of applicableSuites) { + const parameters = suite.parameters; + + const applicableExpectations = filterByParameters( + filterByPlatform(expectations, platform), + parameters + ).reverse(); + + // Add more logging when the GitHub Action Debugging option is set + // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + const githubActionDebugging = process.env['RUNNER_DEBUG'] + ? { + DEBUG: 'puppeteer:*', + EXTRA_LAUNCH_OPTIONS: JSON.stringify({ + extraPrefsFirefox: { + 'remote.log.level': 'Trace', + }, + }), + } + : {}; + + const env = extendProcessEnv([ + ...parameters.map(param => { + return parsedSuitesFile.parameterDefinitions[param]; + }), + { + PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify( + applicableExpectations.map(ex => { + return { + testIdPattern: ex.testIdPattern, + skip: ex.expectations.includes('SKIP'), + }; + }) + ), + }, + githubActionDebugging, + ]); + + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-test-runner-') + ); + const tmpFilename = statsFilename + ? statsFilename + : path.join(tmpDir, 'output.json'); + console.log('Running', JSON.stringify(parameters), tmpFilename); + const reporterArgumentIndex = process.argv.indexOf('--reporter'); + const args = [ + '-u', + path.join(__dirname, 'interface.js'), + '-R', + reporterArgumentIndex === -1 + ? path.join(__dirname, 'reporter.js') + : process.argv[reporterArgumentIndex + 1] || '', + '-O', + 'output=' + tmpFilename, + ]; + const retriesArgumentIndex = process.argv.indexOf('--retries'); + const timeoutArgumentIndex = process.argv.indexOf('--timeout'); + if (retriesArgumentIndex > -1) { + args.push('--retries', process.argv[retriesArgumentIndex + 1] || ''); + } + if (timeoutArgumentIndex > -1) { + args.push('--timeout', process.argv[timeoutArgumentIndex + 1] || ''); + } + if (process.argv.indexOf('--no-parallel')) { + args.push('--no-parallel'); + } + if (process.argv.indexOf('--fullTrace')) { + args.push('--fullTrace'); + } + const spawnArgs: SpawnOptions = { + shell: true, + cwd: process.cwd(), + stdio: 'inherit', + env, + }; + const handle = noCoverage + ? spawn('npx', ['mocha', ...args], spawnArgs) + : spawn( + 'npx', + [ + 'c8', + '--check-coverage', + '--lines', + String(suite.expectedLineCoverage), + 'npx mocha', + ...args, + ], + spawnArgs + ); + await new Promise<void>((resolve, reject) => { + handle.on('error', err => { + reject(err); + }); + handle.on('close', () => { + resolve(); + }); + }); + console.log('Finished', JSON.stringify(parameters)); + try { + const results = readJSON(tmpFilename) as MochaResults; + const updates = getExpectationUpdates(results, applicableExpectations, { + platforms: [os.platform()], + parameters, + }); + results.parameters = parameters; + results.platform = platform; + results.date = new Date().toISOString(); + if (updates.length > 0) { + fail = true; + recommendations.push(...updates); + results.updates = updates; + writeJSON(tmpFilename, results); + } else { + console.log('Test run matches expectations'); + writeJSON(tmpFilename, results); + continue; + } + } catch (err) { + fail = true; + console.error(err); + } + } + } catch (err) { + fail = true; + console.error(err); + } finally { + if (!noSuggestions) { + printSuggestions( + recommendations, + 'add', + 'Add the following to TestExpectations.json to ignore the error:' + ); + printSuggestions( + recommendations, + 'remove', + 'Remove the following from the TestExpectations.json to ignore the error:' + ); + printSuggestions( + recommendations, + 'update', + 'Update the following expectations in the TestExpectations.json to ignore the error:' + ); + } + process.exit(fail ? 1 : 0); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/remote/test/puppeteer/tools/mochaRunner/src/reporter.ts b/remote/test/puppeteer/tools/mochaRunner/src/reporter.ts new file mode 100644 index 0000000000..37ca586215 --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/src/reporter.ts @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Mocha from 'mocha'; + +class SpecJSONReporter extends Mocha.reporters.Spec { + constructor(runner: Mocha.Runner, options?: Mocha.MochaOptions) { + super(runner, options); + Mocha.reporters.JSON.call(this, runner, options); + } +} + +module.exports = SpecJSONReporter; diff --git a/remote/test/puppeteer/tools/mochaRunner/src/test.ts b/remote/test/puppeteer/tools/mochaRunner/src/test.ts new file mode 100644 index 0000000000..1e0328499c --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/src/test.ts @@ -0,0 +1,134 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import assert from 'node:assert/strict'; +import {describe, test} from 'node:test'; + +import {TestExpectation} from './types.js'; +import { + filterByParameters, + getTestResultForFailure, + isWildCardPattern, + testIdMatchesExpectationPattern, +} from './utils.js'; +import {getFilename, extendProcessEnv} from './utils.js'; + +void test('extendProcessEnv', () => { + const env = extendProcessEnv([{TEST: 'TEST'}, {TEST2: 'TEST2'}]); + assert.equal(env['TEST'], 'TEST'); + assert.equal(env['TEST2'], 'TEST2'); +}); + +void test('getFilename', () => { + assert.equal(getFilename('/etc/test.ts'), 'test'); + assert.equal(getFilename('/etc/test.js'), 'test'); +}); + +void test('getTestResultForFailure', () => { + assert.equal( + getTestResultForFailure({err: {code: 'ERR_MOCHA_TIMEOUT'}}), + 'TIMEOUT' + ); + assert.equal(getTestResultForFailure({err: {code: 'ERROR'}}), 'FAIL'); +}); + +void test('filterByParameters', () => { + const expectations: TestExpectation[] = [ + { + testIdPattern: + '[oopif.spec] OOPIF "after all" hook for "should keep track of a frames OOP state"', + platforms: ['darwin'], + parameters: ['firefox', 'headless'], + expectations: ['FAIL'], + }, + ]; + assert.equal( + filterByParameters(expectations, ['firefox', 'headless']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['firefox']).length, 0); + assert.equal( + filterByParameters(expectations, ['firefox', 'headless', 'other']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['other']).length, 0); +}); + +void test('isWildCardPattern', () => { + assert.equal(isWildCardPattern(''), false); + assert.equal(isWildCardPattern('a'), false); + assert.equal(isWildCardPattern('*'), true); + + assert.equal(isWildCardPattern('[queryHandler.spec]'), false); + assert.equal(isWildCardPattern('[queryHandler.spec] *'), true); + assert.equal(isWildCardPattern(' [queryHandler.spec] '), false); + + assert.equal(isWildCardPattern('[queryHandler.spec] Query'), false); + assert.equal(isWildCardPattern('[queryHandler.spec] Page *'), true); + assert.equal(isWildCardPattern('[queryHandler.spec] Page Page.goto *'), true); +}); + +describe('testIdMatchesExpectationPattern', () => { + const expectations: Array<[string, boolean]> = [ + ['', false], + ['*', true], + ['* should work', true], + ['* Page.setContent *', true], + ['* should work as expected', false], + ['Page.setContent *', false], + ['[page.spec]', false], + ['[page.spec] *', true], + ['[page.spec] Page *', true], + ['[page.spec] Page Page.setContent *', true], + ['[page.spec] Page Page.setContent should work', true], + ['[page.spec] Page * should work', true], + ['[page.spec] * Page.setContent *', true], + ['[jshandle.spec] *', false], + ['[jshandle.spec] JSHandle should work', false], + ]; + + void test('with MochaTest', () => { + const test = { + title: 'should work', + file: 'page.spec.ts', + fullTitle() { + return 'Page Page.setContent should work'; + }, + } as any; + + for (const [pattern, expected] of expectations) { + assert.equal( + testIdMatchesExpectationPattern(test, pattern), + expected, + `Expected "${pattern}" to yield "${expected}"` + ); + } + }); + void test('with MochaTestResult', () => { + const test = { + title: 'should work', + file: 'page.spec.ts', + fullTitle: 'Page Page.setContent should work', + } as any; + + for (const [pattern, expected] of expectations) { + assert.equal( + testIdMatchesExpectationPattern(test, pattern), + expected, + `Expected "${pattern}" to yield "${expected}"` + ); + } + }); +}); diff --git a/remote/test/puppeteer/tools/mochaRunner/src/types.ts b/remote/test/puppeteer/tools/mochaRunner/src/types.ts new file mode 100644 index 0000000000..8d8a08ee98 --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/src/types.ts @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {z} from 'zod'; + +import {RecommendedExpectation} from './utils.js'; + +export const zPlatform = z.enum(['win32', 'linux', 'darwin']); + +export type Platform = z.infer<typeof zPlatform>; + +export const zTestSuite = z.object({ + id: z.string(), + platforms: z.array(zPlatform), + parameters: z.array(z.string()), + expectedLineCoverage: z.number(), +}); + +export type TestSuite = z.infer<typeof zTestSuite>; + +export const zTestSuiteFile = z.object({ + testSuites: z.array(zTestSuite), + parameterDefinitions: z.record(z.any()), +}); + +export type TestSuiteFile = z.infer<typeof zTestSuiteFile>; + +export type TestResult = 'PASS' | 'FAIL' | 'TIMEOUT' | 'SKIP'; + +export type TestExpectation = { + testIdPattern: string; + platforms: NodeJS.Platform[]; + parameters: string[]; + expectations: TestResult[]; +}; + +export type MochaTestResult = { + fullTitle: string; + title: string; + file: string; + err?: {code: string}; +}; + +export type MochaResults = { + stats: unknown; + pending: MochaTestResult[]; + passes: MochaTestResult[]; + failures: MochaTestResult[]; + // Added by mochaRunner. + updates?: RecommendedExpectation[]; + parameters?: string[]; + platform?: string; + date?: string; +}; diff --git a/remote/test/puppeteer/tools/mochaRunner/src/utils.ts b/remote/test/puppeteer/tools/mochaRunner/src/utils.ts new file mode 100644 index 0000000000..9fdbf65583 --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/src/utils.ts @@ -0,0 +1,265 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import { + MochaTestResult, + TestExpectation, + MochaResults, + TestResult, +} from './types.js'; + +export function extendProcessEnv(envs: object[]): NodeJS.ProcessEnv { + return envs.reduce( + (acc: object, item: object) => { + Object.assign(acc, item); + return acc; + }, + { + ...process.env, + } + ) as NodeJS.ProcessEnv; +} + +export function getFilename(file: string): string { + return path.basename(file).replace(path.extname(file), ''); +} + +export function readJSON(path: string): unknown { + return JSON.parse(fs.readFileSync(path, 'utf-8')); +} + +export function writeJSON(path: string, json: unknown): unknown { + return fs.writeFileSync(path, JSON.stringify(json, null, 2)); +} + +export function filterByPlatform<T extends {platforms: NodeJS.Platform[]}>( + items: T[], + platform: NodeJS.Platform +): T[] { + return items.filter(item => { + return item.platforms.includes(platform); + }); +} + +export function prettyPrintJSON(json: unknown): void { + console.log(JSON.stringify(json, null, 2)); +} + +export function printSuggestions( + recommendations: RecommendedExpectation[], + action: RecommendedExpectation['action'], + message: string +): void { + const toPrint = recommendations.filter(item => { + return item.action === action; + }); + if (toPrint.length) { + console.log(message); + prettyPrintJSON( + toPrint.map(item => { + return item.expectation; + }) + ); + console.log( + 'The recommendations are based on the following applied expectaions:' + ); + prettyPrintJSON( + toPrint.map(item => { + return item.basedOn; + }) + ); + } +} + +export function filterByParameters( + expectations: TestExpectation[], + parameters: string[] +): TestExpectation[] { + const querySet = new Set(parameters); + return expectations.filter(ex => { + return ex.parameters.every(param => { + return querySet.has(param); + }); + }); +} + +/** + * The last expectation that matches an empty string as all tests pattern + * or the name of the file or the whole name of the test the filter wins. + */ +export function findEffectiveExpectationForTest( + expectations: TestExpectation[], + result: MochaTestResult +): TestExpectation | undefined { + return expectations.find(expectation => { + return testIdMatchesExpectationPattern(result, expectation.testIdPattern); + }); +} + +export type RecommendedExpectation = { + expectation: TestExpectation; + action: 'remove' | 'add' | 'update'; + basedOn?: TestExpectation; +}; + +export function isWildCardPattern(testIdPattern: string): boolean { + return testIdPattern.includes('*'); +} + +export function getExpectationUpdates( + results: MochaResults, + expectations: TestExpectation[], + context: { + platforms: NodeJS.Platform[]; + parameters: string[]; + } +): RecommendedExpectation[] { + const output: Map<string, RecommendedExpectation> = new Map(); + + for (const pass of results.passes) { + // If an error occurs during a hook + // the error not have a file associated with it + if (!pass.file) { + continue; + } + + const expectationEntry = findEffectiveExpectationForTest( + expectations, + pass + ); + if (expectationEntry && !expectationEntry.expectations.includes('PASS')) { + if (isWildCardPattern(expectationEntry.testIdPattern)) { + addEntry({ + expectation: { + testIdPattern: getTestId(pass.file, pass.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: ['PASS'], + }, + action: 'add', + basedOn: expectationEntry, + }); + } else { + addEntry({ + expectation: expectationEntry, + action: 'remove', + basedOn: expectationEntry, + }); + } + } + } + + for (const failure of results.failures) { + // If an error occurs during a hook + // the error not have a file associated with it + if (!failure.file) { + continue; + } + + const expectationEntry = findEffectiveExpectationForTest( + expectations, + failure + ); + if (expectationEntry && !expectationEntry.expectations.includes('SKIP')) { + if ( + !expectationEntry.expectations.includes( + getTestResultForFailure(failure) + ) + ) { + // If the effective explanation is a wildcard, we recommend adding a new + // expectation instead of updating the wildcard that might affect multiple + // tests. + if (isWildCardPattern(expectationEntry.testIdPattern)) { + addEntry({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + action: 'add', + basedOn: expectationEntry, + }); + } else { + addEntry({ + expectation: { + ...expectationEntry, + expectations: [ + ...expectationEntry.expectations, + getTestResultForFailure(failure), + ], + }, + action: 'update', + basedOn: expectationEntry, + }); + } + } + } else if (!expectationEntry) { + addEntry({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + action: 'add', + }); + } + } + + function addEntry(value: RecommendedExpectation) { + const key = JSON.stringify(value); + if (!output.has(key)) { + output.set(key, value); + } + } + + return [...output.values()]; +} + +export function getTestResultForFailure( + test: Pick<MochaTestResult, 'err'> +): TestResult { + return test.err?.code === 'ERR_MOCHA_TIMEOUT' ? 'TIMEOUT' : 'FAIL'; +} + +export function getTestId(file: string, fullTitle?: string): string { + return fullTitle + ? `[${getFilename(file)}] ${fullTitle}` + : `[${getFilename(file)}]`; +} + +export function testIdMatchesExpectationPattern( + test: MochaTestResult | Mocha.Test, + pattern: string +): boolean { + const patternRegExString = pattern + // Replace `*` with non special character + .replace(/\*/g, '--STAR--') + // Escape special characters https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // Replace placeholder with greedy match + .replace(/--STAR--/g, '(.*)?'); + // Match beginning and end explicitly + const patternRegEx = new RegExp(`^${patternRegExString}$`); + const fullTitle = + typeof test.fullTitle === 'string' ? test.fullTitle : test.fullTitle(); + + return patternRegEx.test(getTestId(test.file ?? '', fullTitle)); +} diff --git a/remote/test/puppeteer/tools/mochaRunner/tsconfig.json b/remote/test/puppeteer/tools/mochaRunner/tsconfig.json new file mode 100644 index 0000000000..c2576c2564 --- /dev/null +++ b/remote/test/puppeteer/tools/mochaRunner/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "allowJs": true, + "composite": true, + "module": "CommonJS", + "outDir": "lib", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/remote/test/puppeteer/tools/remove_version_suffix.js b/remote/test/puppeteer/tools/remove_version_suffix.js new file mode 100644 index 0000000000..091a35ec9b --- /dev/null +++ b/remote/test/puppeteer/tools/remove_version_suffix.js @@ -0,0 +1,26 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const json = fs.readFileSync('./package.json', 'utf8').toString(); +const pkg = JSON.parse(json); +const oldVersion = pkg.version; +const version = oldVersion.replace(/-post$/, ''); +const updated = json.replace( + `"version": "${oldVersion}"`, + `"version": "${version}"` +); +fs.writeFileSync('./package.json', updated); diff --git a/remote/test/puppeteer/tools/sort-test-expectations.js b/remote/test/puppeteer/tools/sort-test-expectations.js new file mode 100644 index 0000000000..96e32145e4 --- /dev/null +++ b/remote/test/puppeteer/tools/sort-test-expectations.js @@ -0,0 +1,59 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// TODO: this could be an eslint rule probably. + +const fs = require('fs'); +const path = require('path'); + +const prettier = require('prettier'); + +const source = 'test/TestExpectations.json'; + +const testExpectations = JSON.parse(fs.readFileSync(source, 'utf-8')); + +function getSpecificity(item) { + return ( + item.parameters.length + + (item.testIdPattern.includes('*') + ? item.testIdPattern === '*' + ? 0 + : 1 + : 2) + ); +} + +testExpectations.sort((a, b) => { + const result = getSpecificity(a) - getSpecificity(b); + if (result === 0) { + return a.testIdPattern.localeCompare(b.testIdPattern); + } + return result; +}); + +testExpectations.forEach(item => { + item.parameters.sort(); + item.expectations.sort(); + item.platforms.sort(); +}); + +fs.writeFileSync( + source, + prettier.format(JSON.stringify(testExpectations), { + ...require(path.join(__dirname, '..', '.prettierrc.cjs')), + parser: 'json', + }) +); diff --git a/remote/test/puppeteer/tools/third_party/validate-licenses.ts b/remote/test/puppeteer/tools/third_party/validate-licenses.ts new file mode 100644 index 0000000000..4d1c05497b --- /dev/null +++ b/remote/test/puppeteer/tools/third_party/validate-licenses.ts @@ -0,0 +1,153 @@ +// The MIT License + +// Copyright (c) 2010-2022 Google LLC. http://angular.io/license + +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// Taken and adapted from https://github.com/angular/angular-cli/blob/173823d/scripts/validate-licenses.ts. + +import * as path from 'path'; + +import checker from 'license-checker'; +import spdxSatisfies from 'spdx-satisfies'; + +/** + * A general note on some black listed specific licenses: + * + * - CC0 This is not a valid license. It does not grant copyright of the + * code/asset, and does not resolve patents or other licensed work. The + * different claims also have no standing in court and do not provide + * protection to or from Google and/or third parties. We cannot use nor + * contribute to CC0 licenses. + * - Public Domain Same as CC0, it is not a valid license. + */ +const allowedLicenses = [ + // Regular valid open source licenses supported by Google. + 'MIT', + 'ISC', + 'Apache-2.0', + 'Python-2.0', + 'Artistic-2.0', + + 'BSD-2-Clause', + 'BSD-3-Clause', + 'BSD-4-Clause', + + // All CC-BY licenses have a full copyright grant and attribution section. + 'CC-BY-3.0', + 'CC-BY-4.0', + + // Have a full copyright grant. Validated by opensource team. + 'Unlicense', + 'CC0-1.0', + '0BSD', + + // Combinations. + '(AFL-2.1 OR BSD-2-Clause)', +]; + +// Name variations of SPDX licenses that some packages have. +// Licenses not included in SPDX but accepted will be converted to MIT. +const licenseReplacements: {[key: string]: string} = { + // Just a longer string that our script catches. SPDX official name is the shorter one. + 'Apache License, Version 2.0': 'Apache-2.0', + Apache2: 'Apache-2.0', + 'Apache 2.0': 'Apache-2.0', + 'Apache v2': 'Apache-2.0', + 'AFLv2.1': 'AFL-2.1', + // BSD is BSD-2-clause by default. + BSD: 'BSD-2-Clause', +}; + +// Specific packages to ignore, add a reason in a comment. Format: package-name@version. +const ignoredPackages = [ + // * Development only + 'spdx-license-ids@3.0.5', // CC0 but it's content only (index.json, no code) and not distributed. +]; + +// Check if a license is accepted by an array of accepted licenses +function _passesSpdx(licenses: string[], accepted: string[]) { + try { + return spdxSatisfies(licenses.join(' AND '), accepted.join(' OR ')); + } catch { + return false; + } +} + +function main(): Promise<number> { + return new Promise(resolve => { + const startFolder = path.join(__dirname, '..', '..'); + checker.init( + {start: startFolder, excludePrivatePackages: true}, + (err: Error, json: object) => { + if (err) { + console.error(`Something happened:\n${err.message}`); + resolve(1); + } else { + console.info(`Testing ${Object.keys(json).length} packages.\n`); + + // Packages with bad licenses are those that neither pass SPDX nor are ignored. + const badLicensePackages = Object.keys(json) + .map(key => { + return { + id: key, + licenses: ([] as string[]) + .concat((json[key] as {licenses: string[]}).licenses) + // `*` is used when the license is guessed. + .map(x => { + return x.replace(/\*$/, ''); + }) + .map(x => { + return x in licenseReplacements + ? licenseReplacements[x] + : x; + }), + }; + }) + .filter(pkg => { + return !_passesSpdx(pkg.licenses, allowedLicenses); + }) + .filter(pkg => { + return !ignoredPackages.find(ignored => { + return ignored === pkg.id; + }); + }); + + // Report packages with bad licenses + if (badLicensePackages.length > 0) { + console.error('Invalid package licences found:'); + badLicensePackages.forEach(pkg => { + console.error(`${pkg.id}: ${JSON.stringify(pkg.licenses)}`); + }); + console.error( + `\n${badLicensePackages.length} total packages with invalid licenses.` + ); + resolve(2); + } else { + console.info('All package licenses are valid.'); + resolve(0); + } + } + } + ); + }); +} + +main().then(code => { + return process.exit(code); +}); diff --git a/remote/test/puppeteer/tools/tsconfig.json b/remote/test/puppeteer/tools/tsconfig.json new file mode 100644 index 0000000000..393392c494 --- /dev/null +++ b/remote/test/puppeteer/tools/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../tsconfig.base.json", + "files": ["../package.json"] +} |