diff options
Diffstat (limited to 'remote/test/puppeteer/packages/browsers')
43 files changed, 4488 insertions, 0 deletions
diff --git a/remote/test/puppeteer/packages/browsers/.gitignore b/remote/test/puppeteer/packages/browsers/.gitignore new file mode 100644 index 0000000000..23b2baa7ca --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/.gitignore @@ -0,0 +1 @@ +test/cache
\ No newline at end of file diff --git a/remote/test/puppeteer/packages/browsers/.mocharc.cjs b/remote/test/puppeteer/packages/browsers/.mocharc.cjs new file mode 100644 index 0000000000..4cabbc3232 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/.mocharc.cjs @@ -0,0 +1,7 @@ +module.exports = { + logLevel: 'debug', + spec: 'test/build/**/*.spec.js', + exit: !!process.env.CI, + reporter: 'spec', + timeout: 10_000, +}; diff --git a/remote/test/puppeteer/packages/browsers/CHANGELOG.md b/remote/test/puppeteer/packages/browsers/CHANGELOG.md new file mode 100644 index 0000000000..9da7ab8980 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/CHANGELOG.md @@ -0,0 +1,131 @@ +# Changelog + +## [1.0.0](https://github.com/puppeteer/puppeteer/compare/browsers-v0.5.0...browsers-v1.0.0) (2023-05-02) + + +### ⚠ BREAKING CHANGES + +* drop support for node14 ([#10019](https://github.com/puppeteer/puppeteer/issues/10019)) +* switch to Chrome for Testing instead of Chromium ([#10054](https://github.com/puppeteer/puppeteer/issues/10054)) + +### Features + +* drop support for node14 ([#10019](https://github.com/puppeteer/puppeteer/issues/10019)) ([7405d65](https://github.com/puppeteer/puppeteer/commit/7405d6585aa09b240fbab09aa360674d4442b3d9)) +* switch to Chrome for Testing instead of Chromium ([#10054](https://github.com/puppeteer/puppeteer/issues/10054)) ([df4d60c](https://github.com/puppeteer/puppeteer/commit/df4d60c187aa11c4ad783827242e9511f4ec2aab)) + + +### Bug Fixes + +* add Host header when used with http_proxy ([#10080](https://github.com/puppeteer/puppeteer/issues/10080)) ([edbfff7](https://github.com/puppeteer/puppeteer/commit/edbfff7b04baffc29c01c37c595d6b3355c0dea0)) + +## [0.5.0](https://github.com/puppeteer/puppeteer/compare/browsers-v0.4.1...browsers-v0.5.0) (2023-04-21) + + +### Features + +* **browser:** add a method to get installed browsers ([#10057](https://github.com/puppeteer/puppeteer/issues/10057)) ([e16e2a9](https://github.com/puppeteer/puppeteer/commit/e16e2a97284f5e7ab4073f375254572a6a89e800)) + +## [0.4.1](https://github.com/puppeteer/puppeteer/compare/browsers-v0.4.0...browsers-v0.4.1) (2023-04-13) + + +### Bug Fixes + +* report install errors properly ([#10016](https://github.com/puppeteer/puppeteer/issues/10016)) ([7381229](https://github.com/puppeteer/puppeteer/commit/7381229a164e598e7523862f2438cd0cd1cd796a)) + +## [0.4.0](https://github.com/puppeteer/puppeteer/compare/browsers-v0.3.3...browsers-v0.4.0) (2023-04-06) + + +### Features + +* **browsers:** support downloading chromedriver ([#9990](https://github.com/puppeteer/puppeteer/issues/9990)) ([ef0fb5d](https://github.com/puppeteer/puppeteer/commit/ef0fb5d87299c604af2387ac1c72be317c50316d)) + +## [0.3.3](https://github.com/puppeteer/puppeteer/compare/browsers-v0.3.2...browsers-v0.3.3) (2023-04-06) + + +### Bug Fixes + +* **browsers:** update package json ([#9968](https://github.com/puppeteer/puppeteer/issues/9968)) ([817288c](https://github.com/puppeteer/puppeteer/commit/817288cd901121ddc8a44226eda689bb784cee61)) +* **browsers:** various fixes and improvements ([#9966](https://github.com/puppeteer/puppeteer/issues/9966)) ([f1211cb](https://github.com/puppeteer/puppeteer/commit/f1211cbec091ec669de019aeb7fb4f011a81c1d7)) +* consider downloadHost as baseUrl ([#9973](https://github.com/puppeteer/puppeteer/issues/9973)) ([05a44af](https://github.com/puppeteer/puppeteer/commit/05a44afe5affcac9fe0f0a2e83f17807c99b2f0c)) + +## [0.3.2](https://github.com/puppeteer/puppeteer/compare/browsers-v0.3.1...browsers-v0.3.2) (2023-04-03) + + +### Bug Fixes + +* typo in the browsers package ([#9957](https://github.com/puppeteer/puppeteer/issues/9957)) ([c780384](https://github.com/puppeteer/puppeteer/commit/c7803844cf10b6edaa2da83134029b7acf5b45b2)) + +## [0.3.1](https://github.com/puppeteer/puppeteer/compare/browsers-v0.3.0...browsers-v0.3.1) (2023-03-29) + + +### Bug Fixes + +* bump @puppeteer/browsers ([#9938](https://github.com/puppeteer/puppeteer/issues/9938)) ([2a29d30](https://github.com/puppeteer/puppeteer/commit/2a29d30d1790b47c99f8d196b3844364d351acbd)) + +## [0.3.0](https://github.com/puppeteer/puppeteer/compare/browsers-v0.2.0...browsers-v0.3.0) (2023-03-27) + + +### Features + +* update Chrome browser binaries ([#9917](https://github.com/puppeteer/puppeteer/issues/9917)) ([fcb233c](https://github.com/puppeteer/puppeteer/commit/fcb233ce949f5f716aee39253e910104b04aa000)) + +## [0.2.0](https://github.com/puppeteer/puppeteer/compare/browsers-v0.1.1...browsers-v0.2.0) (2023-03-24) + + +### Features + +* implement a command to clear the cache ([#9868](https://github.com/puppeteer/puppeteer/issues/9868)) ([b8d38cb](https://github.com/puppeteer/puppeteer/commit/b8d38cb05f7eedf554ed46f2f7428b621197d1cc)) + +## [0.1.1](https://github.com/puppeteer/puppeteer/compare/browsers-v0.1.0...browsers-v0.1.1) (2023-03-14) + + +### Bug Fixes + +* export ChromeReleaseChannel ([#9851](https://github.com/puppeteer/puppeteer/issues/9851)) ([3e7a514](https://github.com/puppeteer/puppeteer/commit/3e7a514e556ddb4306aa3c15f24c512beaac65f4)) + +## [0.1.0](https://github.com/puppeteer/puppeteer/compare/browsers-v0.0.5...browsers-v0.1.0) (2023-03-14) + + +### Features + +* implement system channels for chrome in browsers ([#9844](https://github.com/puppeteer/puppeteer/issues/9844)) ([dec48a9](https://github.com/puppeteer/puppeteer/commit/dec48a95923e21a054c1d70d22c14001a0150293)) + + +### Bug Fixes + +* add browsers entry point ([#9846](https://github.com/puppeteer/puppeteer/issues/9846)) ([1a1e79d](https://github.com/puppeteer/puppeteer/commit/1a1e79d046ccad6fe843aa219501c17da08bc498)) + +## [0.0.5](https://github.com/puppeteer/puppeteer/compare/browsers-v0.0.4...browsers-v0.0.5) (2023-03-07) + + +### Bug Fixes + +* change the install output to include the executable path ([#9797](https://github.com/puppeteer/puppeteer/issues/9797)) ([8cca7bb](https://github.com/puppeteer/puppeteer/commit/8cca7bb7a2a1cdf62919d9c7eca62d6774e698db)) + +## [0.0.4](https://github.com/puppeteer/puppeteer/compare/browsers-v0.0.3...browsers-v0.0.4) (2023-03-06) + + +### Features + +* browsers: recognize chromium as a valid browser ([#9760](https://github.com/puppeteer/puppeteer/issues/9760)) ([04247a4](https://github.com/puppeteer/puppeteer/commit/04247a4e00b43683977bd8aa309d493eee663735)) + +## [0.0.3](https://github.com/puppeteer/puppeteer/compare/browsers-v0.0.2...browsers-v0.0.3) (2023-02-22) + + +### Bug Fixes + +* define options per command ([#9733](https://github.com/puppeteer/puppeteer/issues/9733)) ([8bae054](https://github.com/puppeteer/puppeteer/commit/8bae0545b7321d398dae3f522952dd981111587e)) + +## [0.0.2](https://github.com/puppeteer/puppeteer/compare/browsers-v0.0.1...browsers-v0.0.2) (2023-02-22) + + +### Bug Fixes + +* permissions for the browser CLI ([#9731](https://github.com/puppeteer/puppeteer/issues/9731)) ([e944931](https://github.com/puppeteer/puppeteer/commit/e944931de22726f35c5c83052892f8ab4667b035)) + +## 0.0.1 (2023-02-22) + + +### Features + +* initial release of browsers ([#9722](https://github.com/puppeteer/puppeteer/issues/9722)) ([#9727](https://github.com/puppeteer/puppeteer/issues/9727)) ([86a2d1d](https://github.com/puppeteer/puppeteer/commit/86a2d1dd3b2c024b886c6280e08a2d7dc8caabc5)) diff --git a/remote/test/puppeteer/packages/browsers/README.md b/remote/test/puppeteer/packages/browsers/README.md new file mode 100644 index 0000000000..4c40b78e3f --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/README.md @@ -0,0 +1,28 @@ +# @puppeteer/browsers + +Manage and launch browsers/drivers from a CLI or programmatically. + +## CLI + +Use `npx` to run the CLI: + +```bash +npx @puppeteer/browsers --help +``` + +CLI help will provide all documentation you need to use the CLI. + +```bash +npx @puppeteer/browsers --help # help for all commands +npx @puppeteer/browsers install --help # help for the install command +npx @puppeteer/browsers launch --help # help for the launch command +``` + +## Known limitations + +1. We support installing and running Firefox and Chrome/Chromium. The `latest` keyword only works during the installation. For the `launch` command you need to specify an exact build ID. The build ID is provided by the `install` command (see `npx @puppeteer/browsers install --help` for the format). +2. Launching the system browsers is only possible for Chrome/Chromium. + +## API + +The programmatic API allows installing and launching browsers from your code. See the `test` folder for examples on how to use the `install`, `canInstall`, `launch`, `computeExecutablePath`, `computeSystemExecutablePath` and other methods. diff --git a/remote/test/puppeteer/packages/browsers/api-extractor.docs.json b/remote/test/puppeteer/packages/browsers/api-extractor.docs.json new file mode 100644 index 0000000000..6a41a3b59c --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/api-extractor.docs.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "mainEntryPointFilePath": "<projectFolder>/lib/esm/main.d.ts", + + "extends": "./api-extractor.json", + + "dtsRollup": { + "enabled": false + }, + + "docModel": { + "enabled": true, + "apiJsonFilePath": "<projectFolder>/../../docs/<unscopedPackageName>.api.json" + } +} diff --git a/remote/test/puppeteer/packages/browsers/api-extractor.json b/remote/test/puppeteer/packages/browsers/api-extractor.json new file mode 100644 index 0000000000..da1caae622 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/api-extractor.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "mainEntryPointFilePath": "<projectFolder>/lib/esm/main.d.ts", + "bundledPackages": [], + + "apiReport": { + "enabled": false + }, + + "docModel": { + "enabled": false + }, + + "tsdocMetadata": { + "enabled": false + }, + + "messages": { + "compilerMessageReporting": { + "default": { + "logLevel": "warning" + } + }, + + "extractorMessageReporting": { + "ae-internal-missing-underscore": { + "logLevel": "none" + }, + "default": { + "logLevel": "warning" + } + }, + + "tsdocMessageReporting": { + "default": { + "logLevel": "warning" + } + } + } +} diff --git a/remote/test/puppeteer/packages/browsers/package.json b/remote/test/puppeteer/packages/browsers/package.json new file mode 100644 index 0000000000..c28787e197 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/package.json @@ -0,0 +1,123 @@ +{ + "name": "@puppeteer/browsers", + "version": "1.0.0", + "description": "Download and launch browsers", + "scripts": { + "build:docs": "wireit", + "build": "wireit", + "build:test": "wireit", + "clean": "tsc --build --clean && rm -rf lib", + "test": "wireit" + }, + "bin": { + "@puppeteer/browsers": "lib/cjs/main-cli.js" + }, + "main": "./lib/cjs/main.js", + "module": "./lib/esm/main.js", + "type": "commonjs", + "exports": { + ".": { + "import": "./lib/esm/main.js", + "require": "./lib/cjs/main.js" + } + }, + "wireit": { + "build": { + "command": "tsc -b && tsx ../../tools/chmod.ts 755 lib/cjs/main-cli.js lib/esm/main-cli.js", + "files": [ + "src/**/*.ts", + "tsconfig.json" + ], + "clean": "if-file-deleted", + "output": [ + "lib/**", + "!lib/esm/package.json" + ], + "dependencies": [ + "generate:package-json" + ] + }, + "generate:package-json": { + "command": "tsx ../../tools/generate_module_package_json.ts lib/esm/package.json", + "files": [ + "../../tools/generate_module_package_json.ts" + ], + "output": [ + "lib/esm/package.json" + ] + }, + "build:docs": { + "command": "api-extractor run --local --config \"./api-extractor.docs.json\"", + "files": [ + "api-extractor.docs.json", + "lib/esm/main.d.ts", + "tsconfig.json" + ], + "dependencies": [ + "build" + ] + }, + "build:test": { + "command": "tsc -b test/src/tsconfig.json", + "files": [ + "test/**/*.ts", + "test/src/tsconfig.json" + ], + "output": [ + "test/build/**" + ], + "dependencies": [ + "build", + "../testserver:build" + ] + }, + "test": { + "command": "node tools/downloadTestBrowsers.mjs && cross-env DEBUG=puppeteer:* mocha", + "files": [ + ".mocharc.cjs" + ], + "dependencies": [ + "build:test" + ] + } + }, + "keywords": [ + "puppeteer", + "browsers" + ], + "repository": { + "type": "git", + "url": "https://github.com/puppeteer/puppeteer/tree/main/packages/browsers" + }, + "author": "The Chromium Authors", + "license": "Apache-2.0", + "engines": { + "node": ">=16.0.0" + }, + "files": [ + "lib", + "!*.tsbuildinfo" + ], + "dependencies": { + "debug": "4.3.4", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "progress": "2.0.3", + "proxy-from-env": "1.1.0", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "yargs": "17.7.1" + }, + "devDependencies": { + "@types/node": "^14.15.0", + "@types/yargs": "17.0.22" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/CLI.ts b/remote/test/puppeteer/packages/browsers/src/CLI.ts new file mode 100644 index 0000000000..6767002269 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/CLI.ts @@ -0,0 +1,313 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {stdin as input, stdout as output} from 'process'; +import * as readline from 'readline'; + +import ProgressBar from 'progress'; +import type * as Yargs from 'yargs'; +import {hideBin} from 'yargs/helpers'; +import yargs from 'yargs/yargs'; + +import { + resolveBuildId, + Browser, + BrowserPlatform, + ChromeReleaseChannel, +} from './browser-data/browser-data.js'; +import {Cache} from './Cache.js'; +import {detectBrowserPlatform} from './detectPlatform.js'; +import {install} from './install.js'; +import { + computeExecutablePath, + computeSystemExecutablePath, + launch, +} from './launch.js'; + +type InstallArgs = { + browser: { + name: Browser; + buildId: string; + }; + path?: string; + platform?: BrowserPlatform; + baseUrl?: string; +}; + +type LaunchArgs = { + browser: { + name: Browser; + buildId: string; + }; + path?: string; + platform?: BrowserPlatform; + detached: boolean; + system: boolean; +}; + +type ClearArgs = { + path?: string; +}; + +/** + * @public + */ +export class CLI { + #cachePath; + #rl?: readline.Interface; + + constructor(cachePath = process.cwd(), rl?: readline.Interface) { + this.#cachePath = cachePath; + this.#rl = rl; + } + + #defineBrowserParameter(yargs: Yargs.Argv<unknown>): void { + yargs.positional('browser', { + description: + 'Which browser to install <browser>[@<buildId|latest>]. `latest` will try to find the latest available build. `buildId` is a browser-specific identifier such as a version or a revision.', + type: 'string', + coerce: (opt): InstallArgs['browser'] => { + return { + name: this.#parseBrowser(opt), + buildId: this.#parseBuildId(opt), + }; + }, + }); + } + + #definePlatformParameter(yargs: Yargs.Argv<unknown>): void { + yargs.option('platform', { + type: 'string', + desc: 'Platform that the binary needs to be compatible with.', + choices: Object.values(BrowserPlatform), + defaultDescription: 'Auto-detected', + }); + } + + #definePathParameter(yargs: Yargs.Argv<unknown>, required = false): void { + yargs.option('path', { + type: 'string', + desc: 'Path to the root folder for the browser downloads and installation. The installation folder structure is compatible with the cache structure used by Puppeteer.', + defaultDescription: 'Current working directory', + ...(required ? {} : {default: process.cwd()}), + }); + if (required) { + yargs.demandOption('path'); + } + } + + async run(argv: string[]): Promise<void> { + const yargsInstance = yargs(hideBin(argv)); + await yargsInstance + .scriptName('@puppeteer/browsers') + .command( + 'install <browser>', + 'Download and install the specified browser. If successful, the command outputs the actual browser buildId that was installed and the absolute path to the browser executable (format: <browser>@<buildID> <path>).', + yargs => { + this.#defineBrowserParameter(yargs); + this.#definePlatformParameter(yargs); + this.#definePathParameter(yargs); + yargs.option('base-url', { + type: 'string', + desc: 'Base URL to download from', + }); + yargs.example( + '$0 install chrome', + 'Install the latest available build of the Chrome browser.' + ); + yargs.example( + '$0 install chrome@latest', + 'Install the latest available build for the Chrome browser.' + ); + yargs.example( + '$0 install chromium@1083080', + 'Install the revision 1083080 of the Chromium browser.' + ); + yargs.example( + '$0 install firefox', + 'Install the latest available build of the Firefox browser.' + ); + yargs.example( + '$0 install firefox --platform mac', + 'Install the latest Mac (Intel) build of the Firefox browser.' + ); + yargs.example( + '$0 install firefox --path /tmp/my-browser-cache', + 'Install to the specified cache directory.' + ); + }, + async argv => { + const args = argv as unknown as InstallArgs; + args.platform ??= detectBrowserPlatform(); + if (!args.platform) { + throw new Error(`Could not resolve the current platform`); + } + args.browser.buildId = await resolveBuildId( + args.browser.name, + args.platform, + args.browser.buildId + ); + await install({ + browser: args.browser.name, + buildId: args.browser.buildId, + platform: args.platform, + cacheDir: args.path ?? this.#cachePath, + downloadProgressCallback: makeProgressCallback( + args.browser.name, + args.browser.buildId + ), + baseUrl: args.baseUrl, + }); + console.log( + `${args.browser.name}@${ + args.browser.buildId + } ${computeExecutablePath({ + browser: args.browser.name, + buildId: args.browser.buildId, + cacheDir: args.path ?? this.#cachePath, + platform: args.platform, + })}` + ); + } + ) + .command( + 'launch <browser>', + 'Launch the specified browser', + yargs => { + this.#defineBrowserParameter(yargs); + this.#definePlatformParameter(yargs); + this.#definePathParameter(yargs); + yargs.option('detached', { + type: 'boolean', + desc: 'Detach the child process.', + default: false, + }); + yargs.option('system', { + type: 'boolean', + desc: 'Search for a browser installed on the system instead of the cache folder.', + default: false, + }); + yargs.example( + '$0 launch chrome@1083080', + 'Launch the Chrome browser identified by the revision 1083080.' + ); + yargs.example( + '$0 launch firefox@112.0a1', + 'Launch the Firefox browser identified by the milestone 112.0a1.' + ); + yargs.example( + '$0 launch chrome@1083080 --detached', + 'Launch the browser but detach the sub-processes.' + ); + yargs.example( + '$0 launch chrome@canary --system', + 'Try to locate the Canary build of Chrome installed on the system and launch it.' + ); + }, + async argv => { + const args = argv as unknown as LaunchArgs; + const executablePath = args.system + ? computeSystemExecutablePath({ + browser: args.browser.name, + // TODO: throw an error if not a ChromeReleaseChannel is provided. + channel: args.browser.buildId as ChromeReleaseChannel, + platform: args.platform, + }) + : computeExecutablePath({ + browser: args.browser.name, + buildId: args.browser.buildId, + cacheDir: args.path ?? this.#cachePath, + platform: args.platform, + }); + launch({ + executablePath, + detached: args.detached, + }); + } + ) + .command( + 'clear', + 'Removes all installed browsers from the specified cache directory', + yargs => { + this.#definePathParameter(yargs, true); + }, + async argv => { + const args = argv as unknown as ClearArgs; + const cacheDir = args.path ?? this.#cachePath; + const rl = this.#rl ?? readline.createInterface({input, output}); + rl.question( + `Do you want to permanently and recursively delete the content of ${cacheDir} (yes/No)? `, + answer => { + rl.close(); + if (!['y', 'yes'].includes(answer.toLowerCase().trim())) { + console.log('Cancelled.'); + return; + } + const cache = new Cache(cacheDir); + cache.clear(); + console.log(`${cacheDir} cleared.`); + } + ); + } + ) + .demandCommand(1) + .help() + .wrap(Math.min(120, yargsInstance.terminalWidth())) + .parse(); + } + + #parseBrowser(version: string): Browser { + return version.split('@').shift() as Browser; + } + + #parseBuildId(version: string): string { + return version.split('@').pop() ?? 'latest'; + } +} + +/** + * @public + */ +export function makeProgressCallback( + browser: Browser, + buildId: string +): (downloadedBytes: number, totalBytes: number) => void { + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + return (downloadedBytes: number, totalBytes: number) => { + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${browser} r${buildId} - ${toMegabytes( + totalBytes + )} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +function toMegabytes(bytes: number) { + const mb = bytes / 1000 / 1000; + return `${Math.round(mb * 10) / 10} MB`; +} diff --git a/remote/test/puppeteer/packages/browsers/src/Cache.ts b/remote/test/puppeteer/packages/browsers/src/Cache.ts new file mode 100644 index 0000000000..142bceb08e --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/Cache.ts @@ -0,0 +1,119 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import {Browser, BrowserPlatform} from './browser-data/browser-data.js'; + +/** + * @public + */ +export type InstalledBrowser = { + path: string; + browser: Browser; + buildId: string; + platform: BrowserPlatform; +}; + +/** + * The cache used by Puppeteer relies on the following structure: + * + * - rootDir + * -- <browser1> | browserRoot(browser1) + * ---- <platform>-<buildId> | installationDir() + * ------ the browser-platform-buildId + * ------ specific structure. + * -- <browser2> | browserRoot(browser2) + * ---- <platform>-<buildId> | installationDir() + * ------ the browser-platform-buildId + * ------ specific structure. + * @internal + */ +export class Cache { + #rootDir: string; + + constructor(rootDir: string) { + this.#rootDir = rootDir; + } + + browserRoot(browser: Browser): string { + return path.join(this.#rootDir, browser); + } + + installationDir( + browser: Browser, + platform: BrowserPlatform, + buildId: string + ): string { + return path.join(this.browserRoot(browser), `${platform}-${buildId}`); + } + + clear(): void { + fs.rmSync(this.#rootDir, { + force: true, + recursive: true, + maxRetries: 10, + retryDelay: 500, + }); + } + + getInstalledBrowsers(): InstalledBrowser[] { + if (!fs.existsSync(this.#rootDir)) { + return []; + } + const types = fs.readdirSync(this.#rootDir); + const browsers = types.filter((t): t is Browser => { + return (Object.values(Browser) as string[]).includes(t); + }); + return browsers.flatMap(browser => { + const files = fs.readdirSync(this.browserRoot(browser)); + return files + .map(file => { + const result = parseFolderPath( + path.join(this.browserRoot(browser), file) + ); + if (!result) { + return null; + } + return { + path: path.join(this.browserRoot(browser), file), + browser, + platform: result.platform, + buildId: result.buildId, + }; + }) + .filter((item): item is InstalledBrowser => { + return item !== null; + }); + }); + } +} + +function parseFolderPath( + folderPath: string +): {platform: string; buildId: string} | undefined { + const name = path.basename(folderPath); + const splits = name.split('-'); + if (splits.length !== 2) { + return; + } + const [platform, buildId] = splits; + if (!buildId || !platform) { + return; + } + return {platform, buildId}; +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/browser-data.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/browser-data.ts new file mode 100644 index 0000000000..413435453a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/browser-data.ts @@ -0,0 +1,124 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as chrome from './chrome.js'; +import * as chromedriver from './chromedriver.js'; +import * as chromium from './chromium.js'; +import * as firefox from './firefox.js'; +import { + Browser, + BrowserPlatform, + BrowserTag, + ChromeReleaseChannel, + ProfileOptions, +} from './types.js'; + +export {ProfileOptions}; + +export const downloadUrls = { + [Browser.CHROMEDRIVER]: chromedriver.resolveDownloadUrl, + [Browser.CHROME]: chrome.resolveDownloadUrl, + [Browser.CHROMIUM]: chromium.resolveDownloadUrl, + [Browser.FIREFOX]: firefox.resolveDownloadUrl, +}; + +export const downloadPaths = { + [Browser.CHROMEDRIVER]: chromedriver.resolveDownloadPath, + [Browser.CHROME]: chrome.resolveDownloadPath, + [Browser.CHROMIUM]: chromium.resolveDownloadPath, + [Browser.FIREFOX]: firefox.resolveDownloadPath, +}; + +export const executablePathByBrowser = { + [Browser.CHROMEDRIVER]: chromedriver.relativeExecutablePath, + [Browser.CHROME]: chrome.relativeExecutablePath, + [Browser.CHROMIUM]: chromium.relativeExecutablePath, + [Browser.FIREFOX]: firefox.relativeExecutablePath, +}; + +export {Browser, BrowserPlatform, ChromeReleaseChannel}; + +/** + * @public + */ +export async function resolveBuildId( + browser: Browser, + platform: BrowserPlatform, + tag: string +): Promise<string> { + switch (browser) { + case Browser.FIREFOX: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + return await firefox.resolveBuildId('FIREFOX_NIGHTLY'); + } + case Browser.CHROME: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + // In CfT beta is the latest version. + return await chrome.resolveBuildId(platform, 'beta'); + } + case Browser.CHROMEDRIVER: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + return await chromedriver.resolveBuildId('latest'); + } + case Browser.CHROMIUM: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + return await chromium.resolveBuildId(platform, 'latest'); + } + } + // We assume the tag is the buildId if it didn't match any keywords. + return tag; +} + +/** + * @public + */ +export async function createProfile( + browser: Browser, + opts: ProfileOptions +): Promise<void> { + switch (browser) { + case Browser.FIREFOX: + return await firefox.createProfile(opts); + case Browser.CHROME: + case Browser.CHROMIUM: + throw new Error(`Profile creation is not support for ${browser} yet`); + } +} + +/** + * @public + */ +export function resolveSystemExecutablePath( + browser: Browser, + platform: BrowserPlatform, + channel: ChromeReleaseChannel +): string { + switch (browser) { + case Browser.CHROMEDRIVER: + case Browser.FIREFOX: + throw new Error( + `System browser detection is not supported for ${browser} yet.` + ); + case Browser.CHROME: + return chromium.resolveSystemExecutablePath(platform, channel); + case Browser.CHROMIUM: + return chrome.resolveSystemExecutablePath(platform, channel); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/chrome.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/chrome.ts new file mode 100644 index 0000000000..1fbf8c9647 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/chrome.ts @@ -0,0 +1,169 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform, ChromeReleaseChannel} from './types.js'; + +function folder(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'linux64'; + case BrowserPlatform.MAC_ARM: + return 'mac-arm64'; + case BrowserPlatform.MAC: + return 'mac-x64'; + case BrowserPlatform.WIN32: + return 'win32'; + case BrowserPlatform.WIN64: + return 'win64'; + } +} + +function chromiumDashPlatform(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'linux'; + case BrowserPlatform.MAC_ARM: + return 'mac'; + case BrowserPlatform.MAC: + return 'mac'; + case BrowserPlatform.WIN32: + return 'win'; + case BrowserPlatform.WIN64: + return 'win64'; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [buildId, folder(platform), `chrome-${folder(platform)}.zip`]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC: + case BrowserPlatform.MAC_ARM: + return path.join( + 'chrome-' + folder(platform), + 'Google Chrome for Testing.app', + 'Contents', + 'MacOS', + 'Google Chrome for Testing' + ); + case BrowserPlatform.LINUX: + return path.join('chrome-linux64', 'chrome'); + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return path.join('chrome-' + folder(platform), 'chrome.exe'); + } +} +export async function resolveBuildId( + platform: BrowserPlatform, + channel: 'beta' | 'stable' = 'beta' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL( + `https://chromiumdash.appspot.com/fetch_releases?platform=${chromiumDashPlatform( + platform + )}&channel=${channel}` + ), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + const response = JSON.parse(String(data)); + return resolve(response[0].version); + } catch { + return reject(new Error('Chrome version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} + +export function resolveSystemExecutablePath( + platform: BrowserPlatform, + channel: ChromeReleaseChannel +): string { + switch (platform) { + case BrowserPlatform.WIN64: + case BrowserPlatform.WIN32: + switch (channel) { + case ChromeReleaseChannel.STABLE: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome\\Application\\chrome.exe`; + case ChromeReleaseChannel.BETA: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome Beta\\Application\\chrome.exe`; + case ChromeReleaseChannel.CANARY: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome SxS\\Application\\chrome.exe`; + case ChromeReleaseChannel.DEV: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome Dev\\Application\\chrome.exe`; + } + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + switch (channel) { + case ChromeReleaseChannel.STABLE: + return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; + case ChromeReleaseChannel.BETA: + return '/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta'; + case ChromeReleaseChannel.CANARY: + return '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'; + case ChromeReleaseChannel.DEV: + return '/Applications/Google Chrome Dev.app/Contents/MacOS/Google Chrome Dev'; + } + case BrowserPlatform.LINUX: + switch (channel) { + case ChromeReleaseChannel.STABLE: + return '/opt/google/chrome/chrome'; + case ChromeReleaseChannel.BETA: + return '/opt/google/chrome-beta/chrome'; + case ChromeReleaseChannel.DEV: + return '/opt/google/chrome-unstable/chrome'; + } + } + + throw new Error( + `Unable to detect browser executable path for '${channel}' on ${platform}.` + ); +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/chromedriver.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/chromedriver.ts new file mode 100644 index 0000000000..39894d2e86 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/chromedriver.ts @@ -0,0 +1,93 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform} from './types.js'; + +function archive(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'chromedriver_linux64'; + case BrowserPlatform.MAC_ARM: + return 'chromedriver_mac_arm64'; + case BrowserPlatform.MAC: + return 'chromedriver_mac64'; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return 'chromedriver_win32'; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://chromedriver.storage.googleapis.com' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [buildId, `${archive(platform)}.zip`]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC: + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.LINUX: + return 'chromedriver'; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return 'chromedriver.exe'; + } +} +export async function resolveBuildId( + _channel: 'latest' = 'latest' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL(`https://chromedriver.storage.googleapis.com/LATEST_RELEASE`), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + return resolve(String(data)); + } catch { + return reject(new Error('Chrome version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/chromium.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/chromium.ts new file mode 100644 index 0000000000..71fa003e0a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/chromium.ts @@ -0,0 +1,125 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform} from './types.js'; + +export {resolveSystemExecutablePath} from './chrome.js'; + +function archive(platform: BrowserPlatform, buildId: string): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'chrome-linux'; + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + return 'chrome-mac'; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + // Windows archive name changed at r591479. + return parseInt(buildId, 10) > 591479 ? 'chrome-win' : 'chrome-win32'; + } +} + +function folder(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'Linux_x64'; + case BrowserPlatform.MAC_ARM: + return 'Mac_Arm'; + case BrowserPlatform.MAC: + return 'Mac'; + case BrowserPlatform.WIN32: + return 'Win'; + case BrowserPlatform.WIN64: + return 'Win_x64'; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://storage.googleapis.com/chromium-browser-snapshots' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [folder(platform), buildId, `${archive(platform, buildId)}.zip`]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC: + case BrowserPlatform.MAC_ARM: + return path.join( + 'chrome-mac', + 'Chromium.app', + 'Contents', + 'MacOS', + 'Chromium' + ); + case BrowserPlatform.LINUX: + return path.join('chrome-linux', 'chrome'); + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return path.join('chrome-win', 'chrome.exe'); + } +} +export async function resolveBuildId( + platform: BrowserPlatform, + // We will need it for other channels/keywords. + _channel: 'latest' = 'latest' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL( + `https://storage.googleapis.com/chromium-browser-snapshots/${folder( + platform + )}/LAST_CHANGE` + ), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + return resolve(String(data)); + } catch { + return reject(new Error('Chrome version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/firefox.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/firefox.ts new file mode 100644 index 0000000000..c3f337ee5a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/firefox.ts @@ -0,0 +1,355 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform, ProfileOptions} from './types.js'; + +function archive(platform: BrowserPlatform, buildId: string): string { + switch (platform) { + case BrowserPlatform.LINUX: + return `firefox-${buildId}.en-US.${platform}-x86_64.tar.bz2`; + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + return `firefox-${buildId}.en-US.mac.dmg`; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return `firefox-${buildId}.en-US.${platform}.zip`; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [archive(platform, buildId)]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + return path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox'); + case BrowserPlatform.LINUX: + return path.join('firefox', 'firefox'); + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return path.join('firefox', 'firefox.exe'); + } +} + +export async function resolveBuildId( + channel: 'FIREFOX_NIGHTLY' = 'FIREFOX_NIGHTLY' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL('https://product-details.mozilla.org/1.0/firefox_versions.json'), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + const versions = JSON.parse(data); + return resolve(versions[channel]); + } catch { + return reject(new Error('Firefox version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} + +export async function createProfile(options: ProfileOptions): Promise<void> { + if (!fs.existsSync(options.path)) { + await fs.promises.mkdir(options.path, { + recursive: true, + }); + } + await writePreferences({ + preferences: { + ...defaultProfilePreferences(options.preferences), + ...options.preferences, + }, + path: options.path, + }); +} + +function defaultProfilePreferences( + extraPrefs: Record<string, unknown> +): Record<string, unknown> { + const server = 'dummy.test'; + + const defaultPrefs = { + // Make sure Shield doesn't hit the network. + 'app.normandy.api_url': '', + // Disable Firefox old build background check + 'app.update.checkInstallTime': false, + // Disable automatically upgrading Firefox + 'app.update.disabledForTesting': true, + + // Increase the APZ content response timeout to 1 minute + 'apz.content_response_timeout': 60000, + + // Prevent various error message on the console + // jest-puppeteer asserts that no error message is emitted by the console + 'browser.contentblocking.features.standard': + '-tp,tpPrivate,cookieBehavior0,-cm,-fp', + + // Enable the dump function: which sends messages to the system + // console + // https://bugzilla.mozilla.org/show_bug.cgi?id=1543115 + 'browser.dom.window.dump.enabled': true, + // Disable topstories + 'browser.newtabpage.activity-stream.feeds.system.topstories': false, + // Always display a blank page + 'browser.newtabpage.enabled': false, + // Background thumbnails in particular cause grief: and disabling + // thumbnails in general cannot hurt + 'browser.pagethumbnails.capturing_disabled': true, + + // Disable safebrowsing components. + 'browser.safebrowsing.blockedURIs.enabled': false, + 'browser.safebrowsing.downloads.enabled': false, + 'browser.safebrowsing.malware.enabled': false, + 'browser.safebrowsing.passwords.enabled': false, + 'browser.safebrowsing.phishing.enabled': false, + + // Disable updates to search engines. + 'browser.search.update': false, + // Do not restore the last open set of tabs if the browser has crashed + 'browser.sessionstore.resume_from_crash': false, + // Skip check for default browser on startup + 'browser.shell.checkDefaultBrowser': false, + + // Disable newtabpage + 'browser.startup.homepage': 'about:blank', + // Do not redirect user when a milstone upgrade of Firefox is detected + 'browser.startup.homepage_override.mstone': 'ignore', + // Start with a blank page about:blank + 'browser.startup.page': 0, + + // Do not allow background tabs to be zombified on Android: otherwise for + // tests that open additional tabs: the test harness tab itself might get + // unloaded + 'browser.tabs.disableBackgroundZombification': false, + // Do not warn when closing all other open tabs + 'browser.tabs.warnOnCloseOtherTabs': false, + // Do not warn when multiple tabs will be opened + 'browser.tabs.warnOnOpen': false, + + // Disable page translations, which can cause issues with tests. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1836093. + 'browser.translations.enable': false, + + // Disable the UI tour. + 'browser.uitour.enabled': false, + // Turn off search suggestions in the location bar so as not to trigger + // network connections. + 'browser.urlbar.suggest.searches': false, + // Disable first run splash page on Windows 10 + 'browser.usedOnWindows10.introURL': '', + // Do not warn on quitting Firefox + 'browser.warnOnQuit': false, + + // Defensively disable data reporting systems + 'datareporting.healthreport.documentServerURI': `http://${server}/dummy/healthreport/`, + 'datareporting.healthreport.logging.consoleEnabled': false, + 'datareporting.healthreport.service.enabled': false, + 'datareporting.healthreport.service.firstRun': false, + 'datareporting.healthreport.uploadEnabled': false, + + // Do not show datareporting policy notifications which can interfere with tests + 'datareporting.policy.dataSubmissionEnabled': false, + 'datareporting.policy.dataSubmissionPolicyBypassNotification': true, + + // DevTools JSONViewer sometimes fails to load dependencies with its require.js. + // This doesn't affect Puppeteer but spams console (Bug 1424372) + 'devtools.jsonview.enabled': false, + + // Disable popup-blocker + 'dom.disable_open_during_load': false, + + // Enable the support for File object creation in the content process + // Required for |Page.setFileInputFiles| protocol method. + 'dom.file.createInChild': true, + + // Disable the ProcessHangMonitor + 'dom.ipc.reportProcessHangs': false, + + // Disable slow script dialogues + 'dom.max_chrome_script_run_time': 0, + 'dom.max_script_run_time': 0, + + // Only load extensions from the application and user profile + // AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION + 'extensions.autoDisableScopes': 0, + 'extensions.enabledScopes': 5, + + // Disable metadata caching for installed add-ons by default + 'extensions.getAddons.cache.enabled': false, + + // Disable installing any distribution extensions or add-ons. + 'extensions.installDistroAddons': false, + + // Disabled screenshots extension + 'extensions.screenshots.disabled': true, + + // Turn off extension updates so they do not bother tests + 'extensions.update.enabled': false, + + // Turn off extension updates so they do not bother tests + 'extensions.update.notifyUser': false, + + // Make sure opening about:addons will not hit the network + 'extensions.webservice.discoverURL': `http://${server}/dummy/discoveryURL`, + + // Temporarily force disable BFCache in parent (https://bit.ly/bug-1732263) + 'fission.bfcacheInParent': false, + + // Force all web content to use a single content process + 'fission.webContentIsolationStrategy': 0, + + // Allow the application to have focus even it runs in the background + 'focusmanager.testmode': true, + // Disable useragent updates + 'general.useragent.updates.enabled': false, + // Always use network provider for geolocation tests so we bypass the + // macOS dialog raised by the corelocation provider + 'geo.provider.testing': true, + // Do not scan Wifi + 'geo.wifi.scan': false, + // No hang monitor + 'hangmonitor.timeout': 0, + // Show chrome errors and warnings in the error console + 'javascript.options.showInConsole': true, + + // Disable download and usage of OpenH264: and Widevine plugins + 'media.gmp-manager.updateEnabled': false, + // Prevent various error message on the console + // jest-puppeteer asserts that no error message is emitted by the console + 'network.cookie.cookieBehavior': 0, + + // Disable experimental feature that is only available in Nightly + 'network.cookie.sameSite.laxByDefault': false, + + // Do not prompt for temporary redirects + 'network.http.prompt-temp-redirect': false, + + // Disable speculative connections so they are not reported as leaking + // when they are hanging around + 'network.http.speculative-parallel-limit': 0, + + // Do not automatically switch between offline and online + 'network.manage-offline-status': false, + + // Make sure SNTP requests do not hit the network + 'network.sntp.pools': server, + + // Disable Flash. + 'plugin.state.flash': 0, + + 'privacy.trackingprotection.enabled': false, + + // Can be removed once Firefox 89 is no longer supported + // https://bugzilla.mozilla.org/show_bug.cgi?id=1710839 + 'remote.enabled': true, + + // Don't do network connections for mitm priming + 'security.certerrors.mitm.priming.enabled': false, + // Local documents have access to all other local documents, + // including directory listings + 'security.fileuri.strict_origin_policy': false, + // Do not wait for the notification button security delay + 'security.notification_enable_delay': 0, + + // Ensure blocklist updates do not hit the network + 'services.settings.server': `http://${server}/dummy/blocklist/`, + + // Do not automatically fill sign-in forms with known usernames and + // passwords + 'signon.autofillForms': false, + // Disable password capture, so that tests that include forms are not + // influenced by the presence of the persistent doorhanger notification + 'signon.rememberSignons': false, + + // Disable first-run welcome page + 'startup.homepage_welcome_url': 'about:blank', + + // Disable first-run welcome page + 'startup.homepage_welcome_url.additional': '', + + // Disable browser animations (tabs, fullscreen, sliding alerts) + 'toolkit.cosmeticAnimations.enabled': false, + + // Prevent starting into safe mode after application crashes + 'toolkit.startup.max_resumed_crashes': -1, + }; + + return Object.assign(defaultPrefs, extraPrefs); +} + +/** + * Populates the user.js file with custom preferences as needed to allow + * Firefox's CDP support to properly function. These preferences will be + * automatically copied over to prefs.js during startup of Firefox. To be + * able to restore the original values of preferences a backup of prefs.js + * will be created. + * + * @param prefs - List of preferences to add. + * @param profilePath - Firefox profile to write the preferences to. + */ +async function writePreferences(options: ProfileOptions): Promise<void> { + const lines = Object.entries(options.preferences).map(([key, value]) => { + return `user_pref(${JSON.stringify(key)}, ${JSON.stringify(value)});`; + }); + + await fs.promises.writeFile( + path.join(options.path, 'user.js'), + lines.join('\n') + ); + + // Create a backup of the preferences file if it already exitsts. + const prefsPath = path.join(options.path, 'prefs.js'); + if (fs.existsSync(prefsPath)) { + const prefsBackupPath = path.join(options.path, 'prefs.js.puppeteer'); + await fs.promises.copyFile(prefsPath, prefsBackupPath); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/types.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/types.ts new file mode 100644 index 0000000000..f88d2ca098 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/types.ts @@ -0,0 +1,75 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as chrome from './chrome.js'; +import * as firefox from './firefox.js'; + +/** + * Supported browsers. + * + * @public + */ +export enum Browser { + CHROME = 'chrome', + CHROMIUM = 'chromium', + FIREFOX = 'firefox', + CHROMEDRIVER = 'chromedriver', +} + +/** + * Platform names used to identify a OS platfrom x architecture combination in the way + * that is relevant for the browser download. + * + * @public + */ +export enum BrowserPlatform { + LINUX = 'linux', + MAC = 'mac', + MAC_ARM = 'mac_arm', + WIN32 = 'win32', + WIN64 = 'win64', +} + +export const downloadUrls = { + [Browser.CHROME]: chrome.resolveDownloadUrl, + [Browser.CHROMIUM]: chrome.resolveDownloadUrl, + [Browser.FIREFOX]: firefox.resolveDownloadUrl, +}; + +/** + * @public + */ +export enum BrowserTag { + LATEST = 'latest', +} + +/** + * @public + */ +export interface ProfileOptions { + preferences: Record<string, unknown>; + path: string; +} + +/** + * @public + */ +export enum ChromeReleaseChannel { + STABLE = 'stable', + DEV = 'dev', + CANARY = 'canary', + BETA = 'beta', +} diff --git a/remote/test/puppeteer/packages/browsers/src/debug.ts b/remote/test/puppeteer/packages/browsers/src/debug.ts new file mode 100644 index 0000000000..eee0a347e8 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/debug.ts @@ -0,0 +1,19 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import debug from 'debug'; + +export {debug}; diff --git a/remote/test/puppeteer/packages/browsers/src/detectPlatform.ts b/remote/test/puppeteer/packages/browsers/src/detectPlatform.ts new file mode 100644 index 0000000000..fed8c2e2ea --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/detectPlatform.ts @@ -0,0 +1,61 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; + +import {BrowserPlatform} from './browser-data/browser-data.js'; + +/** + * @public + */ +export function detectBrowserPlatform(): BrowserPlatform | undefined { + const platform = os.platform(); + switch (platform) { + case 'darwin': + return os.arch() === 'arm64' + ? BrowserPlatform.MAC_ARM + : BrowserPlatform.MAC; + case 'linux': + return BrowserPlatform.LINUX; + case 'win32': + return os.arch() === 'x64' || + // Windows 11 for ARM supports x64 emulation + (os.arch() === 'arm64' && isWindows11(os.release())) + ? BrowserPlatform.WIN64 + : BrowserPlatform.WIN32; + default: + return undefined; + } +} + +/** + * Windows 11 is identified by the version 10.0.22000 or greater + * @internal + */ +function isWindows11(version: string): boolean { + const parts = version.split('.'); + if (parts.length > 2) { + const major = parseInt(parts[0] as string, 10); + const minor = parseInt(parts[1] as string, 10); + const patch = parseInt(parts[2] as string, 10); + return ( + major > 10 || + (major === 10 && minor > 0) || + (major === 10 && minor === 0 && patch >= 22000) + ); + } + return false; +} diff --git a/remote/test/puppeteer/packages/browsers/src/fileUtil.ts b/remote/test/puppeteer/packages/browsers/src/fileUtil.ts new file mode 100644 index 0000000000..6139accd49 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/fileUtil.ts @@ -0,0 +1,89 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {exec as execChildProcess} from 'child_process'; +import {createReadStream} from 'fs'; +import {mkdir, readdir} from 'fs/promises'; +import * as path from 'path'; +import {promisify} from 'util'; + +import extractZip from 'extract-zip'; +import tar from 'tar-fs'; +import bzip from 'unbzip2-stream'; + +const exec = promisify(execChildProcess); + +/** + * @internal + */ +export async function unpackArchive( + archivePath: string, + folderPath: string +): Promise<void> { + if (archivePath.endsWith('.zip')) { + await extractZip(archivePath, {dir: folderPath}); + } else if (archivePath.endsWith('.tar.bz2')) { + await extractTar(archivePath, folderPath); + } else if (archivePath.endsWith('.dmg')) { + await mkdir(folderPath); + await installDMG(archivePath, folderPath); + } else { + throw new Error(`Unsupported archive format: ${archivePath}`); + } +} + +/** + * @internal + */ +function extractTar(tarPath: string, folderPath: string): Promise<void> { + return new Promise((fulfill, reject) => { + const tarStream = tar.extract(folderPath); + tarStream.on('error', reject); + tarStream.on('finish', fulfill); + const readStream = createReadStream(tarPath); + readStream.pipe(bzip()).pipe(tarStream); + }); +} + +/** + * @internal + */ +async function installDMG(dmgPath: string, folderPath: string): Promise<void> { + const {stdout} = await exec( + `hdiutil attach -nobrowse -noautoopen "${dmgPath}"` + ); + + const volumes = stdout.match(/\/Volumes\/(.*)/m); + if (!volumes) { + throw new Error(`Could not find volume path in ${stdout}`); + } + const mountPath = volumes[0]!; + + try { + const fileNames = await readdir(mountPath); + const appName = fileNames.find(item => { + return typeof item === 'string' && item.endsWith('.app'); + }); + if (!appName) { + throw new Error(`Cannot find app in ${mountPath}`); + } + const mountedPath = path.join(mountPath!, appName); + + await exec(`cp -R "${mountedPath}" "${folderPath}"`); + } finally { + await exec(`hdiutil detach "${mountPath}" -quiet`); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/httpUtil.ts b/remote/test/puppeteer/packages/browsers/src/httpUtil.ts new file mode 100644 index 0000000000..5b6150f734 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/httpUtil.ts @@ -0,0 +1,141 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {createWriteStream} from 'fs'; +import * as http from 'http'; +import * as https from 'https'; +import {URL} from 'url'; + +import createHttpsProxyAgent from 'https-proxy-agent'; +import {getProxyForUrl} from 'proxy-from-env'; + +export function headHttpRequest(url: URL): Promise<boolean> { + return new Promise(resolve => { + const request = httpRequest( + url, + 'HEAD', + response => { + resolve(response.statusCode === 200); + }, + false + ); + request.on('error', () => { + resolve(false); + }); + }); +} + +export function httpRequest( + url: URL, + method: string, + response: (x: http.IncomingMessage) => void, + keepAlive = true +): http.ClientRequest { + const options: http.RequestOptions = { + protocol: url.protocol, + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + method, + headers: keepAlive ? {Connection: 'keep-alive'} : undefined, + }; + + const proxyURL = getProxyForUrl(url.toString()); + if (proxyURL) { + const proxy = new URL(proxyURL); + if (proxy.protocol === 'http:') { + options.path = url.href; + options.hostname = proxy.hostname; + options.protocol = proxy.protocol; + options.port = proxy.port; + options.headers ??= {}; + options.headers['Host'] ||= url.host; + } else { + options.agent = createHttpsProxyAgent({ + host: proxy.host, + path: proxy.pathname, + port: proxy.port, + secureProxy: proxy.protocol === 'https:', + headers: options.headers, + }); + } + } + + const requestCallback = (res: http.IncomingMessage): void => { + if ( + res.statusCode && + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + httpRequest(new URL(res.headers.location), method, response); + } else { + response(res); + } + }; + const request = + options.protocol === 'https:' + ? https.request(options, requestCallback) + : http.request(options, requestCallback); + request.end(); + return request; +} + +/** + * @internal + */ +export function downloadFile( + url: URL, + destinationPath: string, + progressCallback?: (downloadedBytes: number, totalBytes: number) => void +): Promise<void> { + return new Promise<void>((resolve, reject) => { + let downloadedBytes = 0; + let totalBytes = 0; + + function onData(chunk: string): void { + downloadedBytes += chunk.length; + progressCallback!(downloadedBytes, totalBytes); + } + + const request = httpRequest(url, 'GET', response => { + if (response.statusCode !== 200) { + const error = new Error( + `Download failed: server returned code ${response.statusCode}. URL: ${url}` + ); + // consume response data to free up memory + response.resume(); + reject(error); + return; + } + const file = createWriteStream(destinationPath); + file.on('finish', () => { + return resolve(); + }); + file.on('error', error => { + return reject(error); + }); + response.pipe(file); + totalBytes = parseInt(response.headers['content-length']!, 10); + if (progressCallback) { + response.on('data', onData); + } + }); + request.on('error', error => { + return reject(error); + }); + }); +} diff --git a/remote/test/puppeteer/packages/browsers/src/install.ts b/remote/test/puppeteer/packages/browsers/src/install.ts new file mode 100644 index 0000000000..054e048420 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/install.ts @@ -0,0 +1,218 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import {existsSync} from 'fs'; +import {mkdir, unlink} from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { + Browser, + BrowserPlatform, + downloadUrls, +} from './browser-data/browser-data.js'; +import {Cache, InstalledBrowser} from './Cache.js'; +import {debug} from './debug.js'; +import {detectBrowserPlatform} from './detectPlatform.js'; +import {unpackArchive} from './fileUtil.js'; +import {downloadFile, headHttpRequest} from './httpUtil.js'; + +const debugInstall = debug('puppeteer:browsers:install'); + +const times = new Map<string, [number, number]>(); +function debugTime(label: string) { + times.set(label, process.hrtime()); +} + +function debugTimeEnd(label: string) { + const end = process.hrtime(); + const start = times.get(label); + if (!start) { + return; + } + const duration = + end[0] * 1000 + end[1] / 1e6 - (start[0] * 1000 + start[1] / 1e6); // calculate duration in milliseconds + debugInstall(`Duration for ${label}: ${duration}ms`); +} + +/** + * @public + */ +export interface InstallOptions { + /** + * Determines the path to download browsers to. + */ + cacheDir: string; + /** + * Determines which platform the browser will be suited for. + * + * @defaultValue **Auto-detected.** + */ + platform?: BrowserPlatform; + /** + * Determines which browser to install. + */ + browser: Browser; + /** + * Determines which buildId to dowloand. BuildId should uniquely identify + * binaries and they are used for caching. + */ + buildId: string; + /** + * Provides information about the progress of the download. + */ + downloadProgressCallback?: ( + downloadedBytes: number, + totalBytes: number + ) => void; + /** + * Determines the host that will be used for downloading. + * + * @defaultValue Either + * + * - https://storage.googleapis.com/chromium-browser-snapshots or + * - https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central + * + */ + baseUrl?: string; + /** + * Whether to unpack and install browser archives. + * + * @defaultValue `true` + */ + unpack?: boolean; +} + +/** + * @public + */ +export async function install( + options: InstallOptions +): Promise<InstalledBrowser> { + options.platform ??= detectBrowserPlatform(); + options.unpack ??= true; + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + const url = getDownloadUrl( + options.browser, + options.platform, + options.buildId, + options.baseUrl + ); + const fileName = url.toString().split('/').pop(); + assert(fileName, `A malformed download URL was found: ${url}.`); + const structure = new Cache(options.cacheDir); + const browserRoot = structure.browserRoot(options.browser); + const archivePath = path.join(browserRoot, fileName); + if (!existsSync(browserRoot)) { + await mkdir(browserRoot, {recursive: true}); + } + + if (!options.unpack) { + if (existsSync(archivePath)) { + return { + path: archivePath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; + } + debugInstall(`Downloading binary from ${url}`); + debugTime('download'); + await downloadFile(url, archivePath, options.downloadProgressCallback); + debugTimeEnd('download'); + return { + path: archivePath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; + } + + const outputPath = structure.installationDir( + options.browser, + options.platform, + options.buildId + ); + if (existsSync(outputPath)) { + return { + path: outputPath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; + } + try { + debugInstall(`Downloading binary from ${url}`); + try { + debugTime('download'); + await downloadFile(url, archivePath, options.downloadProgressCallback); + } finally { + debugTimeEnd('download'); + } + + debugInstall(`Installing ${archivePath} to ${outputPath}`); + try { + debugTime('extract'); + await unpackArchive(archivePath, outputPath); + } finally { + debugTimeEnd('extract'); + } + } finally { + if (existsSync(archivePath)) { + await unlink(archivePath); + } + } + return { + path: outputPath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; +} + +/** + * @public + */ +export async function canDownload(options: InstallOptions): Promise<boolean> { + options.platform ??= detectBrowserPlatform(); + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + return await headHttpRequest( + getDownloadUrl( + options.browser, + options.platform, + options.buildId, + options.baseUrl + ) + ); +} + +function getDownloadUrl( + browser: Browser, + platform: BrowserPlatform, + buildId: string, + baseUrl?: string +): URL { + return new URL(downloadUrls[browser](platform, buildId, baseUrl)); +} diff --git a/remote/test/puppeteer/packages/browsers/src/launch.ts b/remote/test/puppeteer/packages/browsers/src/launch.ts new file mode 100644 index 0000000000..9f8c8f20ed --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/launch.ts @@ -0,0 +1,494 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import childProcess from 'child_process'; +import {accessSync} from 'fs'; +import os from 'os'; +import path from 'path'; +import readline from 'readline'; + +import { + Browser, + BrowserPlatform, + executablePathByBrowser, + resolveSystemExecutablePath, + ChromeReleaseChannel, +} from './browser-data/browser-data.js'; +import {Cache} from './Cache.js'; +import {debug} from './debug.js'; +import {detectBrowserPlatform} from './detectPlatform.js'; + +const debugLaunch = debug('puppeteer:browsers:launcher'); + +/** + * @public + */ +export interface ComputeExecutablePathOptions { + /** + * Root path to the storage directory. + */ + cacheDir: string; + /** + * Determines which platform the browser will be suited for. + * + * @defaultValue **Auto-detected.** + */ + platform?: BrowserPlatform; + /** + * Determines which browser to launch. + */ + browser: Browser; + /** + * Determines which buildId to download. BuildId should uniquely identify + * binaries and they are used for caching. + */ + buildId: string; +} + +/** + * @public + */ +export function computeExecutablePath( + options: ComputeExecutablePathOptions +): string { + options.platform ??= detectBrowserPlatform(); + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + const installationDir = new Cache(options.cacheDir).installationDir( + options.browser, + options.platform, + options.buildId + ); + return path.join( + installationDir, + executablePathByBrowser[options.browser](options.platform, options.buildId) + ); +} + +/** + * @public + */ +export interface SystemOptions { + /** + * Determines which platform the browser will be suited for. + * + * @defaultValue **Auto-detected.** + */ + platform?: BrowserPlatform; + /** + * Determines which browser to launch. + */ + browser: Browser; + /** + * Release channel to look for on the system. + */ + channel: ChromeReleaseChannel; +} + +/** + * @public + */ +export function computeSystemExecutablePath(options: SystemOptions): string { + options.platform ??= detectBrowserPlatform(); + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + const path = resolveSystemExecutablePath( + options.browser, + options.platform, + options.channel + ); + try { + accessSync(path); + } catch (error) { + throw new Error( + `Could not find Google Chrome executable for channel '${options.channel}' at '${path}'.` + ); + } + return path; +} + +/** + * @public + */ +export type LaunchOptions = { + executablePath: string; + pipe?: boolean; + dumpio?: boolean; + args?: string[]; + env?: Record<string, string | undefined>; + handleSIGINT?: boolean; + handleSIGTERM?: boolean; + handleSIGHUP?: boolean; + detached?: boolean; + onExit?: () => Promise<void>; +}; + +/** + * @public + */ +export function launch(opts: LaunchOptions): Process { + return new Process(opts); +} + +/** + * @public + */ +export const CDP_WEBSOCKET_ENDPOINT_REGEX = + /^DevTools listening on (ws:\/\/.*)$/; + +/** + * @public + */ +export const WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX = + /^WebDriver BiDi listening on (ws:\/\/.*)$/; + +/** + * @public + */ +export class Process { + #executablePath; + #args: string[]; + #browserProcess: childProcess.ChildProcess; + #exited = false; + // The browser process can be closed externally or from the driver process. We + // need to invoke the hooks only once though but we don't know how many times + // we will be invoked. + #hooksRan = false; + #onExitHook = async () => {}; + #browserProcessExiting: Promise<void>; + + constructor(opts: LaunchOptions) { + this.#executablePath = opts.executablePath; + this.#args = opts.args ?? []; + + opts.pipe ??= false; + opts.dumpio ??= false; + opts.handleSIGINT ??= true; + opts.handleSIGTERM ??= true; + opts.handleSIGHUP ??= true; + // On non-windows platforms, `detached: true` makes child process a + // leader of a new process group, making it possible to kill child + // process tree with `.kill(-pid)` command. @see + // https://nodejs.org/api/child_process.html#child_process_options_detached + opts.detached ??= process.platform !== 'win32'; + + const stdio = this.#configureStdio({ + pipe: opts.pipe, + dumpio: opts.dumpio, + }); + + debugLaunch(`Launching ${this.#executablePath} ${this.#args.join(' ')}`, { + detached: opts.detached, + env: opts.env, + stdio, + }); + + this.#browserProcess = childProcess.spawn( + this.#executablePath, + this.#args, + { + detached: opts.detached, + env: opts.env, + stdio, + } + ); + + debugLaunch(`Launched ${this.#browserProcess.pid}`); + if (opts.dumpio) { + this.#browserProcess.stderr?.pipe(process.stderr); + this.#browserProcess.stdout?.pipe(process.stdout); + } + process.on('exit', this.#onDriverProcessExit); + if (opts.handleSIGINT) { + process.on('SIGINT', this.#onDriverProcessSignal); + } + if (opts.handleSIGTERM) { + process.on('SIGTERM', this.#onDriverProcessSignal); + } + if (opts.handleSIGHUP) { + process.on('SIGHUP', this.#onDriverProcessSignal); + } + if (opts.onExit) { + this.#onExitHook = opts.onExit; + } + this.#browserProcessExiting = new Promise((resolve, reject) => { + this.#browserProcess.once('exit', async () => { + debugLaunch(`Browser process ${this.#browserProcess.pid} onExit`); + this.#clearListeners(); + this.#exited = true; + try { + await this.#runHooks(); + } catch (err) { + reject(err); + return; + } + resolve(); + }); + }); + } + + async #runHooks() { + if (this.#hooksRan) { + return; + } + this.#hooksRan = true; + await this.#onExitHook(); + } + + get nodeProcess(): childProcess.ChildProcess { + return this.#browserProcess; + } + + #configureStdio(opts: { + pipe: boolean; + dumpio: boolean; + }): Array<'ignore' | 'pipe'> { + if (opts.pipe) { + if (opts.dumpio) { + return ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']; + } else { + return ['ignore', 'ignore', 'ignore', 'pipe', 'pipe']; + } + } else { + if (opts.dumpio) { + return ['pipe', 'pipe', 'pipe']; + } else { + return ['pipe', 'ignore', 'pipe']; + } + } + } + + #clearListeners(): void { + process.off('exit', this.#onDriverProcessExit); + process.off('SIGINT', this.#onDriverProcessSignal); + process.off('SIGTERM', this.#onDriverProcessSignal); + process.off('SIGHUP', this.#onDriverProcessSignal); + } + + #onDriverProcessExit = (_code: number) => { + this.kill(); + }; + + #onDriverProcessSignal = (signal: string): void => { + switch (signal) { + case 'SIGINT': + this.kill(); + process.exit(130); + case 'SIGTERM': + case 'SIGHUP': + void this.close(); + break; + } + }; + + async close(): Promise<void> { + await this.#runHooks(); + if (!this.#exited) { + this.kill(); + } + return this.#browserProcessExiting; + } + + hasClosed(): Promise<void> { + return this.#browserProcessExiting; + } + + kill(): void { + debugLaunch(`Trying to kill ${this.#browserProcess.pid}`); + // If the process failed to launch (for example if the browser executable path + // is invalid), then the process does not get a pid assigned. A call to + // `proc.kill` would error, as the `pid` to-be-killed can not be found. + if ( + this.#browserProcess && + this.#browserProcess.pid && + pidExists(this.#browserProcess.pid) + ) { + try { + debugLaunch(`Browser process ${this.#browserProcess.pid} exists`); + if (process.platform === 'win32') { + try { + childProcess.execSync( + `taskkill /pid ${this.#browserProcess.pid} /T /F` + ); + } catch (error) { + debugLaunch( + `Killing ${this.#browserProcess.pid} using taskkill failed`, + error + ); + // taskkill can fail to kill the process e.g. due to missing permissions. + // Let's kill the process via Node API. This delays killing of all child + // processes of `this.proc` until the main Node.js process dies. + this.#browserProcess.kill(); + } + } else { + // on linux the process group can be killed with the group id prefixed with + // a minus sign. The process group id is the group leader's pid. + const processGroupId = -this.#browserProcess.pid; + + try { + process.kill(processGroupId, 'SIGKILL'); + } catch (error) { + debugLaunch( + `Killing ${this.#browserProcess.pid} using process.kill failed`, + error + ); + // Killing the process group can fail due e.g. to missing permissions. + // Let's kill the process via Node API. This delays killing of all child + // processes of `this.proc` until the main Node.js process dies. + this.#browserProcess.kill('SIGKILL'); + } + } + } catch (error) { + throw new Error( + `${PROCESS_ERROR_EXPLANATION}\nError cause: ${ + isErrorLike(error) ? error.stack : error + }` + ); + } + } + this.#clearListeners(); + } + + waitForLineOutput(regex: RegExp, timeout?: number): Promise<string> { + if (!this.#browserProcess.stderr) { + throw new Error('`browserProcess` does not have stderr.'); + } + const rl = readline.createInterface(this.#browserProcess.stderr); + let stderr = ''; + + return new Promise((resolve, reject) => { + rl.on('line', onLine); + rl.on('close', onClose); + this.#browserProcess.on('exit', onClose); + this.#browserProcess.on('error', onClose); + const timeoutId = timeout ? setTimeout(onTimeout, timeout) : 0; + + const cleanup = (): void => { + if (timeoutId) { + clearTimeout(timeoutId); + } + rl.off('line', onLine); + rl.off('close', onClose); + this.#browserProcess.off('exit', onClose); + this.#browserProcess.off('error', onClose); + }; + + function onClose(error?: Error): void { + cleanup(); + reject( + new Error( + [ + `Failed to launch the browser process!${ + error ? ' ' + error.message : '' + }`, + stderr, + '', + 'TROUBLESHOOTING: https://pptr.dev/troubleshooting', + '', + ].join('\n') + ) + ); + } + + function onTimeout(): void { + cleanup(); + reject( + new TimeoutError( + `Timed out after ${timeout} ms while waiting for the WS endpoint URL to appear in stdout!` + ) + ); + } + + function onLine(line: string): void { + stderr += line + '\n'; + const match = line.match(regex); + if (!match) { + return; + } + cleanup(); + // The RegExp matches, so this will obviously exist. + resolve(match[1]!); + } + }); + } +} + +const PROCESS_ERROR_EXPLANATION = `Puppeteer was unable to kill the process which ran the browser binary. +This means that, on future Puppeteer launches, Puppeteer might not be able to launch the browser. +Please check your open processes and ensure that the browser processes that Puppeteer launched have been killed. +If you think this is a bug, please report it on the Puppeteer issue tracker.`; + +/** + * @internal + */ +function pidExists(pid: number): boolean { + try { + return process.kill(pid, 0); + } catch (error) { + if (isErrnoException(error)) { + if (error.code && error.code === 'ESRCH') { + return false; + } + } + throw error; + } +} + +/** + * @internal + */ +export interface ErrorLike extends Error { + name: string; + message: string; +} + +/** + * @internal + */ +export function isErrorLike(obj: unknown): obj is ErrorLike { + return ( + typeof obj === 'object' && obj !== null && 'name' in obj && 'message' in obj + ); +} +/** + * @internal + */ +export function isErrnoException(obj: unknown): obj is NodeJS.ErrnoException { + return ( + isErrorLike(obj) && + ('errno' in obj || 'code' in obj || 'path' in obj || 'syscall' in obj) + ); +} + +/** + * @public + */ +export class TimeoutError extends Error { + /** + * @internal + */ + constructor(message?: string) { + super(message); + this.name = this.constructor.name; + Error.captureStackTrace(this, this.constructor); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/main-cli.ts b/remote/test/puppeteer/packages/browsers/src/main-cli.ts new file mode 100644 index 0000000000..a086c1c3b9 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/main-cli.ts @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {CLI} from './CLI.js'; + +void new CLI().run(process.argv); diff --git a/remote/test/puppeteer/packages/browsers/src/main.ts b/remote/test/puppeteer/packages/browsers/src/main.ts new file mode 100644 index 0000000000..14ca6a838a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/main.ts @@ -0,0 +1,40 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + launch, + computeExecutablePath, + computeSystemExecutablePath, + TimeoutError, + LaunchOptions, + ComputeExecutablePathOptions as Options, + SystemOptions, + CDP_WEBSOCKET_ENDPOINT_REGEX, + WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX, + Process, +} from './launch.js'; +export {install, canDownload, InstallOptions} from './install.js'; +export {detectBrowserPlatform} from './detectPlatform.js'; +export { + resolveBuildId, + Browser, + BrowserPlatform, + ChromeReleaseChannel, + createProfile, + ProfileOptions, +} from './browser-data/browser-data.js'; +export {CLI, makeProgressCallback} from './CLI.js'; +export {Cache, InstalledBrowser} from './Cache.js'; diff --git a/remote/test/puppeteer/packages/browsers/src/tsconfig.cjs.json b/remote/test/puppeteer/packages/browsers/src/tsconfig.cjs.json new file mode 100644 index 0000000000..ef01b990b7 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "module": "CommonJS", + "outDir": "../lib/cjs" + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/tsconfig.esm.json b/remote/test/puppeteer/packages/browsers/src/tsconfig.esm.json new file mode 100644 index 0000000000..a824bc8cb8 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/tsconfig.esm.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "../lib/esm" + } +} diff --git a/remote/test/puppeteer/packages/browsers/test/src/chrome/chrome-data.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chrome/chrome-data.spec.ts new file mode 100644 index 0000000000..c0404baad8 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chrome/chrome-data.spec.ts @@ -0,0 +1,120 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import path from 'path'; + +import { + BrowserPlatform, + ChromeReleaseChannel, +} from '../../../lib/cjs/browser-data/browser-data.js'; +import { + resolveDownloadUrl, + relativeExecutablePath, + resolveSystemExecutablePath, +} from '../../../lib/cjs/browser-data/chrome.js'; + +describe('Chrome', () => { + it('should resolve download URLs', () => { + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.LINUX, '113.0.5672.0'), + 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/linux64/chrome-linux64.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC, '113.0.5672.0'), + 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-x64/chrome-mac-x64.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC_ARM, '113.0.5672.0'), + 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-arm64/chrome-mac-arm64.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN32, '113.0.5672.0'), + 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win32/chrome-win32.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN64, '113.0.5672.0'), + 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win64/chrome-win64.zip' + ); + }); + + it('should resolve executable paths', () => { + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.LINUX, '12372323'), + path.join('chrome-linux64', 'chrome') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC, '12372323'), + path.join( + 'chrome-mac-x64', + 'Google Chrome for Testing.app', + 'Contents', + 'MacOS', + 'Google Chrome for Testing' + ) + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC_ARM, '12372323'), + path.join( + 'chrome-mac-arm64', + 'Google Chrome for Testing.app', + 'Contents', + 'MacOS', + 'Google Chrome for Testing' + ) + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN32, '12372323'), + path.join('chrome-win32', 'chrome.exe') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN64, '12372323'), + path.join('chrome-win64', 'chrome.exe') + ); + }); + + it('should resolve system executable path', () => { + process.env['PROGRAMFILES'] = 'C:\\ProgramFiles'; + try { + assert.strictEqual( + resolveSystemExecutablePath( + BrowserPlatform.WIN32, + ChromeReleaseChannel.DEV + ), + 'C:\\ProgramFiles\\Google\\Chrome Dev\\Application\\chrome.exe' + ); + } finally { + delete process.env['PROGRAMFILES']; + } + + assert.strictEqual( + resolveSystemExecutablePath( + BrowserPlatform.MAC, + ChromeReleaseChannel.BETA + ), + '/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta' + ); + assert.throws(() => { + assert.strictEqual( + resolveSystemExecutablePath( + BrowserPlatform.LINUX, + ChromeReleaseChannel.CANARY + ), + path.join('chrome-linux', 'chrome') + ); + }, new Error(`Unable to detect browser executable path for 'canary' on linux.`)); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chrome/cli.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chrome/cli.spec.ts new file mode 100644 index 0000000000..a9a08c9d62 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chrome/cli.spec.ts @@ -0,0 +1,104 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import {CLI} from '../../../lib/cjs/CLI.js'; +import { + createMockedReadlineInterface, + setupTestServer, + getServerUrl, +} from '../utils.js'; +import {testChromeBuildId} from '../versions.js'; + +describe('Chrome CLI', function () { + this.timeout(90000); + + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'puppeteer-browsers-test')); + }); + + afterEach(async () => { + await new CLI(tmpDir, createMockedReadlineInterface('yes')).run([ + 'npx', + '@puppeteer/browsers', + 'clear', + `--path=${tmpDir}`, + `--base-url=${getServerUrl()}`, + ]); + }); + + it('should download Chrome binaries', async () => { + await new CLI(tmpDir).run([ + 'npx', + '@puppeteer/browsers', + 'install', + `chrome@${testChromeBuildId}`, + `--path=${tmpDir}`, + '--platform=linux', + `--base-url=${getServerUrl()}`, + ]); + assert.ok( + fs.existsSync( + path.join( + tmpDir, + 'chrome', + `linux-${testChromeBuildId}`, + 'chrome-linux64', + 'chrome' + ) + ) + ); + + await new CLI(tmpDir, createMockedReadlineInterface('no')).run([ + 'npx', + '@puppeteer/browsers', + 'clear', + `--path=${tmpDir}`, + ]); + assert.ok( + fs.existsSync( + path.join( + tmpDir, + 'chrome', + `linux-${testChromeBuildId}`, + 'chrome-linux64', + 'chrome' + ) + ) + ); + }); + + // Skipped because the current latest is not published yet. + it.skip('should download latest Chrome binaries', async () => { + await new CLI(tmpDir).run([ + 'npx', + '@puppeteer/browsers', + 'install', + `chrome@latest`, + `--path=${tmpDir}`, + '--platform=linux', + `--base-url=${getServerUrl()}`, + ]); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chrome/install.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chrome/install.spec.ts new file mode 100644 index 0000000000..7dc3e7ebaf --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chrome/install.spec.ts @@ -0,0 +1,239 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import http from 'http'; +import https from 'https'; +import os from 'os'; +import path from 'path'; + +import { + install, + canDownload, + Browser, + BrowserPlatform, + Cache, +} from '../../../lib/cjs/main.js'; +import {getServerUrl, setupTestServer} from '../utils.js'; +import {testChromeBuildId} from '../versions.js'; + +/** + * Tests in this spec use real download URLs and unpack live browser archives + * so it requires the network access. + */ +describe('Chrome install', () => { + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'puppeteer-browsers-test')); + }); + + afterEach(() => { + new Cache(tmpDir).clear(); + }); + + it('should check if a buildId can be downloaded', async () => { + assert.ok( + await canDownload({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: testChromeBuildId, + baseUrl: getServerUrl(), + }) + ); + }); + + it('should report if a buildId is not downloadable', async () => { + assert.strictEqual( + await canDownload({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: 'unknown', + baseUrl: getServerUrl(), + }), + false + ); + }); + + it('should download a buildId that is a zip archive', async function () { + this.timeout(60000); + const expectedOutputPath = path.join( + tmpDir, + 'chrome', + `${BrowserPlatform.LINUX}-${testChromeBuildId}` + ); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + let browser = await install({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: testChromeBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + // Second iteration should be no-op. + browser = await install({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: testChromeBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + // Should discover installed browsers. + const cache = new Cache(tmpDir); + const installed = cache.getInstalledBrowsers(); + assert.deepStrictEqual(browser, installed[0]); + }); + + it('throws on invalid URL', async function () { + const expectedOutputPath = path.join( + tmpDir, + 'chrome', + `${BrowserPlatform.LINUX}-${testChromeBuildId}` + ); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + + async function installThatThrows(): Promise<unknown> { + try { + await install({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: testChromeBuildId, + baseUrl: 'https://127.0.0.1', + }); + return undefined; + } catch (err) { + return err; + } + } + assert.ok(await installThatThrows()); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + }); + + describe('with proxy', () => { + const proxyUrl = new URL(`http://localhost:54321`); + let proxyServer: http.Server; + let proxiedRequestUrls: string[] = []; + let proxiedRequestHosts: string[] = []; + + beforeEach(() => { + proxiedRequestUrls = []; + proxiedRequestHosts = []; + proxyServer = http + .createServer( + ( + originalRequest: http.IncomingMessage, + originalResponse: http.ServerResponse + ) => { + const url = originalRequest.url as string; + const proxyRequest = ( + url.startsWith('http:') ? http : https + ).request( + url, + { + method: originalRequest.method, + rejectUnauthorized: false, + }, + proxyResponse => { + originalResponse.writeHead( + proxyResponse.statusCode as number, + proxyResponse.headers + ); + proxyResponse.pipe(originalResponse, {end: true}); + } + ); + originalRequest.pipe(proxyRequest, {end: true}); + proxiedRequestUrls.push(url); + proxiedRequestHosts.push(originalRequest.headers?.host || ''); + } + ) + .listen({ + port: proxyUrl.port, + hostname: proxyUrl.hostname, + }); + + process.env['HTTPS_PROXY'] = proxyUrl.toString(); + process.env['HTTP_PROXY'] = proxyUrl.toString(); + }); + + afterEach(async () => { + await new Promise((resolve, reject) => { + proxyServer.close(error => { + if (error) { + reject(error); + } else { + resolve(undefined); + } + }); + }); + delete process.env['HTTP_PROXY']; + delete process.env['HTTPS_PROXY']; + }); + + it('can send canDownload requests via a proxy', async () => { + assert.strictEqual( + await canDownload({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: testChromeBuildId, + baseUrl: getServerUrl(), + }), + true + ); + assert.deepStrictEqual(proxiedRequestUrls, [ + getServerUrl() + '/113.0.5672.0/linux64/chrome-linux64.zip', + ]); + assert.deepStrictEqual(proxiedRequestHosts, [ + getServerUrl().replace('http://', ''), + ]); + }); + + it('can download via a proxy', async function () { + this.timeout(120000); + const expectedOutputPath = path.join( + tmpDir, + 'chrome', + `${BrowserPlatform.LINUX}-${testChromeBuildId}` + ); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + const browser = await install({ + cacheDir: tmpDir, + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: testChromeBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + assert.deepStrictEqual(proxiedRequestUrls, [ + getServerUrl() + '/113.0.5672.0/linux64/chrome-linux64.zip', + ]); + assert.deepStrictEqual(proxiedRequestHosts, [ + getServerUrl().replace('http://', ''), + ]); + }); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chrome/launch.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chrome/launch.spec.ts new file mode 100644 index 0000000000..cef690a6bb --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chrome/launch.spec.ts @@ -0,0 +1,132 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { + CDP_WEBSOCKET_ENDPOINT_REGEX, + computeExecutablePath, + launch, + install, + Browser, + BrowserPlatform, +} from '../../../lib/cjs/main.js'; +import {getServerUrl, setupTestServer, clearCache} from '../utils.js'; +import {testChromeBuildId} from '../versions.js'; + +describe('Chrome', () => { + it('should compute executable path for Chrome', () => { + assert.strictEqual( + computeExecutablePath({ + browser: Browser.CHROME, + platform: BrowserPlatform.LINUX, + buildId: '123', + cacheDir: 'cache', + }), + path.join('cache', 'chrome', 'linux-123', 'chrome-linux64', 'chrome') + ); + }); + + describe('launcher', function () { + setupTestServer(); + + this.timeout(60000); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(async () => { + tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-browsers-test') + ); + await install({ + cacheDir: tmpDir, + browser: Browser.CHROME, + buildId: testChromeBuildId, + baseUrl: getServerUrl(), + }); + }); + + afterEach(() => { + clearCache(tmpDir); + }); + + function getArgs() { + return [ + '--allow-pre-commit-input', + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-component-update', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-extensions', + '--disable-features=Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints,DialMediaRouteProvider', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--disable-sync', + '--enable-automation', + '--enable-features=NetworkServiceInProcess2', + '--export-tagged-pdf', + '--force-color-profile=srgb', + '--headless=new', + '--metrics-recording-only', + '--no-first-run', + '--password-store=basic', + '--remote-debugging-port=9222', + '--use-mock-keychain', + `--user-data-dir=${path.join(tmpDir, 'profile')}`, + 'about:blank', + ]; + } + + it('should launch a Chrome browser', async () => { + const executablePath = computeExecutablePath({ + cacheDir: tmpDir, + browser: Browser.CHROME, + buildId: testChromeBuildId, + }); + const process = launch({ + executablePath, + args: getArgs(), + }); + await process.close(); + }); + + it('should allow parsing stderr output of the browser process', async () => { + const executablePath = computeExecutablePath({ + cacheDir: tmpDir, + browser: Browser.CHROME, + buildId: testChromeBuildId, + }); + const process = launch({ + executablePath, + args: getArgs(), + }); + const url = await process.waitForLineOutput(CDP_WEBSOCKET_ENDPOINT_REGEX); + await process.close(); + assert.ok(url.startsWith('ws://127.0.0.1:9222/devtools/browser')); + }); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chromedriver/chromedriver-data.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chromedriver/chromedriver-data.spec.ts new file mode 100644 index 0000000000..fb4134a663 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chromedriver/chromedriver-data.spec.ts @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; + +import {BrowserPlatform} from '../../../lib/cjs/browser-data/browser-data.js'; +import { + resolveDownloadUrl, + relativeExecutablePath, +} from '../../../lib/cjs/browser-data/chromedriver.js'; + +describe('ChromeDriver', () => { + it('should resolve download URLs', () => { + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.LINUX, '112.0.5615.49'), + 'https://chromedriver.storage.googleapis.com/112.0.5615.49/chromedriver_linux64.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC, '112.0.5615.49'), + 'https://chromedriver.storage.googleapis.com/112.0.5615.49/chromedriver_mac64.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC_ARM, '112.0.5615.49'), + 'https://chromedriver.storage.googleapis.com/112.0.5615.49/chromedriver_mac_arm64.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN32, '112.0.5615.49'), + 'https://chromedriver.storage.googleapis.com/112.0.5615.49/chromedriver_win32.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN64, '112.0.5615.49'), + 'https://chromedriver.storage.googleapis.com/112.0.5615.49/chromedriver_win32.zip' + ); + }); + + it('should resolve executable paths', () => { + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.LINUX, '12372323'), + 'chromedriver' + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC, '12372323'), + 'chromedriver' + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC_ARM, '12372323'), + 'chromedriver' + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN32, '12372323'), + 'chromedriver.exe' + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN64, '12372323'), + 'chromedriver.exe' + ); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chromedriver/cli.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chromedriver/cli.spec.ts new file mode 100644 index 0000000000..52c23d22c2 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chromedriver/cli.spec.ts @@ -0,0 +1,89 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import {CLI} from '../../../lib/cjs/CLI.js'; +import { + createMockedReadlineInterface, + setupTestServer, + getServerUrl, +} from '../utils.js'; +import {testChromeDriverBuildId} from '../versions.js'; + +describe('ChromeDriver CLI', function () { + this.timeout(90000); + + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'puppeteer-browsers-test')); + }); + + afterEach(async () => { + await new CLI(tmpDir, createMockedReadlineInterface('yes')).run([ + 'npx', + '@puppeteer/browsers', + 'clear', + `--path=${tmpDir}`, + `--base-url=${getServerUrl()}`, + ]); + }); + + it('should download ChromeDriver binaries', async () => { + await new CLI(tmpDir).run([ + 'npx', + '@puppeteer/browsers', + 'install', + `chromedriver@${testChromeDriverBuildId}`, + `--path=${tmpDir}`, + '--platform=linux', + `--base-url=${getServerUrl()}`, + ]); + assert.ok( + fs.existsSync( + path.join( + tmpDir, + 'chromedriver', + `linux-${testChromeDriverBuildId}`, + 'chromedriver' + ) + ) + ); + + await new CLI(tmpDir, createMockedReadlineInterface('no')).run([ + 'npx', + '@puppeteer/browsers', + 'clear', + `--path=${tmpDir}`, + ]); + assert.ok( + fs.existsSync( + path.join( + tmpDir, + 'chromedriver', + `linux-${testChromeDriverBuildId}`, + 'chromedriver' + ) + ) + ); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chromedriver/install.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chromedriver/install.spec.ts new file mode 100644 index 0000000000..fb725de010 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chromedriver/install.spec.ts @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { + install, + canDownload, + Browser, + BrowserPlatform, + Cache, +} from '../../../lib/cjs/main.js'; +import {getServerUrl, setupTestServer} from '../utils.js'; +import {testChromeDriverBuildId} from '../versions.js'; + +/** + * Tests in this spec use real download URLs and unpack live browser archives + * so it requires the network access. + */ +describe('ChromeDriver install', () => { + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'puppeteer-browsers-test')); + }); + + afterEach(() => { + new Cache(tmpDir).clear(); + }); + + it('should check if a buildId can be downloaded', async () => { + assert.ok( + await canDownload({ + cacheDir: tmpDir, + browser: Browser.CHROMEDRIVER, + platform: BrowserPlatform.LINUX, + buildId: testChromeDriverBuildId, + baseUrl: getServerUrl(), + }) + ); + }); + + it('should report if a buildId is not downloadable', async () => { + assert.strictEqual( + await canDownload({ + cacheDir: tmpDir, + browser: Browser.CHROMEDRIVER, + platform: BrowserPlatform.LINUX, + buildId: 'unknown', + baseUrl: getServerUrl(), + }), + false + ); + }); + + it('should download and unpack the binary', async function () { + this.timeout(60000); + const expectedOutputPath = path.join( + tmpDir, + 'chromedriver', + `${BrowserPlatform.LINUX}-${testChromeDriverBuildId}` + ); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + let browser = await install({ + cacheDir: tmpDir, + browser: Browser.CHROMEDRIVER, + platform: BrowserPlatform.LINUX, + buildId: testChromeDriverBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + // Second iteration should be no-op. + browser = await install({ + cacheDir: tmpDir, + browser: Browser.CHROMEDRIVER, + platform: BrowserPlatform.LINUX, + buildId: testChromeDriverBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chromium/chromium-data.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chromium/chromium-data.spec.ts new file mode 100644 index 0000000000..5dae0d0d29 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chromium/chromium-data.spec.ts @@ -0,0 +1,108 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import path from 'path'; + +import { + BrowserPlatform, + ChromeReleaseChannel, +} from '../../../lib/cjs/browser-data/browser-data.js'; +import { + resolveDownloadUrl, + relativeExecutablePath, + resolveSystemExecutablePath, +} from '../../../lib/cjs/browser-data/chromium.js'; + +describe('Chromium', () => { + it('should resolve download URLs', () => { + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.LINUX, '1083080'), + 'https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/1083080/chrome-linux.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC, '1083080'), + 'https://storage.googleapis.com/chromium-browser-snapshots/Mac/1083080/chrome-mac.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC_ARM, '1083080'), + 'https://storage.googleapis.com/chromium-browser-snapshots/Mac_Arm/1083080/chrome-mac.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN32, '1083080'), + 'https://storage.googleapis.com/chromium-browser-snapshots/Win/1083080/chrome-win.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN64, '1083080'), + 'https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/1083080/chrome-win.zip' + ); + }); + + it('should resolve executable paths', () => { + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.LINUX, '12372323'), + path.join('chrome-linux', 'chrome') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC, '12372323'), + path.join('chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC_ARM, '12372323'), + path.join('chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN32, '12372323'), + path.join('chrome-win', 'chrome.exe') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN64, '12372323'), + path.join('chrome-win', 'chrome.exe') + ); + }); + + it('should resolve system executable path', () => { + process.env['PROGRAMFILES'] = 'C:\\ProgramFiles'; + try { + assert.strictEqual( + resolveSystemExecutablePath( + BrowserPlatform.WIN32, + ChromeReleaseChannel.DEV + ), + 'C:\\ProgramFiles\\Google\\Chrome Dev\\Application\\chrome.exe' + ); + } finally { + delete process.env['PROGRAMFILES']; + } + + assert.strictEqual( + resolveSystemExecutablePath( + BrowserPlatform.MAC, + ChromeReleaseChannel.BETA + ), + '/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta' + ); + assert.throws(() => { + assert.strictEqual( + resolveSystemExecutablePath( + BrowserPlatform.LINUX, + ChromeReleaseChannel.CANARY + ), + path.join('chrome-linux', 'chrome') + ); + }, new Error(`Unable to detect browser executable path for 'canary' on linux.`)); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/chromium/launch.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/chromium/launch.spec.ts new file mode 100644 index 0000000000..7fade9e52d --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/chromium/launch.spec.ts @@ -0,0 +1,132 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { + CDP_WEBSOCKET_ENDPOINT_REGEX, + computeExecutablePath, + launch, + install, + Browser, + BrowserPlatform, +} from '../../../lib/cjs/main.js'; +import {getServerUrl, setupTestServer, clearCache} from '../utils.js'; +import {testChromiumBuildId} from '../versions.js'; + +describe('Chromium', () => { + it('should compute executable path for Chromium', () => { + assert.strictEqual( + computeExecutablePath({ + browser: Browser.CHROMIUM, + platform: BrowserPlatform.LINUX, + buildId: '123', + cacheDir: 'cache', + }), + path.join('cache', 'chromium', 'linux-123', 'chrome-linux', 'chrome') + ); + }); + + describe('launcher', function () { + setupTestServer(); + + this.timeout(120000); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(async () => { + tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-browsers-test') + ); + await install({ + cacheDir: tmpDir, + browser: Browser.CHROMIUM, + buildId: testChromiumBuildId, + baseUrl: getServerUrl(), + }); + }); + + afterEach(() => { + clearCache(tmpDir); + }); + + function getArgs() { + return [ + '--allow-pre-commit-input', + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-component-update', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-extensions', + '--disable-features=Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints,DialMediaRouteProvider', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--disable-sync', + '--enable-automation', + '--enable-features=NetworkServiceInProcess2', + '--export-tagged-pdf', + '--force-color-profile=srgb', + '--headless=new', + '--metrics-recording-only', + '--no-first-run', + '--password-store=basic', + '--remote-debugging-port=9222', + '--use-mock-keychain', + `--user-data-dir=${path.join(tmpDir, 'profile')}`, + 'about:blank', + ]; + } + + it('should launch a Chromium browser', async () => { + const executablePath = computeExecutablePath({ + cacheDir: tmpDir, + browser: Browser.CHROMIUM, + buildId: testChromiumBuildId, + }); + const process = launch({ + executablePath, + args: getArgs(), + }); + await process.close(); + }); + + it('should allow parsing stderr output of the browser process', async () => { + const executablePath = computeExecutablePath({ + cacheDir: tmpDir, + browser: Browser.CHROMIUM, + buildId: testChromiumBuildId, + }); + const process = launch({ + executablePath, + args: getArgs(), + }); + const url = await process.waitForLineOutput(CDP_WEBSOCKET_ENDPOINT_REGEX); + await process.close(); + assert.ok(url.startsWith('ws://127.0.0.1:9222/devtools/browser')); + }); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/firefox/cli.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/firefox/cli.spec.ts new file mode 100644 index 0000000000..ec93e0c353 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/firefox/cli.spec.ts @@ -0,0 +1,79 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import {CLI} from '../../../lib/cjs/CLI.js'; +import { + createMockedReadlineInterface, + getServerUrl, + setupTestServer, +} from '../utils.js'; +import {testFirefoxBuildId} from '../versions.js'; + +describe('Firefox CLI', function () { + this.timeout(90000); + + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'puppeteer-browsers-test')); + }); + + afterEach(async () => { + await new CLI(tmpDir, createMockedReadlineInterface('yes')).run([ + 'npx', + '@puppeteer/browsers', + 'clear', + `--path=${tmpDir}`, + `--base-url=${getServerUrl()}`, + ]); + }); + + it('should download Firefox binaries', async () => { + await new CLI(tmpDir).run([ + 'npx', + '@puppeteer/browsers', + 'install', + `firefox@${testFirefoxBuildId}`, + `--path=${tmpDir}`, + '--platform=linux', + `--base-url=${getServerUrl()}`, + ]); + assert.ok( + fs.existsSync( + path.join(tmpDir, 'firefox', `linux-${testFirefoxBuildId}`, 'firefox') + ) + ); + }); + + it('should download latest Firefox binaries', async () => { + await new CLI(tmpDir).run([ + 'npx', + '@puppeteer/browsers', + 'install', + `firefox@latest`, + `--path=${tmpDir}`, + '--platform=linux', + `--base-url=${getServerUrl()}`, + ]); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/firefox/firefox-data.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/firefox/firefox-data.spec.ts new file mode 100644 index 0000000000..cad4ef5fa3 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/firefox/firefox-data.spec.ts @@ -0,0 +1,107 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import {BrowserPlatform} from '../../../lib/cjs/browser-data/browser-data.js'; +import { + createProfile, + relativeExecutablePath, + resolveDownloadUrl, +} from '../../../lib/cjs/browser-data/firefox.js'; + +describe('Firefox', () => { + it('should resolve download URLs', () => { + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.LINUX, '111.0a1'), + 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-111.0a1.en-US.linux-x86_64.tar.bz2' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC, '111.0a1'), + 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-111.0a1.en-US.mac.dmg' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.MAC_ARM, '111.0a1'), + 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-111.0a1.en-US.mac.dmg' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN32, '111.0a1'), + 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-111.0a1.en-US.win32.zip' + ); + assert.strictEqual( + resolveDownloadUrl(BrowserPlatform.WIN64, '111.0a1'), + 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-111.0a1.en-US.win64.zip' + ); + }); + + it('should resolve executable paths', () => { + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.LINUX, '111.0a1'), + path.join('firefox', 'firefox') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC, '111.0a1'), + path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.MAC_ARM, '111.0a1'), + path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN32, '111.0a1'), + path.join('firefox', 'firefox.exe') + ); + assert.strictEqual( + relativeExecutablePath(BrowserPlatform.WIN64, '111.0a1'), + path.join('firefox', 'firefox.exe') + ); + }); + + describe('profile', () => { + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-browsers-test') + ); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { + force: true, + recursive: true, + maxRetries: 5, + }); + }); + + it('should create a profile', async () => { + await createProfile({ + preferences: { + test: 1, + }, + path: tmpDir, + }); + const text = fs.readFileSync(path.join(tmpDir, 'user.js'), 'utf-8'); + assert.ok( + text.includes(`user_pref("toolkit.startup.max_resumed_crashes", -1);`) + ); // default preference. + assert.ok(text.includes(`user_pref("test", 1);`)); // custom preference. + }); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/firefox/install.spec.ts b/remote/test/puppeteer/packages/browsers/test/src/firefox/install.spec.ts new file mode 100644 index 0000000000..29d5974c73 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/firefox/install.spec.ts @@ -0,0 +1,85 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import {install, Browser, BrowserPlatform} from '../../../lib/cjs/main.js'; +import {setupTestServer, getServerUrl, clearCache} from '../utils.js'; +import {testFirefoxBuildId} from '../versions.js'; + +/** + * Tests in this spec use real download URLs and unpack live browser archives + * so it requires the network access. + */ +describe('Firefox install', () => { + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'puppeteer-browsers-test')); + }); + + afterEach(() => { + clearCache(tmpDir); + }); + + it('should download a buildId that is a bzip2 archive', async function () { + this.timeout(90000); + const expectedOutputPath = path.join( + tmpDir, + 'firefox', + `${BrowserPlatform.LINUX}-${testFirefoxBuildId}` + ); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + const browser = await install({ + cacheDir: tmpDir, + browser: Browser.FIREFOX, + platform: BrowserPlatform.LINUX, + buildId: testFirefoxBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + }); + + // install relies on the `hdiutil` utility on MacOS. + // The utility is not available on other platforms. + (os.platform() === 'darwin' ? it : it.skip)( + 'should download a buildId that is a dmg archive', + async function () { + this.timeout(180000); + const expectedOutputPath = path.join( + tmpDir, + 'firefox', + `${BrowserPlatform.MAC}-${testFirefoxBuildId}` + ); + assert.strictEqual(fs.existsSync(expectedOutputPath), false); + const browser = await install({ + cacheDir: tmpDir, + browser: Browser.FIREFOX, + platform: BrowserPlatform.MAC, + buildId: testFirefoxBuildId, + baseUrl: getServerUrl(), + }); + assert.strictEqual(browser.path, expectedOutputPath); + assert.ok(fs.existsSync(expectedOutputPath)); + } + ); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/firefox/launch.ts b/remote/test/puppeteer/packages/browsers/test/src/firefox/launch.ts new file mode 100644 index 0000000000..88388d8d6d --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/firefox/launch.ts @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { + computeExecutablePath, + launch, + install, + Browser, + BrowserPlatform, + createProfile, +} from '../../../lib/cjs/main.js'; +import {setupTestServer, getServerUrl, clearCache} from '../utils.js'; +import {testFirefoxBuildId} from '../versions.js'; + +describe('Firefox', () => { + it('should compute executable path for Firefox', () => { + assert.strictEqual( + computeExecutablePath({ + browser: Browser.FIREFOX, + platform: BrowserPlatform.LINUX, + buildId: '123', + cacheDir: 'cache', + }), + path.join('cache', 'firefox', 'linux-123', 'firefox', 'firefox') + ); + }); + + describe('launcher', function () { + this.timeout(120000); + + setupTestServer(); + + let tmpDir = '/tmp/puppeteer-browsers-test'; + + beforeEach(async () => { + tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-browsers-test') + ); + await install({ + cacheDir: tmpDir, + browser: Browser.FIREFOX, + buildId: testFirefoxBuildId, + baseUrl: getServerUrl(), + }); + }); + + afterEach(() => { + clearCache(tmpDir); + }); + + it('should launch a Firefox browser', async () => { + const userDataDir = path.join(tmpDir, 'profile'); + function getArgs(): string[] { + const firefoxArguments = ['--no-remote']; + switch (os.platform()) { + case 'darwin': + firefoxArguments.push('--foreground'); + break; + case 'win32': + firefoxArguments.push('--wait-for-browser'); + break; + } + firefoxArguments.push('--profile', userDataDir); + firefoxArguments.push('--headless'); + firefoxArguments.push('about:blank'); + return firefoxArguments; + } + await createProfile(Browser.FIREFOX, { + path: userDataDir, + preferences: {}, + }); + const executablePath = computeExecutablePath({ + cacheDir: tmpDir, + browser: Browser.FIREFOX, + buildId: testFirefoxBuildId, + }); + const process = launch({ + executablePath, + args: getArgs(), + }); + await process.close(); + }); + }); +}); diff --git a/remote/test/puppeteer/packages/browsers/test/src/tsconfig.json b/remote/test/puppeteer/packages/browsers/test/src/tsconfig.json new file mode 100644 index 0000000000..63dd3f1eeb --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "module": "CommonJS", + "outDir": "../build" + }, + "references": [{"path": "../../tsconfig.json"}] +} diff --git a/remote/test/puppeteer/packages/browsers/test/src/utils.ts b/remote/test/puppeteer/packages/browsers/test/src/utils.ts new file mode 100644 index 0000000000..9afb1bb763 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/utils.ts @@ -0,0 +1,85 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {execSync} from 'child_process'; +import os from 'os'; +import path from 'path'; +import * as readline from 'readline'; +import {Writable, Readable} from 'stream'; + +import {TestServer} from '@pptr/testserver'; + +import {isErrorLike} from '../../lib/cjs/launch.js'; +import {Cache} from '../../lib/cjs/main.js'; + +export function createMockedReadlineInterface( + input: string +): readline.Interface { + const readable = Readable.from([input]); + const writable = new Writable({ + write(_chunk, _encoding, callback) { + // Suppress the output to keep the test clean + callback(); + }, + }); + + return readline.createInterface({ + input: readable, + output: writable, + }); +} + +const startServer = async () => { + const assetsPath = path.join(__dirname, '..', 'cache', 'server'); + return await TestServer.create(assetsPath); +}; + +interface ServerState { + server: TestServer; +} + +const state: Partial<ServerState> = {}; + +export function setupTestServer(): void { + before(async () => { + state.server = await startServer(); + }); + + after(async () => { + await state.server!.stop(); + state.server = undefined; + }); +} + +export function getServerUrl(): string { + return `http://localhost:${state.server!.port}`; +} + +export function clearCache(tmpDir: string): void { + try { + new Cache(tmpDir).clear(); + } catch (err) { + if (os.platform() === 'win32') { + console.log(execSync('tasklist').toString('utf-8')); + // Sometimes on Windows the folder cannot be removed due to unknown reasons. + // We suppress the error to avoud flakiness. + if (isErrorLike(err) && err.message.includes('EBUSY')) { + return; + } + } + throw err; + } +} diff --git a/remote/test/puppeteer/packages/browsers/test/src/versions.ts b/remote/test/puppeteer/packages/browsers/test/src/versions.ts new file mode 100644 index 0000000000..606827fe3c --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/test/src/versions.ts @@ -0,0 +1,22 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const testChromeBuildId = '113.0.5672.0'; +export const testChromiumBuildId = '1083080'; +// TODO: We can add a Cron job to auto-update on change. +// Firefox keeps only `latest` version of Nightly builds. +export const testFirefoxBuildId = '114.0a1'; +export const testChromeDriverBuildId = '112.0.5615.49'; diff --git a/remote/test/puppeteer/packages/browsers/tools/downloadTestBrowsers.mjs b/remote/test/puppeteer/packages/browsers/tools/downloadTestBrowsers.mjs new file mode 100644 index 0000000000..9ee861f878 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/tools/downloadTestBrowsers.mjs @@ -0,0 +1,81 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Downloads test browser binaries to test/cache/server folder that + * mirrors the structure of the download server. + */ + +import {BrowserPlatform, install} from '@puppeteer/browsers'; +import path from 'path'; +import fs from 'fs'; + +import * as versions from '../test/build/versions.js'; +import {downloadPaths} from '../lib/esm/browser-data/browser-data.js'; + +function getBrowser(str) { + const regex = /test(.+)BuildId/; + const match = str.match(regex); + + if (match && match[1]) { + return match[1].toLowerCase(); + } else { + return null; + } +} + +const cacheDir = path.normalize(path.join('.', 'test', 'cache')); + +for (const version of Object.keys(versions)) { + const browser = getBrowser(version); + + if (!browser) { + continue; + } + + const buildId = versions[version]; + + for (const platform of Object.values(BrowserPlatform)) { + const targetPath = path.join( + cacheDir, + 'server', + ...downloadPaths[browser](platform, buildId) + ); + + if (fs.existsSync(targetPath)) { + continue; + } + + const result = await install({ + browser, + buildId, + platform, + cacheDir: path.join(cacheDir, 'tmp'), + unpack: false, + }); + + fs.mkdirSync(path.dirname(targetPath), { + recursive: true, + }); + fs.copyFileSync(result.path, targetPath); + } +} + +fs.rmSync(path.join(cacheDir, 'tmp'), { + recursive: true, + force: true, + maxRetries: 10, +}); diff --git a/remote/test/puppeteer/packages/browsers/tsconfig.json b/remote/test/puppeteer/packages/browsers/tsconfig.json new file mode 100644 index 0000000000..a219f8b704 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "references": [ + {"path": "src/tsconfig.esm.json"}, + {"path": "src/tsconfig.cjs.json"} + ] +} |