diff options
Diffstat (limited to 'remote/test/puppeteer/packages/browsers/src')
18 files changed, 2469 insertions, 0 deletions
diff --git a/remote/test/puppeteer/packages/browsers/src/CLI.ts b/remote/test/puppeteer/packages/browsers/src/CLI.ts new file mode 100644 index 0000000000..6767002269 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/CLI.ts @@ -0,0 +1,313 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {stdin as input, stdout as output} from 'process'; +import * as readline from 'readline'; + +import ProgressBar from 'progress'; +import type * as Yargs from 'yargs'; +import {hideBin} from 'yargs/helpers'; +import yargs from 'yargs/yargs'; + +import { + resolveBuildId, + Browser, + BrowserPlatform, + ChromeReleaseChannel, +} from './browser-data/browser-data.js'; +import {Cache} from './Cache.js'; +import {detectBrowserPlatform} from './detectPlatform.js'; +import {install} from './install.js'; +import { + computeExecutablePath, + computeSystemExecutablePath, + launch, +} from './launch.js'; + +type InstallArgs = { + browser: { + name: Browser; + buildId: string; + }; + path?: string; + platform?: BrowserPlatform; + baseUrl?: string; +}; + +type LaunchArgs = { + browser: { + name: Browser; + buildId: string; + }; + path?: string; + platform?: BrowserPlatform; + detached: boolean; + system: boolean; +}; + +type ClearArgs = { + path?: string; +}; + +/** + * @public + */ +export class CLI { + #cachePath; + #rl?: readline.Interface; + + constructor(cachePath = process.cwd(), rl?: readline.Interface) { + this.#cachePath = cachePath; + this.#rl = rl; + } + + #defineBrowserParameter(yargs: Yargs.Argv<unknown>): void { + yargs.positional('browser', { + description: + 'Which browser to install <browser>[@<buildId|latest>]. `latest` will try to find the latest available build. `buildId` is a browser-specific identifier such as a version or a revision.', + type: 'string', + coerce: (opt): InstallArgs['browser'] => { + return { + name: this.#parseBrowser(opt), + buildId: this.#parseBuildId(opt), + }; + }, + }); + } + + #definePlatformParameter(yargs: Yargs.Argv<unknown>): void { + yargs.option('platform', { + type: 'string', + desc: 'Platform that the binary needs to be compatible with.', + choices: Object.values(BrowserPlatform), + defaultDescription: 'Auto-detected', + }); + } + + #definePathParameter(yargs: Yargs.Argv<unknown>, required = false): void { + yargs.option('path', { + type: 'string', + desc: 'Path to the root folder for the browser downloads and installation. The installation folder structure is compatible with the cache structure used by Puppeteer.', + defaultDescription: 'Current working directory', + ...(required ? {} : {default: process.cwd()}), + }); + if (required) { + yargs.demandOption('path'); + } + } + + async run(argv: string[]): Promise<void> { + const yargsInstance = yargs(hideBin(argv)); + await yargsInstance + .scriptName('@puppeteer/browsers') + .command( + 'install <browser>', + 'Download and install the specified browser. If successful, the command outputs the actual browser buildId that was installed and the absolute path to the browser executable (format: <browser>@<buildID> <path>).', + yargs => { + this.#defineBrowserParameter(yargs); + this.#definePlatformParameter(yargs); + this.#definePathParameter(yargs); + yargs.option('base-url', { + type: 'string', + desc: 'Base URL to download from', + }); + yargs.example( + '$0 install chrome', + 'Install the latest available build of the Chrome browser.' + ); + yargs.example( + '$0 install chrome@latest', + 'Install the latest available build for the Chrome browser.' + ); + yargs.example( + '$0 install chromium@1083080', + 'Install the revision 1083080 of the Chromium browser.' + ); + yargs.example( + '$0 install firefox', + 'Install the latest available build of the Firefox browser.' + ); + yargs.example( + '$0 install firefox --platform mac', + 'Install the latest Mac (Intel) build of the Firefox browser.' + ); + yargs.example( + '$0 install firefox --path /tmp/my-browser-cache', + 'Install to the specified cache directory.' + ); + }, + async argv => { + const args = argv as unknown as InstallArgs; + args.platform ??= detectBrowserPlatform(); + if (!args.platform) { + throw new Error(`Could not resolve the current platform`); + } + args.browser.buildId = await resolveBuildId( + args.browser.name, + args.platform, + args.browser.buildId + ); + await install({ + browser: args.browser.name, + buildId: args.browser.buildId, + platform: args.platform, + cacheDir: args.path ?? this.#cachePath, + downloadProgressCallback: makeProgressCallback( + args.browser.name, + args.browser.buildId + ), + baseUrl: args.baseUrl, + }); + console.log( + `${args.browser.name}@${ + args.browser.buildId + } ${computeExecutablePath({ + browser: args.browser.name, + buildId: args.browser.buildId, + cacheDir: args.path ?? this.#cachePath, + platform: args.platform, + })}` + ); + } + ) + .command( + 'launch <browser>', + 'Launch the specified browser', + yargs => { + this.#defineBrowserParameter(yargs); + this.#definePlatformParameter(yargs); + this.#definePathParameter(yargs); + yargs.option('detached', { + type: 'boolean', + desc: 'Detach the child process.', + default: false, + }); + yargs.option('system', { + type: 'boolean', + desc: 'Search for a browser installed on the system instead of the cache folder.', + default: false, + }); + yargs.example( + '$0 launch chrome@1083080', + 'Launch the Chrome browser identified by the revision 1083080.' + ); + yargs.example( + '$0 launch firefox@112.0a1', + 'Launch the Firefox browser identified by the milestone 112.0a1.' + ); + yargs.example( + '$0 launch chrome@1083080 --detached', + 'Launch the browser but detach the sub-processes.' + ); + yargs.example( + '$0 launch chrome@canary --system', + 'Try to locate the Canary build of Chrome installed on the system and launch it.' + ); + }, + async argv => { + const args = argv as unknown as LaunchArgs; + const executablePath = args.system + ? computeSystemExecutablePath({ + browser: args.browser.name, + // TODO: throw an error if not a ChromeReleaseChannel is provided. + channel: args.browser.buildId as ChromeReleaseChannel, + platform: args.platform, + }) + : computeExecutablePath({ + browser: args.browser.name, + buildId: args.browser.buildId, + cacheDir: args.path ?? this.#cachePath, + platform: args.platform, + }); + launch({ + executablePath, + detached: args.detached, + }); + } + ) + .command( + 'clear', + 'Removes all installed browsers from the specified cache directory', + yargs => { + this.#definePathParameter(yargs, true); + }, + async argv => { + const args = argv as unknown as ClearArgs; + const cacheDir = args.path ?? this.#cachePath; + const rl = this.#rl ?? readline.createInterface({input, output}); + rl.question( + `Do you want to permanently and recursively delete the content of ${cacheDir} (yes/No)? `, + answer => { + rl.close(); + if (!['y', 'yes'].includes(answer.toLowerCase().trim())) { + console.log('Cancelled.'); + return; + } + const cache = new Cache(cacheDir); + cache.clear(); + console.log(`${cacheDir} cleared.`); + } + ); + } + ) + .demandCommand(1) + .help() + .wrap(Math.min(120, yargsInstance.terminalWidth())) + .parse(); + } + + #parseBrowser(version: string): Browser { + return version.split('@').shift() as Browser; + } + + #parseBuildId(version: string): string { + return version.split('@').pop() ?? 'latest'; + } +} + +/** + * @public + */ +export function makeProgressCallback( + browser: Browser, + buildId: string +): (downloadedBytes: number, totalBytes: number) => void { + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + return (downloadedBytes: number, totalBytes: number) => { + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${browser} r${buildId} - ${toMegabytes( + totalBytes + )} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +function toMegabytes(bytes: number) { + const mb = bytes / 1000 / 1000; + return `${Math.round(mb * 10) / 10} MB`; +} diff --git a/remote/test/puppeteer/packages/browsers/src/Cache.ts b/remote/test/puppeteer/packages/browsers/src/Cache.ts new file mode 100644 index 0000000000..142bceb08e --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/Cache.ts @@ -0,0 +1,119 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import {Browser, BrowserPlatform} from './browser-data/browser-data.js'; + +/** + * @public + */ +export type InstalledBrowser = { + path: string; + browser: Browser; + buildId: string; + platform: BrowserPlatform; +}; + +/** + * The cache used by Puppeteer relies on the following structure: + * + * - rootDir + * -- <browser1> | browserRoot(browser1) + * ---- <platform>-<buildId> | installationDir() + * ------ the browser-platform-buildId + * ------ specific structure. + * -- <browser2> | browserRoot(browser2) + * ---- <platform>-<buildId> | installationDir() + * ------ the browser-platform-buildId + * ------ specific structure. + * @internal + */ +export class Cache { + #rootDir: string; + + constructor(rootDir: string) { + this.#rootDir = rootDir; + } + + browserRoot(browser: Browser): string { + return path.join(this.#rootDir, browser); + } + + installationDir( + browser: Browser, + platform: BrowserPlatform, + buildId: string + ): string { + return path.join(this.browserRoot(browser), `${platform}-${buildId}`); + } + + clear(): void { + fs.rmSync(this.#rootDir, { + force: true, + recursive: true, + maxRetries: 10, + retryDelay: 500, + }); + } + + getInstalledBrowsers(): InstalledBrowser[] { + if (!fs.existsSync(this.#rootDir)) { + return []; + } + const types = fs.readdirSync(this.#rootDir); + const browsers = types.filter((t): t is Browser => { + return (Object.values(Browser) as string[]).includes(t); + }); + return browsers.flatMap(browser => { + const files = fs.readdirSync(this.browserRoot(browser)); + return files + .map(file => { + const result = parseFolderPath( + path.join(this.browserRoot(browser), file) + ); + if (!result) { + return null; + } + return { + path: path.join(this.browserRoot(browser), file), + browser, + platform: result.platform, + buildId: result.buildId, + }; + }) + .filter((item): item is InstalledBrowser => { + return item !== null; + }); + }); + } +} + +function parseFolderPath( + folderPath: string +): {platform: string; buildId: string} | undefined { + const name = path.basename(folderPath); + const splits = name.split('-'); + if (splits.length !== 2) { + return; + } + const [platform, buildId] = splits; + if (!buildId || !platform) { + return; + } + return {platform, buildId}; +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/browser-data.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/browser-data.ts new file mode 100644 index 0000000000..413435453a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/browser-data.ts @@ -0,0 +1,124 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as chrome from './chrome.js'; +import * as chromedriver from './chromedriver.js'; +import * as chromium from './chromium.js'; +import * as firefox from './firefox.js'; +import { + Browser, + BrowserPlatform, + BrowserTag, + ChromeReleaseChannel, + ProfileOptions, +} from './types.js'; + +export {ProfileOptions}; + +export const downloadUrls = { + [Browser.CHROMEDRIVER]: chromedriver.resolveDownloadUrl, + [Browser.CHROME]: chrome.resolveDownloadUrl, + [Browser.CHROMIUM]: chromium.resolveDownloadUrl, + [Browser.FIREFOX]: firefox.resolveDownloadUrl, +}; + +export const downloadPaths = { + [Browser.CHROMEDRIVER]: chromedriver.resolveDownloadPath, + [Browser.CHROME]: chrome.resolveDownloadPath, + [Browser.CHROMIUM]: chromium.resolveDownloadPath, + [Browser.FIREFOX]: firefox.resolveDownloadPath, +}; + +export const executablePathByBrowser = { + [Browser.CHROMEDRIVER]: chromedriver.relativeExecutablePath, + [Browser.CHROME]: chrome.relativeExecutablePath, + [Browser.CHROMIUM]: chromium.relativeExecutablePath, + [Browser.FIREFOX]: firefox.relativeExecutablePath, +}; + +export {Browser, BrowserPlatform, ChromeReleaseChannel}; + +/** + * @public + */ +export async function resolveBuildId( + browser: Browser, + platform: BrowserPlatform, + tag: string +): Promise<string> { + switch (browser) { + case Browser.FIREFOX: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + return await firefox.resolveBuildId('FIREFOX_NIGHTLY'); + } + case Browser.CHROME: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + // In CfT beta is the latest version. + return await chrome.resolveBuildId(platform, 'beta'); + } + case Browser.CHROMEDRIVER: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + return await chromedriver.resolveBuildId('latest'); + } + case Browser.CHROMIUM: + switch (tag as BrowserTag) { + case BrowserTag.LATEST: + return await chromium.resolveBuildId(platform, 'latest'); + } + } + // We assume the tag is the buildId if it didn't match any keywords. + return tag; +} + +/** + * @public + */ +export async function createProfile( + browser: Browser, + opts: ProfileOptions +): Promise<void> { + switch (browser) { + case Browser.FIREFOX: + return await firefox.createProfile(opts); + case Browser.CHROME: + case Browser.CHROMIUM: + throw new Error(`Profile creation is not support for ${browser} yet`); + } +} + +/** + * @public + */ +export function resolveSystemExecutablePath( + browser: Browser, + platform: BrowserPlatform, + channel: ChromeReleaseChannel +): string { + switch (browser) { + case Browser.CHROMEDRIVER: + case Browser.FIREFOX: + throw new Error( + `System browser detection is not supported for ${browser} yet.` + ); + case Browser.CHROME: + return chromium.resolveSystemExecutablePath(platform, channel); + case Browser.CHROMIUM: + return chrome.resolveSystemExecutablePath(platform, channel); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/chrome.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/chrome.ts new file mode 100644 index 0000000000..1fbf8c9647 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/chrome.ts @@ -0,0 +1,169 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform, ChromeReleaseChannel} from './types.js'; + +function folder(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'linux64'; + case BrowserPlatform.MAC_ARM: + return 'mac-arm64'; + case BrowserPlatform.MAC: + return 'mac-x64'; + case BrowserPlatform.WIN32: + return 'win32'; + case BrowserPlatform.WIN64: + return 'win64'; + } +} + +function chromiumDashPlatform(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'linux'; + case BrowserPlatform.MAC_ARM: + return 'mac'; + case BrowserPlatform.MAC: + return 'mac'; + case BrowserPlatform.WIN32: + return 'win'; + case BrowserPlatform.WIN64: + return 'win64'; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [buildId, folder(platform), `chrome-${folder(platform)}.zip`]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC: + case BrowserPlatform.MAC_ARM: + return path.join( + 'chrome-' + folder(platform), + 'Google Chrome for Testing.app', + 'Contents', + 'MacOS', + 'Google Chrome for Testing' + ); + case BrowserPlatform.LINUX: + return path.join('chrome-linux64', 'chrome'); + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return path.join('chrome-' + folder(platform), 'chrome.exe'); + } +} +export async function resolveBuildId( + platform: BrowserPlatform, + channel: 'beta' | 'stable' = 'beta' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL( + `https://chromiumdash.appspot.com/fetch_releases?platform=${chromiumDashPlatform( + platform + )}&channel=${channel}` + ), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + const response = JSON.parse(String(data)); + return resolve(response[0].version); + } catch { + return reject(new Error('Chrome version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} + +export function resolveSystemExecutablePath( + platform: BrowserPlatform, + channel: ChromeReleaseChannel +): string { + switch (platform) { + case BrowserPlatform.WIN64: + case BrowserPlatform.WIN32: + switch (channel) { + case ChromeReleaseChannel.STABLE: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome\\Application\\chrome.exe`; + case ChromeReleaseChannel.BETA: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome Beta\\Application\\chrome.exe`; + case ChromeReleaseChannel.CANARY: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome SxS\\Application\\chrome.exe`; + case ChromeReleaseChannel.DEV: + return `${process.env['PROGRAMFILES']}\\Google\\Chrome Dev\\Application\\chrome.exe`; + } + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + switch (channel) { + case ChromeReleaseChannel.STABLE: + return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; + case ChromeReleaseChannel.BETA: + return '/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta'; + case ChromeReleaseChannel.CANARY: + return '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'; + case ChromeReleaseChannel.DEV: + return '/Applications/Google Chrome Dev.app/Contents/MacOS/Google Chrome Dev'; + } + case BrowserPlatform.LINUX: + switch (channel) { + case ChromeReleaseChannel.STABLE: + return '/opt/google/chrome/chrome'; + case ChromeReleaseChannel.BETA: + return '/opt/google/chrome-beta/chrome'; + case ChromeReleaseChannel.DEV: + return '/opt/google/chrome-unstable/chrome'; + } + } + + throw new Error( + `Unable to detect browser executable path for '${channel}' on ${platform}.` + ); +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/chromedriver.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/chromedriver.ts new file mode 100644 index 0000000000..39894d2e86 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/chromedriver.ts @@ -0,0 +1,93 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform} from './types.js'; + +function archive(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'chromedriver_linux64'; + case BrowserPlatform.MAC_ARM: + return 'chromedriver_mac_arm64'; + case BrowserPlatform.MAC: + return 'chromedriver_mac64'; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return 'chromedriver_win32'; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://chromedriver.storage.googleapis.com' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [buildId, `${archive(platform)}.zip`]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC: + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.LINUX: + return 'chromedriver'; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return 'chromedriver.exe'; + } +} +export async function resolveBuildId( + _channel: 'latest' = 'latest' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL(`https://chromedriver.storage.googleapis.com/LATEST_RELEASE`), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + return resolve(String(data)); + } catch { + return reject(new Error('Chrome version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/chromium.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/chromium.ts new file mode 100644 index 0000000000..71fa003e0a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/chromium.ts @@ -0,0 +1,125 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform} from './types.js'; + +export {resolveSystemExecutablePath} from './chrome.js'; + +function archive(platform: BrowserPlatform, buildId: string): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'chrome-linux'; + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + return 'chrome-mac'; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + // Windows archive name changed at r591479. + return parseInt(buildId, 10) > 591479 ? 'chrome-win' : 'chrome-win32'; + } +} + +function folder(platform: BrowserPlatform): string { + switch (platform) { + case BrowserPlatform.LINUX: + return 'Linux_x64'; + case BrowserPlatform.MAC_ARM: + return 'Mac_Arm'; + case BrowserPlatform.MAC: + return 'Mac'; + case BrowserPlatform.WIN32: + return 'Win'; + case BrowserPlatform.WIN64: + return 'Win_x64'; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://storage.googleapis.com/chromium-browser-snapshots' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [folder(platform), buildId, `${archive(platform, buildId)}.zip`]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC: + case BrowserPlatform.MAC_ARM: + return path.join( + 'chrome-mac', + 'Chromium.app', + 'Contents', + 'MacOS', + 'Chromium' + ); + case BrowserPlatform.LINUX: + return path.join('chrome-linux', 'chrome'); + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return path.join('chrome-win', 'chrome.exe'); + } +} +export async function resolveBuildId( + platform: BrowserPlatform, + // We will need it for other channels/keywords. + _channel: 'latest' = 'latest' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL( + `https://storage.googleapis.com/chromium-browser-snapshots/${folder( + platform + )}/LAST_CHANGE` + ), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + return resolve(String(data)); + } catch { + return reject(new Error('Chrome version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/firefox.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/firefox.ts new file mode 100644 index 0000000000..c3f337ee5a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/firefox.ts @@ -0,0 +1,355 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import {httpRequest} from '../httpUtil.js'; + +import {BrowserPlatform, ProfileOptions} from './types.js'; + +function archive(platform: BrowserPlatform, buildId: string): string { + switch (platform) { + case BrowserPlatform.LINUX: + return `firefox-${buildId}.en-US.${platform}-x86_64.tar.bz2`; + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + return `firefox-${buildId}.en-US.mac.dmg`; + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return `firefox-${buildId}.en-US.${platform}.zip`; + } +} + +export function resolveDownloadUrl( + platform: BrowserPlatform, + buildId: string, + baseUrl = 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central' +): string { + return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`; +} + +export function resolveDownloadPath( + platform: BrowserPlatform, + buildId: string +): string[] { + return [archive(platform, buildId)]; +} + +export function relativeExecutablePath( + platform: BrowserPlatform, + _buildId: string +): string { + switch (platform) { + case BrowserPlatform.MAC_ARM: + case BrowserPlatform.MAC: + return path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox'); + case BrowserPlatform.LINUX: + return path.join('firefox', 'firefox'); + case BrowserPlatform.WIN32: + case BrowserPlatform.WIN64: + return path.join('firefox', 'firefox.exe'); + } +} + +export async function resolveBuildId( + channel: 'FIREFOX_NIGHTLY' = 'FIREFOX_NIGHTLY' +): Promise<string> { + return new Promise((resolve, reject) => { + const request = httpRequest( + new URL('https://product-details.mozilla.org/1.0/firefox_versions.json'), + 'GET', + response => { + let data = ''; + if (response.statusCode && response.statusCode >= 400) { + return reject(new Error(`Got status code ${response.statusCode}`)); + } + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + try { + const versions = JSON.parse(data); + return resolve(versions[channel]); + } catch { + return reject(new Error('Firefox version not found')); + } + }); + }, + false + ); + request.on('error', err => { + reject(err); + }); + }); +} + +export async function createProfile(options: ProfileOptions): Promise<void> { + if (!fs.existsSync(options.path)) { + await fs.promises.mkdir(options.path, { + recursive: true, + }); + } + await writePreferences({ + preferences: { + ...defaultProfilePreferences(options.preferences), + ...options.preferences, + }, + path: options.path, + }); +} + +function defaultProfilePreferences( + extraPrefs: Record<string, unknown> +): Record<string, unknown> { + const server = 'dummy.test'; + + const defaultPrefs = { + // Make sure Shield doesn't hit the network. + 'app.normandy.api_url': '', + // Disable Firefox old build background check + 'app.update.checkInstallTime': false, + // Disable automatically upgrading Firefox + 'app.update.disabledForTesting': true, + + // Increase the APZ content response timeout to 1 minute + 'apz.content_response_timeout': 60000, + + // Prevent various error message on the console + // jest-puppeteer asserts that no error message is emitted by the console + 'browser.contentblocking.features.standard': + '-tp,tpPrivate,cookieBehavior0,-cm,-fp', + + // Enable the dump function: which sends messages to the system + // console + // https://bugzilla.mozilla.org/show_bug.cgi?id=1543115 + 'browser.dom.window.dump.enabled': true, + // Disable topstories + 'browser.newtabpage.activity-stream.feeds.system.topstories': false, + // Always display a blank page + 'browser.newtabpage.enabled': false, + // Background thumbnails in particular cause grief: and disabling + // thumbnails in general cannot hurt + 'browser.pagethumbnails.capturing_disabled': true, + + // Disable safebrowsing components. + 'browser.safebrowsing.blockedURIs.enabled': false, + 'browser.safebrowsing.downloads.enabled': false, + 'browser.safebrowsing.malware.enabled': false, + 'browser.safebrowsing.passwords.enabled': false, + 'browser.safebrowsing.phishing.enabled': false, + + // Disable updates to search engines. + 'browser.search.update': false, + // Do not restore the last open set of tabs if the browser has crashed + 'browser.sessionstore.resume_from_crash': false, + // Skip check for default browser on startup + 'browser.shell.checkDefaultBrowser': false, + + // Disable newtabpage + 'browser.startup.homepage': 'about:blank', + // Do not redirect user when a milstone upgrade of Firefox is detected + 'browser.startup.homepage_override.mstone': 'ignore', + // Start with a blank page about:blank + 'browser.startup.page': 0, + + // Do not allow background tabs to be zombified on Android: otherwise for + // tests that open additional tabs: the test harness tab itself might get + // unloaded + 'browser.tabs.disableBackgroundZombification': false, + // Do not warn when closing all other open tabs + 'browser.tabs.warnOnCloseOtherTabs': false, + // Do not warn when multiple tabs will be opened + 'browser.tabs.warnOnOpen': false, + + // Disable page translations, which can cause issues with tests. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1836093. + 'browser.translations.enable': false, + + // Disable the UI tour. + 'browser.uitour.enabled': false, + // Turn off search suggestions in the location bar so as not to trigger + // network connections. + 'browser.urlbar.suggest.searches': false, + // Disable first run splash page on Windows 10 + 'browser.usedOnWindows10.introURL': '', + // Do not warn on quitting Firefox + 'browser.warnOnQuit': false, + + // Defensively disable data reporting systems + 'datareporting.healthreport.documentServerURI': `http://${server}/dummy/healthreport/`, + 'datareporting.healthreport.logging.consoleEnabled': false, + 'datareporting.healthreport.service.enabled': false, + 'datareporting.healthreport.service.firstRun': false, + 'datareporting.healthreport.uploadEnabled': false, + + // Do not show datareporting policy notifications which can interfere with tests + 'datareporting.policy.dataSubmissionEnabled': false, + 'datareporting.policy.dataSubmissionPolicyBypassNotification': true, + + // DevTools JSONViewer sometimes fails to load dependencies with its require.js. + // This doesn't affect Puppeteer but spams console (Bug 1424372) + 'devtools.jsonview.enabled': false, + + // Disable popup-blocker + 'dom.disable_open_during_load': false, + + // Enable the support for File object creation in the content process + // Required for |Page.setFileInputFiles| protocol method. + 'dom.file.createInChild': true, + + // Disable the ProcessHangMonitor + 'dom.ipc.reportProcessHangs': false, + + // Disable slow script dialogues + 'dom.max_chrome_script_run_time': 0, + 'dom.max_script_run_time': 0, + + // Only load extensions from the application and user profile + // AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION + 'extensions.autoDisableScopes': 0, + 'extensions.enabledScopes': 5, + + // Disable metadata caching for installed add-ons by default + 'extensions.getAddons.cache.enabled': false, + + // Disable installing any distribution extensions or add-ons. + 'extensions.installDistroAddons': false, + + // Disabled screenshots extension + 'extensions.screenshots.disabled': true, + + // Turn off extension updates so they do not bother tests + 'extensions.update.enabled': false, + + // Turn off extension updates so they do not bother tests + 'extensions.update.notifyUser': false, + + // Make sure opening about:addons will not hit the network + 'extensions.webservice.discoverURL': `http://${server}/dummy/discoveryURL`, + + // Temporarily force disable BFCache in parent (https://bit.ly/bug-1732263) + 'fission.bfcacheInParent': false, + + // Force all web content to use a single content process + 'fission.webContentIsolationStrategy': 0, + + // Allow the application to have focus even it runs in the background + 'focusmanager.testmode': true, + // Disable useragent updates + 'general.useragent.updates.enabled': false, + // Always use network provider for geolocation tests so we bypass the + // macOS dialog raised by the corelocation provider + 'geo.provider.testing': true, + // Do not scan Wifi + 'geo.wifi.scan': false, + // No hang monitor + 'hangmonitor.timeout': 0, + // Show chrome errors and warnings in the error console + 'javascript.options.showInConsole': true, + + // Disable download and usage of OpenH264: and Widevine plugins + 'media.gmp-manager.updateEnabled': false, + // Prevent various error message on the console + // jest-puppeteer asserts that no error message is emitted by the console + 'network.cookie.cookieBehavior': 0, + + // Disable experimental feature that is only available in Nightly + 'network.cookie.sameSite.laxByDefault': false, + + // Do not prompt for temporary redirects + 'network.http.prompt-temp-redirect': false, + + // Disable speculative connections so they are not reported as leaking + // when they are hanging around + 'network.http.speculative-parallel-limit': 0, + + // Do not automatically switch between offline and online + 'network.manage-offline-status': false, + + // Make sure SNTP requests do not hit the network + 'network.sntp.pools': server, + + // Disable Flash. + 'plugin.state.flash': 0, + + 'privacy.trackingprotection.enabled': false, + + // Can be removed once Firefox 89 is no longer supported + // https://bugzilla.mozilla.org/show_bug.cgi?id=1710839 + 'remote.enabled': true, + + // Don't do network connections for mitm priming + 'security.certerrors.mitm.priming.enabled': false, + // Local documents have access to all other local documents, + // including directory listings + 'security.fileuri.strict_origin_policy': false, + // Do not wait for the notification button security delay + 'security.notification_enable_delay': 0, + + // Ensure blocklist updates do not hit the network + 'services.settings.server': `http://${server}/dummy/blocklist/`, + + // Do not automatically fill sign-in forms with known usernames and + // passwords + 'signon.autofillForms': false, + // Disable password capture, so that tests that include forms are not + // influenced by the presence of the persistent doorhanger notification + 'signon.rememberSignons': false, + + // Disable first-run welcome page + 'startup.homepage_welcome_url': 'about:blank', + + // Disable first-run welcome page + 'startup.homepage_welcome_url.additional': '', + + // Disable browser animations (tabs, fullscreen, sliding alerts) + 'toolkit.cosmeticAnimations.enabled': false, + + // Prevent starting into safe mode after application crashes + 'toolkit.startup.max_resumed_crashes': -1, + }; + + return Object.assign(defaultPrefs, extraPrefs); +} + +/** + * Populates the user.js file with custom preferences as needed to allow + * Firefox's CDP support to properly function. These preferences will be + * automatically copied over to prefs.js during startup of Firefox. To be + * able to restore the original values of preferences a backup of prefs.js + * will be created. + * + * @param prefs - List of preferences to add. + * @param profilePath - Firefox profile to write the preferences to. + */ +async function writePreferences(options: ProfileOptions): Promise<void> { + const lines = Object.entries(options.preferences).map(([key, value]) => { + return `user_pref(${JSON.stringify(key)}, ${JSON.stringify(value)});`; + }); + + await fs.promises.writeFile( + path.join(options.path, 'user.js'), + lines.join('\n') + ); + + // Create a backup of the preferences file if it already exitsts. + const prefsPath = path.join(options.path, 'prefs.js'); + if (fs.existsSync(prefsPath)) { + const prefsBackupPath = path.join(options.path, 'prefs.js.puppeteer'); + await fs.promises.copyFile(prefsPath, prefsBackupPath); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/browser-data/types.ts b/remote/test/puppeteer/packages/browsers/src/browser-data/types.ts new file mode 100644 index 0000000000..f88d2ca098 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/browser-data/types.ts @@ -0,0 +1,75 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as chrome from './chrome.js'; +import * as firefox from './firefox.js'; + +/** + * Supported browsers. + * + * @public + */ +export enum Browser { + CHROME = 'chrome', + CHROMIUM = 'chromium', + FIREFOX = 'firefox', + CHROMEDRIVER = 'chromedriver', +} + +/** + * Platform names used to identify a OS platfrom x architecture combination in the way + * that is relevant for the browser download. + * + * @public + */ +export enum BrowserPlatform { + LINUX = 'linux', + MAC = 'mac', + MAC_ARM = 'mac_arm', + WIN32 = 'win32', + WIN64 = 'win64', +} + +export const downloadUrls = { + [Browser.CHROME]: chrome.resolveDownloadUrl, + [Browser.CHROMIUM]: chrome.resolveDownloadUrl, + [Browser.FIREFOX]: firefox.resolveDownloadUrl, +}; + +/** + * @public + */ +export enum BrowserTag { + LATEST = 'latest', +} + +/** + * @public + */ +export interface ProfileOptions { + preferences: Record<string, unknown>; + path: string; +} + +/** + * @public + */ +export enum ChromeReleaseChannel { + STABLE = 'stable', + DEV = 'dev', + CANARY = 'canary', + BETA = 'beta', +} diff --git a/remote/test/puppeteer/packages/browsers/src/debug.ts b/remote/test/puppeteer/packages/browsers/src/debug.ts new file mode 100644 index 0000000000..eee0a347e8 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/debug.ts @@ -0,0 +1,19 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import debug from 'debug'; + +export {debug}; diff --git a/remote/test/puppeteer/packages/browsers/src/detectPlatform.ts b/remote/test/puppeteer/packages/browsers/src/detectPlatform.ts new file mode 100644 index 0000000000..fed8c2e2ea --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/detectPlatform.ts @@ -0,0 +1,61 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; + +import {BrowserPlatform} from './browser-data/browser-data.js'; + +/** + * @public + */ +export function detectBrowserPlatform(): BrowserPlatform | undefined { + const platform = os.platform(); + switch (platform) { + case 'darwin': + return os.arch() === 'arm64' + ? BrowserPlatform.MAC_ARM + : BrowserPlatform.MAC; + case 'linux': + return BrowserPlatform.LINUX; + case 'win32': + return os.arch() === 'x64' || + // Windows 11 for ARM supports x64 emulation + (os.arch() === 'arm64' && isWindows11(os.release())) + ? BrowserPlatform.WIN64 + : BrowserPlatform.WIN32; + default: + return undefined; + } +} + +/** + * Windows 11 is identified by the version 10.0.22000 or greater + * @internal + */ +function isWindows11(version: string): boolean { + const parts = version.split('.'); + if (parts.length > 2) { + const major = parseInt(parts[0] as string, 10); + const minor = parseInt(parts[1] as string, 10); + const patch = parseInt(parts[2] as string, 10); + return ( + major > 10 || + (major === 10 && minor > 0) || + (major === 10 && minor === 0 && patch >= 22000) + ); + } + return false; +} diff --git a/remote/test/puppeteer/packages/browsers/src/fileUtil.ts b/remote/test/puppeteer/packages/browsers/src/fileUtil.ts new file mode 100644 index 0000000000..6139accd49 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/fileUtil.ts @@ -0,0 +1,89 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {exec as execChildProcess} from 'child_process'; +import {createReadStream} from 'fs'; +import {mkdir, readdir} from 'fs/promises'; +import * as path from 'path'; +import {promisify} from 'util'; + +import extractZip from 'extract-zip'; +import tar from 'tar-fs'; +import bzip from 'unbzip2-stream'; + +const exec = promisify(execChildProcess); + +/** + * @internal + */ +export async function unpackArchive( + archivePath: string, + folderPath: string +): Promise<void> { + if (archivePath.endsWith('.zip')) { + await extractZip(archivePath, {dir: folderPath}); + } else if (archivePath.endsWith('.tar.bz2')) { + await extractTar(archivePath, folderPath); + } else if (archivePath.endsWith('.dmg')) { + await mkdir(folderPath); + await installDMG(archivePath, folderPath); + } else { + throw new Error(`Unsupported archive format: ${archivePath}`); + } +} + +/** + * @internal + */ +function extractTar(tarPath: string, folderPath: string): Promise<void> { + return new Promise((fulfill, reject) => { + const tarStream = tar.extract(folderPath); + tarStream.on('error', reject); + tarStream.on('finish', fulfill); + const readStream = createReadStream(tarPath); + readStream.pipe(bzip()).pipe(tarStream); + }); +} + +/** + * @internal + */ +async function installDMG(dmgPath: string, folderPath: string): Promise<void> { + const {stdout} = await exec( + `hdiutil attach -nobrowse -noautoopen "${dmgPath}"` + ); + + const volumes = stdout.match(/\/Volumes\/(.*)/m); + if (!volumes) { + throw new Error(`Could not find volume path in ${stdout}`); + } + const mountPath = volumes[0]!; + + try { + const fileNames = await readdir(mountPath); + const appName = fileNames.find(item => { + return typeof item === 'string' && item.endsWith('.app'); + }); + if (!appName) { + throw new Error(`Cannot find app in ${mountPath}`); + } + const mountedPath = path.join(mountPath!, appName); + + await exec(`cp -R "${mountedPath}" "${folderPath}"`); + } finally { + await exec(`hdiutil detach "${mountPath}" -quiet`); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/httpUtil.ts b/remote/test/puppeteer/packages/browsers/src/httpUtil.ts new file mode 100644 index 0000000000..5b6150f734 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/httpUtil.ts @@ -0,0 +1,141 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {createWriteStream} from 'fs'; +import * as http from 'http'; +import * as https from 'https'; +import {URL} from 'url'; + +import createHttpsProxyAgent from 'https-proxy-agent'; +import {getProxyForUrl} from 'proxy-from-env'; + +export function headHttpRequest(url: URL): Promise<boolean> { + return new Promise(resolve => { + const request = httpRequest( + url, + 'HEAD', + response => { + resolve(response.statusCode === 200); + }, + false + ); + request.on('error', () => { + resolve(false); + }); + }); +} + +export function httpRequest( + url: URL, + method: string, + response: (x: http.IncomingMessage) => void, + keepAlive = true +): http.ClientRequest { + const options: http.RequestOptions = { + protocol: url.protocol, + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + method, + headers: keepAlive ? {Connection: 'keep-alive'} : undefined, + }; + + const proxyURL = getProxyForUrl(url.toString()); + if (proxyURL) { + const proxy = new URL(proxyURL); + if (proxy.protocol === 'http:') { + options.path = url.href; + options.hostname = proxy.hostname; + options.protocol = proxy.protocol; + options.port = proxy.port; + options.headers ??= {}; + options.headers['Host'] ||= url.host; + } else { + options.agent = createHttpsProxyAgent({ + host: proxy.host, + path: proxy.pathname, + port: proxy.port, + secureProxy: proxy.protocol === 'https:', + headers: options.headers, + }); + } + } + + const requestCallback = (res: http.IncomingMessage): void => { + if ( + res.statusCode && + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + httpRequest(new URL(res.headers.location), method, response); + } else { + response(res); + } + }; + const request = + options.protocol === 'https:' + ? https.request(options, requestCallback) + : http.request(options, requestCallback); + request.end(); + return request; +} + +/** + * @internal + */ +export function downloadFile( + url: URL, + destinationPath: string, + progressCallback?: (downloadedBytes: number, totalBytes: number) => void +): Promise<void> { + return new Promise<void>((resolve, reject) => { + let downloadedBytes = 0; + let totalBytes = 0; + + function onData(chunk: string): void { + downloadedBytes += chunk.length; + progressCallback!(downloadedBytes, totalBytes); + } + + const request = httpRequest(url, 'GET', response => { + if (response.statusCode !== 200) { + const error = new Error( + `Download failed: server returned code ${response.statusCode}. URL: ${url}` + ); + // consume response data to free up memory + response.resume(); + reject(error); + return; + } + const file = createWriteStream(destinationPath); + file.on('finish', () => { + return resolve(); + }); + file.on('error', error => { + return reject(error); + }); + response.pipe(file); + totalBytes = parseInt(response.headers['content-length']!, 10); + if (progressCallback) { + response.on('data', onData); + } + }); + request.on('error', error => { + return reject(error); + }); + }); +} diff --git a/remote/test/puppeteer/packages/browsers/src/install.ts b/remote/test/puppeteer/packages/browsers/src/install.ts new file mode 100644 index 0000000000..054e048420 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/install.ts @@ -0,0 +1,218 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import {existsSync} from 'fs'; +import {mkdir, unlink} from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { + Browser, + BrowserPlatform, + downloadUrls, +} from './browser-data/browser-data.js'; +import {Cache, InstalledBrowser} from './Cache.js'; +import {debug} from './debug.js'; +import {detectBrowserPlatform} from './detectPlatform.js'; +import {unpackArchive} from './fileUtil.js'; +import {downloadFile, headHttpRequest} from './httpUtil.js'; + +const debugInstall = debug('puppeteer:browsers:install'); + +const times = new Map<string, [number, number]>(); +function debugTime(label: string) { + times.set(label, process.hrtime()); +} + +function debugTimeEnd(label: string) { + const end = process.hrtime(); + const start = times.get(label); + if (!start) { + return; + } + const duration = + end[0] * 1000 + end[1] / 1e6 - (start[0] * 1000 + start[1] / 1e6); // calculate duration in milliseconds + debugInstall(`Duration for ${label}: ${duration}ms`); +} + +/** + * @public + */ +export interface InstallOptions { + /** + * Determines the path to download browsers to. + */ + cacheDir: string; + /** + * Determines which platform the browser will be suited for. + * + * @defaultValue **Auto-detected.** + */ + platform?: BrowserPlatform; + /** + * Determines which browser to install. + */ + browser: Browser; + /** + * Determines which buildId to dowloand. BuildId should uniquely identify + * binaries and they are used for caching. + */ + buildId: string; + /** + * Provides information about the progress of the download. + */ + downloadProgressCallback?: ( + downloadedBytes: number, + totalBytes: number + ) => void; + /** + * Determines the host that will be used for downloading. + * + * @defaultValue Either + * + * - https://storage.googleapis.com/chromium-browser-snapshots or + * - https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central + * + */ + baseUrl?: string; + /** + * Whether to unpack and install browser archives. + * + * @defaultValue `true` + */ + unpack?: boolean; +} + +/** + * @public + */ +export async function install( + options: InstallOptions +): Promise<InstalledBrowser> { + options.platform ??= detectBrowserPlatform(); + options.unpack ??= true; + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + const url = getDownloadUrl( + options.browser, + options.platform, + options.buildId, + options.baseUrl + ); + const fileName = url.toString().split('/').pop(); + assert(fileName, `A malformed download URL was found: ${url}.`); + const structure = new Cache(options.cacheDir); + const browserRoot = structure.browserRoot(options.browser); + const archivePath = path.join(browserRoot, fileName); + if (!existsSync(browserRoot)) { + await mkdir(browserRoot, {recursive: true}); + } + + if (!options.unpack) { + if (existsSync(archivePath)) { + return { + path: archivePath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; + } + debugInstall(`Downloading binary from ${url}`); + debugTime('download'); + await downloadFile(url, archivePath, options.downloadProgressCallback); + debugTimeEnd('download'); + return { + path: archivePath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; + } + + const outputPath = structure.installationDir( + options.browser, + options.platform, + options.buildId + ); + if (existsSync(outputPath)) { + return { + path: outputPath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; + } + try { + debugInstall(`Downloading binary from ${url}`); + try { + debugTime('download'); + await downloadFile(url, archivePath, options.downloadProgressCallback); + } finally { + debugTimeEnd('download'); + } + + debugInstall(`Installing ${archivePath} to ${outputPath}`); + try { + debugTime('extract'); + await unpackArchive(archivePath, outputPath); + } finally { + debugTimeEnd('extract'); + } + } finally { + if (existsSync(archivePath)) { + await unlink(archivePath); + } + } + return { + path: outputPath, + browser: options.browser, + platform: options.platform, + buildId: options.buildId, + }; +} + +/** + * @public + */ +export async function canDownload(options: InstallOptions): Promise<boolean> { + options.platform ??= detectBrowserPlatform(); + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + return await headHttpRequest( + getDownloadUrl( + options.browser, + options.platform, + options.buildId, + options.baseUrl + ) + ); +} + +function getDownloadUrl( + browser: Browser, + platform: BrowserPlatform, + buildId: string, + baseUrl?: string +): URL { + return new URL(downloadUrls[browser](platform, buildId, baseUrl)); +} diff --git a/remote/test/puppeteer/packages/browsers/src/launch.ts b/remote/test/puppeteer/packages/browsers/src/launch.ts new file mode 100644 index 0000000000..9f8c8f20ed --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/launch.ts @@ -0,0 +1,494 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import childProcess from 'child_process'; +import {accessSync} from 'fs'; +import os from 'os'; +import path from 'path'; +import readline from 'readline'; + +import { + Browser, + BrowserPlatform, + executablePathByBrowser, + resolveSystemExecutablePath, + ChromeReleaseChannel, +} from './browser-data/browser-data.js'; +import {Cache} from './Cache.js'; +import {debug} from './debug.js'; +import {detectBrowserPlatform} from './detectPlatform.js'; + +const debugLaunch = debug('puppeteer:browsers:launcher'); + +/** + * @public + */ +export interface ComputeExecutablePathOptions { + /** + * Root path to the storage directory. + */ + cacheDir: string; + /** + * Determines which platform the browser will be suited for. + * + * @defaultValue **Auto-detected.** + */ + platform?: BrowserPlatform; + /** + * Determines which browser to launch. + */ + browser: Browser; + /** + * Determines which buildId to download. BuildId should uniquely identify + * binaries and they are used for caching. + */ + buildId: string; +} + +/** + * @public + */ +export function computeExecutablePath( + options: ComputeExecutablePathOptions +): string { + options.platform ??= detectBrowserPlatform(); + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + const installationDir = new Cache(options.cacheDir).installationDir( + options.browser, + options.platform, + options.buildId + ); + return path.join( + installationDir, + executablePathByBrowser[options.browser](options.platform, options.buildId) + ); +} + +/** + * @public + */ +export interface SystemOptions { + /** + * Determines which platform the browser will be suited for. + * + * @defaultValue **Auto-detected.** + */ + platform?: BrowserPlatform; + /** + * Determines which browser to launch. + */ + browser: Browser; + /** + * Release channel to look for on the system. + */ + channel: ChromeReleaseChannel; +} + +/** + * @public + */ +export function computeSystemExecutablePath(options: SystemOptions): string { + options.platform ??= detectBrowserPlatform(); + if (!options.platform) { + throw new Error( + `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})` + ); + } + const path = resolveSystemExecutablePath( + options.browser, + options.platform, + options.channel + ); + try { + accessSync(path); + } catch (error) { + throw new Error( + `Could not find Google Chrome executable for channel '${options.channel}' at '${path}'.` + ); + } + return path; +} + +/** + * @public + */ +export type LaunchOptions = { + executablePath: string; + pipe?: boolean; + dumpio?: boolean; + args?: string[]; + env?: Record<string, string | undefined>; + handleSIGINT?: boolean; + handleSIGTERM?: boolean; + handleSIGHUP?: boolean; + detached?: boolean; + onExit?: () => Promise<void>; +}; + +/** + * @public + */ +export function launch(opts: LaunchOptions): Process { + return new Process(opts); +} + +/** + * @public + */ +export const CDP_WEBSOCKET_ENDPOINT_REGEX = + /^DevTools listening on (ws:\/\/.*)$/; + +/** + * @public + */ +export const WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX = + /^WebDriver BiDi listening on (ws:\/\/.*)$/; + +/** + * @public + */ +export class Process { + #executablePath; + #args: string[]; + #browserProcess: childProcess.ChildProcess; + #exited = false; + // The browser process can be closed externally or from the driver process. We + // need to invoke the hooks only once though but we don't know how many times + // we will be invoked. + #hooksRan = false; + #onExitHook = async () => {}; + #browserProcessExiting: Promise<void>; + + constructor(opts: LaunchOptions) { + this.#executablePath = opts.executablePath; + this.#args = opts.args ?? []; + + opts.pipe ??= false; + opts.dumpio ??= false; + opts.handleSIGINT ??= true; + opts.handleSIGTERM ??= true; + opts.handleSIGHUP ??= true; + // On non-windows platforms, `detached: true` makes child process a + // leader of a new process group, making it possible to kill child + // process tree with `.kill(-pid)` command. @see + // https://nodejs.org/api/child_process.html#child_process_options_detached + opts.detached ??= process.platform !== 'win32'; + + const stdio = this.#configureStdio({ + pipe: opts.pipe, + dumpio: opts.dumpio, + }); + + debugLaunch(`Launching ${this.#executablePath} ${this.#args.join(' ')}`, { + detached: opts.detached, + env: opts.env, + stdio, + }); + + this.#browserProcess = childProcess.spawn( + this.#executablePath, + this.#args, + { + detached: opts.detached, + env: opts.env, + stdio, + } + ); + + debugLaunch(`Launched ${this.#browserProcess.pid}`); + if (opts.dumpio) { + this.#browserProcess.stderr?.pipe(process.stderr); + this.#browserProcess.stdout?.pipe(process.stdout); + } + process.on('exit', this.#onDriverProcessExit); + if (opts.handleSIGINT) { + process.on('SIGINT', this.#onDriverProcessSignal); + } + if (opts.handleSIGTERM) { + process.on('SIGTERM', this.#onDriverProcessSignal); + } + if (opts.handleSIGHUP) { + process.on('SIGHUP', this.#onDriverProcessSignal); + } + if (opts.onExit) { + this.#onExitHook = opts.onExit; + } + this.#browserProcessExiting = new Promise((resolve, reject) => { + this.#browserProcess.once('exit', async () => { + debugLaunch(`Browser process ${this.#browserProcess.pid} onExit`); + this.#clearListeners(); + this.#exited = true; + try { + await this.#runHooks(); + } catch (err) { + reject(err); + return; + } + resolve(); + }); + }); + } + + async #runHooks() { + if (this.#hooksRan) { + return; + } + this.#hooksRan = true; + await this.#onExitHook(); + } + + get nodeProcess(): childProcess.ChildProcess { + return this.#browserProcess; + } + + #configureStdio(opts: { + pipe: boolean; + dumpio: boolean; + }): Array<'ignore' | 'pipe'> { + if (opts.pipe) { + if (opts.dumpio) { + return ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']; + } else { + return ['ignore', 'ignore', 'ignore', 'pipe', 'pipe']; + } + } else { + if (opts.dumpio) { + return ['pipe', 'pipe', 'pipe']; + } else { + return ['pipe', 'ignore', 'pipe']; + } + } + } + + #clearListeners(): void { + process.off('exit', this.#onDriverProcessExit); + process.off('SIGINT', this.#onDriverProcessSignal); + process.off('SIGTERM', this.#onDriverProcessSignal); + process.off('SIGHUP', this.#onDriverProcessSignal); + } + + #onDriverProcessExit = (_code: number) => { + this.kill(); + }; + + #onDriverProcessSignal = (signal: string): void => { + switch (signal) { + case 'SIGINT': + this.kill(); + process.exit(130); + case 'SIGTERM': + case 'SIGHUP': + void this.close(); + break; + } + }; + + async close(): Promise<void> { + await this.#runHooks(); + if (!this.#exited) { + this.kill(); + } + return this.#browserProcessExiting; + } + + hasClosed(): Promise<void> { + return this.#browserProcessExiting; + } + + kill(): void { + debugLaunch(`Trying to kill ${this.#browserProcess.pid}`); + // If the process failed to launch (for example if the browser executable path + // is invalid), then the process does not get a pid assigned. A call to + // `proc.kill` would error, as the `pid` to-be-killed can not be found. + if ( + this.#browserProcess && + this.#browserProcess.pid && + pidExists(this.#browserProcess.pid) + ) { + try { + debugLaunch(`Browser process ${this.#browserProcess.pid} exists`); + if (process.platform === 'win32') { + try { + childProcess.execSync( + `taskkill /pid ${this.#browserProcess.pid} /T /F` + ); + } catch (error) { + debugLaunch( + `Killing ${this.#browserProcess.pid} using taskkill failed`, + error + ); + // taskkill can fail to kill the process e.g. due to missing permissions. + // Let's kill the process via Node API. This delays killing of all child + // processes of `this.proc` until the main Node.js process dies. + this.#browserProcess.kill(); + } + } else { + // on linux the process group can be killed with the group id prefixed with + // a minus sign. The process group id is the group leader's pid. + const processGroupId = -this.#browserProcess.pid; + + try { + process.kill(processGroupId, 'SIGKILL'); + } catch (error) { + debugLaunch( + `Killing ${this.#browserProcess.pid} using process.kill failed`, + error + ); + // Killing the process group can fail due e.g. to missing permissions. + // Let's kill the process via Node API. This delays killing of all child + // processes of `this.proc` until the main Node.js process dies. + this.#browserProcess.kill('SIGKILL'); + } + } + } catch (error) { + throw new Error( + `${PROCESS_ERROR_EXPLANATION}\nError cause: ${ + isErrorLike(error) ? error.stack : error + }` + ); + } + } + this.#clearListeners(); + } + + waitForLineOutput(regex: RegExp, timeout?: number): Promise<string> { + if (!this.#browserProcess.stderr) { + throw new Error('`browserProcess` does not have stderr.'); + } + const rl = readline.createInterface(this.#browserProcess.stderr); + let stderr = ''; + + return new Promise((resolve, reject) => { + rl.on('line', onLine); + rl.on('close', onClose); + this.#browserProcess.on('exit', onClose); + this.#browserProcess.on('error', onClose); + const timeoutId = timeout ? setTimeout(onTimeout, timeout) : 0; + + const cleanup = (): void => { + if (timeoutId) { + clearTimeout(timeoutId); + } + rl.off('line', onLine); + rl.off('close', onClose); + this.#browserProcess.off('exit', onClose); + this.#browserProcess.off('error', onClose); + }; + + function onClose(error?: Error): void { + cleanup(); + reject( + new Error( + [ + `Failed to launch the browser process!${ + error ? ' ' + error.message : '' + }`, + stderr, + '', + 'TROUBLESHOOTING: https://pptr.dev/troubleshooting', + '', + ].join('\n') + ) + ); + } + + function onTimeout(): void { + cleanup(); + reject( + new TimeoutError( + `Timed out after ${timeout} ms while waiting for the WS endpoint URL to appear in stdout!` + ) + ); + } + + function onLine(line: string): void { + stderr += line + '\n'; + const match = line.match(regex); + if (!match) { + return; + } + cleanup(); + // The RegExp matches, so this will obviously exist. + resolve(match[1]!); + } + }); + } +} + +const PROCESS_ERROR_EXPLANATION = `Puppeteer was unable to kill the process which ran the browser binary. +This means that, on future Puppeteer launches, Puppeteer might not be able to launch the browser. +Please check your open processes and ensure that the browser processes that Puppeteer launched have been killed. +If you think this is a bug, please report it on the Puppeteer issue tracker.`; + +/** + * @internal + */ +function pidExists(pid: number): boolean { + try { + return process.kill(pid, 0); + } catch (error) { + if (isErrnoException(error)) { + if (error.code && error.code === 'ESRCH') { + return false; + } + } + throw error; + } +} + +/** + * @internal + */ +export interface ErrorLike extends Error { + name: string; + message: string; +} + +/** + * @internal + */ +export function isErrorLike(obj: unknown): obj is ErrorLike { + return ( + typeof obj === 'object' && obj !== null && 'name' in obj && 'message' in obj + ); +} +/** + * @internal + */ +export function isErrnoException(obj: unknown): obj is NodeJS.ErrnoException { + return ( + isErrorLike(obj) && + ('errno' in obj || 'code' in obj || 'path' in obj || 'syscall' in obj) + ); +} + +/** + * @public + */ +export class TimeoutError extends Error { + /** + * @internal + */ + constructor(message?: string) { + super(message); + this.name = this.constructor.name; + Error.captureStackTrace(this, this.constructor); + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/main-cli.ts b/remote/test/puppeteer/packages/browsers/src/main-cli.ts new file mode 100644 index 0000000000..a086c1c3b9 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/main-cli.ts @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {CLI} from './CLI.js'; + +void new CLI().run(process.argv); diff --git a/remote/test/puppeteer/packages/browsers/src/main.ts b/remote/test/puppeteer/packages/browsers/src/main.ts new file mode 100644 index 0000000000..14ca6a838a --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/main.ts @@ -0,0 +1,40 @@ +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + launch, + computeExecutablePath, + computeSystemExecutablePath, + TimeoutError, + LaunchOptions, + ComputeExecutablePathOptions as Options, + SystemOptions, + CDP_WEBSOCKET_ENDPOINT_REGEX, + WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX, + Process, +} from './launch.js'; +export {install, canDownload, InstallOptions} from './install.js'; +export {detectBrowserPlatform} from './detectPlatform.js'; +export { + resolveBuildId, + Browser, + BrowserPlatform, + ChromeReleaseChannel, + createProfile, + ProfileOptions, +} from './browser-data/browser-data.js'; +export {CLI, makeProgressCallback} from './CLI.js'; +export {Cache, InstalledBrowser} from './Cache.js'; diff --git a/remote/test/puppeteer/packages/browsers/src/tsconfig.cjs.json b/remote/test/puppeteer/packages/browsers/src/tsconfig.cjs.json new file mode 100644 index 0000000000..ef01b990b7 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "module": "CommonJS", + "outDir": "../lib/cjs" + } +} diff --git a/remote/test/puppeteer/packages/browsers/src/tsconfig.esm.json b/remote/test/puppeteer/packages/browsers/src/tsconfig.esm.json new file mode 100644 index 0000000000..a824bc8cb8 --- /dev/null +++ b/remote/test/puppeteer/packages/browsers/src/tsconfig.esm.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "../lib/esm" + } +} |