diff options
Diffstat (limited to 'remote/test/puppeteer/src/node')
-rw-r--r-- | remote/test/puppeteer/src/node/BrowserFetcher.ts | 602 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/BrowserRunner.ts | 257 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/LaunchOptions.ts | 42 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/Launcher.ts | 673 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/NodeWebSocketTransport.ts | 59 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/PipeTransport.ts | 80 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/Puppeteer.ts | 230 | ||||
-rw-r--r-- | remote/test/puppeteer/src/node/install.ts | 185 |
8 files changed, 2128 insertions, 0 deletions
diff --git a/remote/test/puppeteer/src/node/BrowserFetcher.ts b/remote/test/puppeteer/src/node/BrowserFetcher.ts new file mode 100644 index 0000000000..4653cd230c --- /dev/null +++ b/remote/test/puppeteer/src/node/BrowserFetcher.ts @@ -0,0 +1,602 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as os from 'os'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as util from 'util'; +import * as childProcess from 'child_process'; +import * as https from 'https'; +import * as http from 'http'; + +import { Product } from '../common/Product.js'; +import extractZip from 'extract-zip'; +import { debug } from '../common/Debug.js'; +import { promisify } from 'util'; +import removeRecursive from 'rimraf'; +import * as URL from 'url'; +import ProxyAgent from 'https-proxy-agent'; +import { getProxyForUrl } from 'proxy-from-env'; +import { assert } from '../common/assert.js'; + +const debugFetcher = debug(`puppeteer:fetcher`); + +const downloadURLs = { + chrome: { + linux: '%s/chromium-browser-snapshots/Linux_x64/%d/%s.zip', + mac: '%s/chromium-browser-snapshots/Mac/%d/%s.zip', + win32: '%s/chromium-browser-snapshots/Win/%d/%s.zip', + win64: '%s/chromium-browser-snapshots/Win_x64/%d/%s.zip', + }, + firefox: { + linux: '%s/firefox-%s.en-US.%s-x86_64.tar.bz2', + mac: '%s/firefox-%s.en-US.%s.dmg', + win32: '%s/firefox-%s.en-US.%s.zip', + win64: '%s/firefox-%s.en-US.%s.zip', + }, +} as const; + +const browserConfig = { + chrome: { + host: 'https://storage.googleapis.com', + destination: '.local-chromium', + }, + firefox: { + host: + 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central', + destination: '.local-firefox', + }, +} as const; + +/** + * Supported platforms. + * @public + */ +export type Platform = 'linux' | 'mac' | 'win32' | 'win64'; + +function archiveName( + product: Product, + platform: Platform, + revision: string +): string { + if (product === 'chrome') { + if (platform === 'linux') return 'chrome-linux'; + if (platform === 'mac') return 'chrome-mac'; + if (platform === 'win32' || platform === 'win64') { + // Windows archive name changed at r591479. + return parseInt(revision, 10) > 591479 ? 'chrome-win' : 'chrome-win32'; + } + } else if (product === 'firefox') { + return platform; + } +} + +/** + * @internal + */ +function downloadURL( + product: Product, + platform: Platform, + host: string, + revision: string +): string { + const url = util.format( + downloadURLs[product][platform], + host, + revision, + archiveName(product, platform, revision) + ); + return url; +} + +/** + * @internal + */ +function handleArm64(): void { + fs.stat('/usr/bin/chromium-browser', function (err, stats) { + if (stats === undefined) { + console.error(`The chromium binary is not available for arm64: `); + console.error(`If you are on Ubuntu, you can install with: `); + console.error(`\n apt-get install chromium-browser\n`); + throw new Error(); + } + }); +} +const readdirAsync = promisify(fs.readdir.bind(fs)); +const mkdirAsync = promisify(fs.mkdir.bind(fs)); +const unlinkAsync = promisify(fs.unlink.bind(fs)); +const chmodAsync = promisify(fs.chmod.bind(fs)); + +function existsAsync(filePath: string): Promise<boolean> { + return new Promise((resolve) => { + fs.access(filePath, (err) => resolve(!err)); + }); +} + +/** + * @public + */ +export interface BrowserFetcherOptions { + platform?: Platform; + product?: string; + path?: string; + host?: string; +} + +/** + * @public + */ +export interface BrowserFetcherRevisionInfo { + folderPath: string; + executablePath: string; + url: string; + local: boolean; + revision: string; + product: string; +} +/** + * BrowserFetcher can download and manage different versions of Chromium and Firefox. + * + * @remarks + * BrowserFetcher operates on revision strings that specify a precise version of Chromium, e.g. `"533271"`. Revision strings can be obtained from {@link http://omahaproxy.appspot.com/ | omahaproxy.appspot.com}. + * In the Firefox case, BrowserFetcher downloads Firefox Nightly and + * operates on version numbers such as `"75"`. + * + * @example + * An example of using BrowserFetcher to download a specific version of Chromium + * and running Puppeteer against it: + * + * ```js + * const browserFetcher = puppeteer.createBrowserFetcher(); + * const revisionInfo = await browserFetcher.download('533271'); + * const browser = await puppeteer.launch({executablePath: revisionInfo.executablePath}) + * ``` + * + * **NOTE** BrowserFetcher is not designed to work concurrently with other + * instances of BrowserFetcher that share the same downloads directory. + * + * @public + */ + +export class BrowserFetcher { + private _product: Product; + private _downloadsFolder: string; + private _downloadHost: string; + private _platform: Platform; + + /** + * @internal + */ + constructor(projectRoot: string, options: BrowserFetcherOptions = {}) { + this._product = (options.product || 'chrome').toLowerCase() as Product; + assert( + this._product === 'chrome' || this._product === 'firefox', + `Unknown product: "${options.product}"` + ); + + this._downloadsFolder = + options.path || + path.join(projectRoot, browserConfig[this._product].destination); + this._downloadHost = options.host || browserConfig[this._product].host; + this.setPlatform(options.platform); + assert( + downloadURLs[this._product][this._platform], + 'Unsupported platform: ' + this._platform + ); + } + + private setPlatform(platformFromOptions?: Platform): void { + if (platformFromOptions) { + this._platform = platformFromOptions; + return; + } + + const platform = os.platform(); + if (platform === 'darwin') this._platform = 'mac'; + else if (platform === 'linux') this._platform = 'linux'; + else if (platform === 'win32') + this._platform = os.arch() === 'x64' ? 'win64' : 'win32'; + else assert(this._platform, 'Unsupported platform: ' + os.platform()); + } + + /** + * @returns Returns the current `Platform`. + */ + platform(): Platform { + return this._platform; + } + + /** + * @returns Returns the current `Product`. + */ + product(): Product { + return this._product; + } + + /** + * @returns The download host being used. + */ + host(): string { + return this._downloadHost; + } + + /** + * Initiates a HEAD request to check if the revision is available. + * @remarks + * This method is affected by the current `product`. + * @param revision - The revision to check availability for. + * @returns A promise that resolves to `true` if the revision could be downloaded + * from the host. + */ + canDownload(revision: string): Promise<boolean> { + const url = downloadURL( + this._product, + this._platform, + this._downloadHost, + revision + ); + return new Promise((resolve) => { + const request = httpRequest(url, 'HEAD', (response) => { + resolve(response.statusCode === 200); + }); + request.on('error', (error) => { + console.error(error); + resolve(false); + }); + }); + } + + /** + * Initiates a GET request to download the revision from the host. + * @remarks + * This method is affected by the current `product`. + * @param revision - The revision to download. + * @param progressCallback - A function that will be called with two arguments: + * How many bytes have been downloaded and the total number of bytes of the download. + * @returns A promise with revision information when the revision is downloaded + * and extracted. + */ + async download( + revision: string, + progressCallback: (x: number, y: number) => void = (): void => {} + ): Promise<BrowserFetcherRevisionInfo> { + const url = downloadURL( + this._product, + this._platform, + this._downloadHost, + revision + ); + const fileName = url.split('/').pop(); + const archivePath = path.join(this._downloadsFolder, fileName); + const outputPath = this._getFolderPath(revision); + if (await existsAsync(outputPath)) return this.revisionInfo(revision); + if (!(await existsAsync(this._downloadsFolder))) + await mkdirAsync(this._downloadsFolder); + if (os.arch() === 'arm64') { + handleArm64(); + return; + } + try { + await downloadFile(url, archivePath, progressCallback); + await install(archivePath, outputPath); + } finally { + if (await existsAsync(archivePath)) await unlinkAsync(archivePath); + } + const revisionInfo = this.revisionInfo(revision); + if (revisionInfo) await chmodAsync(revisionInfo.executablePath, 0o755); + return revisionInfo; + } + + /** + * @remarks + * This method is affected by the current `product`. + * @returns A promise with a list of all revision strings (for the current `product`) + * available locally on disk. + */ + async localRevisions(): Promise<string[]> { + if (!(await existsAsync(this._downloadsFolder))) return []; + const fileNames = await readdirAsync(this._downloadsFolder); + return fileNames + .map((fileName) => parseFolderPath(this._product, fileName)) + .filter((entry) => entry && entry.platform === this._platform) + .map((entry) => entry.revision); + } + + /** + * @remarks + * This method is affected by the current `product`. + * @param revision - A revision to remove for the current `product`. + * @returns A promise that resolves when the revision has been removes or + * throws if the revision has not been downloaded. + */ + async remove(revision: string): Promise<void> { + const folderPath = this._getFolderPath(revision); + assert( + await existsAsync(folderPath), + `Failed to remove: revision ${revision} is not downloaded` + ); + await new Promise((fulfill) => removeRecursive(folderPath, fulfill)); + } + + /** + * @param revision - The revision to get info for. + * @returns The revision info for the given revision. + */ + revisionInfo(revision: string): BrowserFetcherRevisionInfo { + const folderPath = this._getFolderPath(revision); + let executablePath = ''; + if (this._product === 'chrome') { + if (this._platform === 'mac') + executablePath = path.join( + folderPath, + archiveName(this._product, this._platform, revision), + 'Chromium.app', + 'Contents', + 'MacOS', + 'Chromium' + ); + else if (this._platform === 'linux') + executablePath = path.join( + folderPath, + archiveName(this._product, this._platform, revision), + 'chrome' + ); + else if (this._platform === 'win32' || this._platform === 'win64') + executablePath = path.join( + folderPath, + archiveName(this._product, this._platform, revision), + 'chrome.exe' + ); + else throw new Error('Unsupported platform: ' + this._platform); + } else if (this._product === 'firefox') { + if (this._platform === 'mac') + executablePath = path.join( + folderPath, + 'Firefox Nightly.app', + 'Contents', + 'MacOS', + 'firefox' + ); + else if (this._platform === 'linux') + executablePath = path.join(folderPath, 'firefox', 'firefox'); + else if (this._platform === 'win32' || this._platform === 'win64') + executablePath = path.join(folderPath, 'firefox', 'firefox.exe'); + else throw new Error('Unsupported platform: ' + this._platform); + } else { + throw new Error('Unsupported product: ' + this._product); + } + const url = downloadURL( + this._product, + this._platform, + this._downloadHost, + revision + ); + const local = fs.existsSync(folderPath); + debugFetcher({ + revision, + executablePath, + folderPath, + local, + url, + product: this._product, + }); + return { + revision, + executablePath, + folderPath, + local, + url, + product: this._product, + }; + } + + /** + * @internal + */ + _getFolderPath(revision: string): string { + return path.join(this._downloadsFolder, this._platform + '-' + revision); + } +} + +function parseFolderPath( + product: Product, + folderPath: string +): { product: string; platform: string; revision: string } | null { + const name = path.basename(folderPath); + const splits = name.split('-'); + if (splits.length !== 2) return null; + const [platform, revision] = splits; + if (!downloadURLs[product][platform]) return null; + return { product, platform, revision }; +} + +/** + * @internal + */ +function downloadFile( + url: string, + destinationPath: string, + progressCallback: (x: number, y: number) => void +): Promise<void> { + debugFetcher(`Downloading binary from ${url}`); + let fulfill, reject; + let downloadedBytes = 0; + let totalBytes = 0; + + const promise = new Promise<void>((x, y) => { + fulfill = x; + reject = y; + }); + + const request = httpRequest(url, 'GET', (response) => { + if (response.statusCode !== 200) { + const error = new Error( + `Download failed: server returned code ${response.statusCode}. URL: ${url}` + ); + // consume response data to free up memory + response.resume(); + reject(error); + return; + } + const file = fs.createWriteStream(destinationPath); + file.on('finish', () => fulfill()); + file.on('error', (error) => reject(error)); + response.pipe(file); + totalBytes = parseInt( + /** @type {string} */ response.headers['content-length'], + 10 + ); + if (progressCallback) response.on('data', onData); + }); + request.on('error', (error) => reject(error)); + return promise; + + function onData(chunk: string): void { + downloadedBytes += chunk.length; + progressCallback(downloadedBytes, totalBytes); + } +} + +function install(archivePath: string, folderPath: string): Promise<unknown> { + debugFetcher(`Installing ${archivePath} to ${folderPath}`); + if (archivePath.endsWith('.zip')) + return extractZip(archivePath, { dir: folderPath }); + else if (archivePath.endsWith('.tar.bz2')) + return extractTar(archivePath, folderPath); + else if (archivePath.endsWith('.dmg')) + return mkdirAsync(folderPath).then(() => + installDMG(archivePath, folderPath) + ); + else throw new Error(`Unsupported archive format: ${archivePath}`); +} + +/** + * @internal + */ +function extractTar(tarPath: string, folderPath: string): Promise<unknown> { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const tar = require('tar-fs'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const bzip = require('unbzip2-stream'); + return new Promise((fulfill, reject) => { + const tarStream = tar.extract(folderPath); + tarStream.on('error', reject); + tarStream.on('finish', fulfill); + const readStream = fs.createReadStream(tarPath); + readStream.pipe(bzip()).pipe(tarStream); + }); +} + +/** + * @internal + */ +function installDMG(dmgPath: string, folderPath: string): Promise<void> { + let mountPath; + + function mountAndCopy(fulfill: () => void, reject: (Error) => void): void { + const mountCommand = `hdiutil attach -nobrowse -noautoopen "${dmgPath}"`; + childProcess.exec(mountCommand, (err, stdout) => { + if (err) return reject(err); + const volumes = stdout.match(/\/Volumes\/(.*)/m); + if (!volumes) + return reject(new Error(`Could not find volume path in ${stdout}`)); + mountPath = volumes[0]; + readdirAsync(mountPath) + .then((fileNames) => { + const appName = fileNames.filter( + (item) => typeof item === 'string' && item.endsWith('.app') + )[0]; + if (!appName) + return reject(new Error(`Cannot find app in ${mountPath}`)); + const copyPath = path.join(mountPath, appName); + debugFetcher(`Copying ${copyPath} to ${folderPath}`); + childProcess.exec(`cp -R "${copyPath}" "${folderPath}"`, (err) => { + if (err) reject(err); + else fulfill(); + }); + }) + .catch(reject); + }); + } + + function unmount(): void { + if (!mountPath) return; + const unmountCommand = `hdiutil detach "${mountPath}" -quiet`; + debugFetcher(`Unmounting ${mountPath}`); + childProcess.exec(unmountCommand, (err) => { + if (err) console.error(`Error unmounting dmg: ${err}`); + }); + } + + return new Promise<void>(mountAndCopy) + .catch((error) => { + console.error(error); + }) + .finally(unmount); +} + +function httpRequest( + url: string, + method: string, + response: (x: http.IncomingMessage) => void +): http.ClientRequest { + const urlParsed = URL.parse(url); + + type Options = Partial<URL.UrlWithStringQuery> & { + method?: string; + agent?: ProxyAgent; + rejectUnauthorized?: boolean; + }; + + let options: Options = { + ...urlParsed, + method, + }; + + const proxyURL = getProxyForUrl(url); + if (proxyURL) { + if (url.startsWith('http:')) { + const proxy = URL.parse(proxyURL); + options = { + path: options.href, + host: proxy.hostname, + port: proxy.port, + }; + } else { + const parsedProxyURL = URL.parse(proxyURL); + + const proxyOptions = { + ...parsedProxyURL, + secureProxy: parsedProxyURL.protocol === 'https:', + } as ProxyAgent.HttpsProxyAgentOptions; + + options.agent = new ProxyAgent(proxyOptions); + options.rejectUnauthorized = false; + } + } + + const requestCallback = (res: http.IncomingMessage): void => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) + httpRequest(res.headers.location, method, response); + else response(res); + }; + const request = + options.protocol === 'https:' + ? https.request(options, requestCallback) + : http.request(options, requestCallback); + request.end(); + return request; +} diff --git a/remote/test/puppeteer/src/node/BrowserRunner.ts b/remote/test/puppeteer/src/node/BrowserRunner.ts new file mode 100644 index 0000000000..e7e11bb143 --- /dev/null +++ b/remote/test/puppeteer/src/node/BrowserRunner.ts @@ -0,0 +1,257 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { debug } from '../common/Debug.js'; + +import removeFolder from 'rimraf'; +import * as childProcess from 'child_process'; +import { assert } from '../common/assert.js'; +import { helper, debugError } from '../common/helper.js'; +import { LaunchOptions } from './LaunchOptions.js'; +import { Connection } from '../common/Connection.js'; +import { NodeWebSocketTransport as WebSocketTransport } from '../node/NodeWebSocketTransport.js'; +import { PipeTransport } from './PipeTransport.js'; +import * as readline from 'readline'; +import { TimeoutError } from '../common/Errors.js'; +import { promisify } from 'util'; + +const removeFolderAsync = promisify(removeFolder); +const debugLauncher = debug('puppeteer:launcher'); +const PROCESS_ERROR_EXPLANATION = `Puppeteer was unable to kill the process which ran the browser binary. +This means that, on future Puppeteer launches, Puppeteer might not be able to launch the browser. +Please check your open processes and ensure that the browser processes that Puppeteer launched have been killed. +If you think this is a bug, please report it on the Puppeteer issue tracker.`; + +export class BrowserRunner { + private _executablePath: string; + private _processArguments: string[]; + private _tempDirectory?: string; + + proc = null; + connection = null; + + private _closed = true; + private _listeners = []; + private _processClosing: Promise<void>; + + constructor( + executablePath: string, + processArguments: string[], + tempDirectory?: string + ) { + this._executablePath = executablePath; + this._processArguments = processArguments; + this._tempDirectory = tempDirectory; + } + + start(options: LaunchOptions): void { + const { + handleSIGINT, + handleSIGTERM, + handleSIGHUP, + dumpio, + env, + pipe, + } = options; + let stdio: Array<'ignore' | 'pipe'> = ['pipe', 'pipe', 'pipe']; + if (pipe) { + if (dumpio) stdio = ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']; + else stdio = ['ignore', 'ignore', 'ignore', 'pipe', 'pipe']; + } + assert(!this.proc, 'This process has previously been started.'); + debugLauncher( + `Calling ${this._executablePath} ${this._processArguments.join(' ')}` + ); + this.proc = childProcess.spawn( + this._executablePath, + this._processArguments, + { + // On non-windows platforms, `detached: true` makes child process a + // leader of a new process group, making it possible to kill child + // process tree with `.kill(-pid)` command. @see + // https://nodejs.org/api/child_process.html#child_process_options_detached + detached: process.platform !== 'win32', + env, + stdio, + } + ); + if (dumpio) { + this.proc.stderr.pipe(process.stderr); + this.proc.stdout.pipe(process.stdout); + } + this._closed = false; + this._processClosing = new Promise((fulfill) => { + this.proc.once('exit', () => { + this._closed = true; + // Cleanup as processes exit. + if (this._tempDirectory) { + removeFolderAsync(this._tempDirectory) + .then(() => fulfill()) + .catch((error) => console.error(error)); + } else { + fulfill(); + } + }); + }); + this._listeners = [ + helper.addEventListener(process, 'exit', this.kill.bind(this)), + ]; + if (handleSIGINT) + this._listeners.push( + helper.addEventListener(process, 'SIGINT', () => { + this.kill(); + process.exit(130); + }) + ); + if (handleSIGTERM) + this._listeners.push( + helper.addEventListener(process, 'SIGTERM', this.close.bind(this)) + ); + if (handleSIGHUP) + this._listeners.push( + helper.addEventListener(process, 'SIGHUP', this.close.bind(this)) + ); + } + + close(): Promise<void> { + if (this._closed) return Promise.resolve(); + if (this._tempDirectory) { + this.kill(); + } else if (this.connection) { + // Attempt to close the browser gracefully + this.connection.send('Browser.close').catch((error) => { + debugError(error); + this.kill(); + }); + } + // Cleanup this listener last, as that makes sure the full callback runs. If we + // perform this earlier, then the previous function calls would not happen. + helper.removeEventListeners(this._listeners); + return this._processClosing; + } + + kill(): void { + // Attempt to remove temporary profile directory to avoid littering. + try { + removeFolder.sync(this._tempDirectory); + } catch (error) {} + + // If the process failed to launch (for example if the browser executable path + // is invalid), then the process does not get a pid assigned. A call to + // `proc.kill` would error, as the `pid` to-be-killed can not be found. + if (this.proc && this.proc.pid && !this.proc.killed) { + try { + this.proc.kill('SIGKILL'); + } catch (error) { + throw new Error( + `${PROCESS_ERROR_EXPLANATION}\nError cause: ${error.stack}` + ); + } + } + // Cleanup this listener last, as that makes sure the full callback runs. If we + // perform this earlier, then the previous function calls would not happen. + helper.removeEventListeners(this._listeners); + } + + async setupConnection(options: { + usePipe?: boolean; + timeout: number; + slowMo: number; + preferredRevision: string; + }): Promise<Connection> { + const { usePipe, timeout, slowMo, preferredRevision } = options; + if (!usePipe) { + const browserWSEndpoint = await waitForWSEndpoint( + this.proc, + timeout, + preferredRevision + ); + const transport = await WebSocketTransport.create(browserWSEndpoint); + this.connection = new Connection(browserWSEndpoint, transport, slowMo); + } else { + // stdio was assigned during start(), and the 'pipe' option there adds the + // 4th and 5th items to stdio array + const { 3: pipeWrite, 4: pipeRead } = this.proc.stdio; + const transport = new PipeTransport( + pipeWrite as NodeJS.WritableStream, + pipeRead as NodeJS.ReadableStream + ); + this.connection = new Connection('', transport, slowMo); + } + return this.connection; + } +} + +function waitForWSEndpoint( + browserProcess: childProcess.ChildProcess, + timeout: number, + preferredRevision: string +): Promise<string> { + return new Promise((resolve, reject) => { + const rl = readline.createInterface({ input: browserProcess.stderr }); + let stderr = ''; + const listeners = [ + helper.addEventListener(rl, 'line', onLine), + helper.addEventListener(rl, 'close', () => onClose()), + helper.addEventListener(browserProcess, 'exit', () => onClose()), + helper.addEventListener(browserProcess, 'error', (error) => + onClose(error) + ), + ]; + const timeoutId = timeout ? setTimeout(onTimeout, timeout) : 0; + + /** + * @param {!Error=} error + */ + function onClose(error?: Error): void { + cleanup(); + reject( + new Error( + [ + 'Failed to launch the browser process!' + + (error ? ' ' + error.message : ''), + stderr, + '', + 'TROUBLESHOOTING: https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md', + '', + ].join('\n') + ) + ); + } + + function onTimeout(): void { + cleanup(); + reject( + new TimeoutError( + `Timed out after ${timeout} ms while trying to connect to the browser! Only Chrome at revision r${preferredRevision} is guaranteed to work.` + ) + ); + } + + function onLine(line: string): void { + stderr += line + '\n'; + const match = line.match(/^DevTools listening on (ws:\/\/.*)$/); + if (!match) return; + cleanup(); + resolve(match[1]); + } + + function cleanup(): void { + if (timeoutId) clearTimeout(timeoutId); + helper.removeEventListeners(listeners); + } + }); +} diff --git a/remote/test/puppeteer/src/node/LaunchOptions.ts b/remote/test/puppeteer/src/node/LaunchOptions.ts new file mode 100644 index 0000000000..0eb99b6980 --- /dev/null +++ b/remote/test/puppeteer/src/node/LaunchOptions.ts @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Launcher options that only apply to Chrome. + * + * @public + */ +export interface ChromeArgOptions { + headless?: boolean; + args?: string[]; + userDataDir?: string; + devtools?: boolean; +} + +/** + * Generic launch options that can be passed when launching any browser. + * @public + */ +export interface LaunchOptions { + executablePath?: string; + ignoreDefaultArgs?: boolean | string[]; + handleSIGINT?: boolean; + handleSIGTERM?: boolean; + handleSIGHUP?: boolean; + timeout?: number; + dumpio?: boolean; + env?: Record<string, string | undefined>; + pipe?: boolean; +} diff --git a/remote/test/puppeteer/src/node/Launcher.ts b/remote/test/puppeteer/src/node/Launcher.ts new file mode 100644 index 0000000000..8ca62db389 --- /dev/null +++ b/remote/test/puppeteer/src/node/Launcher.ts @@ -0,0 +1,673 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as os from 'os'; +import * as path from 'path'; +import * as fs from 'fs'; + +import { BrowserFetcher } from './BrowserFetcher.js'; +import { Browser } from '../common/Browser.js'; +import { BrowserRunner } from './BrowserRunner.js'; +import { promisify } from 'util'; + +const mkdtempAsync = promisify(fs.mkdtemp); +const writeFileAsync = promisify(fs.writeFile); + +import { ChromeArgOptions, LaunchOptions } from './LaunchOptions.js'; +import { BrowserOptions } from '../common/BrowserConnector.js'; +import { Product } from '../common/Product.js'; + +/** + * Describes a launcher - a class that is able to create and launch a browser instance. + * @public + */ +export interface ProductLauncher { + launch(object); + executablePath: () => string; + defaultArgs(object); + product: Product; +} + +/** + * @internal + */ +class ChromeLauncher implements ProductLauncher { + _projectRoot: string; + _preferredRevision: string; + _isPuppeteerCore: boolean; + + constructor( + projectRoot: string, + preferredRevision: string, + isPuppeteerCore: boolean + ) { + this._projectRoot = projectRoot; + this._preferredRevision = preferredRevision; + this._isPuppeteerCore = isPuppeteerCore; + } + + async launch( + options: LaunchOptions & ChromeArgOptions & BrowserOptions = {} + ): Promise<Browser> { + const { + ignoreDefaultArgs = false, + args = [], + dumpio = false, + executablePath = null, + pipe = false, + env = process.env, + handleSIGINT = true, + handleSIGTERM = true, + handleSIGHUP = true, + ignoreHTTPSErrors = false, + defaultViewport = { width: 800, height: 600 }, + slowMo = 0, + timeout = 30000, + } = options; + + const profilePath = path.join(os.tmpdir(), 'puppeteer_dev_chrome_profile-'); + const chromeArguments = []; + if (!ignoreDefaultArgs) chromeArguments.push(...this.defaultArgs(options)); + else if (Array.isArray(ignoreDefaultArgs)) + chromeArguments.push( + ...this.defaultArgs(options).filter( + (arg) => !ignoreDefaultArgs.includes(arg) + ) + ); + else chromeArguments.push(...args); + + let temporaryUserDataDir = null; + + if ( + !chromeArguments.some((argument) => + argument.startsWith('--remote-debugging-') + ) + ) + chromeArguments.push( + pipe ? '--remote-debugging-pipe' : '--remote-debugging-port=0' + ); + if (!chromeArguments.some((arg) => arg.startsWith('--user-data-dir'))) { + temporaryUserDataDir = await mkdtempAsync(profilePath); + chromeArguments.push(`--user-data-dir=${temporaryUserDataDir}`); + } + + let chromeExecutable = executablePath; + if (os.arch() === 'arm64') { + chromeExecutable = '/usr/bin/chromium-browser'; + } else if (!executablePath) { + const { missingText, executablePath } = resolveExecutablePath(this); + if (missingText) throw new Error(missingText); + chromeExecutable = executablePath; + } + + const usePipe = chromeArguments.includes('--remote-debugging-pipe'); + const runner = new BrowserRunner( + chromeExecutable, + chromeArguments, + temporaryUserDataDir + ); + runner.start({ + handleSIGHUP, + handleSIGTERM, + handleSIGINT, + dumpio, + env, + pipe: usePipe, + }); + + try { + const connection = await runner.setupConnection({ + usePipe, + timeout, + slowMo, + preferredRevision: this._preferredRevision, + }); + const browser = await Browser.create( + connection, + [], + ignoreHTTPSErrors, + defaultViewport, + runner.proc, + runner.close.bind(runner) + ); + await browser.waitForTarget((t) => t.type() === 'page'); + return browser; + } catch (error) { + runner.kill(); + throw error; + } + } + + /** + * @param {!Launcher.ChromeArgOptions=} options + * @returns {!Array<string>} + */ + defaultArgs(options: ChromeArgOptions = {}): string[] { + const chromeArguments = [ + '--disable-background-networking', + '--enable-features=NetworkService,NetworkServiceInProcess', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-extensions', + '--disable-features=TranslateUI', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--disable-sync', + '--force-color-profile=srgb', + '--metrics-recording-only', + '--no-first-run', + '--enable-automation', + '--password-store=basic', + '--use-mock-keychain', + // TODO(sadym): remove '--enable-blink-features=IdleDetection' + // once IdleDetection is turned on by default. + '--enable-blink-features=IdleDetection', + ]; + const { + devtools = false, + headless = !devtools, + args = [], + userDataDir = null, + } = options; + if (userDataDir) + chromeArguments.push(`--user-data-dir=${path.resolve(userDataDir)}`); + if (devtools) chromeArguments.push('--auto-open-devtools-for-tabs'); + if (headless) { + chromeArguments.push('--headless', '--hide-scrollbars', '--mute-audio'); + } + if (args.every((arg) => arg.startsWith('-'))) + chromeArguments.push('about:blank'); + chromeArguments.push(...args); + return chromeArguments; + } + + executablePath(): string { + return resolveExecutablePath(this).executablePath; + } + + get product(): Product { + return 'chrome'; + } +} + +/** + * @internal + */ +class FirefoxLauncher implements ProductLauncher { + _projectRoot: string; + _preferredRevision: string; + _isPuppeteerCore: boolean; + + constructor( + projectRoot: string, + preferredRevision: string, + isPuppeteerCore: boolean + ) { + this._projectRoot = projectRoot; + this._preferredRevision = preferredRevision; + this._isPuppeteerCore = isPuppeteerCore; + } + + async launch( + options: LaunchOptions & + ChromeArgOptions & + BrowserOptions & { + extraPrefsFirefox?: { [x: string]: unknown }; + } = {} + ): Promise<Browser> { + const { + ignoreDefaultArgs = false, + args = [], + dumpio = false, + executablePath = null, + pipe = false, + env = process.env, + handleSIGINT = true, + handleSIGTERM = true, + handleSIGHUP = true, + ignoreHTTPSErrors = false, + defaultViewport = { width: 800, height: 600 }, + slowMo = 0, + timeout = 30000, + extraPrefsFirefox = {}, + } = options; + + const firefoxArguments = []; + if (!ignoreDefaultArgs) firefoxArguments.push(...this.defaultArgs(options)); + else if (Array.isArray(ignoreDefaultArgs)) + firefoxArguments.push( + ...this.defaultArgs(options).filter( + (arg) => !ignoreDefaultArgs.includes(arg) + ) + ); + else firefoxArguments.push(...args); + + if ( + !firefoxArguments.some((argument) => + argument.startsWith('--remote-debugging-') + ) + ) + firefoxArguments.push('--remote-debugging-port=0'); + + let temporaryUserDataDir = null; + + if ( + !firefoxArguments.includes('-profile') && + !firefoxArguments.includes('--profile') + ) { + temporaryUserDataDir = await this._createProfile(extraPrefsFirefox); + firefoxArguments.push('--profile'); + firefoxArguments.push(temporaryUserDataDir); + } + + await this._updateRevision(); + let firefoxExecutable = executablePath; + if (!executablePath) { + const { missingText, executablePath } = resolveExecutablePath(this); + if (missingText) throw new Error(missingText); + firefoxExecutable = executablePath; + } + + const runner = new BrowserRunner( + firefoxExecutable, + firefoxArguments, + temporaryUserDataDir + ); + runner.start({ + handleSIGHUP, + handleSIGTERM, + handleSIGINT, + dumpio, + env, + pipe, + }); + + try { + const connection = await runner.setupConnection({ + usePipe: pipe, + timeout, + slowMo, + preferredRevision: this._preferredRevision, + }); + const browser = await Browser.create( + connection, + [], + ignoreHTTPSErrors, + defaultViewport, + runner.proc, + runner.close.bind(runner) + ); + await browser.waitForTarget((t) => t.type() === 'page'); + return browser; + } catch (error) { + runner.kill(); + throw error; + } + } + + executablePath(): string { + return resolveExecutablePath(this).executablePath; + } + + async _updateRevision(): Promise<void> { + // replace 'latest' placeholder with actual downloaded revision + if (this._preferredRevision === 'latest') { + const browserFetcher = new BrowserFetcher(this._projectRoot, { + product: this.product, + }); + const localRevisions = await browserFetcher.localRevisions(); + if (localRevisions[0]) this._preferredRevision = localRevisions[0]; + } + } + + get product(): Product { + return 'firefox'; + } + + defaultArgs(options: ChromeArgOptions = {}): string[] { + const firefoxArguments = ['--no-remote', '--foreground']; + if (os.platform().startsWith('win')) { + firefoxArguments.push('--wait-for-browser'); + } + const { + devtools = false, + headless = !devtools, + args = [], + userDataDir = null, + } = options; + if (userDataDir) { + firefoxArguments.push('--profile'); + firefoxArguments.push(userDataDir); + } + if (headless) firefoxArguments.push('--headless'); + if (devtools) firefoxArguments.push('--devtools'); + if (args.every((arg) => arg.startsWith('-'))) + firefoxArguments.push('about:blank'); + firefoxArguments.push(...args); + return firefoxArguments; + } + + async _createProfile(extraPrefs: { [x: string]: unknown }): Promise<string> { + const profilePath = await mkdtempAsync( + path.join(os.tmpdir(), 'puppeteer_dev_firefox_profile-') + ); + const prefsJS = []; + const userJS = []; + const server = 'dummy.test'; + const defaultPreferences = { + // Make sure Shield doesn't hit the network. + 'app.normandy.api_url': '', + // Disable Firefox old build background check + 'app.update.checkInstallTime': false, + // Disable automatically upgrading Firefox + 'app.update.disabledForTesting': true, + + // Increase the APZ content response timeout to 1 minute + 'apz.content_response_timeout': 60000, + + // Prevent various error message on the console + // jest-puppeteer asserts that no error message is emitted by the console + 'browser.contentblocking.features.standard': + '-tp,tpPrivate,cookieBehavior0,-cm,-fp', + + // Enable the dump function: which sends messages to the system + // console + // https://bugzilla.mozilla.org/show_bug.cgi?id=1543115 + 'browser.dom.window.dump.enabled': true, + // Disable topstories + 'browser.newtabpage.activity-stream.feeds.system.topstories': false, + // Always display a blank page + 'browser.newtabpage.enabled': false, + // Background thumbnails in particular cause grief: and disabling + // thumbnails in general cannot hurt + 'browser.pagethumbnails.capturing_disabled': true, + + // Disable safebrowsing components. + 'browser.safebrowsing.blockedURIs.enabled': false, + 'browser.safebrowsing.downloads.enabled': false, + 'browser.safebrowsing.malware.enabled': false, + 'browser.safebrowsing.passwords.enabled': false, + 'browser.safebrowsing.phishing.enabled': false, + + // Disable updates to search engines. + 'browser.search.update': false, + // Do not restore the last open set of tabs if the browser has crashed + 'browser.sessionstore.resume_from_crash': false, + // Skip check for default browser on startup + 'browser.shell.checkDefaultBrowser': false, + + // Disable newtabpage + 'browser.startup.homepage': 'about:blank', + // Do not redirect user when a milstone upgrade of Firefox is detected + 'browser.startup.homepage_override.mstone': 'ignore', + // Start with a blank page about:blank + 'browser.startup.page': 0, + + // Do not allow background tabs to be zombified on Android: otherwise for + // tests that open additional tabs: the test harness tab itself might get + // unloaded + 'browser.tabs.disableBackgroundZombification': false, + // Do not warn when closing all other open tabs + 'browser.tabs.warnOnCloseOtherTabs': false, + // Do not warn when multiple tabs will be opened + 'browser.tabs.warnOnOpen': false, + + // Disable the UI tour. + 'browser.uitour.enabled': false, + // Turn off search suggestions in the location bar so as not to trigger + // network connections. + 'browser.urlbar.suggest.searches': false, + // Disable first run splash page on Windows 10 + 'browser.usedOnWindows10.introURL': '', + // Do not warn on quitting Firefox + 'browser.warnOnQuit': false, + + // Defensively disable data reporting systems + 'datareporting.healthreport.documentServerURI': `http://${server}/dummy/healthreport/`, + 'datareporting.healthreport.logging.consoleEnabled': false, + 'datareporting.healthreport.service.enabled': false, + 'datareporting.healthreport.service.firstRun': false, + 'datareporting.healthreport.uploadEnabled': false, + + // Do not show datareporting policy notifications which can interfere with tests + 'datareporting.policy.dataSubmissionEnabled': false, + 'datareporting.policy.dataSubmissionPolicyBypassNotification': true, + + // DevTools JSONViewer sometimes fails to load dependencies with its require.js. + // This doesn't affect Puppeteer but spams console (Bug 1424372) + 'devtools.jsonview.enabled': false, + + // Disable popup-blocker + 'dom.disable_open_during_load': false, + + // Enable the support for File object creation in the content process + // Required for |Page.setFileInputFiles| protocol method. + 'dom.file.createInChild': true, + + // Disable the ProcessHangMonitor + 'dom.ipc.reportProcessHangs': false, + + // Disable slow script dialogues + 'dom.max_chrome_script_run_time': 0, + 'dom.max_script_run_time': 0, + + // Only load extensions from the application and user profile + // AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION + 'extensions.autoDisableScopes': 0, + 'extensions.enabledScopes': 5, + + // Disable metadata caching for installed add-ons by default + 'extensions.getAddons.cache.enabled': false, + + // Disable installing any distribution extensions or add-ons. + 'extensions.installDistroAddons': false, + + // Disabled screenshots extension + 'extensions.screenshots.disabled': true, + + // Turn off extension updates so they do not bother tests + 'extensions.update.enabled': false, + + // Turn off extension updates so they do not bother tests + 'extensions.update.notifyUser': false, + + // Make sure opening about:addons will not hit the network + 'extensions.webservice.discoverURL': `http://${server}/dummy/discoveryURL`, + + // Force disable Fission until the Remote Agent is compatible + 'fission.autostart': false, + + // Allow the application to have focus even it runs in the background + 'focusmanager.testmode': true, + // Disable useragent updates + 'general.useragent.updates.enabled': false, + // Always use network provider for geolocation tests so we bypass the + // macOS dialog raised by the corelocation provider + 'geo.provider.testing': true, + // Do not scan Wifi + 'geo.wifi.scan': false, + // No hang monitor + 'hangmonitor.timeout': 0, + // Show chrome errors and warnings in the error console + 'javascript.options.showInConsole': true, + + // Disable download and usage of OpenH264: and Widevine plugins + 'media.gmp-manager.updateEnabled': false, + // Prevent various error message on the console + // jest-puppeteer asserts that no error message is emitted by the console + 'network.cookie.cookieBehavior': 0, + + // Disable experimental feature that is only available in Nightly + 'network.cookie.sameSite.laxByDefault': false, + + // Do not prompt for temporary redirects + 'network.http.prompt-temp-redirect': false, + + // Disable speculative connections so they are not reported as leaking + // when they are hanging around + 'network.http.speculative-parallel-limit': 0, + + // Do not automatically switch between offline and online + 'network.manage-offline-status': false, + + // Make sure SNTP requests do not hit the network + 'network.sntp.pools': server, + + // Disable Flash. + 'plugin.state.flash': 0, + + 'privacy.trackingprotection.enabled': false, + + // Enable Remote Agent + // https://bugzilla.mozilla.org/show_bug.cgi?id=1544393 + 'remote.enabled': true, + + // Don't do network connections for mitm priming + 'security.certerrors.mitm.priming.enabled': false, + // Local documents have access to all other local documents, + // including directory listings + 'security.fileuri.strict_origin_policy': false, + // Do not wait for the notification button security delay + 'security.notification_enable_delay': 0, + + // Ensure blocklist updates do not hit the network + 'services.settings.server': `http://${server}/dummy/blocklist/`, + + // Do not automatically fill sign-in forms with known usernames and + // passwords + 'signon.autofillForms': false, + // Disable password capture, so that tests that include forms are not + // influenced by the presence of the persistent doorhanger notification + 'signon.rememberSignons': false, + + // Disable first-run welcome page + 'startup.homepage_welcome_url': 'about:blank', + + // Disable first-run welcome page + 'startup.homepage_welcome_url.additional': '', + + // Disable browser animations (tabs, fullscreen, sliding alerts) + 'toolkit.cosmeticAnimations.enabled': false, + + // Prevent starting into safe mode after application crashes + 'toolkit.startup.max_resumed_crashes': -1, + }; + + Object.assign(defaultPreferences, extraPrefs); + for (const [key, value] of Object.entries(defaultPreferences)) + userJS.push( + `user_pref(${JSON.stringify(key)}, ${JSON.stringify(value)});` + ); + await writeFileAsync(path.join(profilePath, 'user.js'), userJS.join('\n')); + await writeFileAsync( + path.join(profilePath, 'prefs.js'), + prefsJS.join('\n') + ); + return profilePath; + } +} + +function resolveExecutablePath( + launcher: ChromeLauncher | FirefoxLauncher +): { executablePath: string; missingText?: string } { + let downloadPath: string; + // puppeteer-core doesn't take into account PUPPETEER_* env variables. + if (!launcher._isPuppeteerCore) { + const executablePath = + process.env.PUPPETEER_EXECUTABLE_PATH || + process.env.npm_config_puppeteer_executable_path || + process.env.npm_package_config_puppeteer_executable_path; + if (executablePath) { + const missingText = !fs.existsSync(executablePath) + ? 'Tried to use PUPPETEER_EXECUTABLE_PATH env variable to launch browser but did not find any executable at: ' + + executablePath + : null; + return { executablePath, missingText }; + } + downloadPath = + process.env.PUPPETEER_DOWNLOAD_PATH || + process.env.npm_config_puppeteer_download_path || + process.env.npm_package_config_puppeteer_download_path; + } + const browserFetcher = new BrowserFetcher(launcher._projectRoot, { + product: launcher.product, + path: downloadPath, + }); + if (!launcher._isPuppeteerCore && launcher.product === 'chrome') { + const revision = process.env['PUPPETEER_CHROMIUM_REVISION']; + if (revision) { + const revisionInfo = browserFetcher.revisionInfo(revision); + const missingText = !revisionInfo.local + ? 'Tried to use PUPPETEER_CHROMIUM_REVISION env variable to launch browser but did not find executable at: ' + + revisionInfo.executablePath + : null; + return { executablePath: revisionInfo.executablePath, missingText }; + } + } + const revisionInfo = browserFetcher.revisionInfo(launcher._preferredRevision); + const missingText = !revisionInfo.local + ? `Could not find browser revision ${launcher._preferredRevision}. Run "PUPPETEER_PRODUCT=firefox npm install" or "PUPPETEER_PRODUCT=firefox yarn install" to download a supported Firefox browser binary.` + : null; + return { executablePath: revisionInfo.executablePath, missingText }; +} + +/** + * @internal + */ +export default function Launcher( + projectRoot: string, + preferredRevision: string, + isPuppeteerCore: boolean, + product?: string +): ProductLauncher { + // puppeteer-core doesn't take into account PUPPETEER_* env variables. + if (!product && !isPuppeteerCore) + product = + process.env.PUPPETEER_PRODUCT || + process.env.npm_config_puppeteer_product || + process.env.npm_package_config_puppeteer_product; + switch (product) { + case 'firefox': + return new FirefoxLauncher( + projectRoot, + preferredRevision, + isPuppeteerCore + ); + case 'chrome': + default: + if (typeof product !== 'undefined' && product !== 'chrome') { + /* The user gave us an incorrect product name + * we'll default to launching Chrome, but log to the console + * to let the user know (they've probably typoed). + */ + console.warn( + `Warning: unknown product name ${product}. Falling back to chrome.` + ); + } + return new ChromeLauncher( + projectRoot, + preferredRevision, + isPuppeteerCore + ); + } +} diff --git a/remote/test/puppeteer/src/node/NodeWebSocketTransport.ts b/remote/test/puppeteer/src/node/NodeWebSocketTransport.ts new file mode 100644 index 0000000000..d1077ae7c2 --- /dev/null +++ b/remote/test/puppeteer/src/node/NodeWebSocketTransport.ts @@ -0,0 +1,59 @@ +/** + * Copyright 2018 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConnectionTransport } from '../common/ConnectionTransport.js'; +import NodeWebSocket from 'ws'; + +export class NodeWebSocketTransport implements ConnectionTransport { + static create(url: string): Promise<NodeWebSocketTransport> { + return new Promise((resolve, reject) => { + const ws = new NodeWebSocket(url, [], { + perMessageDeflate: false, + maxPayload: 256 * 1024 * 1024, // 256Mb + }); + + ws.addEventListener('open', () => + resolve(new NodeWebSocketTransport(ws)) + ); + ws.addEventListener('error', reject); + }); + } + + private _ws: NodeWebSocket; + onmessage?: (message: string) => void; + onclose?: () => void; + + constructor(ws: NodeWebSocket) { + this._ws = ws; + this._ws.addEventListener('message', (event) => { + if (this.onmessage) this.onmessage.call(null, event.data); + }); + this._ws.addEventListener('close', () => { + if (this.onclose) this.onclose.call(null); + }); + // Silently ignore all errors - we don't know what to do with them. + this._ws.addEventListener('error', () => {}); + this.onmessage = null; + this.onclose = null; + } + + send(message: string): void { + this._ws.send(message); + } + + close(): void { + this._ws.close(); + } +} diff --git a/remote/test/puppeteer/src/node/PipeTransport.ts b/remote/test/puppeteer/src/node/PipeTransport.ts new file mode 100644 index 0000000000..e66c2c0b47 --- /dev/null +++ b/remote/test/puppeteer/src/node/PipeTransport.ts @@ -0,0 +1,80 @@ +/** + * Copyright 2018 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + helper, + debugError, + PuppeteerEventListener, +} from '../common/helper.js'; +import { ConnectionTransport } from '../common/ConnectionTransport.js'; + +export class PipeTransport implements ConnectionTransport { + _pipeWrite: NodeJS.WritableStream; + _pendingMessage: string; + _eventListeners: PuppeteerEventListener[]; + + onclose?: () => void; + onmessage?: () => void; + + constructor( + pipeWrite: NodeJS.WritableStream, + pipeRead: NodeJS.ReadableStream + ) { + this._pipeWrite = pipeWrite; + this._pendingMessage = ''; + this._eventListeners = [ + helper.addEventListener(pipeRead, 'data', (buffer) => + this._dispatch(buffer) + ), + helper.addEventListener(pipeRead, 'close', () => { + if (this.onclose) this.onclose.call(null); + }), + helper.addEventListener(pipeRead, 'error', debugError), + helper.addEventListener(pipeWrite, 'error', debugError), + ]; + this.onmessage = null; + this.onclose = null; + } + + send(message: string): void { + this._pipeWrite.write(message); + this._pipeWrite.write('\0'); + } + + _dispatch(buffer: Buffer): void { + let end = buffer.indexOf('\0'); + if (end === -1) { + this._pendingMessage += buffer.toString(); + return; + } + const message = this._pendingMessage + buffer.toString(undefined, 0, end); + if (this.onmessage) this.onmessage.call(null, message); + + let start = end + 1; + end = buffer.indexOf('\0', start); + while (end !== -1) { + if (this.onmessage) + this.onmessage.call(null, buffer.toString(undefined, start, end)); + start = end + 1; + end = buffer.indexOf('\0', start); + } + this._pendingMessage = buffer.toString(undefined, start); + } + + close(): void { + this._pipeWrite = null; + helper.removeEventListeners(this._eventListeners); + } +} diff --git a/remote/test/puppeteer/src/node/Puppeteer.ts b/remote/test/puppeteer/src/node/Puppeteer.ts new file mode 100644 index 0000000000..924d2fa96e --- /dev/null +++ b/remote/test/puppeteer/src/node/Puppeteer.ts @@ -0,0 +1,230 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Puppeteer, + CommonPuppeteerSettings, + ConnectOptions, +} from '../common/Puppeteer.js'; +import { BrowserFetcher, BrowserFetcherOptions } from './BrowserFetcher.js'; +import { LaunchOptions, ChromeArgOptions } from './LaunchOptions.js'; +import { BrowserOptions } from '../common/BrowserConnector.js'; +import { Browser } from '../common/Browser.js'; +import Launcher, { ProductLauncher } from './Launcher.js'; +import { PUPPETEER_REVISIONS } from '../revisions.js'; +import { Product } from '../common/Product.js'; + +/** + * Extends the main {@link Puppeteer} class with Node specific behaviour for fetching and + * downloading browsers. + * + * If you're using Puppeteer in a Node environment, this is the class you'll get + * when you run `require('puppeteer')` (or the equivalent ES `import`). + * + * @remarks + * + * The most common method to use is {@link PuppeteerNode.launch | launch}, which + * is used to launch and connect to a new browser instance. + * + * See {@link Puppeteer | the main Puppeteer class} for methods common to all + * environments, such as {@link Puppeteer.connect}. + * + * @example + * The following is a typical example of using Puppeteer to drive automation: + * ```js + * const puppeteer = require('puppeteer'); + * + * (async () => { + * const browser = await puppeteer.launch(); + * const page = await browser.newPage(); + * await page.goto('https://www.google.com'); + * // other actions... + * await browser.close(); + * })(); + * ``` + * + * Once you have created a `page` you have access to a large API to interact + * with the page, navigate, or find certain elements in that page. + * The {@link Page | `page` documentation} lists all the available methods. + * + * @public + */ +export class PuppeteerNode extends Puppeteer { + private _lazyLauncher: ProductLauncher; + private _projectRoot: string; + private __productName?: Product; + /** + * @internal + */ + _preferredRevision: string; + + /** + * @internal + */ + constructor( + settings: { + projectRoot: string; + preferredRevision: string; + productName?: Product; + } & CommonPuppeteerSettings + ) { + const { + projectRoot, + preferredRevision, + productName, + ...commonSettings + } = settings; + super(commonSettings); + this._projectRoot = projectRoot; + this.__productName = productName; + this._preferredRevision = preferredRevision; + } + + /** + * This method attaches Puppeteer to an existing browser instance. + * + * @remarks + * + * @param options - Set of configurable options to set on the browser. + * @returns Promise which resolves to browser instance. + */ + connect(options: ConnectOptions): Promise<Browser> { + if (options.product) this._productName = options.product; + return super.connect(options); + } + + /** + * @internal + */ + get _productName(): Product { + return this.__productName; + } + + // don't need any TSDoc here - because the getter is internal the setter is too. + set _productName(name: Product) { + if (this.__productName !== name) this._changedProduct = true; + this.__productName = name; + } + + /** + * Launches puppeteer and launches a browser instance with given arguments + * and options when specified. + * + * @remarks + * + * @example + * You can use `ignoreDefaultArgs` to filter out `--mute-audio` from default arguments: + * ```js + * const browser = await puppeteer.launch({ + * ignoreDefaultArgs: ['--mute-audio'] + * }); + * ``` + * + * **NOTE** Puppeteer can also be used to control the Chrome browser, + * but it works best with the version of Chromium it is bundled with. + * There is no guarantee it will work with any other version. + * Use `executablePath` option with extreme caution. + * If Google Chrome (rather than Chromium) is preferred, a {@link https://www.google.com/chrome/browser/canary.html | Chrome Canary} or {@link https://www.chromium.org/getting-involved/dev-channel | Dev Channel} build is suggested. + * In `puppeteer.launch([options])`, any mention of Chromium also applies to Chrome. + * See {@link https://www.howtogeek.com/202825/what%E2%80%99s-the-difference-between-chromium-and-chrome/ | this article} for a description of the differences between Chromium and Chrome. {@link https://chromium.googlesource.com/chromium/src/+/lkgr/docs/chromium_browser_vs_google_chrome.md | This article} describes some differences for Linux users. + * + * @param options - Set of configurable options to set on the browser. + * @returns Promise which resolves to browser instance. + */ + launch( + options: LaunchOptions & + ChromeArgOptions & + BrowserOptions & { + product?: Product; + extraPrefsFirefox?: Record<string, unknown>; + } = {} + ): Promise<Browser> { + if (options.product) this._productName = options.product; + return this._launcher.launch(options); + } + + /** + * @remarks + * + * **NOTE** `puppeteer.executablePath()` is affected by the `PUPPETEER_EXECUTABLE_PATH` + * and `PUPPETEER_CHROMIUM_REVISION` environment variables. + * + * @returns A path where Puppeteer expects to find the bundled browser. + * The browser binary might not be there if the download was skipped with + * the `PUPPETEER_SKIP_DOWNLOAD` environment variable. + */ + executablePath(): string { + return this._launcher.executablePath(); + } + + /** + * @internal + */ + get _launcher(): ProductLauncher { + if ( + !this._lazyLauncher || + this._lazyLauncher.product !== this._productName || + this._changedProduct + ) { + switch (this._productName) { + case 'firefox': + this._preferredRevision = PUPPETEER_REVISIONS.firefox; + break; + case 'chrome': + default: + this._preferredRevision = PUPPETEER_REVISIONS.chromium; + } + this._changedProduct = false; + this._lazyLauncher = Launcher( + this._projectRoot, + this._preferredRevision, + this._isPuppeteerCore, + this._productName + ); + } + return this._lazyLauncher; + } + + /** + * The name of the browser that is under automation (`"chrome"` or `"firefox"`) + * + * @remarks + * The product is set by the `PUPPETEER_PRODUCT` environment variable or the `product` + * option in `puppeteer.launch([options])` and defaults to `chrome`. + * Firefox support is experimental. + */ + get product(): string { + return this._launcher.product; + } + + /** + * + * @param options - Set of configurable options to set on the browser. + * @returns The default flags that Chromium will be launched with. + */ + defaultArgs(options: ChromeArgOptions = {}): string[] { + return this._launcher.defaultArgs(options); + } + + /** + * @param options - Set of configurable options to specify the settings + * of the BrowserFetcher. + * @returns A new BrowserFetcher instance. + */ + createBrowserFetcher(options: BrowserFetcherOptions): BrowserFetcher { + return new BrowserFetcher(this._projectRoot, options); + } +} diff --git a/remote/test/puppeteer/src/node/install.ts b/remote/test/puppeteer/src/node/install.ts new file mode 100644 index 0000000000..41f2834d80 --- /dev/null +++ b/remote/test/puppeteer/src/node/install.ts @@ -0,0 +1,185 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import https from 'https'; +import ProgressBar from 'progress'; +import puppeteer from '../node.js'; +import { PUPPETEER_REVISIONS } from '../revisions.js'; +import { PuppeteerNode } from './Puppeteer.js'; + +const supportedProducts = { + chrome: 'Chromium', + firefox: 'Firefox Nightly', +} as const; + +export async function downloadBrowser() { + const downloadHost = + process.env.PUPPETEER_DOWNLOAD_HOST || + process.env.npm_config_puppeteer_download_host || + process.env.npm_package_config_puppeteer_download_host; + const product = + process.env.PUPPETEER_PRODUCT || + process.env.npm_config_puppeteer_product || + process.env.npm_package_config_puppeteer_product || + 'chrome'; + const downloadPath = + process.env.PUPPETEER_DOWNLOAD_PATH || + process.env.npm_config_puppeteer_download_path || + process.env.npm_package_config_puppeteer_download_path; + const browserFetcher = (puppeteer as PuppeteerNode).createBrowserFetcher({ + product, + host: downloadHost, + path: downloadPath, + }); + const revision = await getRevision(); + await fetchBinary(revision); + + function getRevision() { + if (product === 'chrome') { + return ( + process.env.PUPPETEER_CHROMIUM_REVISION || + process.env.npm_config_puppeteer_chromium_revision || + PUPPETEER_REVISIONS.chromium + ); + } else if (product === 'firefox') { + (puppeteer as PuppeteerNode)._preferredRevision = + PUPPETEER_REVISIONS.firefox; + return getFirefoxNightlyVersion().catch((error) => { + console.error(error); + process.exit(1); + }); + } else { + throw new Error(`Unsupported product ${product}`); + } + } + + function fetchBinary(revision) { + const revisionInfo = browserFetcher.revisionInfo(revision); + + // Do nothing if the revision is already downloaded. + if (revisionInfo.local) { + logPolitely( + `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.` + ); + return; + } + + // Override current environment proxy settings with npm configuration, if any. + const NPM_HTTPS_PROXY = + process.env.npm_config_https_proxy || process.env.npm_config_proxy; + const NPM_HTTP_PROXY = + process.env.npm_config_http_proxy || process.env.npm_config_proxy; + const NPM_NO_PROXY = process.env.npm_config_no_proxy; + + if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; + if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY; + if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY; + + function onSuccess(localRevisions: string[]): void { + if (os.arch() !== 'arm64') { + logPolitely( + `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}` + ); + } + localRevisions = localRevisions.filter( + (revision) => revision !== revisionInfo.revision + ); + const cleanupOldVersions = localRevisions.map((revision) => + browserFetcher.remove(revision) + ); + Promise.all([...cleanupOldVersions]); + } + + function onError(error: Error) { + console.error( + `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` + ); + console.error(error); + process.exit(1); + } + + let progressBar = null; + let lastDownloadedBytes = 0; + function onProgress(downloadedBytes, totalBytes) { + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${ + supportedProducts[product] + } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + } + + return browserFetcher + .download(revisionInfo.revision, onProgress) + .then(() => browserFetcher.localRevisions()) + .then(onSuccess) + .catch(onError); + } + + function toMegabytes(bytes) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; + } + + function getFirefoxNightlyVersion() { + const firefoxVersions = + 'https://product-details.mozilla.org/1.0/firefox_versions.json'; + + const promise = new Promise((resolve, reject) => { + let data = ''; + logPolitely( + `Requesting latest Firefox Nightly version from ${firefoxVersions}` + ); + https + .get(firefoxVersions, (r) => { + if (r.statusCode >= 400) + return reject(new Error(`Got status code ${r.statusCode}`)); + r.on('data', (chunk) => { + data += chunk; + }); + r.on('end', () => { + try { + const versions = JSON.parse(data); + return resolve(versions.FIREFOX_NIGHTLY); + } catch { + return reject(new Error('Firefox version not found')); + } + }); + }) + .on('error', reject); + }); + return promise; + } +} + +export function logPolitely(toBeLogged) { + const logLevel = process.env.npm_config_loglevel; + const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1; + + // eslint-disable-next-line no-console + if (!logLevelDisplay) console.log(toBeLogged); +} |