From 43a97878ce14b72f0981164f87f2e35e14151312 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 11:22:09 +0200 Subject: Adding upstream version 110.0.1. Signed-off-by: Daniel Baumann --- remote/test/puppeteer/src/node/BrowserFetcher.ts | 701 +++++++++++++++++++++++ 1 file changed, 701 insertions(+) create mode 100644 remote/test/puppeteer/src/node/BrowserFetcher.ts (limited to 'remote/test/puppeteer/src/node/BrowserFetcher.ts') diff --git a/remote/test/puppeteer/src/node/BrowserFetcher.ts b/remote/test/puppeteer/src/node/BrowserFetcher.ts new file mode 100644 index 0000000000..7c425c2dbb --- /dev/null +++ b/remote/test/puppeteer/src/node/BrowserFetcher.ts @@ -0,0 +1,701 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as os from 'os'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as util from 'util'; +import * as childProcess from 'child_process'; +import * as https from 'https'; +import * as http from 'http'; + +import {Product} from '../common/Product.js'; +import extractZip from 'extract-zip'; +import {debug} from '../common/Debug.js'; +import {promisify} from 'util'; +import removeRecursive from 'rimraf'; +import * as URL from 'url'; +import createHttpsProxyAgent, { + HttpsProxyAgent, + HttpsProxyAgentOptions, +} from 'https-proxy-agent'; +import {getProxyForUrl} from 'proxy-from-env'; +import {assert} from '../util/assert.js'; + +import tar from 'tar-fs'; +import bzip from 'unbzip2-stream'; + +const {PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM} = process.env; + +const debugFetcher = debug('puppeteer:fetcher'); + +const downloadURLs: Record>> = { + chrome: { + linux: '%s/chromium-browser-snapshots/Linux_x64/%d/%s.zip', + mac: '%s/chromium-browser-snapshots/Mac/%d/%s.zip', + mac_arm: '%s/chromium-browser-snapshots/Mac_Arm/%d/%s.zip', + win32: '%s/chromium-browser-snapshots/Win/%d/%s.zip', + win64: '%s/chromium-browser-snapshots/Win_x64/%d/%s.zip', + }, + firefox: { + linux: '%s/firefox-%s.en-US.%s-x86_64.tar.bz2', + mac: '%s/firefox-%s.en-US.%s.dmg', + win32: '%s/firefox-%s.en-US.%s.zip', + win64: '%s/firefox-%s.en-US.%s.zip', + }, +}; + +const browserConfig = { + chrome: { + host: 'https://storage.googleapis.com', + destination: '.local-chromium', + }, + firefox: { + host: 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central', + destination: '.local-firefox', + }, +} as const; + +/** + * Supported platforms. + * + * @public + */ +export type Platform = 'linux' | 'mac' | 'mac_arm' | 'win32' | 'win64'; + +function archiveName( + product: Product, + platform: Platform, + revision: string +): string { + switch (product) { + case 'chrome': + switch (platform) { + case 'linux': + return 'chrome-linux'; + case 'mac_arm': + case 'mac': + return 'chrome-mac'; + case 'win32': + case 'win64': + // Windows archive name changed at r591479. + return parseInt(revision, 10) > 591479 + ? 'chrome-win' + : 'chrome-win32'; + } + case 'firefox': + return platform; + } +} + +function downloadURL( + product: Product, + platform: Platform, + host: string, + revision: string +): string { + const url = util.format( + downloadURLs[product][platform], + host, + revision, + archiveName(product, platform, revision) + ); + return url; +} + +function handleArm64(): void { + let exists = fs.existsSync('/usr/bin/chromium-browser'); + if (exists) { + return; + } + exists = fs.existsSync('/usr/bin/chromium'); + if (exists) { + return; + } + console.error( + 'The chromium binary is not available for arm64.' + + '\nIf you are on Ubuntu, you can install with: ' + + '\n\n sudo apt install chromium\n' + + '\n\n sudo apt install chromium-browser\n' + ); + throw new Error(); +} + +const readdirAsync = promisify(fs.readdir.bind(fs)); +const mkdirAsync = promisify(fs.mkdir.bind(fs)); +const unlinkAsync = promisify(fs.unlink.bind(fs)); +const chmodAsync = promisify(fs.chmod.bind(fs)); + +function existsAsync(filePath: string): Promise { + return new Promise(resolve => { + fs.access(filePath, err => { + return resolve(!err); + }); + }); +} + +/** + * @public + */ +export interface BrowserFetcherOptions { + platform?: Platform; + product?: string; + path?: string; + host?: string; +} + +/** + * @public + */ +export interface BrowserFetcherRevisionInfo { + folderPath: string; + executablePath: string; + url: string; + local: boolean; + revision: string; + product: string; +} +/** + * BrowserFetcher can download and manage different versions of Chromium and Firefox. + * + * @remarks + * BrowserFetcher operates on revision strings that specify a precise version of Chromium, e.g. `"533271"`. Revision strings can be obtained from {@link http://omahaproxy.appspot.com/ | omahaproxy.appspot.com}. + * In the Firefox case, BrowserFetcher downloads Firefox Nightly and + * operates on version numbers such as `"75"`. + * + * @example + * An example of using BrowserFetcher to download a specific version of Chromium + * and running Puppeteer against it: + * + * ```ts + * const browserFetcher = puppeteer.createBrowserFetcher(); + * const revisionInfo = await browserFetcher.download('533271'); + * const browser = await puppeteer.launch({ + * executablePath: revisionInfo.executablePath, + * }); + * ``` + * + * **NOTE** BrowserFetcher is not designed to work concurrently with other + * instances of BrowserFetcher that share the same downloads directory. + * + * @public + */ + +export class BrowserFetcher { + #product: Product; + #downloadsFolder: string; + #downloadHost: string; + #platform: Platform; + + /** + * @internal + */ + constructor(projectRoot: string, options: BrowserFetcherOptions = {}) { + this.#product = (options.product || 'chrome').toLowerCase() as Product; + assert( + this.#product === 'chrome' || this.#product === 'firefox', + `Unknown product: "${options.product}"` + ); + + this.#downloadsFolder = + options.path || + path.join(projectRoot, browserConfig[this.#product].destination); + this.#downloadHost = options.host || browserConfig[this.#product].host; + + if (options.platform) { + this.#platform = options.platform; + } else { + const platform = os.platform(); + switch (platform) { + case 'darwin': + switch (this.#product) { + case 'chrome': + this.#platform = + os.arch() === 'arm64' && PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM + ? 'mac_arm' + : 'mac'; + break; + case 'firefox': + this.#platform = 'mac'; + break; + } + break; + case 'linux': + this.#platform = 'linux'; + break; + case 'win32': + this.#platform = os.arch() === 'x64' ? 'win64' : 'win32'; + return; + default: + assert(false, 'Unsupported platform: ' + platform); + } + } + + assert( + downloadURLs[this.#product][this.#platform], + 'Unsupported platform: ' + this.#platform + ); + } + + /** + * @returns Returns the current `Platform`, which is one of `mac`, `linux`, + * `win32` or `win64`. + */ + platform(): Platform { + return this.#platform; + } + + /** + * @returns Returns the current `Product`, which is one of `chrome` or + * `firefox`. + */ + product(): Product { + return this.#product; + } + + /** + * @returns The download host being used. + */ + host(): string { + return this.#downloadHost; + } + + /** + * Initiates a HEAD request to check if the revision is available. + * @remarks + * This method is affected by the current `product`. + * @param revision - The revision to check availability for. + * @returns A promise that resolves to `true` if the revision could be downloaded + * from the host. + */ + canDownload(revision: string): Promise { + const url = downloadURL( + this.#product, + this.#platform, + this.#downloadHost, + revision + ); + return new Promise(resolve => { + const request = httpRequest( + url, + 'HEAD', + response => { + resolve(response.statusCode === 200); + }, + false + ); + request.on('error', error => { + console.error(error); + resolve(false); + }); + }); + } + + /** + * Initiates a GET request to download the revision from the host. + * @remarks + * This method is affected by the current `product`. + * @param revision - The revision to download. + * @param progressCallback - A function that will be called with two arguments: + * How many bytes have been downloaded and the total number of bytes of the download. + * @returns A promise with revision information when the revision is downloaded + * and extracted. + */ + async download( + revision: string, + progressCallback: (x: number, y: number) => void = (): void => {} + ): Promise { + const url = downloadURL( + this.#product, + this.#platform, + this.#downloadHost, + revision + ); + const fileName = url.split('/').pop(); + assert(fileName, `A malformed download URL was found: ${url}.`); + const archivePath = path.join(this.#downloadsFolder, fileName); + const outputPath = this.#getFolderPath(revision); + if (await existsAsync(outputPath)) { + return this.revisionInfo(revision); + } + if (!(await existsAsync(this.#downloadsFolder))) { + await mkdirAsync(this.#downloadsFolder); + } + + // Use system Chromium builds on Linux ARM devices + if (os.platform() !== 'darwin' && os.arch() === 'arm64') { + handleArm64(); + return; + } + try { + await _downloadFile(url, archivePath, progressCallback); + await install(archivePath, outputPath); + } finally { + if (await existsAsync(archivePath)) { + await unlinkAsync(archivePath); + } + } + const revisionInfo = this.revisionInfo(revision); + if (revisionInfo) { + await chmodAsync(revisionInfo.executablePath, 0o755); + } + return revisionInfo; + } + + /** + * @remarks + * This method is affected by the current `product`. + * @returns A promise with a list of all revision strings (for the current `product`) + * available locally on disk. + */ + async localRevisions(): Promise { + if (!(await existsAsync(this.#downloadsFolder))) { + return []; + } + const fileNames = await readdirAsync(this.#downloadsFolder); + return fileNames + .map(fileName => { + return parseFolderPath(this.#product, fileName); + }) + .filter( + ( + entry + ): entry is {product: string; platform: string; revision: string} => { + return (entry && entry.platform === this.#platform) ?? false; + } + ) + .map(entry => { + return entry.revision; + }); + } + + /** + * @remarks + * This method is affected by the current `product`. + * @param revision - A revision to remove for the current `product`. + * @returns A promise that resolves when the revision has been removes or + * throws if the revision has not been downloaded. + */ + async remove(revision: string): Promise { + const folderPath = this.#getFolderPath(revision); + assert( + await existsAsync(folderPath), + `Failed to remove: revision ${revision} is not downloaded` + ); + await new Promise(fulfill => { + return removeRecursive(folderPath, fulfill); + }); + } + + /** + * @param revision - The revision to get info for. + * @returns The revision info for the given revision. + */ + revisionInfo(revision: string): BrowserFetcherRevisionInfo { + const folderPath = this.#getFolderPath(revision); + let executablePath = ''; + if (this.#product === 'chrome') { + if (this.#platform === 'mac' || this.#platform === 'mac_arm') { + executablePath = path.join( + folderPath, + archiveName(this.#product, this.#platform, revision), + 'Chromium.app', + 'Contents', + 'MacOS', + 'Chromium' + ); + } else if (this.#platform === 'linux') { + executablePath = path.join( + folderPath, + archiveName(this.#product, this.#platform, revision), + 'chrome' + ); + } else if (this.#platform === 'win32' || this.#platform === 'win64') { + executablePath = path.join( + folderPath, + archiveName(this.#product, this.#platform, revision), + 'chrome.exe' + ); + } else { + throw new Error('Unsupported platform: ' + this.#platform); + } + } else if (this.#product === 'firefox') { + if (this.#platform === 'mac' || this.#platform === 'mac_arm') { + executablePath = path.join( + folderPath, + 'Firefox Nightly.app', + 'Contents', + 'MacOS', + 'firefox' + ); + } else if (this.#platform === 'linux') { + executablePath = path.join(folderPath, 'firefox', 'firefox'); + } else if (this.#platform === 'win32' || this.#platform === 'win64') { + executablePath = path.join(folderPath, 'firefox', 'firefox.exe'); + } else { + throw new Error('Unsupported platform: ' + this.#platform); + } + } else { + throw new Error('Unsupported product: ' + this.#product); + } + const url = downloadURL( + this.#product, + this.#platform, + this.#downloadHost, + revision + ); + const local = fs.existsSync(folderPath); + debugFetcher({ + revision, + executablePath, + folderPath, + local, + url, + product: this.#product, + }); + return { + revision, + executablePath, + folderPath, + local, + url, + product: this.#product, + }; + } + + #getFolderPath(revision: string): string { + return path.resolve(this.#downloadsFolder, `${this.#platform}-${revision}`); + } +} + +function parseFolderPath( + product: Product, + folderPath: string +): {product: string; platform: string; revision: string} | undefined { + const name = path.basename(folderPath); + const splits = name.split('-'); + if (splits.length !== 2) { + return; + } + const [platform, revision] = splits; + if (!revision || !platform || !(platform in downloadURLs[product])) { + return; + } + return {product, platform, revision}; +} + +/** + * @internal + */ +function _downloadFile( + url: string, + destinationPath: string, + progressCallback?: (x: number, y: number) => void +): Promise { + debugFetcher(`Downloading binary from ${url}`); + let fulfill: (value: void | PromiseLike) => void; + let reject: (err: Error) => void; + const promise = new Promise((x, y) => { + fulfill = x; + reject = y; + }); + + let downloadedBytes = 0; + let totalBytes = 0; + + const request = httpRequest(url, 'GET', response => { + if (response.statusCode !== 200) { + const error = new Error( + `Download failed: server returned code ${response.statusCode}. URL: ${url}` + ); + // consume response data to free up memory + response.resume(); + reject(error); + return; + } + const file = fs.createWriteStream(destinationPath); + file.on('finish', () => { + return fulfill(); + }); + file.on('error', error => { + return reject(error); + }); + response.pipe(file); + totalBytes = parseInt(response.headers['content-length']!, 10); + if (progressCallback) { + response.on('data', onData); + } + }); + request.on('error', error => { + return reject(error); + }); + return promise; + + function onData(chunk: string): void { + downloadedBytes += chunk.length; + progressCallback!(downloadedBytes, totalBytes); + } +} + +function install(archivePath: string, folderPath: string): Promise { + debugFetcher(`Installing ${archivePath} to ${folderPath}`); + if (archivePath.endsWith('.zip')) { + return extractZip(archivePath, {dir: folderPath}); + } else if (archivePath.endsWith('.tar.bz2')) { + return _extractTar(archivePath, folderPath); + } else if (archivePath.endsWith('.dmg')) { + return mkdirAsync(folderPath).then(() => { + return _installDMG(archivePath, folderPath); + }); + } else { + throw new Error(`Unsupported archive format: ${archivePath}`); + } +} + +/** + * @internal + */ +function _extractTar(tarPath: string, folderPath: string): Promise { + return new Promise((fulfill, reject) => { + const tarStream = tar.extract(folderPath); + tarStream.on('error', reject); + tarStream.on('finish', fulfill); + const readStream = fs.createReadStream(tarPath); + readStream.pipe(bzip()).pipe(tarStream); + }); +} + +/** + * @internal + */ +function _installDMG(dmgPath: string, folderPath: string): Promise { + let mountPath: string | undefined; + + return new Promise((fulfill, reject): void => { + const mountCommand = `hdiutil attach -nobrowse -noautoopen "${dmgPath}"`; + childProcess.exec(mountCommand, (err, stdout) => { + if (err) { + return reject(err); + } + const volumes = stdout.match(/\/Volumes\/(.*)/m); + if (!volumes) { + return reject(new Error(`Could not find volume path in ${stdout}`)); + } + mountPath = volumes[0]!; + readdirAsync(mountPath) + .then(fileNames => { + const appName = fileNames.find(item => { + return typeof item === 'string' && item.endsWith('.app'); + }); + if (!appName) { + return reject(new Error(`Cannot find app in ${mountPath}`)); + } + const copyPath = path.join(mountPath!, appName); + debugFetcher(`Copying ${copyPath} to ${folderPath}`); + childProcess.exec(`cp -R "${copyPath}" "${folderPath}"`, err => { + if (err) { + reject(err); + } else { + fulfill(); + } + }); + }) + .catch(reject); + }); + }) + .catch(error => { + console.error(error); + }) + .finally((): void => { + if (!mountPath) { + return; + } + const unmountCommand = `hdiutil detach "${mountPath}" -quiet`; + debugFetcher(`Unmounting ${mountPath}`); + childProcess.exec(unmountCommand, err => { + if (err) { + console.error(`Error unmounting dmg: ${err}`); + } + }); + }); +} + +function httpRequest( + url: string, + method: string, + response: (x: http.IncomingMessage) => void, + keepAlive = true +): http.ClientRequest { + const urlParsed = URL.parse(url); + + type Options = Partial & { + method?: string; + agent?: HttpsProxyAgent; + rejectUnauthorized?: boolean; + headers?: http.OutgoingHttpHeaders | undefined; + }; + + let options: Options = { + ...urlParsed, + method, + headers: keepAlive + ? { + Connection: 'keep-alive', + } + : undefined, + }; + + const proxyURL = getProxyForUrl(url); + if (proxyURL) { + if (url.startsWith('http:')) { + const proxy = URL.parse(proxyURL); + options = { + path: options.href, + host: proxy.hostname, + port: proxy.port, + }; + } else { + const parsedProxyURL = URL.parse(proxyURL); + + const proxyOptions = { + ...parsedProxyURL, + secureProxy: parsedProxyURL.protocol === 'https:', + } as HttpsProxyAgentOptions; + + options.agent = createHttpsProxyAgent(proxyOptions); + options.rejectUnauthorized = false; + } + } + + const requestCallback = (res: http.IncomingMessage): void => { + if ( + res.statusCode && + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + httpRequest(res.headers.location, method, response); + } else { + response(res); + } + }; + const request = + options.protocol === 'https:' + ? https.request(options, requestCallback) + : http.request(options, requestCallback); + request.end(); + return request; +} -- cgit v1.2.3