Puppeteer: staging commit.

This commit is contained in:
Andrey Lushnikov
2017-05-11 00:06:41 -07:00
parent ebac211411
commit 2cda8c18d1
180 changed files with 19038 additions and 2 deletions

142
lib/Browser.js Normal file
View File

@@ -0,0 +1,142 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var CDP = require('chrome-remote-interface');
var http = require('http');
var path = require('path');
var removeRecursive = require('rimraf').sync;
var Page = require('./Page');
var childProcess = require('child_process');
var Downloader = require('./Downloader');
var CHROME_PROFILE_PATH = path.resolve(__dirname, '..', '.dev_profile');
var browserId = 0;
var DEFAULT_ARGS = [
'--disable-background-timer-throttling',
'--no-first-run',
];
class Browser {
/**
* @param {(!Object|undefined)} options
*/
constructor(options) {
options = options || {};
++browserId;
this._userDataDir = CHROME_PROFILE_PATH + browserId;
this._remoteDebuggingPort = 9229;
if (typeof options.remoteDebuggingPort === 'number')
this._remoteDebuggingPort = options.remoteDebuggingPort;
this._chromeArguments = DEFAULT_ARGS.concat([
`--user-data-dir=${this._userDataDir}`,
`--remote-debugging-port=${this._remoteDebuggingPort}`,
]);
if (typeof options.headless !== 'boolean' || options.headless) {
this._chromeArguments.push(...[
`--headless`,
`--disable-gpu`,
]);
}
if (typeof options.executablePath === 'string') {
this._chromeExecutable = options.executablePath;
} else {
var chromiumRevision = require('../package.json').puppeteer.chromium_revision;
this._chromeExecutable = Downloader.executablePath(chromiumRevision);
}
if (Array.isArray(options.args))
this._chromeArguments.push(...options.args);
this._chromeProcess = null;
this._tabSymbol = Symbol('Browser.TabSymbol');
}
/**
* @return {!Promise<!Page>}
*/
async newPage() {
await this._ensureChromeIsRunning();
if (!this._chromeProcess || this._chromeProcess.killed)
throw new Error('ERROR: this chrome instance is not alive any more!');
var tab = await CDP.New({port: this._remoteDebuggingPort});
var client = await CDP({tab: tab, port: this._remoteDebuggingPort});
var page = await Page.create(this, client);
page[this._tabSymbol] = tab;
return page;
}
/**
* @param {!Page} page
*/
async closePage(page) {
if (!this._chromeProcess || this._chromeProcess.killed)
throw new Error('ERROR: this chrome instance is not running');
var tab = page[this._tabSymbol];
if (!tab)
throw new Error('ERROR: page does not belong to this chrome instance');
await CDP.Close({id: tab.id, port: this._remoteDebuggingPort});
}
/**
* @return {string}
*/
async version() {
await this._ensureChromeIsRunning();
var version = await CDP.Version({port: this._remoteDebuggingPort});
return version.Browser;
}
async _ensureChromeIsRunning() {
if (this._chromeProcess)
return;
this._chromeProcess = childProcess.spawn(this._chromeExecutable, this._chromeArguments, {});
// Cleanup as processes exit.
process.on('exit', () => this._chromeProcess.kill());
this._chromeProcess.on('exit', () => removeRecursive(this._userDataDir));
await waitForChromeResponsive(this._remoteDebuggingPort);
}
close() {
if (!this._chromeProcess)
return;
this._chromeProcess.kill();
}
}
module.exports = Browser;
function waitForChromeResponsive(remoteDebuggingPort) {
var fulfill;
var promise = new Promise(x => fulfill = x);
var options = {
method: 'GET',
host: 'localhost',
port: remoteDebuggingPort,
path: '/json/list'
};
var probeTimeout = 100;
var probeAttempt = 1;
sendRequest();
return promise;
function sendRequest() {
var req = http.request(options, res => {
fulfill()
});
req.on('error', e => setTimeout(sendRequest, probeTimeout));
req.end();
}
}

123
lib/Downloader.js Normal file
View File

@@ -0,0 +1,123 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var os = require('os');
var https = require('https');
var fs = require('fs');
var path = require('path');
var extract = require('extract-zip');
var util = require('util');
var CHROMIUM_PATH = path.join(__dirname, '..', '.local-chromium');
var downloadURLs = {
linux: 'https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip',
darwin: 'https://storage.googleapis.com/chromium-browser-snapshots/Mac/%d/chrome-mac.zip',
win32: 'https://storage.googleapis.com/chromium-browser-snapshots/Win/%d/chrome-win32.zip',
win64: 'https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/%d/chrome-win32.zip',
};
module.exports = {
downloadChromium,
executablePath,
};
/**
* @param {string} revision
* @param {?function(number, number)} progressCallback
* @return {!Promise}
*/
async function downloadChromium(revision, progressCallback) {
var url = null;
var platform = os.platform();
if (platform === 'darwin')
url = downloadURLs.darwin;
else if (platform === 'linux')
url = downloadURLs.linux;
else if (platform === 'win32')
url = os.arch() === 'x64' ? downloadURLs.win64 : downloadURLs.win32;
console.assert(url, `Unsupported platform: ${platform}`);
url = util.format(url, revision);
var zipPath = path.join(CHROMIUM_PATH, `download-${revision}.zip`);
var folderPath = path.join(CHROMIUM_PATH, revision);
if (fs.existsSync(folderPath))
return;
try {
if (!fs.existsSync(CHROMIUM_PATH))
fs.mkdirSync(CHROMIUM_PATH);
await downloadFile(url, zipPath, progressCallback);
await extractZip(zipPath, folderPath);
} finally {
if (fs.existsSync(zipPath))
fs.unlinkSync(zipPath);
}
}
/**
* @return {string}
*/
function executablePath(revision) {
var platform = os.platform();
if (platform === 'darwin')
return path.join(CHROMIUM_PATH, revision, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium');
if (platform === 'linux')
return path.join(CHROMIUM_PATH, revision, 'chrome-linux', 'chrome');
if (platform === 'win32')
return path.join(CHROMIUM_PATH, revision, 'chrome-win32', 'chrome.exe');
throw new Error(`Unsupported platform: ${platform}`);
}
/**
* @param {string} url
* @param {string} destinationPath
* @param {?function(number, number)} progressCallback
* @return {!Promise}
*/
function downloadFile(url, destinationPath, progressCallback) {
var fulfill, reject;
var promise = new Promise((x, y) => { fulfill = x; reject = y; });
var request = https.get(url, response => {
if (response.statusCode !== 200) {
var error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
// consume response data to free up memory
response.resume();
reject(error);
return;
}
var file = fs.createWriteStream(destinationPath);
file.on('finish', () => fulfill());
file.on('error', error => reject(error));
response.pipe(file);
var totalBytes = parseInt(response.headers['content-length'], 10);
if (progressCallback)
response.on('data', onData.bind(null, totalBytes));
});
request.on('error', error => reject(error));
return promise;
function onData(totalBytes, chunk) {
progressCallback(totalBytes, chunk.length);
}
}
/**
* @param {string} zipPath
* @param {string} folderPath
* @return {!Promise<?Error>}
*/
function extractZip(zipPath, folderPath) {
return new Promise(fulfill => extract(zipPath, {dir: folderPath}, fulfill));
}

350
lib/Page.js Normal file
View File

@@ -0,0 +1,350 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var fs = require('fs');
var EventEmitter = require('events');
var helpers = require('./helpers');
class Page extends EventEmitter {
/**
* @param {!Browser} browser
* @param {!CDP} client
* @return {!Promise<!Page>}
*/
static async create(browser, client) {
await Promise.all([
client.send('Network.enable', {}),
client.send('Page.enable', {}),
client.send('Runtime.enable', {}),
client.send('Security.enable', {}),
]);
var screenDPI = await helpers.evaluate(client, () => window.devicePixelRatio, []);
var page = new Page(browser, client, screenDPI.result.value);
// Initialize default page size.
await page.setSize({width: 400, height: 300});
return page;
}
/**
* @param {!Browser} browser
* @param {!CDP} client
* @param {number} screenDPI
*/
constructor(browser, client, screenDPI) {
super();
this._browser = browser;
this._client = client;
this._screenDPI = screenDPI;
this._extraHeaders = {};
/** @type {!Map<string, function()>} */
this._scriptIdToPageCallback = new Map();
/** @type {!Map<string, string>} */
this._scriptIdToCallbackName = new Map();
client.on('Debugger.paused', event => this._onDebuggerPaused(event));
client.on('Network.responseReceived', event => this.emit(Page.Events.ResponseReceived, event.response));
client.on('Network.loadingFailed', event => this.emit(Page.Events.ResourceLoadingFailed, event));
client.on('Runtime.consoleAPICalled', event => this._onConsoleAPI(event));
client.on('Page.javascriptDialogOpening', dialog => this.emit(Page.Events.DialogOpened, dialog));
client.on('Runtime.exceptionThrown', exception => this._handleException(exception.exceptionDetails));
}
/**
* @param {string} name
* @param {function(?)} callback
*/
async setInPageCallback(name, callback) {
var hasCallback = await this.evaluate(function(name) {
return !!window[name];
}, name);
if (hasCallback)
throw new Error(`Failed to set in-page callback with name ${name}: window['${name}'] already exists!`);
var sourceURL = '__in_page_callback__' + name;
// Ensure debugger is enabled.
await this._client.send('Debugger.enable', {});
var scriptPromise = helpers.waitForScriptWithURL(this._client, sourceURL);
helpers.evaluate(this._client, inPageCallback, [name], false /* awaitPromise */, sourceURL);
var script = await scriptPromise;
if (!script)
throw new Error(`Failed to set in-page callback with name "${name}"`);
this._scriptIdToPageCallback.set(script.scriptId, callback);
this._scriptIdToCallbackName.set(script.scriptId, name);
function inPageCallback(callbackName) {
window[callbackName] = (...args) => {
window[callbackName].__args = args;
window[callbackName].__result = undefined;
debugger;
return window[callbackName].__result;
};
}
}
/**
* @param {string} scriptId
*/
async _handleInPageCallback(scriptId) {
var name = /** @type {string} */ (this._scriptIdToCallbackName.get(scriptId));
var callback = /** @type {function()} */ (this._scriptIdToPageCallback.get(scriptId));
var args = await this.evaluate(callbackName => window[callbackName].__args, name);
var result = callback.apply(null, args);
await this.evaluate(assignResult, name, result);
this._client.send('Debugger.resume');
/**
* @param {string} callbackName
* @param {string} callbackResult
*/
function assignResult(callbackName, callbackResult) {
window[callbackName].__result = callbackResult;
}
}
_onDebuggerPaused(event) {
var location = event.callFrames[0] ? event.callFrames[0].location : null;
if (location && this._scriptIdToPageCallback.has(location.scriptId)) {
this._handleInPageCallback(location.scriptId);
return;
}
this._client.send('Debugger.resume');
}
/**
* @param {!Object} headers
* @return {!Promise}
*/
async setExtraHTTPHeaders(headers) {
this._extraHeaders = {};
// Note: header names are case-insensitive.
for (var key of Object.keys(headers))
this._extraHeaders[key.toLowerCase()] = headers[key];
return this._client.send('Network.setExtraHTTPHeaders', { headers });
}
/**
* @return {!Object}
*/
extraHTTPHeaders() {
return Object.assign({}, this._extraHeaders);
}
/**
* @param {string} userAgent
* @return {!Promise}
*/
async setUserAgentOverride(userAgent) {
this._userAgent = userAgent;
return this._client.send('Network.setUserAgentOverride', { userAgent });
}
/**
* @return {string}
*/
userAgentOverride() {
return this._userAgent;
}
_handleException(exceptionDetails) {
var stack = [];
if (exceptionDetails.stackTrace) {
stack = exceptionDetails.stackTrace.callFrames.map(cf => cf.url);
}
var stackTrace = exceptionDetails.stackTrace;
this.emit(Page.Events.ExceptionThrown, exceptionDetails.exception.description, stack);
}
_onConsoleAPI(event) {
var values = event.args.map(arg => arg.value || arg.description || '');
this.emit(Page.Events.ConsoleMessageAdded, values.join(' '));
}
/**
* @param {boolean} accept
* @param {string} promptText
* @return {!Promise}
*/
async handleDialog(accept, promptText) {
return this._client.send('Page.handleJavaScriptDialog', {accept, promptText});
}
/**
* @return {!Promise<string>}
*/
async url() {
return this.evaluate(function() {
return window.location.href;
});
}
/**
* @param {string} html
* @return {!Promise}
*/
async setContent(html) {
var resourceTree = await this._client.send('Page.getResourceTree', {});
await this._client.send('Page.setDocumentContent', {
frameId: resourceTree.frameTree.frame.id,
html: html
});
}
/**
* @param {string} html
* @return {!Promise<boolean>}
*/
async navigate(url) {
var loadPromise = new Promise(fulfill => this._client.once('Page.loadEventFired', fulfill)).then(() => true);
var interstitialPromise = new Promise(fulfill => this._client.once('Security.certificateError', fulfill)).then(() => false);
var referrer = this._extraHeaders.referer;
// Await for the command to throw exception in case of illegal arguments.
await this._client.send('Page.navigate', {url, referrer});
return await Promise.race([loadPromise, interstitialPromise]);
}
/**
* @param {!{width: number, height: number}} size
* @return {!Promise}
*/
async setSize(size) {
this._size = size;
var width = size.width;
var height = size.height;
var zoom = this._screenDPI;
return Promise.all([
this._client.send('Emulation.setDeviceMetricsOverride', {
width,
height,
deviceScaleFactor: 1,
scale: 1 / zoom,
mobile: false,
fitWindow: false
}),
this._client.send('Emulation.setVisibleSize', {
width: width / zoom,
height: height / zoom,
})
]);
}
/**
* @return {!{width: number, height: number}}
*/
size() {
return this._size;
}
/**
* @param {function()} fun
* @param {!Array<*>} args
* @return {!Promise<(!Object|udndefined)>}
*/
async evaluate(fun, ...args) {
var response = await helpers.evaluate(this._client, fun, args, false /* awaitPromise */);
if (response.exceptionDetails) {
this._handleException(response.exceptionDetails);
return;
}
return response.result.value;
}
/**
* @param {function()} fun
* @param {!Array<*>} args
* @return {!Promise<(!Object|udndefined)>}
*/
async evaluateAsync(fun, ...args) {
var response = await helpers.evaluate(this._client, fun, args, true /* awaitPromise */);
if (response.exceptionDetails) {
this._handleException(response.exceptionDetails);
return;
}
return response.result.value;
}
/**
* @param {!Page.ScreenshotType} screenshotType
* @param {?{x: number, y: number, width: number, height: number}} clipRect
* @return {!Promise<!Buffer>}
*/
async screenshot(screenshotType, clipRect) {
if (clipRect) {
await Promise.all([
this._client.send('Emulation.setVisibleSize', {
width: clipRect.width / this._screenDPI,
height: clipRect.height / this._screenDPI,
}),
this._client.send('Emulation.forceViewport', {
x: clipRect.x / this._screenDPI,
y: clipRect.y / this._screenDPI,
scale: 1,
})
]);
}
var result = await this._client.send('Page.captureScreenshot', {
fromSurface: true,
format: screenshotType,
});
if (clipRect) {
await Promise.all([
this.setSize(this.size()),
this._client.send('Emulation.resetViewport')
]);
}
return new Buffer(result.data, 'base64');
}
/**
* @return {!Promise<string>}
*/
async plainText() {
return this.evaluate(function() {
return document.body.innerText;
});
}
/**
* @return {!Promise<string>}
*/
async title() {
return this.evaluate(function() {
return document.title;
});
}
/**
* @return {!Promise}
*/
async close() {
return this._browser.closePage(this);
}
}
/** @enum {string} */
Page.ScreenshotTypes = {
PNG: "png",
JPG: "jpeg",
};
Page.Events = {
ConsoleMessageAdded: 'Page.Events.ConsoleMessageAdded',
DialogOpened: 'Page.Events.DialogOpened',
ExceptionThrown: 'Page.Events.ExceptionThrown',
ResourceLoadingFailed: 'Page.Events.ResourceLoadingFailed',
ResponseReceived: 'Page.Events.ResponseReceived',
};
module.exports = Page;

68
lib/helpers.js Normal file
View File

@@ -0,0 +1,68 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
module.exports = {
/**
* @param {!CDP} client
* @param {string} url
* @return {!Promise<?Object>}
*/
waitForScriptWithURL: function(client, url) {
var fulfill;
var promise = new Promise(x => fulfill = x);
client.on('Debugger.scriptParsed', onScriptParsed);
client.on('Debugger.scriptFailedToParse', onScriptFailedToParse);
return promise;
function onScriptParsed(event) {
if (event.url !== url)
return;
client.removeListener('Debugger.scriptParsed', onScriptParsed);
client.removeListener('Debugger.scriptFailedToParse', onScriptFailedToParse);
fulfill(event);
}
function onScriptFailedToParse(event) {
if (event.url !== url)
return;
client.removeListener('Debugger.scriptParsed', onScriptParsed);
client.removeListener('Debugger.scriptFailedToParse', onScriptFailedToParse);
fulfill(null);
}
},
/**
* @param {!CDP} client
* @param {function()} fun
* @param {!Array<*>} args
* @param {boolean} awaitPromise
* @param {string=} sourceURL
* @return {!Promise<!Object>}
*/
evaluate: function(client, fun, args, awaitPromise, sourceURL) {
var argsString = args.map(x => JSON.stringify(x)).join(',');
var code = `(${fun.toString()})(${argsString})`;
if (awaitPromise)
code = `Promise.resolve(${code})`;
if (sourceURL)
code += `\n//# sourceURL=${sourceURL}`;
return client.send('Runtime.evaluate', {
expression: code,
awaitPromise: awaitPromise,
returnByValue: true
});
},
}