diff options
Diffstat (limited to 'platform/dig')
| -rw-r--r-- | platform/dig/package.json | 28 | ||||
| -rw-r--r-- | platform/dig/snfe.js | 389 |
2 files changed, 417 insertions, 0 deletions
diff --git a/platform/dig/package.json b/platform/dig/package.json new file mode 100644 index 0000000..d15b78c --- /dev/null +++ b/platform/dig/package.json @@ -0,0 +1,28 @@ +{ + "name": "@gorhill/ubo-dig", + "version": "0.1.0", + "description": "To investigate code changes (not for publication)", + "type": "module", + "main": "index.js", + "scripts": { + "build": "node build.js", + "snfe": "node snfe.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/gorhill/uBlock.git" + }, + "author": "Raymond Hill", + "license": "GPL-3.0-or-later", + "bugs": { + "url": "https://github.com/gorhill/uBlock/issues" + }, + "homepage": "https://github.com/gorhill/uBlock#readme", + "engines": { + "node": ">=14.0.0", + "npm": ">=6.14.4" + }, + "devDependencies": { + "scaling-palm-tree": "github:mjethani/scaling-palm-tree#15cf1ab" + } +} diff --git a/platform/dig/snfe.js b/platform/dig/snfe.js new file mode 100644 index 0000000..7090828 --- /dev/null +++ b/platform/dig/snfe.js @@ -0,0 +1,389 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* eslint-disable-next-line no-redeclare */ +/* globals process */ + +'use strict'; + +/******************************************************************************/ + +import { strict as assert } from 'assert'; +import { createRequire } from 'module'; +import { readFile, writeFile, mkdir } from 'fs/promises'; +import { dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +import { enableWASM, StaticNetFilteringEngine } from './index.js'; + +/******************************************************************************/ + +const FLAGS = process.argv.slice(2); +const COMPARE = FLAGS.includes('compare'); +const MAXCOST = FLAGS.includes('maxcost'); +const MEDCOST = FLAGS.includes('medcost'); +const MINCOST = FLAGS.includes('mincost'); +const MODIFIERS = FLAGS.includes('modifiers'); +const RECORD = FLAGS.includes('record'); +const WASM = FLAGS.includes('wasm'); +const NEED_RESULTS = COMPARE || MAXCOST || MEDCOST || MINCOST || RECORD; + +// This maps puppeteer types to WebRequest types +const WEBREQUEST_OPTIONS = { + // Consider document requests as sub_document. This is because the request + // dataset does not contain sub_frame or main_frame but only 'document' and + // different blockers have different behaviours. + document: 'sub_frame', + stylesheet: 'stylesheet', + image: 'image', + media: 'media', + font: 'font', + script: 'script', + xhr: 'xmlhttprequest', + fetch: 'xmlhttprequest', + websocket: 'websocket', + ping: 'ping', + // other + other: 'other', + eventsource: 'other', + manifest: 'other', + texttrack: 'other', +}; + +/******************************************************************************/ + +function nanoToMilli(bigint) { + return (Number(bigint) / 1000000).toFixed(2) + ' ms'; +} + +function nanoToMicro(bigint) { + return (Number(bigint) / 1000).toFixed(2) + ' µs'; +} + +async function read(path) { + return readFile(path, 'utf8'); +} + +async function write(path, data) { + await mkdir(dirname(path), { recursive: true }); + return writeFile(path, data, 'utf8'); +} + +/******************************************************************************/ + +async function matchRequests(engine, requests) { + const results = []; + const details = { + r: 0, + f: undefined, + type: '', + url: '', + originURL: '', + t: 0, + }; + + let notBlockedCount = 0; + let blockedCount = 0; + let unblockedCount = 0; + + const start = process.hrtime.bigint(); + + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + const reqstart = process.hrtime.bigint(); + details.type = WEBREQUEST_OPTIONS[request.cpt]; + details.url = request.url; + details.originURL = request.frameUrl; + const r = engine.matchRequest(details); + if ( r === 0 ) { + notBlockedCount += 1; + } else if ( r === 1 ) { + blockedCount += 1; + } else { + unblockedCount += 1; + } + if ( NEED_RESULTS !== true ) { continue; } + const reqstop = process.hrtime.bigint(); + details.r = r; + details.f = r !== 0 ? engine.toLogData().raw : undefined; + details.t = Math.round(Number(reqstop - reqstart) / 10) / 100; + results.push([ i, Object.assign({}, details) ]); + } + + const stop = process.hrtime.bigint(); + + console.log(`Matched ${requests.length} requests in ${nanoToMilli(stop - start)}`); + console.log(`\tNot blocked: ${notBlockedCount} requests`); + console.log(`\tBlocked: ${blockedCount} requests`); + console.log(`\tUnblocked: ${unblockedCount} requests`); + console.log(`\tAverage: ${nanoToMicro((stop - start) / BigInt(requests.length))} per request`); + + if ( RECORD ) { + write('data/snfe.json', JSON.stringify(results, null, 2)); + } + + if ( COMPARE ) { + const diffs = await compare(results); + if ( Array.isArray(diffs) ) { + write('data/snfe.diffs.json', JSON.stringify(diffs, null, 2)); + } + console.log(`Compare: ${diffs.length} requests differ`); + } + + if ( MAXCOST ) { + const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(0, 1000); + write('data/snfe.maxcost.json', JSON.stringify(costly, null, 2)); + } + + if ( MEDCOST ) { + const median = results.length >>> 1; + const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(median - 500, median + 500); + write('data/snfe.medcost.json', JSON.stringify(costly, null, 2)); + } + + if ( MINCOST ) { + const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(-1000); + write('data/snfe.mincost.json', JSON.stringify(costly, null, 2)); + } +} + +async function compare(results) { + let before; + try { + const raw = await read('data/snfe.json'); + before = new Map(JSON.parse(raw)); + } catch(ex) { + console.log(ex); + console.log('Nothing to compare'); + return; + } + const after = new Map(results); + const diffs = []; + for ( let i = 0; i < results.length; i++ ) { + const a = before.get(i); + const b = after.get(i); + if ( a.r === b.r ) { continue; } + diffs.push([ i, { + type: a.type, + url: a.url, + originURL: a.originURL, + before: { r: a.r, f: a.f, t: a.t }, + after: { r: b.r, f: b.f, t: b.t }, + }]); + } + return diffs; +} + +/******************************************************************************/ + +async function matchRequestModifiers(engine, requests) { + const results = { + 'csp': [], + 'redirect-rule': [], + 'removeparam': [], + }; + + const details = { + f: undefined, + type: '', + url: '', + originURL: '', + t: 0, + }; + + let modifiedCount = 0; + + const start = process.hrtime.bigint(); + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + details.type = WEBREQUEST_OPTIONS[request.cpt]; + details.url = request.url; + details.originURL = request.frameUrl; + const r = engine.matchRequest(details); + let modified = false; + if ( r !== 1 && details.type === 'sub_frame' ) { + const reqstart = process.hrtime.bigint(); + const directives = engine.matchAndFetchModifiers(details, 'csp'); + if ( directives !== undefined ) { + modified = true; + if ( NEED_RESULTS ) { + const reqstop = process.hrtime.bigint(); + details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort(); + details.t = Math.round(Number(reqstop - reqstart) / 10) / 100; + results['csp'].push([ i, Object.assign({}, details) ]); + } + } + } + if ( r === 1 ) { + const reqstart = process.hrtime.bigint(); + const directives = engine.matchAndFetchModifiers(details, 'redirect-rule'); + if ( directives !== undefined ) { + modified = true; + if ( NEED_RESULTS ) { + const reqstop = process.hrtime.bigint(); + details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort(); + details.t = Math.round(Number(reqstop - reqstart) / 10) / 100; + results['redirect-rule'].push([ i, Object.assign({}, details) ]); + } + } + } + if ( r !== 1 && engine.hasQuery(details) ) { + const reqstart = process.hrtime.bigint(); + const directives = engine.matchAndFetchModifiers(details, 'removeparam'); + if ( directives !== undefined ) { + modified = true; + if ( NEED_RESULTS ) { + const reqstop = process.hrtime.bigint(); + details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort(); + details.t = Math.round(Number(reqstop - reqstart) / 10) / 100; + results['removeparam'].push([ i, Object.assign({}, details) ]); + } + } + } + if ( modified ) { + modifiedCount += 1; + } + } + const stop = process.hrtime.bigint(); + + console.log(`Matched-modified ${requests.length} requests in ${nanoToMilli(stop - start)}`); + console.log(`\t${modifiedCount} modifiers found`); + console.log(`\tAverage: ${nanoToMicro((stop - start) / BigInt(requests.length))} per request`); + + if ( RECORD ) { + write('data/snfe.modifiers.json', JSON.stringify(results, null, 2)); + } + + if ( COMPARE ) { + const diffs = await compareModifiers(results); + if ( Array.isArray(diffs) ) { + write('data/snfe.modifiers.diffs.json', JSON.stringify(diffs, null, 2)); + } + console.log(`Compare: ${diffs.length} modified requests differ`); + } +} + +async function compareModifiers(afterResults) { + let beforeResults; + try { + const raw = await read('data/snfe.modifiers.json'); + beforeResults = JSON.parse(raw); + } catch(ex) { + console.log(ex); + console.log('Nothing to compare'); + return; + } + const diffs = []; + for ( const modifier of [ 'csp', 'redirect-rule', 'removeparam' ] ) { + const before = new Map(beforeResults[modifier]); + const after = new Map(afterResults[modifier]); + for ( const [ i, b ] of before ) { + const a = after.get(i); + if ( a !== undefined && JSON.stringify(a.f) === JSON.stringify(b.f) ) { continue; } + diffs.push([ i, { + type: b.type, + url: b.url, + originURL: b.originURL, + before: { f: b.f, t: b.t }, + after: a !== undefined ? { f: a.f, t: a.t } : null, + }]); + } + for ( const [ i, a ] of after ) { + const b = before.get(i); + if ( b !== undefined ) { continue; } + diffs.push([ i, { + type: a.type, + url: a.url, + originURL: a.originURL, + before: null, + after: { f: a.f, t: a.t }, + }]); + } + } + return diffs; +} + +/******************************************************************************/ + +async function bench() { + if ( WASM ) { + try { + const result = await enableWASM(); + if ( result === true ) { + console.log('WASM code paths enabled'); + } + } catch(ex) { + console.log(ex); + } + } + + const require = createRequire(import.meta.url); // jshint ignore:line + const requests = await require('./node_modules/scaling-palm-tree/requests.json'); + const engine = await StaticNetFilteringEngine.create(); + + let start = process.hrtime.bigint(); + await engine.useLists([ + read('assets/ublock/filters.min.txt') + .then(raw => ({ name: 'filters', raw })), + read('assets/ublock/badware.txt') + .then(raw => ({ name: 'badware', raw })), + read('assets/ublock/privacy.min.txt') + .then(raw => ({ name: 'privacy', raw })), + read('assets/ublock/quick-fixes.txt') + .then(raw => ({ name: 'quick-fixes.txt', raw })), + read('assets/ublock/unbreak.txt') + .then(raw => ({ name: 'unbreak.txt', raw })), + read('assets/thirdparties/easylist/easylist.txt') + .then(raw => ({ name: 'easylist', raw })), + read('assets/thirdparties/easylist/easyprivacy.txt') + .then(raw => ({ name: 'easyprivacy', raw })), + read('assets/thirdparties/pgl.yoyo.org/as/serverlist') + .then(raw => ({ name: 'PGL', raw })), + read('assets/thirdparties/urlhaus-filter/urlhaus-filter-online.txt') + .then(raw => ({ name: 'urlhaus', raw })), + ]); + let stop = process.hrtime.bigint(); + console.log(`Filter lists parsed-compiled-loaded in ${nanoToMilli(stop - start)}`); + + // Dry run to let JS engine optimize hot JS code paths + for ( let i = 0; i < requests.length; i += 8 ) { + const request = requests[i]; + void engine.matchRequest({ + type: WEBREQUEST_OPTIONS[request.cpt], + url: request.url, + originURL: request.frameUrl, + }); + } + + if ( MODIFIERS === false ) { + matchRequests(engine, requests); + } else { + matchRequestModifiers(engine, requests); + } + + StaticNetFilteringEngine.release(); +} + +bench(); + +/******************************************************************************/ |