Adding upstream version 1.55.0+dfsg.upstream/1.55.0+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 417 insertions, 0 deletions

diff --git a/platform/dig/package.json b/platform/dig/package.json
new file mode 100644
index 0000000..d15b78c
--- /dev/null
+++ b/platform/dig/package.json
@@ -0,0 +1,28 @@
+{
+  "name": "@gorhill/ubo-dig",
+  "version": "0.1.0",
+  "description": "To investigate code changes (not for publication)",
+  "type": "module",
+  "main": "index.js",
+  "scripts": {
+    "build": "node build.js",
+    "snfe": "node snfe.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/gorhill/uBlock.git"
+  },
+  "author": "Raymond Hill",
+  "license": "GPL-3.0-or-later",
+  "bugs": {
+    "url": "https://github.com/gorhill/uBlock/issues"
+  },
+  "homepage": "https://github.com/gorhill/uBlock#readme",
+  "engines": {
+    "node": ">=14.0.0",
+    "npm": ">=6.14.4"
+  },
+  "devDependencies": {
+    "scaling-palm-tree": "github:mjethani/scaling-palm-tree#15cf1ab"
+  }
+}
diff --git a/platform/dig/snfe.js b/platform/dig/snfe.js
new file mode 100644
index 0000000..7090828
--- /dev/null
+++ b/platform/dig/snfe.js
@@ -0,0 +1,389 @@
+/*******************************************************************************
+
+    uBlock Origin - a comprehensive, efficient content blocker
+    Copyright (C) 2014-present Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+/* eslint-disable-next-line no-redeclare */
+/* globals process */
+
+'use strict';
+
+/******************************************************************************/
+
+import { strict as assert } from 'assert';
+import { createRequire } from 'module';
+import { readFile, writeFile, mkdir } from 'fs/promises';
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+import { enableWASM, StaticNetFilteringEngine } from './index.js';
+
+/******************************************************************************/
+
+const FLAGS = process.argv.slice(2);
+const COMPARE = FLAGS.includes('compare');
+const MAXCOST = FLAGS.includes('maxcost');
+const MEDCOST = FLAGS.includes('medcost');
+const MINCOST = FLAGS.includes('mincost');
+const MODIFIERS = FLAGS.includes('modifiers');
+const RECORD = FLAGS.includes('record');
+const WASM = FLAGS.includes('wasm');
+const NEED_RESULTS = COMPARE || MAXCOST || MEDCOST || MINCOST || RECORD;
+
+// This maps puppeteer types to WebRequest types
+const WEBREQUEST_OPTIONS = {
+    // Consider document requests as sub_document. This is because the request
+    // dataset does not contain sub_frame or main_frame but only 'document' and
+    // different blockers have different behaviours.
+    document: 'sub_frame',
+    stylesheet: 'stylesheet',
+    image: 'image',
+    media: 'media',
+    font: 'font',
+    script: 'script',
+    xhr: 'xmlhttprequest',
+    fetch: 'xmlhttprequest',
+    websocket: 'websocket',
+    ping: 'ping',
+    // other
+    other: 'other',
+    eventsource: 'other',
+    manifest: 'other',
+    texttrack: 'other',
+};
+
+/******************************************************************************/
+
+function nanoToMilli(bigint) {
+    return (Number(bigint) / 1000000).toFixed(2) + ' ms';
+}
+
+function nanoToMicro(bigint) {
+    return (Number(bigint) / 1000).toFixed(2) + ' µs';
+}
+
+async function read(path) {
+    return readFile(path, 'utf8');
+}
+
+async function write(path, data) {
+    await mkdir(dirname(path), { recursive: true });
+    return writeFile(path, data, 'utf8');
+}
+
+/******************************************************************************/
+
+async function matchRequests(engine, requests) {
+    const results = [];
+    const details = {
+        r: 0,
+        f: undefined,
+        type: '',
+        url: '',
+        originURL: '',
+        t: 0,
+    };
+
+    let notBlockedCount = 0;
+    let blockedCount = 0;
+    let unblockedCount = 0;
+
+    const start = process.hrtime.bigint();
+
+    for ( let i = 0; i < requests.length; i++ ) {
+        const request = requests[i];
+        const reqstart = process.hrtime.bigint();
+        details.type = WEBREQUEST_OPTIONS[request.cpt];
+        details.url = request.url;
+        details.originURL = request.frameUrl;
+        const r = engine.matchRequest(details);
+        if ( r === 0 ) {
+            notBlockedCount += 1;
+        } else if ( r === 1 ) {
+            blockedCount += 1;
+        } else {
+            unblockedCount += 1;
+        }
+        if ( NEED_RESULTS !== true ) { continue; }
+        const reqstop = process.hrtime.bigint();
+        details.r = r;
+        details.f = r !== 0 ? engine.toLogData().raw : undefined;
+        details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
+        results.push([ i, Object.assign({}, details) ]);
+    }
+
+    const stop = process.hrtime.bigint();
+
+    console.log(`Matched ${requests.length} requests in ${nanoToMilli(stop - start)}`);
+    console.log(`\tNot blocked: ${notBlockedCount} requests`);
+    console.log(`\tBlocked: ${blockedCount} requests`);
+    console.log(`\tUnblocked: ${unblockedCount} requests`);
+    console.log(`\tAverage: ${nanoToMicro((stop - start) / BigInt(requests.length))} per request`);
+
+    if ( RECORD ) {
+        write('data/snfe.json', JSON.stringify(results, null, 2));
+    }
+
+    if ( COMPARE ) {
+        const diffs = await compare(results);
+        if ( Array.isArray(diffs) ) {
+            write('data/snfe.diffs.json', JSON.stringify(diffs, null, 2));
+        }
+        console.log(`Compare: ${diffs.length} requests differ`);
+    }
+
+    if ( MAXCOST ) {
+        const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(0, 1000);
+        write('data/snfe.maxcost.json', JSON.stringify(costly, null, 2));
+    }
+
+    if ( MEDCOST ) {
+        const median = results.length >>> 1;
+        const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(median - 500, median + 500);
+        write('data/snfe.medcost.json', JSON.stringify(costly, null, 2));
+    }
+
+    if ( MINCOST ) {
+        const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(-1000);
+        write('data/snfe.mincost.json', JSON.stringify(costly, null, 2));
+    }
+}
+
+async function compare(results) {
+    let before;
+    try {
+        const raw = await read('data/snfe.json');
+        before = new Map(JSON.parse(raw));
+    } catch(ex) {
+        console.log(ex);
+        console.log('Nothing to compare');
+        return;
+    }
+    const after = new Map(results);
+    const diffs = [];
+    for ( let i = 0; i < results.length; i++ ) {
+        const a = before.get(i);
+        const b = after.get(i);
+        if ( a.r === b.r ) { continue; }
+        diffs.push([ i, {
+            type: a.type,
+            url: a.url,
+            originURL: a.originURL,
+            before: { r: a.r, f: a.f, t: a.t },
+            after: { r: b.r, f: b.f, t: b.t },
+        }]);
+    }
+    return diffs;
+}
+
+/******************************************************************************/
+
+async function matchRequestModifiers(engine, requests) {
+    const results = {
+        'csp': [],
+        'redirect-rule': [],
+        'removeparam': [],
+    };
+
+    const details = {
+        f: undefined,
+        type: '',
+        url: '',
+        originURL: '',
+        t: 0,
+    };
+
+    let modifiedCount = 0;
+
+    const start = process.hrtime.bigint();
+    for ( let i = 0; i < requests.length; i++ ) {
+        const request = requests[i];
+        details.type = WEBREQUEST_OPTIONS[request.cpt];
+        details.url = request.url;
+        details.originURL = request.frameUrl;
+        const r = engine.matchRequest(details);
+        let modified = false;
+        if ( r !== 1 && details.type === 'sub_frame' ) {
+            const reqstart = process.hrtime.bigint();
+            const directives = engine.matchAndFetchModifiers(details, 'csp');
+            if ( directives !== undefined ) {
+                modified = true;
+                if ( NEED_RESULTS ) {
+                    const reqstop = process.hrtime.bigint();
+                    details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort();
+                    details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
+                    results['csp'].push([ i, Object.assign({}, details) ]);
+                }
+            }
+        }
+        if ( r === 1 ) {
+            const reqstart = process.hrtime.bigint();
+            const directives = engine.matchAndFetchModifiers(details, 'redirect-rule');
+            if ( directives !== undefined ) {
+                modified = true;
+                if ( NEED_RESULTS ) {
+                    const reqstop = process.hrtime.bigint();
+                    details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort();
+                    details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
+                    results['redirect-rule'].push([ i, Object.assign({}, details) ]);
+                }
+            }
+        }
+        if ( r !== 1 && engine.hasQuery(details) ) {
+            const reqstart = process.hrtime.bigint();
+            const directives = engine.matchAndFetchModifiers(details, 'removeparam');
+            if ( directives !== undefined ) {
+                modified = true;
+                if ( NEED_RESULTS ) {
+                    const reqstop = process.hrtime.bigint();
+                    details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort();
+                    details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
+                    results['removeparam'].push([ i, Object.assign({}, details) ]);
+                }
+            }
+        }
+        if ( modified ) {
+            modifiedCount += 1;
+        }
+    }
+    const stop = process.hrtime.bigint();
+
+    console.log(`Matched-modified ${requests.length} requests in ${nanoToMilli(stop - start)}`);
+    console.log(`\t${modifiedCount} modifiers found`);
+    console.log(`\tAverage: ${nanoToMicro((stop - start) / BigInt(requests.length))} per request`);
+
+    if ( RECORD ) {
+        write('data/snfe.modifiers.json', JSON.stringify(results, null, 2));
+    }
+
+    if ( COMPARE ) {
+        const diffs = await compareModifiers(results);
+        if ( Array.isArray(diffs) ) {
+            write('data/snfe.modifiers.diffs.json', JSON.stringify(diffs, null, 2));
+        }
+        console.log(`Compare: ${diffs.length} modified requests differ`);
+    }
+}
+
+async function compareModifiers(afterResults) {
+    let beforeResults;
+    try {
+        const raw = await read('data/snfe.modifiers.json');
+        beforeResults = JSON.parse(raw);
+    } catch(ex) {
+        console.log(ex);
+        console.log('Nothing to compare');
+        return;
+    }
+    const diffs = [];
+    for ( const modifier of [ 'csp', 'redirect-rule', 'removeparam' ] ) {
+        const before = new Map(beforeResults[modifier]);
+        const after = new Map(afterResults[modifier]);
+        for ( const [ i, b ] of before ) {
+            const a = after.get(i);
+            if ( a !== undefined && JSON.stringify(a.f) === JSON.stringify(b.f) ) { continue; }
+            diffs.push([ i, {
+                type: b.type,
+                url: b.url,
+                originURL: b.originURL,
+                before: { f: b.f, t: b.t },
+                after: a !== undefined ? { f: a.f, t: a.t } : null,
+            }]);
+        }
+        for ( const [ i, a ] of after ) {
+            const b = before.get(i);
+            if ( b !== undefined ) { continue; }
+            diffs.push([ i, {
+                type: a.type,
+                url: a.url,
+                originURL: a.originURL,
+                before: null,
+                after: { f: a.f, t: a.t },
+            }]);
+        }
+    }
+    return diffs;
+}
+
+/******************************************************************************/
+
+async function bench() {
+    if ( WASM ) {
+        try {
+            const result = await enableWASM();
+            if ( result === true ) {
+                console.log('WASM code paths enabled');
+            }
+        } catch(ex) {
+            console.log(ex);
+        }
+    }
+
+    const require = createRequire(import.meta.url); // jshint ignore:line
+    const requests = await require('./node_modules/scaling-palm-tree/requests.json');
+    const engine = await StaticNetFilteringEngine.create();
+
+    let start = process.hrtime.bigint();
+    await engine.useLists([
+        read('assets/ublock/filters.min.txt')
+            .then(raw => ({ name: 'filters', raw })),
+        read('assets/ublock/badware.txt')
+            .then(raw => ({ name: 'badware', raw })),
+        read('assets/ublock/privacy.min.txt')
+            .then(raw => ({ name: 'privacy', raw })),
+        read('assets/ublock/quick-fixes.txt')
+            .then(raw => ({ name: 'quick-fixes.txt', raw })),
+        read('assets/ublock/unbreak.txt')
+            .then(raw => ({ name: 'unbreak.txt', raw })),
+        read('assets/thirdparties/easylist/easylist.txt')
+            .then(raw => ({ name: 'easylist', raw })),
+        read('assets/thirdparties/easylist/easyprivacy.txt')
+            .then(raw => ({ name: 'easyprivacy', raw })),
+        read('assets/thirdparties/pgl.yoyo.org/as/serverlist')
+            .then(raw => ({ name: 'PGL', raw })),
+        read('assets/thirdparties/urlhaus-filter/urlhaus-filter-online.txt')
+            .then(raw => ({ name: 'urlhaus', raw })),
+    ]);
+    let stop = process.hrtime.bigint();
+    console.log(`Filter lists parsed-compiled-loaded in ${nanoToMilli(stop - start)}`);
+
+    // Dry run to let JS engine optimize hot JS code paths
+    for ( let i = 0; i < requests.length; i += 8 ) {
+        const request = requests[i];
+        void engine.matchRequest({
+            type: WEBREQUEST_OPTIONS[request.cpt],
+            url: request.url,
+            originURL: request.frameUrl,
+        });
+    }
+
+    if ( MODIFIERS === false ) {
+        matchRequests(engine, requests);
+    } else {
+        matchRequestModifiers(engine, requests);
+    }
+
+    StaticNetFilteringEngine.release();
+}
+
+bench();
+
+/******************************************************************************/