diff options
Diffstat (limited to 'src/js')
98 files changed, 52793 insertions, 0 deletions
diff --git a/src/js/1p-filters.js b/src/js/1p-filters.js new file mode 100644 index 0000000..fc50b50 --- /dev/null +++ b/src/js/1p-filters.js @@ -0,0 +1,337 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uBlockDashboard */ + +'use strict'; + +import { onBroadcast } from './broadcast.js'; +import { dom, qs$ } from './dom.js'; +import { i18n$ } from './i18n.js'; +import './codemirror/ubo-static-filtering.js'; + +/******************************************************************************/ + +const cmEditor = new CodeMirror(qs$('#userFilters'), { + autoCloseBrackets: true, + autofocus: true, + extraKeys: { + 'Ctrl-Space': 'autocomplete', + 'Tab': 'toggleComment', + }, + foldGutter: true, + gutters: [ + 'CodeMirror-linenumbers', + { className: 'CodeMirror-lintgutter', style: 'width: 11px' }, + ], + lineNumbers: true, + lineWrapping: true, + matchBrackets: true, + maxScanLines: 1, + styleActiveLine: { + nonEmpty: true, + }, +}); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +let cachedUserFilters = ''; + +/******************************************************************************/ + +// Add auto-complete ability to the editor. Polling is used as the suggested +// hints also depend on the tabs currently opened. + +{ + let hintUpdateToken = 0; + + const getHints = async function() { + const hints = await vAPI.messaging.send('dashboard', { + what: 'getAutoCompleteDetails', + hintUpdateToken + }); + if ( hints instanceof Object === false ) { return; } + if ( hints.hintUpdateToken !== undefined ) { + cmEditor.setOption('uboHints', hints); + hintUpdateToken = hints.hintUpdateToken; + } + timer.on(2503); + }; + + const timer = vAPI.defer.create(( ) => { + getHints(); + }); + + getHints(); +} + +vAPI.messaging.send('dashboard', { + what: 'getTrustedScriptletTokens', +}).then(tokens => { + cmEditor.setOption('trustedScriptletTokens', tokens); +}); + +/******************************************************************************/ + +function getEditorText() { + const text = cmEditor.getValue().replace(/\s+$/, ''); + return text === '' ? text : text + '\n'; +} + +function setEditorText(text) { + cmEditor.setValue(text.replace(/\s+$/, '') + '\n\n'); +} + +/******************************************************************************/ + +function userFiltersChanged(changed) { + if ( typeof changed !== 'boolean' ) { + changed = self.hasUnsavedData(); + } + qs$('#userFiltersApply').disabled = !changed; + qs$('#userFiltersRevert').disabled = !changed; +} + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/3704 +// Merge changes to user filters occurring in the background with changes +// made in the editor. The code assumes that no deletion occurred in the +// background. + +function threeWayMerge(newContent) { + const prvContent = cachedUserFilters.trim().split(/\n/); + const differ = new self.diff_match_patch(); + const newChanges = differ.diff( + prvContent, + newContent.trim().split(/\n/) + ); + const usrChanges = differ.diff( + prvContent, + getEditorText().trim().split(/\n/) + ); + const out = []; + let i = 0, j = 0, k = 0; + while ( i < prvContent.length ) { + for ( ; j < newChanges.length; j++ ) { + const change = newChanges[j]; + if ( change[0] !== 1 ) { break; } + out.push(change[1]); + } + for ( ; k < usrChanges.length; k++ ) { + const change = usrChanges[k]; + if ( change[0] !== 1 ) { break; } + out.push(change[1]); + } + if ( k === usrChanges.length || usrChanges[k][0] !== -1 ) { + out.push(prvContent[i]); + } + i += 1; j += 1; k += 1; + } + for ( ; j < newChanges.length; j++ ) { + const change = newChanges[j]; + if ( change[0] !== 1 ) { continue; } + out.push(change[1]); + } + for ( ; k < usrChanges.length; k++ ) { + const change = usrChanges[k]; + if ( change[0] !== 1 ) { continue; } + out.push(change[1]); + } + return out.join('\n'); +} + +/******************************************************************************/ + +async function renderUserFilters(merge = false) { + const details = await vAPI.messaging.send('dashboard', { + what: 'readUserFilters', + }); + if ( details instanceof Object === false || details.error ) { return; } + + cmEditor.setOption('trustedSource', details.trustedSource === true); + + const newContent = details.content.trim(); + + if ( merge && self.hasUnsavedData() ) { + setEditorText(threeWayMerge(newContent)); + userFiltersChanged(true); + } else { + setEditorText(newContent); + userFiltersChanged(false); + } + + cachedUserFilters = newContent; +} + +/******************************************************************************/ + +function handleImportFilePicker(ev) { + const file = ev.target.files[0]; + if ( file === undefined || file.name === '' ) { return; } + if ( file.type.indexOf('text') !== 0 ) { return; } + const fr = new FileReader(); + fr.onload = function() { + if ( typeof fr.result !== 'string' ) { return; } + const content = uBlockDashboard.mergeNewLines(getEditorText(), fr.result); + cmEditor.operation(( ) => { + const cmPos = cmEditor.getCursor(); + setEditorText(content); + cmEditor.setCursor(cmPos); + cmEditor.focus(); + }); + }; + fr.readAsText(file); +} + +dom.on('#importFilePicker', 'change', handleImportFilePicker); + +function startImportFilePicker() { + const input = qs$('#importFilePicker'); + // Reset to empty string, this will ensure an change event is properly + // triggered if the user pick a file, even if it is the same as the last + // one picked. + input.value = ''; + input.click(); +} + +dom.on('#importUserFiltersFromFile', 'click', startImportFilePicker); + +/******************************************************************************/ + +function exportUserFiltersToFile() { + const val = getEditorText(); + if ( val === '' ) { return; } + const filename = i18n$('1pExportFilename') + .replace('{{datetime}}', uBlockDashboard.dateNowToSensibleString()) + .replace(/ +/g, '_'); + vAPI.download({ + 'url': 'data:text/plain;charset=utf-8,' + encodeURIComponent(val + '\n'), + 'filename': filename + }); +} + +/******************************************************************************/ + +async function applyChanges() { + const details = await vAPI.messaging.send('dashboard', { + what: 'writeUserFilters', + content: getEditorText(), + }); + if ( details instanceof Object === false || details.error ) { return; } + + cachedUserFilters = details.content.trim(); + userFiltersChanged(false); + vAPI.messaging.send('dashboard', { + what: 'reloadAllFilters', + }); +} + +function revertChanges() { + setEditorText(cachedUserFilters); +} + +/******************************************************************************/ + +function getCloudData() { + return getEditorText(); +} + +function setCloudData(data, append) { + if ( typeof data !== 'string' ) { return; } + if ( append ) { + data = uBlockDashboard.mergeNewLines(getEditorText(), data); + } + cmEditor.setValue(data); +} + +self.cloud.onPush = getCloudData; +self.cloud.onPull = setCloudData; + +/******************************************************************************/ + +self.hasUnsavedData = function() { + return getEditorText().trim() !== cachedUserFilters; +}; + +/******************************************************************************/ + +// Handle user interaction +dom.on('#exportUserFiltersToFile', 'click', exportUserFiltersToFile); +dom.on('#userFiltersApply', 'click', ( ) => { applyChanges(); }); +dom.on('#userFiltersRevert', 'click', revertChanges); + +(async ( ) => { + await renderUserFilters(); + + cmEditor.clearHistory(); + + // https://github.com/gorhill/uBlock/issues/3706 + // Save/restore cursor position + { + const line = await vAPI.localStorage.getItemAsync('myFiltersCursorPosition'); + if ( typeof line === 'number' ) { + cmEditor.setCursor(line, 0); + } + cmEditor.focus(); + } + + // https://github.com/gorhill/uBlock/issues/3706 + // Save/restore cursor position + { + let curline = 0; + cmEditor.on('cursorActivity', ( ) => { + if ( timer.ongoing() ) { return; } + if ( cmEditor.getCursor().line === curline ) { return; } + timer.on(701); + }); + const timer = vAPI.defer.create(( ) => { + curline = cmEditor.getCursor().line; + vAPI.localStorage.setItem('myFiltersCursorPosition', curline); + }); + } + + // https://github.com/gorhill/uBlock/issues/3704 + // Merge changes to user filters occurring in the background + onBroadcast(msg => { + switch ( msg.what ) { + case 'userFiltersUpdated': { + cmEditor.startOperation(); + const scroll = cmEditor.getScrollInfo(); + const selections = cmEditor.listSelections(); + renderUserFilters(true).then(( ) => { + cmEditor.clearHistory(); + cmEditor.setSelection(selections[0].anchor, selections[0].head); + cmEditor.scrollTo(scroll.left, scroll.top); + cmEditor.endOperation(); + }); + break; + } + default: + break; + } + }); +})(); + +cmEditor.on('changes', userFiltersChanged); +CodeMirror.commands.save = applyChanges; + +/******************************************************************************/ diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js new file mode 100644 index 0000000..c59365f --- /dev/null +++ b/src/js/3p-filters.js @@ -0,0 +1,861 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { onBroadcast } from './broadcast.js'; +import { dom, qs$, qsa$ } from './dom.js'; +import { i18n, i18n$ } from './i18n.js'; + +/******************************************************************************/ + +const lastUpdateTemplateString = i18n$('3pLastUpdate'); +const obsoleteTemplateString = i18n$('3pExternalListObsolete'); +const reValidExternalList = /^[a-z-]+:\/\/(?:\S+\/\S*|\/\S+)/m; +const recentlyUpdated = 1 * 60 * 60 * 1000; // 1 hour + +let listsetDetails = {}; + +/******************************************************************************/ + +onBroadcast(msg => { + switch ( msg.what ) { + case 'assetUpdated': + updateAssetStatus(msg); + break; + case 'assetsUpdated': + dom.cl.remove(dom.body, 'updating'); + renderWidgets(); + break; + case 'staticFilteringDataChanged': + renderFilterLists(); + break; + default: + break; + } +}); + +/******************************************************************************/ + +const renderNumber = value => { + return value.toLocaleString(); +}; + +const listStatsTemplate = i18n$('3pListsOfBlockedHostsPerListStats'); + +const renderLeafStats = (used, total) => { + if ( isNaN(used) || isNaN(total) ) { return ''; } + return listStatsTemplate + .replace('{{used}}', renderNumber(used)) + .replace('{{total}}', renderNumber(total)); +}; + +const renderNodeStats = (used, total) => { + if ( isNaN(used) || isNaN(total) ) { return ''; } + return `${used.toLocaleString()}/${total.toLocaleString()}`; +}; + +const i18nGroupName = name => { + return i18n$('3pGroup' + name.charAt(0).toUpperCase() + name.slice(1)); +}; + +/******************************************************************************/ + +const renderFilterLists = ( ) => { + // Assemble a pretty list name if possible + const listNameFromListKey = listkey => { + const list = listsetDetails.current[listkey] || listsetDetails.available[listkey]; + const title = list && list.title || ''; + if ( title !== '' ) { return title; } + return listkey; + }; + + const initializeListEntry = (listDetails, listEntry) => { + const listkey = listEntry.dataset.key; + const listEntryPrevious = + qs$(`[data-key="${listDetails.group}"] [data-key="${listkey}"]`); + if ( listEntryPrevious !== null ) { + if ( dom.cl.has(listEntryPrevious, 'checked') ) { + dom.cl.add(listEntry, 'checked'); + } + if ( dom.cl.has(listEntryPrevious, 'stickied') ) { + dom.cl.add(listEntry, 'stickied'); + } + if ( dom.cl.has(listEntryPrevious, 'toRemove') ) { + dom.cl.add(listEntry, 'toRemove'); + } + if ( dom.cl.has(listEntryPrevious, 'searchMatch') ) { + dom.cl.add(listEntry, 'searchMatch'); + } + } else { + dom.cl.toggle(listEntry, 'checked', listDetails.off !== true); + } + const on = dom.cl.has(listEntry, 'checked'); + dom.prop(qs$(listEntry, ':scope > .detailbar input'), 'checked', on); + let elem = qs$(listEntry, ':scope > .detailbar a.content'); + dom.attr(elem, 'href', 'asset-viewer.html?url=' + encodeURIComponent(listkey)); + dom.attr(elem, 'type', 'text/html'); + dom.cl.remove(listEntry, 'toRemove'); + if ( listDetails.supportName ) { + elem = qs$(listEntry, ':scope > .detailbar a.support'); + dom.attr(elem, 'href', listDetails.supportURL || '#'); + dom.attr(elem, 'title', listDetails.supportName); + } + if ( listDetails.external ) { + dom.cl.add(listEntry, 'external'); + } else { + dom.cl.remove(listEntry, 'external'); + } + if ( listDetails.instructionURL ) { + elem = qs$(listEntry, ':scope > .detailbar a.mustread'); + dom.attr(elem, 'href', listDetails.instructionURL || '#'); + } + dom.cl.toggle(listEntry, 'isDefault', + listDetails.isDefault === true || + listDetails.isImportant === true || + listkey === 'user-filters' + ); + elem = qs$(listEntry, '.leafstats'); + dom.text(elem, renderLeafStats(on ? listDetails.entryUsedCount : 0, listDetails.entryCount)); + // https://github.com/chrisaljoudi/uBlock/issues/104 + const asset = listsetDetails.cache[listkey] || {}; + const remoteURL = asset.remoteURL; + dom.cl.toggle(listEntry, 'unsecure', + typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 + ); + dom.cl.toggle(listEntry, 'failed', asset.error !== undefined); + dom.cl.toggle(listEntry, 'obsolete', asset.obsolete === true); + const lastUpdateString = lastUpdateTemplateString.replace('{{ago}}', + i18n.renderElapsedTimeToString(asset.writeTime || 0) + ); + if ( asset.obsolete === true ) { + let title = obsoleteTemplateString; + if ( asset.cached && asset.writeTime !== 0 ) { + title += '\n' + lastUpdateString; + } + dom.attr(qs$(listEntry, ':scope > .detailbar .status.obsolete'), 'title', title); + } + if ( asset.cached === true ) { + dom.cl.add(listEntry, 'cached'); + dom.attr(qs$(listEntry, ':scope > .detailbar .status.cache'), 'title', lastUpdateString); + const timeSinceLastUpdate = Date.now() - asset.writeTime; + dom.cl.toggle(listEntry, 'recent', timeSinceLastUpdate < recentlyUpdated); + } else { + dom.cl.remove(listEntry, 'cached'); + } + }; + + const createListEntry = (listDetails, depth) => { + if ( listDetails.lists === undefined ) { + return dom.clone('#templates .listEntry[data-role="leaf"]'); + } + if ( depth !== 0 ) { + return dom.clone('#templates .listEntry[data-role="node"]'); + } + return dom.clone('#templates .listEntry[data-role="node"][data-parent="root"]'); + }; + + const createListEntries = (parentkey, listTree, depth = 0) => { + const listEntries = dom.clone('#templates .listEntries'); + const treeEntries = Object.entries(listTree); + if ( depth !== 0 ) { + const reEmojis = /\p{Emoji}+/gu; + treeEntries.sort((a ,b) => { + const as = (a[1].title || a[0]).replace(reEmojis, ''); + const bs = (b[1].title || b[0]).replace(reEmojis, ''); + return as.localeCompare(bs); + }); + } + for ( const [ listkey, listDetails ] of treeEntries ) { + const listEntry = createListEntry(listDetails, depth); + if ( dom.cl.has(dom.root, 'mobile') ) { + const leafStats = qs$(listEntry, '.leafstats'); + if ( leafStats ) { + listEntry.append(leafStats); + } + } + listEntry.dataset.key = listkey; + listEntry.dataset.parent = parentkey; + qs$(listEntry, ':scope > .detailbar .listname').append( + i18n.patchUnicodeFlags(listDetails.title) + ); + if ( listDetails.lists !== undefined ) { + listEntry.append(createListEntries(listEntry.dataset.key, listDetails.lists, depth+1)); + dom.cl.toggle(listEntry, 'expanded', listIsExpanded(listkey)); + updateListNode(listEntry); + } else { + initializeListEntry(listDetails, listEntry); + } + listEntries.append(listEntry); + } + return listEntries; + }; + + const onListsReceived = response => { + // Store in global variable + listsetDetails = response; + hashFromListsetDetails(); + + // Build list tree + const listTree = {}; + const groupKeys = [ + 'user', + 'default', + 'ads', + 'privacy', + 'malware', + 'multipurpose', + 'annoyances', + 'regions', + 'custom' + ]; + for ( const key of groupKeys ) { + listTree[key] = { + title: i18nGroupName(key), + lists: {}, + }; + } + for ( const [ listkey, listDetails ] of Object.entries(response.available) ) { + let groupKey = listDetails.group; + if ( groupKey === 'social' ) { + groupKey = 'annoyances'; + } + const groupDetails = listTree[groupKey]; + if ( listDetails.parent !== undefined ) { + let lists = groupDetails.lists; + for ( const parent of listDetails.parent.split('|') ) { + if ( lists[parent] === undefined ) { + lists[parent] = { title: parent, lists: {} }; + } + lists = lists[parent].lists; + } + lists[listkey] = listDetails; + } else { + listDetails.title = listNameFromListKey(listkey); + groupDetails.lists[listkey] = listDetails; + } + } + const listEntries = createListEntries('root', listTree); + qs$('#lists .listEntries').replaceWith(listEntries); + + qs$('#autoUpdate').checked = listsetDetails.autoUpdate === true; + dom.text( + '#listsOfBlockedHostsPrompt', + i18n$('3pListsOfBlockedHostsPrompt') + .replace('{{netFilterCount}}', renderNumber(response.netFilterCount)) + .replace('{{cosmeticFilterCount}}', renderNumber(response.cosmeticFilterCount)) + ); + qs$('#parseCosmeticFilters').checked = + listsetDetails.parseCosmeticFilters === true; + qs$('#ignoreGenericCosmeticFilters').checked = + listsetDetails.ignoreGenericCosmeticFilters === true; + qs$('#suspendUntilListsAreLoaded').checked = + listsetDetails.suspendUntilListsAreLoaded === true; + + // https://github.com/gorhill/uBlock/issues/2394 + dom.cl.toggle(dom.body, 'updating', listsetDetails.isUpdating); + + renderWidgets(); + }; + + return vAPI.messaging.send('dashboard', { + what: 'getLists', + }).then(response => { + onListsReceived(response); + }); +}; + +/******************************************************************************/ + +const renderWidgets = ( ) => { + dom.cl.toggle('#buttonApply', 'disabled', + filteringSettingsHash === hashFromCurrentFromSettings() + ); + const updating = dom.cl.has(dom.body, 'updating'); + dom.cl.toggle('#buttonUpdate', 'active', updating); + dom.cl.toggle('#buttonUpdate', 'disabled', + updating === false && + qs$('#lists .listEntry.checked.obsolete:not(.toRemove)') === null + ); +}; + +/******************************************************************************/ + +const updateAssetStatus = details => { + const listEntry = qs$(`#lists .listEntry[data-key="${details.key}"]`); + if ( listEntry === null ) { return; } + dom.cl.toggle(listEntry, 'failed', !!details.failed); + dom.cl.toggle(listEntry, 'obsolete', !details.cached); + dom.cl.toggle(listEntry, 'cached', !!details.cached); + if ( details.cached ) { + dom.attr(qs$(listEntry, '.status.cache'), 'title', + lastUpdateTemplateString.replace('{{ago}}', i18n.renderElapsedTimeToString(Date.now())) + ); + dom.cl.add(listEntry, 'recent'); + } + updateAncestorListNodes(listEntry, ancestor => { + updateListNode(ancestor); + }); + renderWidgets(); +}; + +/******************************************************************************* + + Compute a hash from all the settings affecting how filter lists are loaded + in memory. + +**/ + +let filteringSettingsHash = ''; + +const hashFromListsetDetails = ( ) => { + const hashParts = [ + listsetDetails.parseCosmeticFilters === true, + listsetDetails.ignoreGenericCosmeticFilters === true, + ]; + const listHashes = []; + for ( const [ listkey, listDetails ] of Object.entries(listsetDetails.available) ) { + if ( listDetails.off === true ) { continue; } + listHashes.push(listkey); + } + hashParts.push( listHashes.sort().join(), '', false); + filteringSettingsHash = hashParts.join(); +}; + +const hashFromCurrentFromSettings = ( ) => { + const hashParts = [ + qs$('#parseCosmeticFilters').checked, + qs$('#ignoreGenericCosmeticFilters').checked, + ]; + const listHashes = []; + const listEntries = qsa$('#lists .listEntry[data-key]:not(.toRemove)'); + for ( const liEntry of listEntries ) { + if ( liEntry.dataset.role !== 'leaf' ) { continue; } + if ( dom.cl.has(liEntry, 'checked') === false ) { continue; } + listHashes.push(liEntry.dataset.key); + } + const textarea = qs$('#lists .listEntry[data-role="import"].expanded textarea'); + hashParts.push( + listHashes.sort().join(), + textarea !== null && textarea.value.trim() || '', + qs$('#lists .listEntry.toRemove') !== null + ); + return hashParts.join(); +}; + +/******************************************************************************/ + +const onListsetChanged = ev => { + const input = ev.target.closest('input'); + if ( input === null ) { return; } + toggleFilterList(input, input.checked, true); +}; + +dom.on('#lists', 'change', '.listEntry > .detailbar input', onListsetChanged); + +const toggleFilterList = (elem, on, ui = false) => { + const listEntry = elem.closest('.listEntry'); + if ( listEntry === null ) { return; } + if ( listEntry.dataset.parent === 'root' ) { return; } + const searchMode = dom.cl.has('#lists', 'searchMode'); + const input = qs$(listEntry, ':scope > .detailbar input'); + if ( on === undefined ) { + on = input.checked === false; + } + input.checked = on; + dom.cl.toggle(listEntry, 'checked', on); + dom.cl.toggle(listEntry, 'stickied', ui && !on && !searchMode); + // Select/unselect descendants. Twist: if in search-mode, select only + // search-matched descendants. + const childListEntries = searchMode + ? qsa$(listEntry, '.listEntry.searchMatch') + : qsa$(listEntry, '.listEntry'); + for ( const descendantList of childListEntries ) { + dom.cl.toggle(descendantList, 'checked', on); + qs$(descendantList, ':scope > .detailbar input').checked = on; + } + updateAncestorListNodes(listEntry, ancestor => { + updateListNode(ancestor); + }); + onFilteringSettingsChanged(); +}; + +const updateListNode = listNode => { + if ( listNode === null ) { return; } + if ( listNode.dataset.role !== 'node' ) { return; } + const checkedListLeaves = qsa$(listNode, '.listEntry[data-role="leaf"].checked'); + const allListLeaves = qsa$(listNode, '.listEntry[data-role="leaf"]'); + dom.text(qs$(listNode, '.nodestats'), + renderNodeStats(checkedListLeaves.length, allListLeaves.length) + ); + dom.cl.toggle(listNode, 'searchMatch', + qs$(listNode, ':scope > .listEntries > .listEntry.searchMatch') !== null + ); + if ( listNode.dataset.parent === 'root' ) { return; } + let usedFilterCount = 0; + let totalFilterCount = 0; + let isCached = false; + let isObsolete = false; + let latestWriteTime = 0; + let oldestWriteTime = Number.MAX_SAFE_INTEGER; + for ( const listLeaf of checkedListLeaves ) { + const listkey = listLeaf.dataset.key; + const listDetails = listsetDetails.available[listkey]; + usedFilterCount += listDetails.off ? 0 : listDetails.entryUsedCount || 0; + totalFilterCount += listDetails.entryCount || 0; + const assetCache = listsetDetails.cache[listkey] || {}; + isCached = isCached || dom.cl.has(listLeaf, 'cached'); + isObsolete = isObsolete || dom.cl.has(listLeaf, 'obsolete'); + latestWriteTime = Math.max(latestWriteTime, assetCache.writeTime || 0); + oldestWriteTime = Math.min(oldestWriteTime, assetCache.writeTime || Number.MAX_SAFE_INTEGER); + } + dom.cl.toggle(listNode, 'checked', checkedListLeaves.length !== 0); + dom.cl.toggle(qs$(listNode, ':scope > .detailbar .checkbox'), + 'partial', + checkedListLeaves.length !== allListLeaves.length + ); + dom.prop(qs$(listNode, ':scope > .detailbar input'), + 'checked', + checkedListLeaves.length !== 0 + ); + dom.text(qs$(listNode, '.leafstats'), + renderLeafStats(usedFilterCount, totalFilterCount) + ); + const firstLeaf = qs$(listNode, '.listEntry[data-role="leaf"]'); + if ( firstLeaf !== null ) { + dom.attr(qs$(listNode, ':scope > .detailbar a.support'), 'href', + dom.attr(qs$(firstLeaf, ':scope > .detailbar a.support'), 'href') || '#' + ); + dom.attr(qs$(listNode, ':scope > .detailbar a.mustread'), 'href', + dom.attr(qs$(firstLeaf, ':scope > .detailbar a.mustread'), 'href') || '#' + ); + } + dom.cl.toggle(listNode, 'cached', isCached); + dom.cl.toggle(listNode, 'obsolete', isObsolete); + if ( isCached ) { + dom.attr(qs$(listNode, ':scope > .detailbar .cache'), 'title', + lastUpdateTemplateString.replace('{{ago}}', i18n.renderElapsedTimeToString(latestWriteTime)) + ); + dom.cl.toggle(listNode, 'recent', (Date.now() - oldestWriteTime) < recentlyUpdated); + } + if ( qs$(listNode, '.listEntry.isDefault') !== null ) { + dom.cl.add(listNode, 'isDefault'); + } + if ( qs$(listNode, '.listEntry.stickied') !== null ) { + dom.cl.add(listNode, 'stickied'); + } +}; + +const updateAncestorListNodes = (listEntry, fn) => { + while ( listEntry !== null ) { + fn(listEntry); + listEntry = qs$(`.listEntry[data-key="${listEntry.dataset.parent}"]`); + } +}; + +/******************************************************************************/ + +const onFilteringSettingsChanged = ( ) => { + renderWidgets(); +}; + +dom.on('#parseCosmeticFilters', 'change', onFilteringSettingsChanged); +dom.on('#ignoreGenericCosmeticFilters', 'change', onFilteringSettingsChanged); +dom.on('#lists', 'input', '[data-role="import"] textarea', onFilteringSettingsChanged); + +/******************************************************************************/ + +const onRemoveExternalList = ev => { + const listEntry = ev.target.closest('[data-key]'); + if ( listEntry === null ) { return; } + dom.cl.toggle(listEntry, 'toRemove'); + renderWidgets(); +}; + +dom.on('#lists', 'click', '.listEntry .remove', onRemoveExternalList); + +/******************************************************************************/ + +const onPurgeClicked = ev => { + const liEntry = ev.target.closest('[data-key]'); + const listkey = liEntry.dataset.key || ''; + if ( listkey === '' ) { return; } + + const assetKeys = [ listkey ]; + for ( const listLeaf of qsa$(liEntry, '[data-role="leaf"]') ) { + assetKeys.push(listLeaf.dataset.key); + dom.cl.add(listLeaf, 'obsolete'); + dom.cl.remove(listLeaf, 'cached'); + } + + vAPI.messaging.send('dashboard', { + what: 'listsUpdateNow', + assetKeys, + preferOrigin: ev.shiftKey, + }); + + // If the cached version is purged, the installed version must be assumed + // to be obsolete. + // https://github.com/gorhill/uBlock/issues/1733 + // An external filter list must not be marked as obsolete, they will + // always be fetched anyways if there is no cached copy. + dom.cl.add(dom.body, 'updating'); + dom.cl.add(liEntry, 'obsolete'); + + if ( qs$(liEntry, 'input[type="checkbox"]').checked ) { + renderWidgets(); + } +}; + +dom.on('#lists', 'click', 'span.cache', onPurgeClicked); + +/******************************************************************************/ + +const selectFilterLists = async ( ) => { + // Cosmetic filtering switch + let checked = qs$('#parseCosmeticFilters').checked; + vAPI.messaging.send('dashboard', { + what: 'userSettings', + name: 'parseAllABPHideFilters', + value: checked, + }); + listsetDetails.parseCosmeticFilters = checked; + + checked = qs$('#ignoreGenericCosmeticFilters').checked; + vAPI.messaging.send('dashboard', { + what: 'userSettings', + name: 'ignoreGenericCosmeticFilters', + value: checked, + }); + listsetDetails.ignoreGenericCosmeticFilters = checked; + + // Filter lists to remove/select + const toSelect = []; + const toRemove = []; + for ( const liEntry of qsa$('#lists .listEntry[data-role="leaf"]') ) { + const listkey = liEntry.dataset.key; + if ( listsetDetails.available.hasOwnProperty(listkey) === false ) { + continue; + } + const listDetails = listsetDetails.available[listkey]; + if ( dom.cl.has(liEntry, 'toRemove') ) { + toRemove.push(listkey); + listDetails.off = true; + continue; + } + if ( dom.cl.has(liEntry, 'checked') ) { + toSelect.push(listkey); + listDetails.off = false; + } else { + listDetails.off = true; + } + } + + // External filter lists to import + const textarea = qs$('#lists .listEntry[data-role="import"].expanded textarea'); + const toImport = textarea !== null && textarea.value.trim() || ''; + if ( textarea !== null ) { + dom.cl.remove(textarea.closest('.expandable'), 'expanded'); + textarea.value = ''; + } + + hashFromListsetDetails(); + + await vAPI.messaging.send('dashboard', { + what: 'applyFilterListSelection', + toSelect, + toImport, + toRemove, + }); +}; + +/******************************************************************************/ + +const buttonApplyHandler = async ( ) => { + await selectFilterLists(); + dom.cl.add(dom.body, 'working'); + dom.cl.remove('#lists .listEntry.stickied', 'stickied'); + renderWidgets(); + await vAPI.messaging.send('dashboard', { what: 'reloadAllFilters' }); + dom.cl.remove(dom.body, 'working'); +}; + +dom.on('#buttonApply', 'click', ( ) => { buttonApplyHandler(); }); + +/******************************************************************************/ + +const buttonUpdateHandler = async ( ) => { + dom.cl.remove('#lists .listEntry.stickied', 'stickied'); + await selectFilterLists(); + dom.cl.add(dom.body, 'updating'); + renderWidgets(); + vAPI.messaging.send('dashboard', { what: 'updateNow' }); +}; + +dom.on('#buttonUpdate', 'click', ( ) => { buttonUpdateHandler(); }); + +/******************************************************************************/ + +const userSettingCheckboxChanged = ( ) => { + const target = event.target; + vAPI.messaging.send('dashboard', { + what: 'userSettings', + name: target.id, + value: target.checked, + }); + listsetDetails[target.id] = target.checked; +}; + +dom.on('#autoUpdate', 'change', userSettingCheckboxChanged); +dom.on('#suspendUntilListsAreLoaded', 'change', userSettingCheckboxChanged); + +/******************************************************************************/ + +const searchFilterLists = ( ) => { + const pattern = dom.prop('.searchbar input', 'value') || ''; + dom.cl.toggle('#lists', 'searchMode', pattern !== ''); + if ( pattern === '' ) { return; } + const reflectSearchMatches = listEntry => { + if ( listEntry.dataset.role !== 'node' ) { return; } + dom.cl.toggle(listEntry, 'searchMatch', + qs$(listEntry, ':scope > .listEntries > .listEntry.searchMatch') !== null + ); + }; + const toI18n = tags => { + if ( tags === '' ) { return ''; } + return tags.toLowerCase().split(/\s+/).reduce((a, v) => { + let s = i18n$(v); + if ( s === '' ) { + s = i18nGroupName(v); + if ( s === '' ) { return a; } + } + return `${a} ${s}`.trim(); + }, ''); + }; + const re = new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'i'); + for ( const listEntry of qsa$('#lists [data-role="leaf"]') ) { + const listkey = listEntry.dataset.key; + const listDetails = listsetDetails.available[listkey]; + if ( listDetails === undefined ) { continue; } + let haystack = perListHaystack.get(listDetails); + if ( haystack === undefined ) { + haystack = [ + listDetails.title, + listDetails.group || '', + i18nGroupName(listDetails.group || ''), + listDetails.tags || '', + toI18n(listDetails.tags || ''), + ].join(' ').trim(); + perListHaystack.set(listDetails, haystack); + } + dom.cl.toggle(listEntry, 'searchMatch', re.test(haystack)); + updateAncestorListNodes(listEntry, reflectSearchMatches); + } +}; + +const perListHaystack = new WeakMap(); + +dom.on('.searchbar input', 'input', searchFilterLists); + +/******************************************************************************/ + +const expandedListSet = new Set([ + 'uBlock filters', + 'AdGuard – Annoyances', + 'EasyList – Annoyances', +]); + +const listIsExpanded = which => { + return expandedListSet.has(which); +}; + +const applyListExpansion = listkeys => { + if ( listkeys === undefined ) { + listkeys = Array.from(expandedListSet); + } + expandedListSet.clear(); + dom.cl.remove('#lists [data-role="node"]', 'expanded'); + listkeys.forEach(which => { + expandedListSet.add(which); + dom.cl.add(`#lists [data-key="${which}"]`, 'expanded'); + }); +}; + +const toggleListExpansion = which => { + const isExpanded = expandedListSet.has(which); + if ( which === '*' ) { + if ( isExpanded ) { + expandedListSet.clear(); + dom.cl.remove('#lists .expandable', 'expanded'); + dom.cl.remove('#lists .stickied', 'stickied'); + } else { + expandedListSet.clear(); + expandedListSet.add('*'); + dom.cl.add('#lists .rootstats', 'expanded'); + for ( const expandable of qsa$('#lists > .listEntries .expandable') ) { + const listkey = expandable.dataset.key || ''; + if ( listkey === '' ) { continue; } + expandedListSet.add(listkey); + dom.cl.add(expandable, 'expanded'); + } + } + } else { + if ( isExpanded ) { + expandedListSet.delete(which); + const listNode = qs$(`#lists > .listEntries [data-key="${which}"]`); + dom.cl.remove(listNode, 'expanded'); + if ( listNode.dataset.parent === 'root' ) { + dom.cl.remove(qsa$(listNode, '.stickied'), 'stickied'); + } + } else { + expandedListSet.add(which); + dom.cl.add(`#lists > .listEntries [data-key="${which}"]`, 'expanded'); + } + } + vAPI.localStorage.setItem('expandedListSet', Array.from(expandedListSet)); + vAPI.localStorage.removeItem('hideUnusedFilterLists'); +}; + +dom.on('#listsOfBlockedHostsPrompt', 'click', ( ) => { + toggleListExpansion('*'); +}); + +dom.on('#lists', 'click', '.listExpander', ev => { + const expandable = ev.target.closest('.expandable'); + if ( expandable === null ) { return; } + const which = expandable.dataset.key; + if ( which !== undefined ) { + toggleListExpansion(which); + } else { + dom.cl.toggle(expandable, 'expanded'); + if ( expandable.dataset.role === 'import' ) { + onFilteringSettingsChanged(); + } + } + ev.preventDefault(); +}); + +dom.on('#lists', 'click', '[data-parent="root"] > .detailbar .listname', ev => { + const listEntry = ev.target.closest('.listEntry'); + if ( listEntry === null ) { return; } + const listkey = listEntry.dataset.key; + if ( listkey === undefined ) { return; } + toggleListExpansion(listkey); + ev.preventDefault(); +}); + +dom.on('#lists', 'click', '[data-role="import"] > .detailbar .listname', ev => { + const expandable = ev.target.closest('.listEntry'); + if ( expandable === null ) { return; } + dom.cl.toggle(expandable, 'expanded'); + ev.preventDefault(); +}); + +dom.on('#lists', 'click', '.listEntry > .detailbar .nodestats', ev => { + const listEntry = ev.target.closest('.listEntry'); + if ( listEntry === null ) { return; } + const listkey = listEntry.dataset.key; + if ( listkey === undefined ) { return; } + toggleListExpansion(listkey); + ev.preventDefault(); +}); + +// Initialize from saved state. +vAPI.localStorage.getItemAsync('expandedListSet').then(listkeys => { + if ( Array.isArray(listkeys) === false ) { return; } + applyListExpansion(listkeys); +}); + +/******************************************************************************/ + +// Cloud storage-related. + +self.cloud.onPush = function toCloudData() { + const bin = { + parseCosmeticFilters: qs$('#parseCosmeticFilters').checked, + ignoreGenericCosmeticFilters: qs$('#ignoreGenericCosmeticFilters').checked, + selectedLists: [] + }; + + const liEntries = qsa$('#lists .listEntry.checked[data-role="leaf"]'); + for ( const liEntry of liEntries ) { + bin.selectedLists.push(liEntry.dataset.key); + } + + return bin; +}; + +self.cloud.onPull = function fromCloudData(data, append) { + if ( typeof data !== 'object' || data === null ) { return; } + + let elem = qs$('#parseCosmeticFilters'); + let checked = data.parseCosmeticFilters === true || append && elem.checked; + elem.checked = listsetDetails.parseCosmeticFilters = checked; + + elem = qs$('#ignoreGenericCosmeticFilters'); + checked = data.ignoreGenericCosmeticFilters === true || append && elem.checked; + elem.checked = listsetDetails.ignoreGenericCosmeticFilters = checked; + + const selectedSet = new Set(data.selectedLists); + for ( const listEntry of qsa$('#lists .listEntry[data-role="leaf"]') ) { + const listkey = listEntry.dataset.key; + const mustEnable = selectedSet.has(listkey); + selectedSet.delete(listkey); + if ( mustEnable === false && append ) { continue; } + toggleFilterList(listEntry, mustEnable); + } + + // If there are URL-like list keys left in the selected set, import them. + for ( const listkey of selectedSet ) { + if ( reValidExternalList.test(listkey) ) { continue; } + selectedSet.delete(listkey); + } + if ( selectedSet.size !== 0 ) { + const textarea = qs$('#lists .listEntry[data-role="import"] textarea'); + const lines = append + ? textarea.value.split(/[\n\r]+/) + : []; + lines.push(...selectedSet); + if ( lines.length !== 0 ) { lines.push(''); } + textarea.value = lines.join('\n'); + dom.cl.toggle('#lists .listEntry[data-role="import"]', 'expanded', textarea.value !== ''); + } + + renderWidgets(); +}; + +/******************************************************************************/ + +self.hasUnsavedData = function() { + return hashFromCurrentFromSettings() !== filteringSettingsHash; +}; + +/******************************************************************************/ + +renderFilterLists().then(( ) => { + const buttonUpdate = qs$('#buttonUpdate'); + if ( dom.cl.has(buttonUpdate, 'active') ) { return; } + if ( dom.cl.has(buttonUpdate, 'disabled') ) { return; } + if ( listsetDetails.autoUpdate !== true ) { return; } + buttonUpdateHandler(); +}); + +/******************************************************************************/ diff --git a/src/js/about.js b/src/js/about.js new file mode 100644 index 0000000..680fab1 --- /dev/null +++ b/src/js/about.js @@ -0,0 +1,34 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { dom } from './dom.js'; + +/******************************************************************************/ + +(async ( ) => { + const appData = await vAPI.messaging.send('dashboard', { + what: 'getAppData', + }); + + dom.text('#aboutNameVer', appData.name + ' ' + appData.version); +})(); diff --git a/src/js/advanced-settings.js b/src/js/advanced-settings.js new file mode 100644 index 0000000..c21346f --- /dev/null +++ b/src/js/advanced-settings.js @@ -0,0 +1,194 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2016-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uBlockDashboard */ + +'use strict'; + +import { dom, qs$ } from './dom.js'; + +/******************************************************************************/ + +let defaultSettings = new Map(); +let adminSettings = new Map(); +let beforeHash = ''; + +/******************************************************************************/ + +CodeMirror.defineMode('raw-settings', function() { + let lastSetting = ''; + + return { + token: function(stream) { + if ( stream.sol() ) { + stream.eatSpace(); + const match = stream.match(/\S+/); + if ( match !== null && defaultSettings.has(match[0]) ) { + lastSetting = match[0]; + return adminSettings.has(match[0]) + ? 'readonly keyword' + : 'keyword'; + } + stream.skipToEnd(); + return 'line-cm-error'; + } + stream.eatSpace(); + const match = stream.match(/.*$/); + if ( match !== null ) { + if ( match[0].trim() !== defaultSettings.get(lastSetting) ) { + return 'line-cm-strong'; + } + if ( adminSettings.has(lastSetting) ) { + return 'readonly'; + } + } + stream.skipToEnd(); + return null; + } + }; +}); + +const cmEditor = new CodeMirror(qs$('#advancedSettings'), { + autofocus: true, + lineNumbers: true, + lineWrapping: false, + styleActiveLine: true +}); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +/******************************************************************************/ + +const hashFromAdvancedSettings = function(raw) { + const aa = typeof raw === 'string' + ? arrayFromString(raw) + : arrayFromObject(raw); + aa.sort((a, b) => a[0].localeCompare(b[0])); + return JSON.stringify(aa); +}; + +/******************************************************************************/ + +const arrayFromObject = function(o) { + const out = []; + for ( const k in o ) { + if ( o.hasOwnProperty(k) === false ) { continue; } + out.push([ k, `${o[k]}` ]); + } + return out; +}; + +const arrayFromString = function(s) { + const out = []; + for ( let line of s.split(/[\n\r]+/) ) { + line = line.trim(); + if ( line === '' ) { continue; } + const pos = line.indexOf(' '); + let k, v; + if ( pos !== -1 ) { + k = line.slice(0, pos); + v = line.slice(pos + 1); + } else { + k = line; + v = ''; + } + out.push([ k.trim(), v.trim() ]); + } + return out; +}; + +/******************************************************************************/ + +const advancedSettingsChanged = (( ) => { + const handler = ( ) => { + const changed = hashFromAdvancedSettings(cmEditor.getValue()) !== beforeHash; + qs$('#advancedSettingsApply').disabled = !changed; + CodeMirror.commands.save = changed ? applyChanges : function(){}; + }; + + const timer = vAPI.defer.create(handler); + + return function() { + timer.offon(200); + }; +})(); + +cmEditor.on('changes', advancedSettingsChanged); + +/******************************************************************************/ + +const renderAdvancedSettings = async function(first) { + const details = await vAPI.messaging.send('dashboard', { + what: 'readHiddenSettings', + }); + defaultSettings = new Map(arrayFromObject(details.default)); + adminSettings = new Map(arrayFromObject(details.admin)); + beforeHash = hashFromAdvancedSettings(details.current); + const pretty = []; + const roLines = []; + const entries = arrayFromObject(details.current); + let max = 0; + for ( const [ k ] of entries ) { + if ( k.length > max ) { max = k.length; } + } + for ( let i = 0; i < entries.length; i++ ) { + const [ k, v ] = entries[i]; + pretty.push(' '.repeat(max - k.length) + `${k} ${v}`); + if ( adminSettings.has(k) ) { + roLines.push(i); + } + } + pretty.push(''); + cmEditor.setValue(pretty.join('\n')); + if ( first ) { + cmEditor.clearHistory(); + } + for ( const line of roLines ) { + cmEditor.markText( + { line, ch: 0 }, + { line: line + 1, ch: 0 }, + { readOnly: true } + ); + } + advancedSettingsChanged(); + cmEditor.focus(); +}; + +/******************************************************************************/ + +const applyChanges = async function() { + await vAPI.messaging.send('dashboard', { + what: 'writeHiddenSettings', + content: cmEditor.getValue(), + }); + renderAdvancedSettings(); +}; + +/******************************************************************************/ + +dom.on('#advancedSettings', 'input', advancedSettingsChanged); +dom.on('#advancedSettingsApply', 'click', ( ) => { + applyChanges(); +}); + +renderAdvancedSettings(true); + +/******************************************************************************/ diff --git a/src/js/asset-viewer.js b/src/js/asset-viewer.js new file mode 100644 index 0000000..eabe136 --- /dev/null +++ b/src/js/asset-viewer.js @@ -0,0 +1,112 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uBlockDashboard */ + +'use strict'; + +/******************************************************************************/ + +import { dom, qs$ } from './dom.js'; +import './codemirror/ubo-static-filtering.js'; + +/******************************************************************************/ + +(async ( ) => { + const subscribeURL = new URL(document.location); + const subscribeParams = subscribeURL.searchParams; + const assetKey = subscribeParams.get('url'); + if ( assetKey === null ) { return; } + + const subscribeElem = subscribeParams.get('subscribe') !== null + ? qs$('#subscribe') + : null; + if ( subscribeElem !== null && subscribeURL.hash !== '#subscribed' ) { + const title = subscribeParams.get('title'); + const promptElem = qs$('#subscribePrompt'); + dom.text(promptElem.children[0], title); + const a = promptElem.children[1]; + dom.text(a, assetKey); + dom.attr(a, 'href', assetKey); + dom.cl.remove(subscribeElem, 'hide'); + } + + const cmEditor = new CodeMirror(qs$('#content'), { + autofocus: true, + foldGutter: true, + gutters: [ + 'CodeMirror-linenumbers', + { className: 'CodeMirror-lintgutter', style: 'width: 11px' }, + ], + lineNumbers: true, + lineWrapping: true, + matchBrackets: true, + maxScanLines: 1, + readOnly: true, + styleActiveLine: { + nonEmpty: true, + }, + }); + + uBlockDashboard.patchCodeMirrorEditor(cmEditor); + + vAPI.messaging.send('dashboard', { + what: 'getAutoCompleteDetails' + }).then(hints => { + if ( hints instanceof Object === false ) { return; } + cmEditor.setOption('uboHints', hints); + }); + + vAPI.messaging.send('dashboard', { + what: 'getTrustedScriptletTokens', + }).then(tokens => { + cmEditor.setOption('trustedScriptletTokens', tokens); + }); + + const details = await vAPI.messaging.send('default', { + what : 'getAssetContent', + url: assetKey, + }); + cmEditor.setOption('trustedSource', details.trustedSource === true); + cmEditor.setValue(details && details.content || ''); + + if ( subscribeElem !== null ) { + dom.on('#subscribeButton', 'click', ( ) => { + dom.cl.add(subscribeElem, 'hide'); + vAPI.messaging.send('scriptlets', { + what: 'applyFilterListSelection', + toImport: assetKey, + }).then(( ) => { + vAPI.messaging.send('scriptlets', { + what: 'reloadAllFilters' + }); + }); + }, { once: true }); + } + + if ( details.sourceURL ) { + const a = qs$('.cm-search-widget .sourceURL'); + dom.attr(a, 'href', details.sourceURL); + dom.attr(a, 'title', details.sourceURL); + } + + dom.cl.remove(dom.body, 'loading'); +})(); diff --git a/src/js/assets.js b/src/js/assets.js new file mode 100644 index 0000000..69c2ef3 --- /dev/null +++ b/src/js/assets.js @@ -0,0 +1,1478 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; +import { broadcast } from './broadcast.js'; +import cacheStorage from './cachestorage.js'; +import { ubolog } from './console.js'; +import { i18n$ } from './i18n.js'; +import logger from './logger.js'; +import * as sfp from './static-filtering-parser.js'; +import { orphanizeString, } from './text-utils.js'; + +/******************************************************************************/ + +const reIsExternalPath = /^(?:[a-z-]+):\/\//; +const reIsUserAsset = /^user-/; +const errorCantConnectTo = i18n$('errorCantConnectTo'); +const MS_PER_HOUR = 60 * 60 * 1000; +const MS_PER_DAY = 24 * MS_PER_HOUR; +const MINUTES_PER_DAY = 24 * 60; +const EXPIRES_DEFAULT = 7; + +const assets = {}; + +// A hint for various pieces of code to take measures if possible to save +// bandwidth of remote servers. +let remoteServerFriendly = false; + +/******************************************************************************/ + +const stringIsNotEmpty = s => typeof s === 'string' && s !== ''; + +const parseExpires = s => { + const matches = s.match(/(\d+)\s*([dhm]?)/i); + if ( matches === null ) { return; } + let updateAfter = parseInt(matches[1], 10); + if ( matches[2] === 'h' ) { + updateAfter = Math.max(updateAfter, 4) / 24; + } else if ( matches[2] === 'm' ) { + updateAfter = Math.max(updateAfter, 240) / 1440; + } + return updateAfter; +}; + +const extractMetadataFromList = (content, fields) => { + const out = {}; + const head = content.slice(0, 1024); + for ( let field of fields ) { + field = field.replace(/\s+/g, '-'); + const re = new RegExp(`^(?:! *|# +)${field.replace(/-/g, '(?: +|-)')}: *(.+)$`, 'im'); + const match = re.exec(head); + let value = match && match[1].trim() || undefined; + if ( value !== undefined && value.startsWith('%') ) { + value = undefined; + } + field = field.toLowerCase().replace( + /-[a-z]/g, s => s.charAt(1).toUpperCase() + ); + out[field] = value && orphanizeString(value); + } + // Pre-process known fields + if ( out.lastModified ) { + out.lastModified = (new Date(out.lastModified)).getTime() || 0; + } + if ( out.expires ) { + out.expires = parseExpires(out.expires); + } + if ( out.diffExpires ) { + out.diffExpires = parseExpires(out.diffExpires); + } + return out; +}; +assets.extractMetadataFromList = extractMetadataFromList; + +const resourceTimeFromXhr = xhr => { + if ( typeof xhr.response !== 'string' ) { return 0; } + const metadata = extractMetadataFromList(xhr.response, [ + 'Last-Modified' + ]); + return metadata.lastModified || 0; +}; + +const resourceTimeFromParts = (parts, time) => { + const goodParts = parts.filter(part => typeof part === 'object'); + return goodParts.reduce((acc, part) => + ((part.resourceTime || 0) > acc ? part.resourceTime : acc), + time + ); +}; + +const resourceIsStale = (networkDetails, cacheDetails) => { + if ( typeof networkDetails.resourceTime !== 'number' ) { return false; } + if ( networkDetails.resourceTime === 0 ) { return false; } + if ( typeof cacheDetails.resourceTime !== 'number' ) { return false; } + if ( cacheDetails.resourceTime === 0 ) { return false; } + if ( networkDetails.resourceTime < cacheDetails.resourceTime ) { + ubolog(`Skip ${networkDetails.url}\n\tolder than ${cacheDetails.remoteURL}`); + return true; + } + return false; +}; + +const getUpdateAfterTime = (assetKey, diff = false) => { + const entry = assetCacheRegistry[assetKey]; + if ( entry ) { + if ( diff && typeof entry.diffExpires === 'number' ) { + return entry.diffExpires * MS_PER_DAY; + } + if ( typeof entry.expires === 'number' ) { + return entry.expires * MS_PER_DAY; + } + } + if ( assetSourceRegistry ) { + const entry = assetSourceRegistry[assetKey]; + if ( entry && typeof entry.updateAfter === 'number' ) { + return entry.updateAfter * MS_PER_DAY; + } + } + return EXPIRES_DEFAULT * MS_PER_DAY; // default to 7-day +}; + +const getWriteTime = assetKey => { + const entry = assetCacheRegistry[assetKey]; + if ( entry ) { return entry.writeTime || 0; } + return 0; +}; + +const isDiffUpdatableAsset = content => { + if ( typeof content !== 'string' ) { return false; } + const data = extractMetadataFromList(content, [ + 'Diff-Path', + ]); + return typeof data.diffPath === 'string' && + data.diffPath.startsWith('%') === false; +}; + +const computedPatchUpdateTime = assetKey => { + const entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { return 0; } + if ( typeof entry.diffPath !== 'string' ) { return 0; } + if ( typeof entry.diffExpires !== 'number' ) { return 0; } + const match = /(\d+)\.(\d+)\.(\d+)\.(\d+)/.exec(entry.diffPath); + if ( match === null ) { return getWriteTime(); } + const date = new Date(); + date.setUTCFullYear( + parseInt(match[1], 10), + parseInt(match[2], 10) - 1, + parseInt(match[3], 10) + ); + date.setUTCHours(0, parseInt(match[4], 10) + entry.diffExpires * MINUTES_PER_DAY, 0, 0); + return date.getTime(); +}; + +/******************************************************************************/ + +// favorLocal: avoid making network requests whenever possible +// favorOrigin: avoid using CDN URLs whenever possible + +const getContentURLs = (assetKey, options = {}) => { + const contentURLs = []; + const entry = assetSourceRegistry[assetKey]; + if ( entry instanceof Object === false ) { return contentURLs; } + if ( typeof entry.contentURL === 'string' ) { + contentURLs.push(entry.contentURL); + } else if ( Array.isArray(entry.contentURL) ) { + contentURLs.push(...entry.contentURL); + } else if ( reIsExternalPath.test(assetKey) ) { + contentURLs.push(assetKey); + } + if ( options.favorLocal ) { + contentURLs.sort((a, b) => { + if ( reIsExternalPath.test(a) ) { return 1; } + if ( reIsExternalPath.test(b) ) { return -1; } + return 0; + }); + } + if ( options.favorOrigin !== true && Array.isArray(entry.cdnURLs) ) { + const cdnURLs = entry.cdnURLs.slice(); + for ( let i = 0, n = cdnURLs.length; i < n; i++ ) { + const j = Math.floor(Math.random() * n); + if ( j === i ) { continue; } + [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ]; + } + if ( options.favorLocal ) { + contentURLs.push(...cdnURLs); + } else { + contentURLs.unshift(...cdnURLs); + } + } + return contentURLs; +}; + +/******************************************************************************/ + +const observers = []; + +assets.addObserver = function(observer) { + if ( observers.indexOf(observer) === -1 ) { + observers.push(observer); + } +}; + +assets.removeObserver = function(observer) { + let pos; + while ( (pos = observers.indexOf(observer)) !== -1 ) { + observers.splice(pos, 1); + } +}; + +const fireNotification = function(topic, details) { + let result; + for ( const observer of observers ) { + const r = observer(topic, details); + if ( r !== undefined ) { result = r; } + } + return result; +}; + +/******************************************************************************/ + +assets.fetch = function(url, options = {}) { + return new Promise((resolve, reject) => { + // Start of executor + + const timeoutAfter = µb.hiddenSettings.assetFetchTimeout || 30; + const xhr = new XMLHttpRequest(); + let contentLoaded = 0; + + const cleanup = function() { + xhr.removeEventListener('load', onLoadEvent); + xhr.removeEventListener('error', onErrorEvent); + xhr.removeEventListener('abort', onErrorEvent); + xhr.removeEventListener('progress', onProgressEvent); + timeoutTimer.off(); + }; + + const fail = function(details, msg) { + logger.writeOne({ + realm: 'message', + type: 'error', + text: msg, + }); + details.content = ''; + details.error = msg; + reject(details); + }; + + // https://github.com/gorhill/uMatrix/issues/15 + const onLoadEvent = function() { + cleanup(); + // xhr for local files gives status 0, but actually succeeds + const details = { + url, + statusCode: this.status || 200, + statusText: this.statusText || '' + }; + if ( details.statusCode < 200 || details.statusCode >= 300 ) { + return fail(details, `${url}: ${details.statusCode} ${details.statusText}`); + } + details.content = this.response; + details.resourceTime = resourceTimeFromXhr(this); + resolve(details); + }; + + const onErrorEvent = function() { + cleanup(); + fail({ url }, errorCantConnectTo.replace('{{msg}}', url)); + }; + + const onTimeout = function() { + xhr.abort(); + }; + + // https://github.com/gorhill/uBlock/issues/2526 + // - Timeout only when there is no progress. + const onProgressEvent = function(ev) { + if ( ev.loaded === contentLoaded ) { return; } + contentLoaded = ev.loaded; + timeoutTimer.offon({ sec: timeoutAfter }); + }; + + const timeoutTimer = vAPI.defer.create(onTimeout); + + // Be ready for thrown exceptions: + // I am pretty sure it used to work, but now using a URL such as + // `file:///` on Chromium 40 results in an exception being thrown. + try { + xhr.open('get', url, true); + xhr.addEventListener('load', onLoadEvent); + xhr.addEventListener('error', onErrorEvent); + xhr.addEventListener('abort', onErrorEvent); + xhr.addEventListener('progress', onProgressEvent); + xhr.responseType = options.responseType || 'text'; + xhr.send(); + timeoutTimer.on({ sec: timeoutAfter }); + } catch (e) { + onErrorEvent.call(xhr); + } + + // End of executor + }); +}; + +/******************************************************************************/ + +assets.fetchText = async function(url) { + const isExternal = reIsExternalPath.test(url); + let actualUrl = isExternal ? url : vAPI.getURL(url); + + // https://github.com/gorhill/uBlock/issues/2592 + // Force browser cache to be bypassed, but only for resources which have + // been fetched more than one hour ago. + // https://github.com/uBlockOrigin/uBlock-issues/issues/682#issuecomment-515197130 + // Provide filter list authors a way to completely bypass + // the browser cache. + // https://github.com/gorhill/uBlock/commit/048bfd251c9b#r37972005 + // Use modulo prime numbers to avoid generating the same token at the + // same time across different days. + // Do not bypass browser cache if we are asked to be gentle on remote + // servers. + if ( isExternal && remoteServerFriendly !== true ) { + const cacheBypassToken = + µb.hiddenSettings.updateAssetBypassBrowserCache + ? Math.floor(Date.now() / 1000) % 86413 + : Math.floor(Date.now() / 3600000) % 13; + const queryValue = `_=${cacheBypassToken}`; + if ( actualUrl.indexOf('?') === -1 ) { + actualUrl += '?'; + } else { + actualUrl += '&'; + } + actualUrl += queryValue; + } + + let details = { content: '' }; + try { + details = await assets.fetch(actualUrl); + + // Consider an empty result to be an error + if ( stringIsNotEmpty(details.content) === false ) { + details.content = ''; + } + + // We never download anything else than plain text: discard if + // response appears to be a HTML document: could happen when server + // serves some kind of error page for example. + const text = details.content.trim(); + if ( text.startsWith('<') && text.endsWith('>') ) { + details.content = ''; + details.error = 'assets.fetchText(): Not a text file'; + } + } catch(ex) { + details = ex; + } + + // We want to return the caller's URL, not our internal one which may + // differ from the caller's one. + details.url = url; + + return details; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/3331 +// Support the seamless loading of sublists. + +assets.fetchFilterList = async function(mainlistURL) { + const toParsedURL = url => { + try { + return new URL(url.trim()); + } catch (ex) { + } + }; + + // https://github.com/NanoAdblocker/NanoCore/issues/239 + // Anything under URL's root directory is allowed to be fetched. The + // URL of a sublist will always be relative to the URL of the parent + // list (instead of the URL of the root list). + let rootDirectoryURL = toParsedURL( + reIsExternalPath.test(mainlistURL) + ? mainlistURL + : vAPI.getURL(mainlistURL) + ); + if ( rootDirectoryURL !== undefined ) { + const pos = rootDirectoryURL.pathname.lastIndexOf('/'); + if ( pos !== -1 ) { + rootDirectoryURL.pathname = + rootDirectoryURL.pathname.slice(0, pos + 1); + } else { + rootDirectoryURL = undefined; + } + } + + const sublistURLs = new Set(); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1113 + // Process only `!#include` directives which are not excluded by an + // `!#if` directive. + const processIncludeDirectives = function(results) { + const out = []; + const reInclude = /^!#include +(\S+)[^\n\r]*(?:[\n\r]+|$)/gm; + for ( const result of results ) { + if ( typeof result === 'string' ) { + out.push(result); + continue; + } + if ( result instanceof Object === false ) { continue; } + const content = result.content; + const slices = sfp.utils.preparser.splitter( + content, + vAPI.webextFlavor.env + ); + for ( let i = 0, n = slices.length - 1; i < n; i++ ) { + const slice = content.slice(slices[i+0], slices[i+1]); + if ( (i & 1) !== 0 ) { + out.push(slice); + continue; + } + let lastIndex = 0; + for (;;) { + if ( rootDirectoryURL === undefined ) { break; } + const match = reInclude.exec(slice); + if ( match === null ) { break; } + if ( toParsedURL(match[1]) !== undefined ) { continue; } + if ( match[1].indexOf('..') !== -1 ) { continue; } + // Compute nested list path relative to parent list path + const pos = result.url.lastIndexOf('/'); + if ( pos === -1 ) { continue; } + const subURL = result.url.slice(0, pos + 1) + match[1].trim(); + if ( sublistURLs.has(subURL) ) { continue; } + sublistURLs.add(subURL); + out.push( + slice.slice(lastIndex, match.index + match[0].length), + `! >>>>>>>> ${subURL}\n`, + assets.fetchText(subURL), + `! <<<<<<<< ${subURL}\n` + ); + lastIndex = reInclude.lastIndex; + } + out.push(lastIndex === 0 ? slice : slice.slice(lastIndex)); + } + } + return out; + }; + + // https://github.com/AdguardTeam/FiltersRegistry/issues/82 + // Not checking for `errored` status was causing repeated notifications + // to the caller. This can happen when more than one out of multiple + // sublists can't be fetched. + + let allParts = [ + this.fetchText(mainlistURL) + ]; + // Abort processing `include` directives if at least one included sublist + // can't be fetched. + let resourceTime = 0; + do { + allParts = await Promise.all(allParts); + const part = allParts + .find(part => typeof part === 'object' && part.error !== undefined); + if ( part !== undefined ) { + return { url: mainlistURL, content: '', error: part.error }; + } + resourceTime = resourceTimeFromParts(allParts, resourceTime); + // Skip pre-parser directives for diff-updatable assets + if ( allParts.length === 1 && allParts[0] instanceof Object ) { + if ( isDiffUpdatableAsset(allParts[0].content) ) { + allParts[0] = allParts[0].content; + break; + } + } + allParts = processIncludeDirectives(allParts); + } while ( allParts.some(part => typeof part !== 'string') ); + // If we reach this point, this means all fetches were successful. + return { + url: mainlistURL, + resourceTime, + content: allParts.length === 1 + ? allParts[0] + : allParts.join('') + '\n' + }; +}; + +/******************************************************************************* + + The purpose of the asset source registry is to keep key detail information + about an asset: + - Where to load it from: this may consist of one or more URLs, either local + or remote. + - After how many days an asset should be deemed obsolete -- i.e. in need of + an update. + - The origin and type of an asset. + - The last time an asset was registered. + +**/ + +let assetSourceRegistryPromise; +let assetSourceRegistry = Object.create(null); + +function getAssetSourceRegistry() { + if ( assetSourceRegistryPromise === undefined ) { + assetSourceRegistryPromise = cacheStorage.get( + 'assetSourceRegistry' + ).then(bin => { + if ( + bin instanceof Object && + bin.assetSourceRegistry instanceof Object + ) { + assetSourceRegistry = bin.assetSourceRegistry; + return assetSourceRegistry; + } + return assets.fetchText( + µb.assetsBootstrapLocation || µb.assetsJsonPath + ).then(details => { + return details.content !== '' + ? details + : assets.fetchText(µb.assetsJsonPath); + }).then(details => { + updateAssetSourceRegistry(details.content, true); + return assetSourceRegistry; + }); + }); + } + + return assetSourceRegistryPromise; +} + +function registerAssetSource(assetKey, newDict) { + const currentDict = assetSourceRegistry[assetKey] || {}; + for ( const [ k, v ] of Object.entries(newDict) ) { + if ( v === undefined || v === null ) { + delete currentDict[k]; + } else { + currentDict[k] = newDict[k]; + } + } + let contentURL = newDict.contentURL; + if ( contentURL !== undefined ) { + if ( typeof contentURL === 'string' ) { + contentURL = currentDict.contentURL = [ contentURL ]; + } else if ( Array.isArray(contentURL) === false ) { + contentURL = currentDict.contentURL = []; + } + let remoteURLCount = 0; + for ( let i = 0; i < contentURL.length; i++ ) { + if ( reIsExternalPath.test(contentURL[i]) ) { + remoteURLCount += 1; + } + } + currentDict.hasLocalURL = remoteURLCount !== contentURL.length; + currentDict.hasRemoteURL = remoteURLCount !== 0; + } else if ( currentDict.contentURL === undefined ) { + currentDict.contentURL = []; + } + if ( currentDict.submitter ) { + currentDict.submitTime = Date.now(); // To detect stale entries + } + assetSourceRegistry[assetKey] = currentDict; +} + +function unregisterAssetSource(assetKey) { + assetCacheRemove(assetKey); + delete assetSourceRegistry[assetKey]; +} + +const saveAssetSourceRegistry = (( ) => { + const save = ( ) => { + timer.off(); + cacheStorage.set({ assetSourceRegistry }); + }; + const timer = vAPI.defer.create(save); + return function(lazily) { + if ( lazily ) { + timer.offon(500); + } else { + save(); + } + }; +})(); + +async function assetSourceGetDetails(assetKey) { + await getAssetSourceRegistry(); + const entry = assetSourceRegistry[assetKey]; + if ( entry === undefined ) { return; } + return entry; +} + +function updateAssetSourceRegistry(json, silent = false) { + let newDict; + try { + newDict = JSON.parse(json); + newDict['assets.json'].defaultListset = + Array.from(Object.entries(newDict)) + .filter(a => a[1].content === 'filters' && a[1].off === undefined) + .map(a => a[0]); + } catch (ex) { + } + if ( newDict instanceof Object === false ) { return; } + + const oldDict = assetSourceRegistry; + + fireNotification('assets.json-updated', { newDict, oldDict }); + + // Remove obsolete entries (only those which were built-in). + for ( const assetKey in oldDict ) { + if ( + newDict[assetKey] === undefined && + oldDict[assetKey].submitter === undefined + ) { + unregisterAssetSource(assetKey); + } + } + // Add/update existing entries. Notify of new asset sources. + for ( const assetKey in newDict ) { + if ( oldDict[assetKey] === undefined && !silent ) { + fireNotification( + 'builtin-asset-source-added', + { assetKey: assetKey, entry: newDict[assetKey] } + ); + } + registerAssetSource(assetKey, newDict[assetKey]); + } + saveAssetSourceRegistry(); +} + +assets.registerAssetSource = async function(assetKey, details) { + await getAssetSourceRegistry(); + registerAssetSource(assetKey, details); + saveAssetSourceRegistry(true); +}; + +assets.unregisterAssetSource = async function(assetKey) { + await getAssetSourceRegistry(); + unregisterAssetSource(assetKey); + saveAssetSourceRegistry(true); +}; + +/******************************************************************************* + + The purpose of the asset cache registry is to keep track of all assets + which have been persisted into the local cache. + +**/ + +const assetCacheRegistryStartTime = Date.now(); +let assetCacheRegistryPromise; +let assetCacheRegistry = {}; + +function getAssetCacheRegistry() { + if ( assetCacheRegistryPromise === undefined ) { + assetCacheRegistryPromise = cacheStorage.get( + 'assetCacheRegistry' + ).then(bin => { + if ( + bin instanceof Object && + bin.assetCacheRegistry instanceof Object + ) { + if ( Object.keys(assetCacheRegistry).length === 0 ) { + assetCacheRegistry = bin.assetCacheRegistry; + } else { + console.error( + 'getAssetCacheRegistry(): assetCacheRegistry reassigned!' + ); + if ( + Object.keys(bin.assetCacheRegistry).sort().join() !== + Object.keys(assetCacheRegistry).sort().join() + ) { + console.error( + 'getAssetCacheRegistry(): assetCacheRegistry changes overwritten!' + ); + } + } + } + return assetCacheRegistry; + }); + } + + return assetCacheRegistryPromise; +} + +const saveAssetCacheRegistry = (( ) => { + const save = function() { + timer.off(); + cacheStorage.set({ assetCacheRegistry }); + }; + const timer = vAPI.defer.create(save); + return function(lazily) { + if ( lazily ) { + timer.offon({ sec: 30 }); + } else { + save(); + } + }; +})(); + +async function assetCacheRead(assetKey, updateReadTime = false) { + const t0 = Date.now(); + const internalKey = `cache/${assetKey}`; + + const reportBack = function(content) { + if ( content instanceof Blob ) { content = ''; } + const details = { assetKey, content }; + if ( content === '' ) { details.error = 'ENOTFOUND'; } + return details; + }; + + const [ , bin ] = await Promise.all([ + getAssetCacheRegistry(), + cacheStorage.get(internalKey), + ]); + + if ( µb.readyToFilter !== true ) { + µb.supportStats.maxAssetCacheWait = Math.max( + Date.now() - t0, + parseInt(µb.supportStats.maxAssetCacheWait, 10) || 0 + ) + ' ms'; + } + + if ( + bin instanceof Object === false || + bin.hasOwnProperty(internalKey) === false + ) { + return reportBack(''); + } + + const entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + return reportBack(''); + } + + entry.readTime = Date.now(); + if ( updateReadTime ) { + saveAssetCacheRegistry(true); + } + + return reportBack(bin[internalKey]); +} + +async function assetCacheWrite(assetKey, details) { + let content = ''; + let options = {}; + if ( typeof details === 'string' ) { + content = details; + } else if ( details instanceof Object ) { + content = details.content || ''; + options = details; + } + + if ( content === '' ) { + return assetCacheRemove(assetKey); + } + + const cacheDict = await getAssetCacheRegistry(); + + let entry = cacheDict[assetKey]; + if ( entry === undefined ) { + entry = cacheDict[assetKey] = {}; + } + entry.writeTime = entry.readTime = Date.now(); + entry.resourceTime = options.resourceTime || 0; + if ( typeof options.url === 'string' ) { + entry.remoteURL = options.url; + } + cacheStorage.set({ + assetCacheRegistry, + [`cache/${assetKey}`]: content + }); + + const result = { assetKey, content }; + // https://github.com/uBlockOrigin/uBlock-issues/issues/248 + if ( options.silent !== true ) { + fireNotification('after-asset-updated', result); + } + return result; +} + +async function assetCacheRemove(pattern) { + const cacheDict = await getAssetCacheRegistry(); + const removedEntries = []; + const removedContent = []; + for ( const assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; + } + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + removedEntries.push(assetKey); + removedContent.push('cache/' + assetKey); + delete cacheDict[assetKey]; + } + if ( removedContent.length !== 0 ) { + await Promise.all([ + cacheStorage.remove(removedContent), + cacheStorage.set({ assetCacheRegistry }), + ]); + } + for ( let i = 0; i < removedEntries.length; i++ ) { + fireNotification('after-asset-updated', { + assetKey: removedEntries[i] + }); + } +} + +async function assetCacheGetDetails(assetKey) { + const cacheDict = await getAssetCacheRegistry(); + const entry = cacheDict[assetKey]; + if ( entry === undefined ) { return; } + return entry; +} + +async function assetCacheSetDetails(assetKey, details) { + const cacheDict = await getAssetCacheRegistry(); + const entry = cacheDict[assetKey]; + if ( entry === undefined ) { return; } + let modified = false; + for ( const [ k, v ] of Object.entries(details) ) { + if ( v === undefined ) { + if ( entry[k] !== undefined ) { + delete entry[k]; + modified = true; + continue; + } + } + if ( v !== entry[k] ) { + entry[k] = v; + modified = true; + } + } + if ( modified ) { + saveAssetCacheRegistry(); + } +} + +async function assetCacheMarkAsDirty(pattern, exclude) { + const cacheDict = await getAssetCacheRegistry(); + let mustSave = false; + for ( const assetKey in cacheDict ) { + if ( pattern instanceof RegExp ) { + if ( pattern.test(assetKey) === false ) { continue; } + } else if ( typeof pattern === 'string' ) { + if ( assetKey !== pattern ) { continue; } + } else if ( Array.isArray(pattern) ) { + if ( pattern.indexOf(assetKey) === -1 ) { continue; } + } + if ( exclude instanceof RegExp ) { + if ( exclude.test(assetKey) ) { continue; } + } else if ( typeof exclude === 'string' ) { + if ( assetKey === exclude ) { continue; } + } else if ( Array.isArray(exclude) ) { + if ( exclude.indexOf(assetKey) !== -1 ) { continue; } + } + const cacheEntry = cacheDict[assetKey]; + if ( !cacheEntry.writeTime ) { continue; } + cacheDict[assetKey].writeTime = 0; + mustSave = true; + } + if ( mustSave ) { + cacheStorage.set({ assetCacheRegistry }); + } +} + +/******************************************************************************* + + User assets are NOT persisted in the cache storage. User assets are + recognized by the asset key which always starts with 'user-'. + + TODO(seamless migration): + Can remove instances of old user asset keys when I am confident all users + are using uBO v1.11 and beyond. + +**/ + +/******************************************************************************* + + User assets are NOT persisted in the cache storage. User assets are + recognized by the asset key which always starts with 'user-'. + +**/ + +const readUserAsset = async function(assetKey) { + const bin = await vAPI.storage.get(assetKey); + const content = + bin instanceof Object && typeof bin[assetKey] === 'string' + ? bin[assetKey] + : ''; + return { assetKey, content }; +}; + +const saveUserAsset = function(assetKey, content) { + return vAPI.storage.set({ [assetKey]: content }).then(( ) => { + return { assetKey, content }; + }); +}; + +/******************************************************************************/ + +assets.get = async function(assetKey, options = {}) { + if ( assetKey === µb.userFiltersPath ) { + return readUserAsset(assetKey); + } + + let assetDetails = {}; + + const reportBack = (content, url = '', err = undefined) => { + const details = { assetKey, content }; + if ( err !== undefined ) { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; + } + if ( options.needSourceURL ) { + if ( + url === '' && + assetCacheRegistry instanceof Object && + assetCacheRegistry[assetKey] instanceof Object + ) { + details.sourceURL = assetCacheRegistry[assetKey].remoteURL; + } + if ( reIsExternalPath.test(url) ) { + details.sourceURL = url; + } + } + return details; + }; + + // Skip read-time property for non-updatable assets: the property is + // completely unused for such assets and thus there is no point incurring + // storage write overhead at launch when reading compiled or selfie assets. + const updateReadTime = /^(?:compiled|selfie)\//.test(assetKey) === false; + + const details = await assetCacheRead(assetKey, updateReadTime); + if ( details.content !== '' ) { + return reportBack(details.content); + } + + const assetRegistry = await getAssetSourceRegistry(); + + assetDetails = assetRegistry[assetKey] || {}; + + const contentURLs = getContentURLs(assetKey, options); + if ( contentURLs.length === 0 && reIsExternalPath.test(assetKey) ) { + assetDetails.content = 'filters'; + contentURLs.push(assetKey); + } + + let error = 'ENOTFOUND'; + for ( const contentURL of contentURLs ) { + const details = assetDetails.content === 'filters' + ? await assets.fetchFilterList(contentURL) + : await assets.fetchText(contentURL); + if ( details.error !== undefined ) { + error = details.error; + } + if ( details.content === '' ) { continue; } + if ( reIsExternalPath.test(contentURL) && options.dontCache !== true ) { + assetCacheWrite(assetKey, { + content: details.content, + url: contentURL, + silent: options.silent === true, + }); + registerAssetSource(assetKey, { error: undefined }); + if ( assetDetails.content === 'filters' ) { + const metadata = extractMetadataFromList(details.content, [ + 'Last-Modified', + 'Expires', + 'Diff-Name', + 'Diff-Path', + 'Diff-Expires', + ]); + metadata.diffUpdated = undefined; + assetCacheSetDetails(assetKey, metadata); + } + } + return reportBack(details.content, contentURL); + } + if ( assetRegistry[assetKey] !== undefined ) { + registerAssetSource(assetKey, { + error: { time: Date.now(), error } + }); + } + return reportBack('', '', error); +}; + +/******************************************************************************/ + +async function getRemote(assetKey, options = {}) { + const [ + assetDetails = {}, + cacheDetails = {}, + ] = await Promise.all([ + assetSourceGetDetails(assetKey), + assetCacheGetDetails(assetKey), + ]); + + let error; + let stale = false; + + const reportBack = function(content, url = '', err = '') { + const details = { assetKey, content, url }; + if ( err !== '') { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; + } + return details; + }; + + for ( const contentURL of getContentURLs(assetKey, options) ) { + if ( reIsExternalPath.test(contentURL) === false ) { continue; } + + const result = assetDetails.content === 'filters' + ? await assets.fetchFilterList(contentURL) + : await assets.fetchText(contentURL); + + // Failure + if ( stringIsNotEmpty(result.content) === false ) { + error = result.statusText; + if ( result.statusCode === 0 ) { + error = 'network error'; + } + continue; + } + + error = undefined; + + // If fetched resource is older than cached one, ignore + if ( options.favorOrigin !== true ) { + stale = resourceIsStale(result, cacheDetails); + if ( stale ) { continue; } + } + + // Success + assetCacheWrite(assetKey, { + content: result.content, + url: contentURL, + resourceTime: result.resourceTime || 0, + }); + + if ( assetDetails.content === 'filters' ) { + const metadata = extractMetadataFromList(result.content, [ + 'Last-Modified', + 'Expires', + 'Diff-Name', + 'Diff-Path', + 'Diff-Expires', + ]); + metadata.diffUpdated = undefined; + assetCacheSetDetails(assetKey, metadata); + } + + registerAssetSource(assetKey, { birthtime: undefined, error: undefined }); + return reportBack(result.content, contentURL); + } + + if ( error !== undefined ) { + registerAssetSource(assetKey, { error: { time: Date.now(), error } }); + return reportBack('', '', 'ENOTFOUND'); + } + + if ( stale ) { + assetCacheSetDetails(assetKey, { writeTime: cacheDetails.resourceTime }); + } + + return reportBack(''); +} + +/******************************************************************************/ + +assets.put = async function(assetKey, content) { + return reIsUserAsset.test(assetKey) + ? await saveUserAsset(assetKey, content) + : await assetCacheWrite(assetKey, content); +}; + +/******************************************************************************/ + +assets.metadata = async function() { + await Promise.all([ + getAssetSourceRegistry(), + getAssetCacheRegistry(), + ]); + + const assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)); + const cacheDict = assetCacheRegistry; + const now = Date.now(); + for ( const assetKey in assetDict ) { + const assetEntry = assetDict[assetKey]; + const cacheEntry = cacheDict[assetKey]; + if ( + assetEntry.content === 'filters' && + assetEntry.external !== true + ) { + assetEntry.isDefault = + assetEntry.off === undefined || + assetEntry.off === true && + µb.listMatchesEnvironment(assetEntry); + } + if ( cacheEntry ) { + assetEntry.cached = true; + assetEntry.writeTime = cacheEntry.writeTime; + const obsoleteAfter = cacheEntry.writeTime + getUpdateAfterTime(assetKey); + assetEntry.obsolete = obsoleteAfter < now; + assetEntry.remoteURL = cacheEntry.remoteURL; + if ( cacheEntry.diffUpdated ) { + assetEntry.diffUpdated = cacheEntry.diffUpdated; + } + } else if ( + assetEntry.contentURL && + assetEntry.contentURL.length !== 0 + ) { + assetEntry.writeTime = 0; + assetEntry.obsolete = true; + } + } + + return assetDict; +}; + +/******************************************************************************/ + +assets.purge = assetCacheMarkAsDirty; + +assets.remove = function(pattern) { + return assetCacheRemove(pattern); +}; + +assets.rmrf = function() { + return assetCacheRemove(/./); +}; + +/******************************************************************************/ + +assets.getUpdateAges = async function(conditions = {}) { + const assetDict = await assets.metadata(); + const now = Date.now(); + const out = []; + for ( const [ assetKey, asset ] of Object.entries(assetDict) ) { + if ( asset.hasRemoteURL !== true ) { continue; } + const tokens = conditions[asset.content]; + if ( Array.isArray(tokens) === false ) { continue; } + if ( tokens.includes('*') === false ) { + if ( tokens.includes(assetKey) === false ) { continue; } + } + const age = now - (asset.writeTime || 0); + out.push({ + assetKey, + age, + ageNormalized: age / Math.max(1, getUpdateAfterTime(assetKey)), + }); + } + return out; +}; + +/******************************************************************************/ + +// Asset updater area. +const updaterAssetDelayDefault = 120000; +const updaterUpdated = []; +const updaterFetched = new Set(); + +let updaterStatus; +let updaterAssetDelay = updaterAssetDelayDefault; +let updaterAuto = false; + +const getAssetDiffDetails = assetKey => { + const out = { assetKey }; + const cacheEntry = assetCacheRegistry[assetKey]; + if ( cacheEntry === undefined ) { return; } + out.patchPath = cacheEntry.diffPath; + if ( out.patchPath === undefined ) { return; } + const match = /#.+$/.exec(out.patchPath); + if ( match !== null ) { + out.diffName = match[0].slice(1); + } else { + out.diffName = cacheEntry.diffName; + } + if ( out.diffName === undefined ) { return; } + out.diffExpires = getUpdateAfterTime(assetKey, true); + out.lastModified = cacheEntry.lastModified; + out.writeTime = cacheEntry.writeTime; + const assetEntry = assetSourceRegistry[assetKey]; + if ( assetEntry === undefined ) { return; } + if ( assetEntry.content !== 'filters' ) { return; } + if ( Array.isArray(assetEntry.cdnURLs) ) { + out.cdnURLs = assetEntry.cdnURLs.slice(); + } else if ( reIsExternalPath.test(assetKey) ) { + out.cdnURLs = [ assetKey ]; + } else if ( typeof assetEntry.contentURL === 'string' ) { + out.cdnURLs = [ assetEntry.contentURL ]; + } else if ( Array.isArray(assetEntry.contentURL) ) { + out.cdnURLs = assetEntry.contentURL.slice(0).filter(url => + reIsExternalPath.test(url) + ); + } + if ( Array.isArray(out.cdnURLs) === false ) { return; } + if ( out.cdnURLs.length === 0 ) { return; } + return out; +}; + +async function diffUpdater() { + if ( updaterAuto === false ) { return; } + if ( µb.hiddenSettings.differentialUpdate === false ) { return; } + const toUpdate = await getUpdateCandidates(); + const now = Date.now(); + const toHardUpdate = []; + const toSoftUpdate = []; + while ( toUpdate.length !== 0 ) { + const assetKey = toUpdate.shift(); + const assetDetails = getAssetDiffDetails(assetKey); + if ( assetDetails === undefined ) { continue; } + assetDetails.what = 'update'; + const computedUpdateTime = computedPatchUpdateTime(assetKey); + if ( computedUpdateTime !== 0 && computedUpdateTime <= now ) { + assetDetails.fetch = true; + toHardUpdate.push(assetDetails); + } else { + assetDetails.fetch = false; + toSoftUpdate.push(assetDetails); + } + } + if ( toHardUpdate.length === 0 ) { return; } + ubolog('Diff updater: cycle start'); + return new Promise(resolve => { + let pendingOps = 0; + const bc = new globalThis.BroadcastChannel('diffUpdater'); + const terminate = error => { + worker.terminate(); + bc.close(); + resolve(); + if ( typeof error !== 'string' ) { return; } + ubolog(`Diff updater: terminate because ${error}`); + }; + const checkAndCorrectDiffPath = data => { + if ( typeof data.text !== 'string' ) { return; } + if ( data.text === '' ) { return; } + const metadata = extractMetadataFromList(data.text, [ 'Diff-Path' ]); + if ( metadata instanceof Object === false ) { return; } + if ( metadata.diffPath === data.patchPath ) { return; } + assetCacheSetDetails(data.assetKey, metadata); + }; + bc.onmessage = ev => { + const data = ev.data || {}; + if ( data.what === 'ready' ) { + ubolog('Diff updater: hard updating', toHardUpdate.map(v => v.assetKey).join()); + while ( toHardUpdate.length !== 0 ) { + const assetDetails = toHardUpdate.shift(); + assetDetails.fetch = true; + bc.postMessage(assetDetails); + pendingOps += 1; + } + return; + } + if ( data.what === 'broken' ) { + terminate(data.error); + return; + } + if ( data.status === 'needtext' ) { + ubolog('Diff updater: need text for', data.assetKey); + assetCacheRead(data.assetKey).then(result => { + data.text = result.content; + data.status = undefined; + checkAndCorrectDiffPath(data); + bc.postMessage(data); + }); + return; + } + if ( data.status === 'updated' ) { + ubolog(`Diff updater: successfully patched ${data.assetKey} using ${data.patchURL} (${data.patchSize})`); + const metadata = extractMetadataFromList(data.text, [ + 'Last-Modified', + 'Expires', + 'Diff-Name', + 'Diff-Path', + 'Diff-Expires', + ]); + assetCacheWrite(data.assetKey, { + content: data.text, + resourceTime: metadata.lastModified || 0, + }); + metadata.diffUpdated = true; + assetCacheSetDetails(data.assetKey, metadata); + updaterUpdated.push(data.assetKey); + } else if ( data.error ) { + ubolog(`Diff updater: failed to update ${data.assetKey} using ${data.patchPath}\n\treason: ${data.error}`); + } else if ( data.status === 'nopatch-yet' || data.status === 'nodiff' ) { + ubolog(`Diff updater: skip update of ${data.assetKey} using ${data.patchPath}\n\treason: ${data.status}`); + assetCacheSetDetails(data.assetKey, { writeTime: data.writeTime }); + broadcast({ + what: 'assetUpdated', + key: data.assetKey, + cached: true, + }); + } else { + ubolog(`Diff updater: ${data.assetKey} / ${data.patchPath} / ${data.status}`); + } + pendingOps -= 1; + if ( pendingOps === 0 && toSoftUpdate.length !== 0 ) { + ubolog('Diff updater: soft updating', toSoftUpdate.map(v => v.assetKey).join()); + while ( toSoftUpdate.length !== 0 ) { + bc.postMessage(toSoftUpdate.shift()); + pendingOps += 1; + } + } + if ( pendingOps !== 0 ) { return; } + ubolog('Diff updater: cycle complete'); + terminate(); + }; + const worker = new Worker('js/diff-updater.js'); + }); +} + +function updateFirst() { + ubolog('Updater: cycle start'); + ubolog('Updater: prefer', updaterAuto ? 'CDNs' : 'origin'); + updaterStatus = 'updating'; + updaterFetched.clear(); + updaterUpdated.length = 0; + diffUpdater().catch(reason => { + ubolog(reason); + }).finally(( ) => { + updateNext(); + }); +} + +async function getUpdateCandidates() { + const [ assetDict, cacheDict ] = await Promise.all([ + getAssetSourceRegistry(), + getAssetCacheRegistry(), + ]); + const toUpdate = []; + for ( const assetKey in assetDict ) { + const assetEntry = assetDict[assetKey]; + if ( assetEntry.hasRemoteURL !== true ) { continue; } + if ( updaterFetched.has(assetKey) ) { continue; } + const cacheEntry = cacheDict[assetKey]; + if ( + fireNotification('before-asset-updated', { + assetKey, + type: assetEntry.content + }) === true + ) { + toUpdate.push(assetKey); + continue; + } + // This will remove a cached asset when it's no longer in use. + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { + assetCacheRemove(assetKey); + } + } + // https://github.com/uBlockOrigin/uBlock-issues/issues/1165 + // Update most obsolete asset first. + toUpdate.sort((a, b) => { + const ta = cacheDict[a] !== undefined ? cacheDict[a].writeTime : 0; + const tb = cacheDict[b] !== undefined ? cacheDict[b].writeTime : 0; + return ta - tb; + }); + return toUpdate; +} + +async function updateNext() { + const toUpdate = await getUpdateCandidates(); + const now = Date.now(); + const toHardUpdate = []; + + while ( toUpdate.length !== 0 ) { + const assetKey = toUpdate.shift(); + const writeTime = getWriteTime(assetKey); + const updateDelay = getUpdateAfterTime(assetKey); + if ( (writeTime + updateDelay) > now ) { continue; } + toHardUpdate.push(assetKey); + } + if ( toHardUpdate.length === 0 ) { + return updateDone(); + } + + const assetKey = toHardUpdate.pop(); + updaterFetched.add(assetKey); + + // In auto-update context, be gentle on remote servers. + remoteServerFriendly = updaterAuto; + + let result; + if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) { + result = await getRemote(assetKey, { favorOrigin: updaterAuto === false }); + } else { + result = await assets.fetchText(µb.assetsJsonPath); + result.assetKey = 'assets.json'; + } + + remoteServerFriendly = false; + + if ( result.error ) { + ubolog(`Full updater: failed to update ${assetKey}`); + fireNotification('asset-update-failed', { assetKey: result.assetKey }); + } else { + ubolog(`Full updater: successfully updated ${assetKey}`); + updaterUpdated.push(result.assetKey); + if ( result.assetKey === 'assets.json' && result.content !== '' ) { + updateAssetSourceRegistry(result.content); + } + } + + updaterTimer.on(updaterAssetDelay); +} + +const updaterTimer = vAPI.defer.create(updateNext); + +function updateDone() { + const assetKeys = updaterUpdated.slice(0); + updaterFetched.clear(); + updaterUpdated.length = 0; + updaterStatus = undefined; + updaterAuto = false; + updaterAssetDelay = updaterAssetDelayDefault; + ubolog('Updater: cycle end'); + if ( assetKeys.length ) { + ubolog(`Updater: ${assetKeys.join()} were updated`); + } + fireNotification('after-assets-updated', { assetKeys }); +} + +assets.updateStart = function(details) { + const oldUpdateDelay = updaterAssetDelay; + const newUpdateDelay = typeof details.fetchDelay === 'number' + ? details.fetchDelay + : updaterAssetDelayDefault; + updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); + updaterAuto = details.auto === true; + if ( updaterStatus !== undefined ) { + if ( newUpdateDelay < oldUpdateDelay ) { + updaterTimer.offon(updaterAssetDelay); + } + return; + } + updateFirst(); +}; + +assets.updateStop = function() { + updaterTimer.off(); + if ( updaterStatus !== undefined ) { + updateDone(); + } +}; + +assets.isUpdating = function() { + return updaterStatus === 'updating' && + updaterAssetDelay <= µb.hiddenSettings.manualUpdateAssetFetchPeriod; +}; + +/******************************************************************************/ + +export default assets; + +/******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js new file mode 100644 index 0000000..578d8a6 --- /dev/null +++ b/src/js/background.js @@ -0,0 +1,410 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +import logger from './logger.js'; +import { FilteringContext } from './filtering-context.js'; +import { ubologSet } from './console.js'; + +import { + domainFromHostname, + hostnameFromURI, + originFromURI, +} from './uri-utils.js'; + +/******************************************************************************/ + +// Not all platforms may have properly declared vAPI.webextFlavor. + +if ( vAPI.webextFlavor === undefined ) { + vAPI.webextFlavor = { major: 0, soup: new Set([ 'ublock' ]) }; +} + +/******************************************************************************/ + +const hiddenSettingsDefault = { + allowGenericProceduralFilters: false, + assetFetchTimeout: 30, + autoCommentFilterTemplate: '{{date}} {{origin}}', + autoUpdateAssetFetchPeriod: 15, + autoUpdateDelayAfterLaunch: 105, + autoUpdatePeriod: 1, + benchmarkDatasetURL: 'unset', + blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', + cacheStorageAPI: 'unset', + cacheStorageCompression: true, + cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', + cloudStorageCompression: true, + cnameIgnoreList: 'unset', + cnameIgnore1stParty: true, + cnameIgnoreExceptions: true, + cnameIgnoreRootDocument: true, + cnameMaxTTL: 120, + cnameReplayFullURL: false, + cnameUncloakProxied: false, + consoleLogLevel: 'unset', + debugAssetsJson: false, + debugScriptlets: false, + debugScriptletInjector: false, + differentialUpdate: true, + disableWebAssembly: false, + extensionUpdateForceReload: false, + filterAuthorMode: false, + loggerPopupType: 'popup', + manualUpdateAssetFetchPeriod: 500, + modifyWebextFlavor: 'unset', + popupFontSize: 'unset', + popupPanelDisabledSections: 0, + popupPanelLockedSections: 0, + popupPanelHeightMode: 0, + requestJournalProcessPeriod: 1000, + selfieAfter: 2, + strictBlockingBypassDuration: 120, + toolbarWarningTimeout: 60, + trustedListPrefixes: 'ublock-', + uiPopupConfig: 'unset', + uiStyles: 'unset', + updateAssetBypassBrowserCache: false, + userResourcesLocation: 'unset', +}; + +if ( vAPI.webextFlavor.soup.has('devbuild') ) { + hiddenSettingsDefault.consoleLogLevel = 'info'; + hiddenSettingsDefault.trustedListPrefixes += ' user-'; + ubologSet(true); +} + +const userSettingsDefault = { + advancedUserEnabled: false, + alwaysDetachLogger: true, + autoUpdate: true, + cloudStorageEnabled: false, + cnameUncloakEnabled: true, + collapseBlocked: true, + colorBlindFriendly: false, + contextMenuEnabled: true, + uiAccentCustom: false, + uiAccentCustom0: '#aca0f7', + uiTheme: 'auto', + externalLists: '', + firewallPaneMinimized: true, + hyperlinkAuditingDisabled: true, + ignoreGenericCosmeticFilters: vAPI.webextFlavor.soup.has('mobile'), + importedLists: [], + largeMediaSize: 50, + parseAllABPHideFilters: true, + popupPanelSections: 0b111, + prefetchingDisabled: true, + requestLogMaxEntries: 1000, + showIconBadge: true, + suspendUntilListsAreLoaded: vAPI.Net.canSuspend(), + tooltipsDisabled: false, + webrtcIPAddressHidden: false, +}; + +const dynamicFilteringDefault = [ + 'behind-the-scene * * noop', + 'behind-the-scene * image noop', + 'behind-the-scene * 3p noop', + 'behind-the-scene * inline-script noop', + 'behind-the-scene * 1p-script noop', + 'behind-the-scene * 3p-script noop', + 'behind-the-scene * 3p-frame noop', +]; + +const hostnameSwitchesDefault = [ + 'no-large-media: behind-the-scene false', +]; +// https://github.com/LiCybora/NanoDefenderFirefox/issues/196 +if ( vAPI.webextFlavor.soup.has('firefox') ) { + hostnameSwitchesDefault.push('no-csp-reports: * true'); +} + +const µBlock = { // jshint ignore:line + wakeupReason: '', + + userSettingsDefault, + userSettings: Object.assign({}, userSettingsDefault), + + hiddenSettingsDefault, + hiddenSettingsAdmin: {}, + hiddenSettings: Object.assign({}, hiddenSettingsDefault), + + dynamicFilteringDefault, + hostnameSwitchesDefault, + + noDashboard: false, + + // Features detection. + privacySettingsSupported: vAPI.browserSettings instanceof Object, + cloudStorageSupported: vAPI.cloud instanceof Object, + canFilterResponseData: typeof browser.webRequest.filterResponseData === 'function', + + // https://github.com/chrisaljoudi/uBlock/issues/180 + // Whitelist directives need to be loaded once the PSL is available + netWhitelist: new Map(), + netWhitelistModifyTime: 0, + netWhitelistDefault: [ + 'about-scheme', + 'chrome-extension-scheme', + 'chrome-scheme', + 'edge-scheme', + 'moz-extension-scheme', + 'opera-scheme', + 'vivaldi-scheme', + 'wyciwyg-scheme', // Firefox's "What-You-Cache-Is-What-You-Get" + ], + + localSettings: { + blockedRequestCount: 0, + allowedRequestCount: 0, + }, + localSettingsLastModified: 0, + + // Read-only + systemSettings: { + compiledMagic: 57, // Increase when compiled format changes + selfieMagic: 57, // Increase when selfie format changes + }, + + // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501 + // The assumption is that cache storage state reflects whether + // compiled or selfie assets are available or not. The properties + // below is to no longer rely on this assumption -- though it's still + // not clear how the assumption could be wrong, and it's still not + // clear whether relying on those properties will really solve the + // issue. It's just an attempt at hardening. + compiledFormatChanged: false, + selfieIsInvalid: false, + + restoreBackupSettings: { + lastRestoreFile: '', + lastRestoreTime: 0, + lastBackupFile: '', + lastBackupTime: 0, + }, + + commandShortcuts: new Map(), + + // Allows to fully customize uBO's assets, typically set through admin + // settings. The content of 'assets.json' will also tell which filter + // lists to enable by default when uBO is first installed. + assetsBootstrapLocation: undefined, + + assetsJsonPath: vAPI.webextFlavor.soup.has('devbuild') + ? '/assets/assets.dev.json' + : '/assets/assets.json', + userFiltersPath: 'user-filters', + pslAssetKey: 'public_suffix_list.dat', + + selectedFilterLists: [], + availableFilterLists: {}, + badLists: new Map(), + + inMemoryFilters: [], + inMemoryFiltersCompiled: '', + + // https://github.com/uBlockOrigin/uBlock-issues/issues/974 + // This can be used to defer filtering decision-making. + readyToFilter: false, + + supportStats: { + allReadyAfter: '?', + maxAssetCacheWait: '?', + }, + + pageStores: new Map(), + pageStoresToken: 0, + + storageQuota: vAPI.storage.QUOTA_BYTES, + storageUsed: 0, + + noopFunc: function(){}, + + apiErrorCount: 0, + + maybeGoodPopup: { + tabId: 0, + url: '', + }, + + epickerArgs: { + eprom: null, + mouse: false, + target: '', + zap: false, + }, + + scriptlets: {}, + + cspNoInlineScript: "script-src 'unsafe-eval' * blob: data:", + cspNoScripting: 'script-src http: https:', + cspNoInlineFont: 'font-src *', + + liveBlockingProfiles: [], + blockingProfileColorCache: new Map(), + parsedTrustedListPrefixes: [], + uiAccentStylesheet: '', +}; + +µBlock.isReadyPromise = new Promise(resolve => { + µBlock.isReadyResolve = resolve; +}); + +µBlock.domainFromHostname = domainFromHostname; +µBlock.hostnameFromURI = hostnameFromURI; + +µBlock.FilteringContext = class extends FilteringContext { + duplicate() { + return (new µBlock.FilteringContext(this)); + } + + fromTabId(tabId) { + const tabContext = µBlock.tabContextManager.mustLookup(tabId); + this.tabOrigin = tabContext.origin; + this.tabHostname = tabContext.rootHostname; + this.tabDomain = tabContext.rootDomain; + this.tabId = tabContext.tabId; + return this; + } + + maybeFromDocumentURL(documentUrl) { + if ( documentUrl === undefined ) { return; } + if ( documentUrl.startsWith(this.tabOrigin) ) { return; } + this.tabOrigin = originFromURI(µBlock.normalizeTabURL(0, documentUrl)); + this.tabHostname = hostnameFromURI(this.tabOrigin); + this.tabDomain = domainFromHostname(this.tabHostname); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/459 + // In case of a request for frame and if ever no context is specified, + // assume the origin of the context is the same as the request itself. + fromWebrequestDetails(details) { + const tabId = details.tabId; + this.type = details.type; + const isMainFrame = this.itype === this.MAIN_FRAME; + if ( isMainFrame && tabId > 0 ) { + µBlock.tabContextManager.push(tabId, details.url); + } + this.fromTabId(tabId); // Must be called AFTER tab context management + this.realm = ''; + this.id = details.requestId; + this.setMethod(details.method); + this.setURL(details.url); + this.aliasURL = details.aliasURL || undefined; + this.redirectURL = undefined; + this.filter = undefined; + if ( this.itype !== this.SUB_FRAME ) { + this.docId = details.frameId; + this.frameId = -1; + } else { + this.docId = details.parentFrameId; + this.frameId = details.frameId; + } + if ( this.tabId > 0 ) { + if ( this.docId === 0 ) { + if ( isMainFrame === false ) { + this.maybeFromDocumentURL(details.documentUrl); + } + this.docOrigin = this.tabOrigin; + this.docHostname = this.tabHostname; + this.docDomain = this.tabDomain; + return this; + } + if ( details.documentUrl !== undefined ) { + this.setDocOriginFromURL(details.documentUrl); + return this; + } + const pageStore = µBlock.pageStoreFromTabId(this.tabId); + const docStore = pageStore && pageStore.getFrameStore(this.docId); + if ( docStore ) { + this.setDocOriginFromURL(docStore.rawURL); + } else { + this.setDocOrigin(this.tabOrigin); + } + return this; + } + if ( details.documentUrl !== undefined ) { + const origin = originFromURI( + µBlock.normalizeTabURL(0, details.documentUrl) + ); + this.setDocOrigin(origin).setTabOrigin(origin); + return this; + } + const origin = (this.itype & this.FRAME_ANY) !== 0 + ? originFromURI(this.url) + : this.tabOrigin; + this.setDocOrigin(origin).setTabOrigin(origin); + return this; + } + + getTabOrigin() { + if ( this.tabOrigin === undefined ) { + const tabContext = µBlock.tabContextManager.mustLookup(this.tabId); + this.tabOrigin = tabContext.origin; + this.tabHostname = tabContext.rootHostname; + this.tabDomain = tabContext.rootDomain; + } + return super.getTabOrigin(); + } + + toLogger() { + const details = { + id: this.id, + tstamp: Date.now(), + realm: this.realm, + method: this.getMethodName(), + type: this.stype, + tabId: this.tabId, + tabDomain: this.getTabDomain(), + tabHostname: this.getTabHostname(), + docDomain: this.getDocDomain(), + docHostname: this.getDocHostname(), + domain: this.getDomain(), + hostname: this.getHostname(), + url: this.url, + aliasURL: this.aliasURL, + filter: undefined, + }; + // Many filters may have been applied to the current context + if ( Array.isArray(this.filter) === false ) { + details.filter = this.filter; + return logger.writeOne(details); + } + for ( const filter of this.filter ) { + details.filter = filter; + logger.writeOne(details); + } + } +}; + +µBlock.filteringContext = new µBlock.FilteringContext(); + +self.µBlock = µBlock; + +/******************************************************************************/ + +export default µBlock; diff --git a/src/js/base64-custom.js b/src/js/base64-custom.js new file mode 100644 index 0000000..34141b8 --- /dev/null +++ b/src/js/base64-custom.js @@ -0,0 +1,246 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// Custom base64 codecs. These codecs are meant to encode/decode typed arrays +// to/from strings. + +// https://github.com/uBlockOrigin/uBlock-issues/issues/461 +// Provide a fallback encoding for Chromium 59 and less by issuing a plain +// JSON string. The fallback can be removed once min supported version is +// above 59. + +// TODO: rename µBlock.base64 to µBlock.SparseBase64, now that +// µBlock.DenseBase64 has been introduced. +// TODO: Should no longer need to test presence of TextEncoder/TextDecoder. + +const valToDigit = new Uint8Array(64); +const digitToVal = new Uint8Array(128); +{ + const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@%'; + for ( let i = 0, n = chars.length; i < n; i++ ) { + const c = chars.charCodeAt(i); + valToDigit[i] = c; + digitToVal[c] = i; + } +} + +// The sparse base64 codec is best for buffers which contains a lot of +// small u32 integer values. Those small u32 integer values are better +// represented with stringified integers, because small values can be +// represented with fewer bits than the usual base64 codec. For example, +// 0 become '0 ', i.e. 16 bits instead of 48 bits with official base64 +// codec. + +const sparseBase64 = { + magic: 'Base64_1', + + encode: function(arrbuf, arrlen) { + const inputLength = (arrlen + 3) >>> 2; + const inbuf = new Uint32Array(arrbuf, 0, inputLength); + const outputLength = this.magic.length + 7 + inputLength * 7; + const outbuf = new Uint8Array(outputLength); + // magic bytes + let j = 0; + for ( let i = 0; i < this.magic.length; i++ ) { + outbuf[j++] = this.magic.charCodeAt(i); + } + // array size + let v = inputLength; + do { + outbuf[j++] = valToDigit[v & 0b111111]; + v >>>= 6; + } while ( v !== 0 ); + outbuf[j++] = 0x20 /* ' ' */; + // array content + for ( let i = 0; i < inputLength; i++ ) { + v = inbuf[i]; + do { + outbuf[j++] = valToDigit[v & 0b111111]; + v >>>= 6; + } while ( v !== 0 ); + outbuf[j++] = 0x20 /* ' ' */; + } + if ( typeof TextDecoder === 'undefined' ) { + return JSON.stringify( + Array.from(new Uint32Array(outbuf.buffer, 0, j >>> 2)) + ); + } + const textDecoder = new TextDecoder(); + return textDecoder.decode(new Uint8Array(outbuf.buffer, 0, j)); + }, + + decode: function(instr, arrbuf) { + if ( instr.charCodeAt(0) === 0x5B /* '[' */ ) { + const inbuf = JSON.parse(instr); + if ( arrbuf instanceof ArrayBuffer === false ) { + return new Uint32Array(inbuf); + } + const outbuf = new Uint32Array(arrbuf); + outbuf.set(inbuf); + return outbuf; + } + if ( instr.startsWith(this.magic) === false ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + const inputLength = instr.length; + const outputLength = this.decodeSize(instr) >> 2; + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint32Array(outputLength) + : new Uint32Array(arrbuf); + let i = instr.indexOf(' ', this.magic.length) + 1; + if ( i === -1 ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + // array content + let j = 0; + for (;;) { + if ( j === outputLength || i >= inputLength ) { break; } + let v = 0, l = 0; + for (;;) { + const c = instr.charCodeAt(i++); + if ( c === 0x20 /* ' ' */ ) { break; } + v += digitToVal[c] << l; + l += 6; + } + outbuf[j++] = v; + } + if ( i < inputLength || j < outputLength ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + return outbuf; + }, + + decodeSize: function(instr) { + if ( instr.startsWith(this.magic) === false ) { return 0; } + let v = 0, l = 0, i = this.magic.length; + for (;;) { + const c = instr.charCodeAt(i++); + if ( c === 0x20 /* ' ' */ ) { break; } + v += digitToVal[c] << l; + l += 6; + } + return v << 2; + }, +}; + +// The dense base64 codec is best for typed buffers which values are +// more random. For example, buffer contents as a result of compression +// contain less repetitive values and thus the content is more +// random-looking. + +// TODO: Investigate that in Firefox, creating a new Uint8Array from the +// ArrayBuffer fails, the content of the resulting Uint8Array is +// non-sensical. WASM-related? + +const denseBase64 = { + magic: 'DenseBase64_1', + + encode: function(input) { + const m = input.length % 3; + const n = input.length - m; + let outputLength = n / 3 * 4; + if ( m !== 0 ) { + outputLength += m + 1; + } + const output = new Uint8Array(outputLength); + let j = 0; + for ( let i = 0; i < n; i += 3) { + const i1 = input[i+0]; + const i2 = input[i+1]; + const i3 = input[i+2]; + output[j+0] = valToDigit[ i1 >>> 2]; + output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; + output[j+2] = valToDigit[i2 << 2 & 0b111100 | i3 >>> 6]; + output[j+3] = valToDigit[i3 & 0b111111 ]; + j += 4; + } + if ( m !== 0 ) { + const i1 = input[n]; + output[j+0] = valToDigit[i1 >>> 2]; + if ( m === 1 ) { // 1 value + output[j+1] = valToDigit[i1 << 4 & 0b110000]; + } else { // 2 values + const i2 = input[n+1]; + output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; + output[j+2] = valToDigit[i2 << 2 & 0b111100 ]; + } + } + const textDecoder = new TextDecoder(); + const b64str = textDecoder.decode(output); + return this.magic + b64str; + }, + + decode: function(instr, arrbuf) { + if ( instr.startsWith(this.magic) === false ) { + throw new Error('Invalid µBlock.denseBase64 encoding'); + } + const outputLength = this.decodeSize(instr); + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint8Array(outputLength) + : new Uint8Array(arrbuf); + const inputLength = instr.length - this.magic.length; + let i = this.magic.length; + let j = 0; + const m = inputLength & 3; + const n = i + inputLength - m; + while ( i < n ) { + const i1 = digitToVal[instr.charCodeAt(i+0)]; + const i2 = digitToVal[instr.charCodeAt(i+1)]; + const i3 = digitToVal[instr.charCodeAt(i+2)]; + const i4 = digitToVal[instr.charCodeAt(i+3)]; + i += 4; + outbuf[j+0] = i1 << 2 | i2 >>> 4; + outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; + outbuf[j+2] = i3 << 6 & 0b11000000 | i4; + j += 3; + } + if ( m !== 0 ) { + const i1 = digitToVal[instr.charCodeAt(i+0)]; + const i2 = digitToVal[instr.charCodeAt(i+1)]; + outbuf[j+0] = i1 << 2 | i2 >>> 4; + if ( m === 3 ) { + const i3 = digitToVal[instr.charCodeAt(i+2)]; + outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; + } + } + return outbuf; + }, + + decodeSize: function(instr) { + if ( instr.startsWith(this.magic) === false ) { return 0; } + const inputLength = instr.length - this.magic.length; + const m = inputLength & 3; + const n = inputLength - m; + let outputLength = (n >>> 2) * 3; + if ( m !== 0 ) { + outputLength += m - 1; + } + return outputLength; + }, +}; + +/******************************************************************************/ + +export { denseBase64, sparseBase64 }; diff --git a/src/js/benchmarks.js b/src/js/benchmarks.js new file mode 100644 index 0000000..8792f03 --- /dev/null +++ b/src/js/benchmarks.js @@ -0,0 +1,421 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import io from './assets.js'; +import scriptletFilteringEngine from './scriptlet-filtering.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import µb from './background.js'; +import webRequest from './traffic.js'; +import { FilteringContext } from './filtering-context.js'; +import { LineIterator } from './text-utils.js'; +import { sessionFirewall } from './filtering-engines.js'; + +import { + domainFromHostname, + entityFromDomain, + hostnameFromURI, +} from './uri-utils.js'; + +/******************************************************************************/ + +// The requests.json.gz file can be downloaded from: +// https://cdn.cliqz.com/adblocking/requests_top500.json.gz +// +// Which is linked from: +// https://whotracks.me/blog/adblockers_performance_study.html +// +// Copy the file into ./tmp/requests.json.gz +// +// If the file is present when you build uBO using `make-[target].sh` from +// the shell, the resulting package will have `./assets/requests.json`, which +// will be looked-up by the method below to launch a benchmark session. +// +// From uBO's dev console, launch the benchmark: +// µBlock.staticNetFilteringEngine.benchmark(); +// +// The usual browser dev tools can be used to obtain useful profiling +// data, i.e. start the profiler, call the benchmark method from the +// console, then stop the profiler when it completes. +// +// Keep in mind that the measurements at the blog post above where obtained +// with ONLY EasyList. The CPU reportedly used was: +// https://www.cpubenchmark.net/cpu.php?cpu=Intel+Core+i7-6600U+%40+2.60GHz&id=2608 +// +// Rename ./tmp/requests.json.gz to something else if you no longer want +// ./assets/requests.json in the build. + +const loadBenchmarkDataset = (( ) => { + let datasetPromise; + + const ttlTimer = vAPI.defer.create(( ) => { + datasetPromise = undefined; + }); + + return function() { + ttlTimer.offon({ min: 5 }); + + if ( datasetPromise !== undefined ) { + return datasetPromise; + } + + const datasetURL = µb.hiddenSettings.benchmarkDatasetURL; + if ( datasetURL === 'unset' ) { + console.info(`No benchmark dataset available.`); + return Promise.resolve(); + } + console.info(`Loading benchmark dataset...`); + datasetPromise = io.fetchText(datasetURL).then(details => { + console.info(`Parsing benchmark dataset...`); + let requests = []; + if ( details.content.startsWith('[') ) { + try { + requests = JSON.parse(details.content); + } catch(ex) { + } + } else { + const lineIter = new LineIterator(details.content); + const parsed = []; + while ( lineIter.eot() === false ) { + const line = lineIter.next().trim(); + if ( line === '' ) { continue; } + try { + parsed.push(JSON.parse(line)); + } catch(ex) { + parsed.length = 0; + break; + } + } + requests = parsed; + } + if ( requests.length === 0 ) { return; } + const out = []; + for ( const request of requests ) { + if ( request instanceof Object === false ) { continue; } + if ( !request.frameUrl || !request.url ) { continue; } + if ( request.cpt === 'document' ) { + request.cpt = 'main_frame'; + } else if ( request.cpt === 'xhr' ) { + request.cpt = 'xmlhttprequest'; + } + out.push(request); + } + return out; + }).catch(details => { + console.info(`Not found: ${details.url}`); + datasetPromise = undefined; + }); + + return datasetPromise; + }; +})(); + +/******************************************************************************/ + +// action: 1=test + +µb.benchmarkStaticNetFiltering = async function(options = {}) { + const { target, redirectEngine } = options; + + const requests = await loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + const text = 'No dataset found to benchmark'; + console.info(text); + return text; + } + + console.info(`Benchmarking staticNetFilteringEngine.matchRequest()...`); + + const fctxt = new FilteringContext(); + + if ( typeof target === 'number' ) { + const request = requests[target]; + fctxt.setURL(request.url); + fctxt.setDocOriginFromURL(request.frameUrl); + fctxt.setType(request.cpt); + const r = staticNetFilteringEngine.matchRequest(fctxt); + console.info(`Result=${r}:`); + console.info(`\ttype=${fctxt.type}`); + console.info(`\turl=${fctxt.url}`); + console.info(`\tdocOrigin=${fctxt.getDocOrigin()}`); + if ( r !== 0 ) { + console.info(staticNetFilteringEngine.toLogData()); + } + return; + } + + const t0 = performance.now(); + let matchCount = 0; + let blockCount = 0; + let allowCount = 0; + let redirectCount = 0; + let removeparamCount = 0; + let cspCount = 0; + let permissionsCount = 0; + let replaceCount = 0; + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + fctxt.setURL(request.url); + fctxt.setDocOriginFromURL(request.frameUrl); + fctxt.setType(request.cpt); + staticNetFilteringEngine.redirectURL = undefined; + const r = staticNetFilteringEngine.matchRequest(fctxt); + matchCount += 1; + if ( r === 1 ) { blockCount += 1; } + else if ( r === 2 ) { allowCount += 1; } + if ( r !== 1 ) { + if ( staticNetFilteringEngine.transformRequest(fctxt) ) { + redirectCount += 1; + } + if ( fctxt.redirectURL !== undefined && staticNetFilteringEngine.hasQuery(fctxt) ) { + if ( staticNetFilteringEngine.filterQuery(fctxt, 'removeparam') ) { + removeparamCount += 1; + } + } + if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) { + if ( staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'csp') ) { + cspCount += 1; + } + if ( staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'permissions') ) { + permissionsCount += 1; + } + } + staticNetFilteringEngine.matchHeaders(fctxt, []); + if ( staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'replace') ) { + replaceCount += 1; + } + } else if ( redirectEngine !== undefined ) { + if ( staticNetFilteringEngine.redirectRequest(redirectEngine, fctxt) ) { + redirectCount += 1; + } + } + } + const t1 = performance.now(); + const dur = t1 - t0; + + const output = [ + 'Benchmarked static network filtering engine:', + `\tEvaluated ${matchCount} match calls in ${dur.toFixed(0)} ms`, + `\tAverage: ${(dur / matchCount).toFixed(3)} ms per request`, + `\tNot blocked: ${matchCount - blockCount - allowCount}`, + `\tBlocked: ${blockCount}`, + `\tUnblocked: ${allowCount}`, + `\tredirect=: ${redirectCount}`, + `\tremoveparam=: ${removeparamCount}`, + `\tcsp=: ${cspCount}`, + `\tpermissions=: ${permissionsCount}`, + `\treplace=: ${replaceCount}`, + ]; + const s = output.join('\n'); + console.info(s); + return s; +}; + +/******************************************************************************/ + +µb.tokenHistograms = async function() { + const requests = await loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + console.info('No requests found to benchmark'); + return; + } + + console.info(`Computing token histograms...`); + + const fctxt = new FilteringContext(); + const missTokenMap = new Map(); + const hitTokenMap = new Map(); + const reTokens = /[0-9a-z%]{2,}/g; + + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + fctxt.setURL(request.url); + fctxt.setDocOriginFromURL(request.frameUrl); + fctxt.setType(request.cpt); + const r = staticNetFilteringEngine.matchRequest(fctxt); + for ( let [ keyword ] of request.url.toLowerCase().matchAll(reTokens) ) { + const token = keyword.slice(0, 7); + if ( r === 0 ) { + missTokenMap.set(token, (missTokenMap.get(token) || 0) + 1); + } else if ( r === 1 ) { + hitTokenMap.set(token, (hitTokenMap.get(token) || 0) + 1); + } + } + } + const customSort = (a, b) => b[1] - a[1]; + const topmisses = Array.from(missTokenMap).sort(customSort).slice(0, 100); + for ( const [ token ] of topmisses ) { + hitTokenMap.delete(token); + } + const tophits = Array.from(hitTokenMap).sort(customSort).slice(0, 100); + console.info('Misses:', JSON.stringify(topmisses)); + console.info('Hits:', JSON.stringify(tophits)); +}; + +/******************************************************************************/ + +µb.benchmarkDynamicNetFiltering = async function() { + const requests = await loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + console.info('No requests found to benchmark'); + return; + } + console.info(`Benchmarking sessionFirewall.evaluateCellZY()...`); + const fctxt = new FilteringContext(); + const t0 = performance.now(); + for ( const request of requests ) { + fctxt.setURL(request.url); + fctxt.setTabOriginFromURL(request.frameUrl); + fctxt.setType(request.cpt); + sessionFirewall.evaluateCellZY( + fctxt.getTabHostname(), + fctxt.getHostname(), + fctxt.type + ); + } + const t1 = performance.now(); + const dur = t1 - t0; + console.info(`Evaluated ${requests.length} requests in ${dur.toFixed(0)} ms`); + console.info(`\tAverage: ${(dur / requests.length).toFixed(3)} ms per request`); +}; + +/******************************************************************************/ + +µb.benchmarkCosmeticFiltering = async function() { + const requests = await loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + console.info('No requests found to benchmark'); + return; + } + console.info('Benchmarking cosmeticFilteringEngine.retrieveSpecificSelectors()...'); + const details = { + tabId: undefined, + frameId: undefined, + hostname: '', + domain: '', + entity: '', + }; + const options = { + noSpecificCosmeticFiltering: false, + noGenericCosmeticFiltering: false, + }; + let count = 0; + const t0 = performance.now(); + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + if ( request.cpt !== 'main_frame' ) { continue; } + count += 1; + details.hostname = hostnameFromURI(request.url); + details.domain = domainFromHostname(details.hostname); + details.entity = entityFromDomain(details.domain); + void cosmeticFilteringEngine.retrieveSpecificSelectors(details, options); + } + const t1 = performance.now(); + const dur = t1 - t0; + console.info(`Evaluated ${count} requests in ${dur.toFixed(0)} ms`); + console.info(`\tAverage: ${(dur / count).toFixed(3)} ms per request`); +}; + +/******************************************************************************/ + +µb.benchmarkScriptletFiltering = async function() { + const requests = await loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + console.info('No requests found to benchmark'); + return; + } + console.info('Benchmarking scriptletFilteringEngine.retrieve()...'); + const details = { + domain: '', + entity: '', + hostname: '', + tabId: 0, + url: '', + }; + let count = 0; + const t0 = performance.now(); + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + if ( request.cpt !== 'main_frame' ) { continue; } + count += 1; + details.url = request.url; + details.hostname = hostnameFromURI(request.url); + details.domain = domainFromHostname(details.hostname); + details.entity = entityFromDomain(details.domain); + void scriptletFilteringEngine.retrieve(details); + } + const t1 = performance.now(); + const dur = t1 - t0; + console.info(`Evaluated ${count} requests in ${dur.toFixed(0)} ms`); + console.info(`\tAverage: ${(dur / count).toFixed(3)} ms per request`); +}; + +/******************************************************************************/ + +µb.benchmarkOnBeforeRequest = async function() { + const requests = await loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + console.info('No requests found to benchmark'); + return; + } + const mappedTypes = new Map([ + [ 'document', 'main_frame' ], + [ 'subdocument', 'sub_frame' ], + ]); + console.info('webRequest.onBeforeRequest()...'); + const t0 = self.performance.now(); + const promises = []; + const details = { + documentUrl: '', + tabId: -1, + parentFrameId: -1, + frameId: 0, + type: '', + url: '', + }; + for ( const request of requests ) { + details.documentUrl = request.frameUrl; + details.tabId = -1; + details.parentFrameId = -1; + details.frameId = 0; + details.type = mappedTypes.get(request.cpt) || request.cpt; + details.url = request.url; + if ( details.type === 'main_frame' ) { continue; } + promises.push(webRequest.onBeforeRequest(details)); + } + return Promise.all(promises).then(results => { + let blockCount = 0; + for ( const r of results ) { + if ( r !== undefined ) { blockCount += 1; } + } + const t1 = self.performance.now(); + const dur = t1 - t0; + console.info(`Evaluated ${requests.length} requests in ${dur.toFixed(0)} ms`); + console.info(`\tBlocked ${blockCount} requests`); + console.info(`\tAverage: ${(dur / requests.length).toFixed(3)} ms per request`); + }); +}; + +/******************************************************************************/ diff --git a/src/js/biditrie.js b/src/js/biditrie.js new file mode 100644 index 0000000..d0f64ee --- /dev/null +++ b/src/js/biditrie.js @@ -0,0 +1,947 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2019-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals WebAssembly, vAPI */ + +'use strict'; + +/******************************************************************************* + + A BidiTrieContainer is mostly a large buffer in which distinct but related + tries are stored. The memory layout of the buffer is as follow: + + 0-2047: haystack section + 2048-2051: number of significant characters in the haystack + 2052-2055: offset to start of trie data section (=> trie0) + 2056-2059: offset to end of trie data section (=> trie1) + 2060-2063: offset to start of character data section (=> char0) + 2064-2067: offset to end of character data section (=> char1) + 2068: start of trie data section + + +--------------+ + Normal cell: | And | If "Segment info" matches: + (aka CELL) +--------------+ Goto "And" + | Or | Else + +--------------+ Goto "Or" + | Segment info | + +--------------+ + + +--------------+ + Boundary cell: | Right And | "Right And" and/or "Left And" + (aka BCELL) +--------------+ can be 0 in last-segment condition. + | Left And | + +--------------+ + | 0 | + +--------------+ + + Given following filters and assuming token is "ad" for all of them: + + -images/ad- + /google_ad. + /images_ad. + _images/ad. + + We get the following internal representation: + + +-----------+ +-----------+ +---+ + | |---->| |---->| 0 | + +-----------+ +-----------+ +---+ +-----------+ + | 0 | +--| | | |---->| 0 | + +-----------+ | +-----------+ +---+ +-----------+ + | ad | | | - | | 0 | | 0 | + +-----------+ | +-----------+ +---+ +-----------+ + | | -images/ | + | +-----------+ +---+ +-----------+ + +->| |---->| 0 | + +-----------+ +---+ +-----------+ +-----------+ + | 0 | | |---->| |---->| 0 | + +-----------+ +---+ +-----------+ +-----------+ + | . | | 0 | +--| | +--| | + +-----------+ +---+ | +-----------+ | +-----------+ + | | _ | | | /google | + | +-----------+ | +-----------+ + | | + | | +-----------+ + | +->| 0 | + | +-----------+ + | | 0 | + | +-----------+ + | | /images | + | +-----------+ + | + | +-----------+ + +->| 0 | + +-----------+ + | 0 | + +-----------+ + | _images/ | + +-----------+ + +*/ + +const PAGE_SIZE = 65536*2; +const HAYSTACK_START = 0; +const HAYSTACK_SIZE = 2048; // i32 / i8 +const HAYSTACK_SIZE_SLOT = HAYSTACK_SIZE >>> 2; // 512 / 2048 +const TRIE0_SLOT = HAYSTACK_SIZE_SLOT + 1; // 513 / 2052 +const TRIE1_SLOT = HAYSTACK_SIZE_SLOT + 2; // 514 / 2056 +const CHAR0_SLOT = HAYSTACK_SIZE_SLOT + 3; // 515 / 2060 +const CHAR1_SLOT = HAYSTACK_SIZE_SLOT + 4; // 516 / 2064 +const RESULT_L_SLOT = HAYSTACK_SIZE_SLOT + 5; // 517 / 2068 +const RESULT_R_SLOT = HAYSTACK_SIZE_SLOT + 6; // 518 / 2072 +const RESULT_IU_SLOT = HAYSTACK_SIZE_SLOT + 7; // 519 / 2076 +const TRIE0_START = HAYSTACK_SIZE_SLOT + 8 << 2; // 2080 + +const CELL_BYTE_LENGTH = 12; +const MIN_FREE_CELL_BYTE_LENGTH = CELL_BYTE_LENGTH * 8; + +const CELL_AND = 0; +const CELL_OR = 1; +const SEGMENT_INFO = 2; +const BCELL_NEXT_AND = 0; +const BCELL_ALT_AND = 1; +const BCELL_EXTRA = 2; +const BCELL_EXTRA_MAX = 0x00FFFFFF; + +const toSegmentInfo = (aL, l, r) => ((r - l) << 24) | (aL + l); +const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); + + +class BidiTrieContainer { + + constructor(extraHandler) { + const len = PAGE_SIZE * 4; + this.buf8 = new Uint8Array(len); + this.buf32 = new Uint32Array(this.buf8.buffer); + this.buf32[TRIE0_SLOT] = TRIE0_START; + this.buf32[TRIE1_SLOT] = this.buf32[TRIE0_SLOT]; + this.buf32[CHAR0_SLOT] = len >>> 1; + this.buf32[CHAR1_SLOT] = this.buf32[CHAR0_SLOT]; + this.haystack = this.buf8.subarray( + HAYSTACK_START, + HAYSTACK_START + HAYSTACK_SIZE + ); + this.extraHandler = extraHandler; + this.textDecoder = null; + this.wasmMemory = null; + + this.lastStored = ''; + this.lastStoredLen = this.lastStoredIndex = 0; + } + + //-------------------------------------------------------------------------- + // Public methods + //-------------------------------------------------------------------------- + + get haystackLen() { + return this.buf32[HAYSTACK_SIZE_SLOT]; + } + + set haystackLen(v) { + this.buf32[HAYSTACK_SIZE_SLOT] = v; + } + + reset(details) { + if ( + details instanceof Object && + typeof details.byteLength === 'number' && + typeof details.char0 === 'number' + ) { + if ( details.byteLength > this.buf8.byteLength ) { + this.reallocateBuf(details.byteLength); + } + this.buf32[CHAR0_SLOT] = details.char0; + } + this.buf32[TRIE1_SLOT] = this.buf32[TRIE0_SLOT]; + this.buf32[CHAR1_SLOT] = this.buf32[CHAR0_SLOT]; + + this.lastStored = ''; + this.lastStoredLen = this.lastStoredIndex = 0; + } + + createTrie() { + // grow buffer if needed + if ( (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < CELL_BYTE_LENGTH ) { + this.growBuf(CELL_BYTE_LENGTH, 0); + } + const iroot = this.buf32[TRIE1_SLOT] >>> 2; + this.buf32[TRIE1_SLOT] += CELL_BYTE_LENGTH; + this.buf32[iroot+CELL_OR] = 0; + this.buf32[iroot+CELL_AND] = 0; + this.buf32[iroot+SEGMENT_INFO] = 0; + return iroot; + } + + matches(icell, ai) { + const buf32 = this.buf32; + const buf8 = this.buf8; + const char0 = buf32[CHAR0_SLOT]; + const aR = buf32[HAYSTACK_SIZE_SLOT]; + let al = ai, x = 0, y = 0; + for (;;) { + x = buf8[al]; + al += 1; + // find matching segment + for (;;) { + y = buf32[icell+SEGMENT_INFO]; + let bl = char0 + (y & 0x00FFFFFF); + if ( buf8[bl] === x ) { + y = (y >>> 24) - 1; + if ( y !== 0 ) { + x = al + y; + if ( x > aR ) { return 0; } + for (;;) { + bl += 1; + if ( buf8[bl] !== buf8[al] ) { return 0; } + al += 1; + if ( al === x ) { break; } + } + } + break; + } + icell = buf32[icell+CELL_OR]; + if ( icell === 0 ) { return 0; } + } + // next segment + icell = buf32[icell+CELL_AND]; + x = buf32[icell+BCELL_EXTRA]; + if ( x <= BCELL_EXTRA_MAX ) { + if ( x !== 0 && this.matchesExtra(ai, al, x) !== 0 ) { + return 1; + } + x = buf32[icell+BCELL_ALT_AND]; + if ( x !== 0 && this.matchesLeft(x, ai, al) !== 0 ) { + return 1; + } + icell = buf32[icell+BCELL_NEXT_AND]; + if ( icell === 0 ) { return 0; } + } + if ( al === aR ) { return 0; } + } + return 0; // eslint-disable-line no-unreachable + } + + matchesLeft(icell, ar, r) { + const buf32 = this.buf32; + const buf8 = this.buf8; + const char0 = buf32[CHAR0_SLOT]; + let x = 0, y = 0; + for (;;) { + if ( ar === 0 ) { return 0; } + ar -= 1; + x = buf8[ar]; + // find first segment with a first-character match + for (;;) { + y = buf32[icell+SEGMENT_INFO]; + let br = char0 + (y & 0x00FFFFFF); + y = (y >>> 24) - 1; + br += y; + if ( buf8[br] === x ) { // all characters in segment must match + if ( y !== 0 ) { + x = ar - y; + if ( x < 0 ) { return 0; } + for (;;) { + ar -= 1; br -= 1; + if ( buf8[ar] !== buf8[br] ) { return 0; } + if ( ar === x ) { break; } + } + } + break; + } + icell = buf32[icell+CELL_OR]; + if ( icell === 0 ) { return 0; } + } + // next segment + icell = buf32[icell+CELL_AND]; + x = buf32[icell+BCELL_EXTRA]; + if ( x <= BCELL_EXTRA_MAX ) { + if ( x !== 0 && this.matchesExtra(ar, r, x) !== 0 ) { + return 1; + } + icell = buf32[icell+BCELL_NEXT_AND]; + if ( icell === 0 ) { return 0; } + } + } + return 0; // eslint-disable-line no-unreachable + } + + matchesExtra(l, r, ix) { + let iu = 0; + if ( ix !== 1 ) { + iu = this.extraHandler(l, r, ix); + if ( iu === 0 ) { return 0; } + } else { + iu = -1; + } + this.buf32[RESULT_IU_SLOT] = iu; + this.buf32[RESULT_L_SLOT] = l; + this.buf32[RESULT_R_SLOT] = r; + return 1; + } + + get $l() { return this.buf32[RESULT_L_SLOT] | 0; } + get $r() { return this.buf32[RESULT_R_SLOT] | 0; } + get $iu() { return this.buf32[RESULT_IU_SLOT] | 0; } + + add(iroot, aL0, n, pivot = 0) { + const aR = n; + if ( aR === 0 ) { return 0; } + // Grow buffer if needed. The characters are already in our character + // data buffer, so we do not need to grow character data buffer. + if ( + (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < + MIN_FREE_CELL_BYTE_LENGTH + ) { + this.growBuf(MIN_FREE_CELL_BYTE_LENGTH, 0); + } + const buf32 = this.buf32; + const char0 = buf32[CHAR0_SLOT]; + let icell = iroot; + let aL = char0 + aL0; + // special case: first node in trie + if ( buf32[icell+SEGMENT_INFO] === 0 ) { + buf32[icell+SEGMENT_INFO] = toSegmentInfo(aL0, pivot, aR); + return this.addLeft(icell, aL0, pivot); + } + const buf8 = this.buf8; + let al = pivot; + let inext; + // find a matching cell: move down + for (;;) { + const binfo = buf32[icell+SEGMENT_INFO]; + // length of segment + const bR = binfo >>> 24; + // skip boundary cells + if ( bR === 0 ) { + icell = buf32[icell+BCELL_NEXT_AND]; + continue; + } + let bl = char0 + (binfo & 0x00FFFFFF); + // if first character is no match, move to next descendant + if ( buf8[bl] !== buf8[aL+al] ) { + inext = buf32[icell+CELL_OR]; + if ( inext === 0 ) { + inext = this.addCell(0, 0, toSegmentInfo(aL0, al, aR)); + buf32[icell+CELL_OR] = inext; + return this.addLeft(inext, aL0, pivot); + } + icell = inext; + continue; + } + // 1st character was tested + let bi = 1; + al += 1; + // find 1st mismatch in rest of segment + if ( bR !== 1 ) { + for (;;) { + if ( bi === bR ) { break; } + if ( al === aR ) { break; } + if ( buf8[bl+bi] !== buf8[aL+al] ) { break; } + bi += 1; + al += 1; + } + } + // all segment characters matched + if ( bi === bR ) { + // needle remainder: no + if ( al === aR ) { + return this.addLeft(icell, aL0, pivot); + } + // needle remainder: yes + inext = buf32[icell+CELL_AND]; + if ( buf32[inext+CELL_AND] !== 0 ) { + icell = inext; + continue; + } + // add needle remainder + icell = this.addCell(0, 0, toSegmentInfo(aL0, al, aR)); + buf32[inext+CELL_AND] = icell; + return this.addLeft(icell, aL0, pivot); + } + // some characters matched + // split current segment + bl -= char0; + buf32[icell+SEGMENT_INFO] = bi << 24 | bl; + inext = this.addCell( + buf32[icell+CELL_AND], 0, bR - bi << 24 | bl + bi + ); + buf32[icell+CELL_AND] = inext; + // needle remainder: no = need boundary cell + if ( al === aR ) { + return this.addLeft(icell, aL0, pivot); + } + // needle remainder: yes = need new cell for remaining characters + icell = this.addCell(0, 0, toSegmentInfo(aL0, al, aR)); + buf32[inext+CELL_OR] = icell; + return this.addLeft(icell, aL0, pivot); + } + } + + addLeft(icell, aL0, pivot) { + const buf32 = this.buf32; + const char0 = buf32[CHAR0_SLOT]; + let aL = aL0 + char0; + // fetch boundary cell + let iboundary = buf32[icell+CELL_AND]; + // add boundary cell if none exist + if ( + iboundary === 0 || + buf32[iboundary+SEGMENT_INFO] > BCELL_EXTRA_MAX + ) { + const inext = iboundary; + iboundary = this.allocateCell(); + buf32[icell+CELL_AND] = iboundary; + buf32[iboundary+BCELL_NEXT_AND] = inext; + if ( pivot === 0 ) { return iboundary; } + } + // shortest match with no extra conditions will always win + if ( buf32[iboundary+BCELL_EXTRA] === 1 ) { + return iboundary; + } + // bail out if no left segment + if ( pivot === 0 ) { return iboundary; } + // fetch root cell of left segment + icell = buf32[iboundary+BCELL_ALT_AND]; + if ( icell === 0 ) { + icell = this.allocateCell(); + buf32[iboundary+BCELL_ALT_AND] = icell; + } + // special case: first node in trie + if ( buf32[icell+SEGMENT_INFO] === 0 ) { + buf32[icell+SEGMENT_INFO] = toSegmentInfo(aL0, 0, pivot); + iboundary = this.allocateCell(); + buf32[icell+CELL_AND] = iboundary; + return iboundary; + } + const buf8 = this.buf8; + let ar = pivot, inext; + // find a matching cell: move down + for (;;) { + const binfo = buf32[icell+SEGMENT_INFO]; + // skip boundary cells + if ( binfo <= BCELL_EXTRA_MAX ) { + inext = buf32[icell+CELL_AND]; + if ( inext !== 0 ) { + icell = inext; + continue; + } + iboundary = this.allocateCell(); + buf32[icell+CELL_AND] = + this.addCell(iboundary, 0, toSegmentInfo(aL0, 0, ar)); + // TODO: boundary cell might be last + // add remainder + boundary cell + return iboundary; + } + const bL = char0 + (binfo & 0x00FFFFFF); + const bR = bL + (binfo >>> 24); + let br = bR; + // if first character is no match, move to next descendant + if ( buf8[br-1] !== buf8[aL+ar-1] ) { + inext = buf32[icell+CELL_OR]; + if ( inext === 0 ) { + iboundary = this.allocateCell(); + inext = this.addCell( + iboundary, 0, toSegmentInfo(aL0, 0, ar) + ); + buf32[icell+CELL_OR] = inext; + return iboundary; + } + icell = inext; + continue; + } + // 1st character was tested + br -= 1; + ar -= 1; + // find 1st mismatch in rest of segment + if ( br !== bL ) { + for (;;) { + if ( br === bL ) { break; } + if ( ar === 0 ) { break; } + if ( buf8[br-1] !== buf8[aL+ar-1] ) { break; } + br -= 1; + ar -= 1; + } + } + // all segment characters matched + // a: ...vvvvvvv + // b: vvvvvvv + if ( br === bL ) { + inext = buf32[icell+CELL_AND]; + // needle remainder: no + // a: vvvvvvv + // b: vvvvvvv + // r: 0 & vvvvvvv + if ( ar === 0 ) { + // boundary cell already present + if ( buf32[inext+BCELL_EXTRA] <= BCELL_EXTRA_MAX ) { + return inext; + } + // need boundary cell + iboundary = this.allocateCell(); + buf32[iboundary+CELL_AND] = inext; + buf32[icell+CELL_AND] = iboundary; + return iboundary; + } + // needle remainder: yes + // a: yyyyyyyvvvvvvv + // b: vvvvvvv + else { + if ( inext !== 0 ) { + icell = inext; + continue; + } + // TODO: we should never reach here because there will + // always be a boundary cell. + // eslint-disable-next-line no-debugger + debugger; // jshint ignore:line + // boundary cell + needle remainder + inext = this.addCell(0, 0, 0); + buf32[icell+CELL_AND] = inext; + buf32[inext+CELL_AND] = + this.addCell(0, 0, toSegmentInfo(aL0, 0, ar)); + } + } + // some segment characters matched + // a: ...vvvvvvv + // b: yyyyyyyvvvvvvv + else { + // split current cell + buf32[icell+SEGMENT_INFO] = (bR - br) << 24 | (br - char0); + inext = this.addCell( + buf32[icell+CELL_AND], + 0, + (br - bL) << 24 | (bL - char0) + ); + // needle remainder: no = need boundary cell + // a: vvvvvvv + // b: yyyyyyyvvvvvvv + // r: yyyyyyy & 0 & vvvvvvv + if ( ar === 0 ) { + iboundary = this.allocateCell(); + buf32[icell+CELL_AND] = iboundary; + buf32[iboundary+CELL_AND] = inext; + return iboundary; + } + // needle remainder: yes = need new cell for remaining + // characters + // a: wwwwvvvvvvv + // b: yyyyyyyvvvvvvv + // r: (0 & wwww | yyyyyyy) & vvvvvvv + else { + buf32[icell+CELL_AND] = inext; + iboundary = this.allocateCell(); + buf32[inext+CELL_OR] = this.addCell( + iboundary, 0, toSegmentInfo(aL0, 0, ar) + ); + return iboundary; + } + } + //debugger; // jshint ignore:line + } + } + + getExtra(iboundary) { + return this.buf32[iboundary+BCELL_EXTRA]; + } + + setExtra(iboundary, v) { + this.buf32[iboundary+BCELL_EXTRA] = v; + } + + optimize(shrink = false) { + if ( shrink ) { + this.shrinkBuf(); + } + return { + byteLength: this.buf8.byteLength, + char0: this.buf32[CHAR0_SLOT], + }; + } + + serialize(encoder) { + if ( encoder instanceof Object ) { + return encoder.encode( + this.buf32.buffer, + this.buf32[CHAR1_SLOT] + ); + } + return Array.from( + new Uint32Array( + this.buf32.buffer, + 0, + this.buf32[CHAR1_SLOT] + 3 >>> 2 + ) + ); + } + + unserialize(selfie, decoder) { + const shouldDecode = typeof selfie === 'string'; + let byteLength = shouldDecode + ? decoder.decodeSize(selfie) + : selfie.length << 2; + if ( byteLength === 0 ) { return false; } + this.reallocateBuf(byteLength); + if ( shouldDecode ) { + decoder.decode(selfie, this.buf8.buffer); + } else { + this.buf32.set(selfie); + } + return true; + } + + storeString(s) { + const n = s.length; + if ( n === this.lastStoredLen && s === this.lastStored ) { + return this.lastStoredIndex; + } + this.lastStored = s; + this.lastStoredLen = n; + if ( (this.buf8.length - this.buf32[CHAR1_SLOT]) < n ) { + this.growBuf(0, n); + } + const offset = this.buf32[CHAR1_SLOT]; + this.buf32[CHAR1_SLOT] = offset + n; + const buf8 = this.buf8; + for ( let i = 0; i < n; i++ ) { + buf8[offset+i] = s.charCodeAt(i); + } + return (this.lastStoredIndex = offset - this.buf32[CHAR0_SLOT]); + } + + extractString(i, n) { + if ( this.textDecoder === null ) { + this.textDecoder = new TextDecoder(); + } + const offset = this.buf32[CHAR0_SLOT] + i; + return this.textDecoder.decode( + this.buf8.subarray(offset, offset + n) + ); + } + + // WASMable. + startsWith(haystackLeft, haystackRight, needleLeft, needleLen) { + if ( haystackLeft < 0 || (haystackLeft + needleLen) > haystackRight ) { + return 0; + } + const charCodes = this.buf8; + needleLeft += this.buf32[CHAR0_SLOT]; + const needleRight = needleLeft + needleLen; + while ( charCodes[haystackLeft] === charCodes[needleLeft] ) { + needleLeft += 1; + if ( needleLeft === needleRight ) { return 1; } + haystackLeft += 1; + } + return 0; + } + + // Find the left-most instance of substring in main string + // WASMable. + indexOf(haystackLeft, haystackEnd, needleLeft, needleLen) { + if ( needleLen === 0 ) { return haystackLeft; } + haystackEnd -= needleLen; + if ( haystackEnd < haystackLeft ) { return -1; } + needleLeft += this.buf32[CHAR0_SLOT]; + const needleRight = needleLeft + needleLen; + const charCodes = this.buf8; + for (;;) { + let i = haystackLeft; + let j = needleLeft; + while ( charCodes[i] === charCodes[j] ) { + j += 1; + if ( j === needleRight ) { return haystackLeft; } + i += 1; + } + haystackLeft += 1; + if ( haystackLeft > haystackEnd ) { break; } + } + return -1; + } + + // Find the right-most instance of substring in main string. + // WASMable. + lastIndexOf(haystackBeg, haystackEnd, needleLeft, needleLen) { + if ( needleLen === 0 ) { return haystackBeg; } + let haystackLeft = haystackEnd - needleLen; + if ( haystackLeft < haystackBeg ) { return -1; } + needleLeft += this.buf32[CHAR0_SLOT]; + const needleRight = needleLeft + needleLen; + const charCodes = this.buf8; + for (;;) { + let i = haystackLeft; + let j = needleLeft; + while ( charCodes[i] === charCodes[j] ) { + j += 1; + if ( j === needleRight ) { return haystackLeft; } + i += 1; + } + if ( haystackLeft === haystackBeg ) { break; } + haystackLeft -= 1; + } + return -1; + } + + dumpTrie(iroot) { + for ( const s of this.trieIterator(iroot) ) { + console.log(s); + } + } + + trieIterator(iroot) { + return { + value: undefined, + done: false, + next() { + if ( this.icell === 0 ) { + if ( this.forks.length === 0 ) { + this.value = undefined; + this.done = true; + return this; + } + this.pattern = this.forks.pop(); + this.dir = this.forks.pop(); + this.icell = this.forks.pop(); + } + const buf32 = this.container.buf32; + const buf8 = this.container.buf8; + for (;;) { + const ialt = buf32[this.icell+CELL_OR]; + const v = buf32[this.icell+SEGMENT_INFO]; + const offset = v & 0x00FFFFFF; + let i0 = buf32[CHAR0_SLOT] + offset; + const len = v >>> 24; + for ( let i = 0; i < len; i++ ) { + this.charBuf[i] = buf8[i0+i]; + } + if ( len !== 0 && ialt !== 0 ) { + this.forks.push(ialt, this.dir, this.pattern); + } + const inext = buf32[this.icell+CELL_AND]; + if ( len !== 0 ) { + const s = this.textDecoder.decode( + new Uint8Array(this.charBuf.buffer, 0, len) + ); + if ( this.dir > 0 ) { + this.pattern += s; + } else if ( this.dir < 0 ) { + this.pattern = s + this.pattern; + } + } + this.icell = inext; + if ( len !== 0 ) { continue; } + // boundary cell + if ( ialt !== 0 ) { + if ( inext === 0 ) { + this.icell = ialt; + this.dir = -1; + } else { + this.forks.push(ialt, -1, this.pattern); + } + } + if ( offset !== 0 ) { + this.value = { pattern: this.pattern, iextra: offset }; + return this; + } + } + }, + container: this, + icell: iroot, + charBuf: new Uint8Array(256), + pattern: '', + dir: 1, + forks: [], + textDecoder: new TextDecoder(), + [Symbol.iterator]() { return this; }, + }; + } + + async enableWASM(wasmModuleFetcher, path) { + if ( typeof WebAssembly !== 'object' ) { return false; } + if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; } + const module = await getWasmModule(wasmModuleFetcher, path); + if ( module instanceof WebAssembly.Module === false ) { return false; } + const memory = new WebAssembly.Memory({ + initial: roundToPageSize(this.buf8.length) >>> 16 + }); + const instance = await WebAssembly.instantiate(module, { + imports: { memory, extraHandler: this.extraHandler } + }); + if ( instance instanceof WebAssembly.Instance === false ) { + return false; + } + this.wasmMemory = memory; + const curPageCount = memory.buffer.byteLength >>> 16; + const newPageCount = roundToPageSize(this.buf8.byteLength) >>> 16; + if ( newPageCount > curPageCount ) { + memory.grow(newPageCount - curPageCount); + } + const buf8 = new Uint8Array(memory.buffer); + buf8.set(this.buf8); + this.buf8 = buf8; + this.buf32 = new Uint32Array(this.buf8.buffer); + this.haystack = this.buf8.subarray( + HAYSTACK_START, + HAYSTACK_START + HAYSTACK_SIZE + ); + this.matches = instance.exports.matches; + this.startsWith = instance.exports.startsWith; + this.indexOf = instance.exports.indexOf; + this.lastIndexOf = instance.exports.lastIndexOf; + return true; + } + + dumpInfo() { + return [ + `Buffer size (Uint8Array): ${this.buf32[CHAR1_SLOT].toLocaleString('en')}`, + `WASM: ${this.wasmMemory === null ? 'disabled' : 'enabled'}`, + ].join('\n'); + } + + //-------------------------------------------------------------------------- + // Private methods + //-------------------------------------------------------------------------- + + allocateCell() { + let icell = this.buf32[TRIE1_SLOT]; + this.buf32[TRIE1_SLOT] = icell + CELL_BYTE_LENGTH; + icell >>>= 2; + this.buf32[icell+0] = 0; + this.buf32[icell+1] = 0; + this.buf32[icell+2] = 0; + return icell; + } + + addCell(iand, ior, v) { + const icell = this.allocateCell(); + this.buf32[icell+CELL_AND] = iand; + this.buf32[icell+CELL_OR] = ior; + this.buf32[icell+SEGMENT_INFO] = v; + return icell; + } + + growBuf(trieGrow, charGrow) { + const char0 = Math.max( + roundToPageSize(this.buf32[TRIE1_SLOT] + trieGrow), + this.buf32[CHAR0_SLOT] + ); + const char1 = char0 + this.buf32[CHAR1_SLOT] - this.buf32[CHAR0_SLOT]; + const bufLen = Math.max( + roundToPageSize(char1 + charGrow), + this.buf8.length + ); + if ( bufLen > this.buf8.length ) { + this.reallocateBuf(bufLen); + } + if ( char0 !== this.buf32[CHAR0_SLOT] ) { + this.buf8.copyWithin( + char0, + this.buf32[CHAR0_SLOT], + this.buf32[CHAR1_SLOT] + ); + this.buf32[CHAR0_SLOT] = char0; + this.buf32[CHAR1_SLOT] = char1; + } + } + + shrinkBuf() { + const char0 = this.buf32[TRIE1_SLOT] + MIN_FREE_CELL_BYTE_LENGTH; + const char1 = char0 + this.buf32[CHAR1_SLOT] - this.buf32[CHAR0_SLOT]; + const bufLen = char1 + 256; + if ( char0 !== this.buf32[CHAR0_SLOT] ) { + this.buf8.copyWithin( + char0, + this.buf32[CHAR0_SLOT], + this.buf32[CHAR1_SLOT] + ); + this.buf32[CHAR0_SLOT] = char0; + this.buf32[CHAR1_SLOT] = char1; + } + if ( bufLen < this.buf8.length ) { + this.reallocateBuf(bufLen); + } + } + + reallocateBuf(newSize) { + newSize = roundToPageSize(newSize); + if ( newSize === this.buf8.length ) { return; } + if ( this.wasmMemory === null ) { + const newBuf = new Uint8Array(newSize); + newBuf.set( + newBuf.length < this.buf8.length + ? this.buf8.subarray(0, newBuf.length) + : this.buf8 + ); + this.buf8 = newBuf; + } else { + const growBy = + ((newSize + 0xFFFF) >>> 16) - (this.buf8.length >>> 16); + if ( growBy <= 0 ) { return; } + this.wasmMemory.grow(growBy); + this.buf8 = new Uint8Array(this.wasmMemory.buffer); + } + this.buf32 = new Uint32Array(this.buf8.buffer); + this.haystack = this.buf8.subarray( + HAYSTACK_START, + HAYSTACK_START + HAYSTACK_SIZE + ); + } +} + +/******************************************************************************/ + +// Code below is to attempt to load a WASM module which implements: +// +// - BidiTrieContainer.startsWith() +// +// The WASM module is entirely optional, the JS implementations will be +// used should the WASM module be unavailable for whatever reason. + +const getWasmModule = (( ) => { + let wasmModulePromise; + + return async function(wasmModuleFetcher, path) { + if ( wasmModulePromise instanceof Promise ) { + return wasmModulePromise; + } + + if ( typeof WebAssembly !== 'object' ) { return; } + + // Soft-dependency on vAPI so that the code here can be used outside of + // uBO (i.e. tests, benchmarks) + if ( typeof vAPI === 'object' && vAPI.canWASM !== true ) { return; } + + // The wasm module will work only if CPU is natively little-endian, + // as we use native uint32 array in our js code. + const uint32s = new Uint32Array(1); + const uint8s = new Uint8Array(uint32s.buffer); + uint32s[0] = 1; + if ( uint8s[0] !== 1 ) { return; } + + wasmModulePromise = wasmModuleFetcher(`${path}biditrie`).catch(reason => { + console.info(reason); + }); + + return wasmModulePromise; + }; +})(); + +/******************************************************************************/ + +export default BidiTrieContainer; diff --git a/src/js/broadcast.js b/src/js/broadcast.js new file mode 100644 index 0000000..0bef46c --- /dev/null +++ b/src/js/broadcast.js @@ -0,0 +1,75 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +// Broadcast a message to all uBO contexts + +let broadcastChannel; + +export function broadcast(message) { + if ( broadcastChannel === undefined ) { + broadcastChannel = new self.BroadcastChannel('uBO'); + } + broadcastChannel.postMessage(message); +} + +/******************************************************************************/ + +// Broadcast a message to all uBO contexts and all uBO's content scripts + +export async function broadcastToAll(message) { + broadcast(message); + const tabs = await vAPI.tabs.query({ + discarded: false, + }); + const bcmessage = Object.assign({ broadcast: true }, message); + for ( const tab of tabs ) { + browser.tabs.sendMessage(tab.id, bcmessage); + } +} + +/******************************************************************************/ + +export function onBroadcast(listener) { + const bc = new self.BroadcastChannel('uBO'); + bc.onmessage = ev => listener(ev.data || {}); + return bc; +} + +/******************************************************************************/ + +export function filteringBehaviorChanged(details = {}) { + if ( typeof details.direction !== 'number' || details.direction >= 0 ) { + filteringBehaviorChanged.throttle.offon(727); + } + broadcast(Object.assign({ what: 'filteringBehaviorChanged' }, details)); +} + +filteringBehaviorChanged.throttle = vAPI.defer.create(( ) => { + vAPI.net.handlerBehaviorChanged(); +}); + +/******************************************************************************/ diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js new file mode 100644 index 0000000..ef056af --- /dev/null +++ b/src/js/cachestorage.js @@ -0,0 +1,533 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2016-present The uBlock Origin authors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global browser, IDBDatabase, indexedDB */ + +'use strict'; + +/******************************************************************************/ + +import lz4Codec from './lz4.js'; +import µb from './background.js'; +import webext from './webext.js'; + +/******************************************************************************/ + +// The code below has been originally manually imported from: +// Commit: https://github.com/nikrolls/uBlock-Edge/commit/d1538ea9bea89d507219d3219592382eee306134 +// Commit date: 29 October 2016 +// Commit author: https://github.com/nikrolls +// Commit message: "Implement cacheStorage using IndexedDB" + +// The original imported code has been subsequently modified as it was not +// compatible with Firefox. +// (a Promise thing, see https://github.com/dfahlander/Dexie.js/issues/317) +// Furthermore, code to migrate from browser.storage.local to vAPI.storage +// has been added, for seamless migration of cache-related entries into +// indexedDB. + +// https://bugzilla.mozilla.org/show_bug.cgi?id=1371255 +// Firefox-specific: we use indexedDB because browser.storage.local() has +// poor performance in Firefox. +// https://github.com/uBlockOrigin/uBlock-issues/issues/328 +// Use IndexedDB for Chromium as well, to take advantage of LZ4 +// compression. +// https://github.com/uBlockOrigin/uBlock-issues/issues/399 +// Revert Chromium support of IndexedDB, use advanced setting to force +// IndexedDB. +// https://github.com/uBlockOrigin/uBlock-issues/issues/409 +// Allow forcing the use of webext storage on Firefox. + +const STORAGE_NAME = 'uBlock0CacheStorage'; + +// Default to webext storage. +const storageLocal = webext.storage.local; + +let storageReadyResolve; +const storageReadyPromise = new Promise(resolve => { + storageReadyResolve = resolve; +}); + +const cacheStorage = { + name: 'browser.storage.local', + get(...args) { + return storageReadyPromise.then(( ) => + storageLocal.get(...args).catch(reason => { + console.log(reason); + }) + ); + }, + set(...args) { + return storageReadyPromise.then(( ) => + storageLocal.set(...args).catch(reason => { + console.log(reason); + }) + ); + }, + remove(...args) { + return storageReadyPromise.then(( ) => + storageLocal.remove(...args).catch(reason => { + console.log(reason); + }) + ); + }, + clear(...args) { + return storageReadyPromise.then(( ) => + storageLocal.clear(...args).catch(reason => { + console.log(reason); + }) + ); + }, + select: function(selectedBackend) { + let actualBackend = selectedBackend; + if ( actualBackend === undefined || actualBackend === 'unset' ) { + actualBackend = vAPI.webextFlavor.soup.has('firefox') + ? 'indexedDB' + : 'browser.storage.local'; + } + if ( actualBackend === 'indexedDB' ) { + return selectIDB().then(success => { + if ( success || selectedBackend === 'indexedDB' ) { + clearWebext(); + storageReadyResolve(); + return 'indexedDB'; + } + clearIDB(); + storageReadyResolve(); + return 'browser.storage.local'; + }); + } + if ( actualBackend === 'browser.storage.local' ) { + clearIDB(); + } + storageReadyResolve(); + return Promise.resolve('browser.storage.local'); + + }, + error: undefined +}; + +// Not all platforms support getBytesInUse +if ( storageLocal.getBytesInUse instanceof Function ) { + cacheStorage.getBytesInUse = function(...args) { + return storageLocal.getBytesInUse(...args).catch(reason => { + console.log(reason); + }); + }; +} + +// Reassign API entries to that of indexedDB-based ones +const selectIDB = async function() { + let db; + let dbPromise; + + const noopfn = function () { + }; + + const disconnect = function() { + dbTimer.off(); + if ( db instanceof IDBDatabase ) { + db.close(); + db = undefined; + } + }; + + const dbTimer = vAPI.defer.create(( ) => { + disconnect(); + }); + + const keepAlive = function() { + dbTimer.offon(Math.max( + µb.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000, + 180000 + )); + }; + + // https://github.com/gorhill/uBlock/issues/3156 + // I have observed that no event was fired in Tor Browser 7.0.7 + + // medium security level after the request to open the database was + // created. When this occurs, I have also observed that the `error` + // property was already set, so this means uBO can detect here whether + // the database can be opened successfully. A try-catch block is + // necessary when reading the `error` property because we are not + // allowed to read this property outside of event handlers in newer + // implementation of IDBRequest (my understanding). + + const getDb = function() { + keepAlive(); + if ( db !== undefined ) { + return Promise.resolve(db); + } + if ( dbPromise !== undefined ) { + return dbPromise; + } + dbPromise = new Promise(resolve => { + let req; + try { + req = indexedDB.open(STORAGE_NAME, 1); + if ( req.error ) { + console.log(req.error); + req = undefined; + } + } catch(ex) { + } + if ( req === undefined ) { + db = null; + dbPromise = undefined; + return resolve(null); + } + req.onupgradeneeded = function(ev) { + // https://github.com/uBlockOrigin/uBlock-issues/issues/2725 + // If context Firefox + incognito mode, fall back to + // browser.storage.local for cache storage purpose. + if ( + vAPI.webextFlavor.soup.has('firefox') && + browser.extension.inIncognitoContext === true + ) { + return req.onerror(); + } + if ( ev.oldVersion === 1 ) { return; } + try { + const db = ev.target.result; + db.createObjectStore(STORAGE_NAME, { keyPath: 'key' }); + } catch(ex) { + req.onerror(); + } + }; + req.onsuccess = function(ev) { + if ( resolve === undefined ) { return; } + req = undefined; + db = ev.target.result; + dbPromise = undefined; + resolve(db); + resolve = undefined; + }; + req.onerror = req.onblocked = function() { + if ( resolve === undefined ) { return; } + req = undefined; + console.log(this.error); + db = null; + dbPromise = undefined; + resolve(null); + resolve = undefined; + }; + vAPI.defer.once(5000).then(( ) => { + if ( resolve === undefined ) { return; } + db = null; + dbPromise = undefined; + resolve(null); + resolve = undefined; + }); + }); + return dbPromise; + }; + + const fromBlob = function(data) { + if ( data instanceof Blob === false ) { + return Promise.resolve(data); + } + return new Promise(resolve => { + const blobReader = new FileReader(); + blobReader.onloadend = ev => { + resolve(new Uint8Array(ev.target.result)); + }; + blobReader.readAsArrayBuffer(data); + }); + }; + + const toBlob = function(data) { + const value = data instanceof Uint8Array + ? new Blob([ data ]) + : data; + return Promise.resolve(value); + }; + + const compress = function(store, key, data) { + return lz4Codec.encode(data, toBlob).then(value => { + store.push({ key, value }); + }); + }; + + const decompress = function(store, key, data) { + return lz4Codec.decode(data, fromBlob).then(data => { + store[key] = data; + }); + }; + + const getFromDb = async function(keys, keyvalStore, callback) { + if ( typeof callback !== 'function' ) { return; } + if ( keys.length === 0 ) { return callback(keyvalStore); } + const promises = []; + const gotOne = function() { + if ( typeof this.result !== 'object' ) { return; } + const { key, value } = this.result; + keyvalStore[key] = value; + if ( value instanceof Blob === false ) { return; } + promises.push(decompress(keyvalStore, key, value)); + }; + try { + const db = await getDb(); + if ( !db ) { return callback(); } + const transaction = db.transaction(STORAGE_NAME, 'readonly'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = ( ) => { + Promise.all(promises).then(( ) => { + callback(keyvalStore); + }); + }; + const table = transaction.objectStore(STORAGE_NAME); + for ( const key of keys ) { + const req = table.get(key); + req.onsuccess = gotOne; + req.onerror = noopfn; + } + } + catch(reason) { + console.info(`cacheStorage.getFromDb() failed: ${reason}`); + callback(); + } + }; + + const visitAllFromDb = async function(visitFn) { + const db = await getDb(); + if ( !db ) { return visitFn(); } + const transaction = db.transaction(STORAGE_NAME, 'readonly'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = ( ) => visitFn(); + const table = transaction.objectStore(STORAGE_NAME); + const req = table.openCursor(); + req.onsuccess = function(ev) { + let cursor = ev.target && ev.target.result; + if ( !cursor ) { return; } + let entry = cursor.value; + visitFn(entry); + cursor.continue(); + }; + }; + + const getAllFromDb = function(callback) { + if ( typeof callback !== 'function' ) { return; } + const promises = []; + const keyvalStore = {}; + visitAllFromDb(entry => { + if ( entry === undefined ) { + Promise.all(promises).then(( ) => { + callback(keyvalStore); + }); + return; + } + const { key, value } = entry; + keyvalStore[key] = value; + if ( entry.value instanceof Blob === false ) { return; } + promises.push(decompress(keyvalStore, key, value)); + }).catch(reason => { + console.info(`cacheStorage.getAllFromDb() failed: ${reason}`); + callback(); + }); + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/141 + // Mind that IDBDatabase.transaction() and IDBObjectStore.put() + // can throw: + // https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction + // https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put + + const putToDb = async function(keyvalStore, callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + const keys = Object.keys(keyvalStore); + if ( keys.length === 0 ) { return callback(); } + const promises = [ getDb() ]; + const entries = []; + const dontCompress = + µb.hiddenSettings.cacheStorageCompression !== true; + for ( const key of keys ) { + const value = keyvalStore[key]; + const isString = typeof value === 'string'; + if ( isString === false || dontCompress ) { + entries.push({ key, value }); + continue; + } + promises.push(compress(entries, key, value)); + } + const finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + const results = await Promise.all(promises); + const db = results[0]; + if ( !db ) { return callback(); } + const transaction = db.transaction( + STORAGE_NAME, + 'readwrite' + ); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = finish; + const table = transaction.objectStore(STORAGE_NAME); + for ( const entry of entries ) { + table.put(entry); + } + } catch (ex) { + finish(); + } + }; + + const deleteFromDb = async function(input, callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + const keys = Array.isArray(input) ? input.slice() : [ input ]; + if ( keys.length === 0 ) { return callback(); } + const finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + const db = await getDb(); + if ( !db ) { return callback(); } + const transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = finish; + const table = transaction.objectStore(STORAGE_NAME); + for ( const key of keys ) { + table.delete(key); + } + } catch (ex) { + finish(); + } + }; + + const clearDb = async function(callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + try { + const db = await getDb(); + if ( !db ) { return callback(); } + const transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = ( ) => { + callback(); + }; + transaction.objectStore(STORAGE_NAME).clear(); + } + catch(reason) { + console.info(`cacheStorage.clearDb() failed: ${reason}`); + callback(); + } + }; + + await getDb(); + if ( !db ) { return false; } + + cacheStorage.name = 'indexedDB'; + cacheStorage.get = function get(keys) { + return storageReadyPromise.then(( ) => + new Promise(resolve => { + if ( keys === null ) { + return getAllFromDb(bin => resolve(bin)); + } + let toRead, output = {}; + if ( typeof keys === 'string' ) { + toRead = [ keys ]; + } else if ( Array.isArray(keys) ) { + toRead = keys; + } else /* if ( typeof keys === 'object' ) */ { + toRead = Object.keys(keys); + output = keys; + } + getFromDb(toRead, output, bin => resolve(bin)); + }) + ); + }; + cacheStorage.set = function set(keys) { + return storageReadyPromise.then(( ) => + new Promise(resolve => { + putToDb(keys, details => resolve(details)); + }) + ); + }; + cacheStorage.remove = function remove(keys) { + return storageReadyPromise.then(( ) => + new Promise(resolve => { + deleteFromDb(keys, ( ) => resolve()); + }) + ); + }; + cacheStorage.clear = function clear() { + return storageReadyPromise.then(( ) => + new Promise(resolve => { + clearDb(( ) => resolve()); + }) + ); + }; + cacheStorage.getBytesInUse = function getBytesInUse() { + return Promise.resolve(0); + }; + return true; +}; + +// https://github.com/uBlockOrigin/uBlock-issues/issues/328 +// Delete cache-related entries from webext storage. +const clearWebext = async function() { + let bin; + try { + bin = await webext.storage.local.get('assetCacheRegistry'); + } catch(ex) { + console.error(ex); + } + if ( bin instanceof Object === false ) { return; } + if ( bin.assetCacheRegistry instanceof Object === false ) { return; } + const toRemove = [ + 'assetCacheRegistry', + 'assetSourceRegistry', + ]; + for ( const key in bin.assetCacheRegistry ) { + if ( bin.assetCacheRegistry.hasOwnProperty(key) ) { + toRemove.push('cache/' + key); + } + } + webext.storage.local.remove(toRemove); +}; + +const clearIDB = function() { + try { + indexedDB.deleteDatabase(STORAGE_NAME); + } catch(ex) { + } +}; + +/******************************************************************************/ + +export default cacheStorage; + +/******************************************************************************/ diff --git a/src/js/click2load.js b/src/js/click2load.js new file mode 100644 index 0000000..42b7525 --- /dev/null +++ b/src/js/click2load.js @@ -0,0 +1,60 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ +/******************************************************************************/ + +(( ) => { + +/******************************************************************************/ + +if ( typeof vAPI !== 'object' ) { return; } + +const url = new URL(self.location.href); +const actualURL = url.searchParams.get('url'); +const frameURL = url.searchParams.get('aliasURL') || actualURL; +const frameURLElem = document.getElementById('frameURL'); + +frameURLElem.children[0].textContent = actualURL; + +frameURLElem.children[1].href = frameURL; +frameURLElem.children[1].title = frameURL; + +document.body.setAttribute('title', actualURL); + +document.body.addEventListener('click', ev => { + if ( ev.isTrusted === false ) { return; } + if ( ev.target.closest('#frameURL') !== null ) { return; } + vAPI.messaging.send('default', { + what: 'clickToLoad', + frameURL, + }).then(ok => { + if ( ok ) { + self.location.replace(frameURL); + } + }); +}); + +/******************************************************************************/ + +})(); diff --git a/src/js/cloud-ui.js b/src/js/cloud-ui.js new file mode 100644 index 0000000..228f114 --- /dev/null +++ b/src/js/cloud-ui.js @@ -0,0 +1,238 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global faIconsInit */ + +'use strict'; + +import { i18n, i18n$ } from './i18n.js'; +import { dom, qs$ } from './dom.js'; +import { faIconsInit } from './fa-icons.js'; + +/******************************************************************************/ + +(( ) => { + +/******************************************************************************/ + +self.cloud = { + options: {}, + datakey: '', + data: undefined, + onPush: null, + onPull: null, +}; + +/******************************************************************************/ + +const widget = qs$('#cloudWidget'); +if ( widget === null ) { return; } + +self.cloud.datakey = dom.attr(widget, 'data-cloud-entry') || ''; +if ( self.cloud.datakey === '' ) { return; } + +/******************************************************************************/ + +const fetchStorageUsed = async function() { + let elem = qs$(widget, '#cloudCapacity'); + if ( dom.cl.has(elem, 'hide') ) { return; } + const result = await vAPI.messaging.send('cloudWidget', { + what: 'cloudUsed', + datakey: self.cloud.datakey, + }); + if ( result instanceof Object === false ) { + dom.cl.add(elem, 'hide'); + return; + } + const units = ' ' + i18n$('genericBytes'); + elem.title = result.max.toLocaleString() + units; + const total = (result.total / result.max * 100).toFixed(1); + elem = elem.firstElementChild; + elem.style.width = `${total}%`; + elem.title = result.total.toLocaleString() + units; + const used = (result.used / result.total * 100).toFixed(1); + elem = elem.firstElementChild; + elem.style.width = `${used}%`; + elem.title = result.used.toLocaleString() + units; +}; + +/******************************************************************************/ + +const fetchCloudData = async function() { + const info = qs$(widget, '#cloudInfo'); + + const entry = await vAPI.messaging.send('cloudWidget', { + what: 'cloudPull', + datakey: self.cloud.datakey, + }); + + const hasData = entry instanceof Object; + if ( hasData === false ) { + dom.attr('#cloudPull', 'disabled', ''); + dom.attr('#cloudPullAndMerge', 'disabled', ''); + info.textContent = '...\n...'; + return entry; + } + + self.cloud.data = entry.data; + + dom.attr('#cloudPull', 'disabled', null); + dom.attr('#cloudPullAndMerge', 'disabled', null); + + const timeOptions = { + weekday: 'short', + year: 'numeric', + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: 'numeric', + second: 'numeric', + timeZoneName: 'short' + }; + + const time = new Date(entry.tstamp); + info.textContent = + entry.source + '\n' + + time.toLocaleString('fullwide', timeOptions); +}; + +/******************************************************************************/ + +const pushData = async function() { + if ( typeof self.cloud.onPush !== 'function' ) { return; } + + const error = await vAPI.messaging.send('cloudWidget', { + what: 'cloudPush', + datakey: self.cloud.datakey, + data: self.cloud.onPush(), + }); + const failed = typeof error === 'string'; + dom.cl.toggle('#cloudPush', 'error', failed); + dom.text('#cloudError', failed ? error : ''); + if ( failed ) { return; } + fetchCloudData(); + fetchStorageUsed(); +}; + +/******************************************************************************/ + +const pullData = function() { + if ( typeof self.cloud.onPull === 'function' ) { + self.cloud.onPull(self.cloud.data, false); + } + dom.cl.remove('#cloudPush', 'error'); + dom.text('#cloudError', ''); +}; + +/******************************************************************************/ + +const pullAndMergeData = function() { + if ( typeof self.cloud.onPull === 'function' ) { + self.cloud.onPull(self.cloud.data, true); + } +}; + +/******************************************************************************/ + +const openOptions = function() { + const input = qs$('#cloudDeviceName'); + input.value = self.cloud.options.deviceName; + dom.attr(input, 'placeholder', self.cloud.options.defaultDeviceName); + dom.cl.add('#cloudOptions', 'show'); +}; + +/******************************************************************************/ + +const closeOptions = function(ev) { + const root = qs$('#cloudOptions'); + if ( ev.target !== root ) { return; } + dom.cl.remove(root, 'show'); +}; + +/******************************************************************************/ + +const submitOptions = async function() { + dom.cl.remove('#cloudOptions', 'show'); + + const options = await vAPI.messaging.send('cloudWidget', { + what: 'cloudSetOptions', + options: { + deviceName: qs$('#cloudDeviceName').value + }, + }); + if ( options instanceof Object ) { + self.cloud.options = options; + } +}; + +/******************************************************************************/ + +const onInitialize = function(options) { + if ( options instanceof Object === false ) { return; } + if ( options.enabled !== true ) { return; } + self.cloud.options = options; + + const xhr = new XMLHttpRequest(); + xhr.open('GET', 'cloud-ui.html', true); + xhr.overrideMimeType('text/html;charset=utf-8'); + xhr.responseType = 'text'; + xhr.onload = function() { + this.onload = null; + const parser = new DOMParser(), + parsed = parser.parseFromString(this.responseText, 'text/html'), + fromParent = parsed.body; + while ( fromParent.firstElementChild !== null ) { + widget.appendChild( + document.adoptNode(fromParent.firstElementChild) + ); + } + + faIconsInit(widget); + + i18n.render(widget); + dom.cl.remove(widget, 'hide'); + + dom.on('#cloudPush', 'click', ( ) => { pushData(); }); + dom.on('#cloudPull', 'click', pullData); + dom.on('#cloudPullAndMerge', 'click', pullAndMergeData); + dom.on('#cloudCog', 'click', openOptions); + dom.on('#cloudOptions', 'click', closeOptions); + dom.on('#cloudOptionsSubmit', 'click', ( ) => { submitOptions(); }); + + fetchCloudData().then(result => { + if ( typeof result !== 'string' ) { return; } + dom.cl.add('#cloudPush', 'error'); + dom.text('#cloudError', result); + }); + fetchStorageUsed(); + }; + xhr.send(); +}; + +vAPI.messaging.send('cloudWidget', { + what: 'cloudGetOptions', +}).then(options => { + onInitialize(options); +}); + +/******************************************************************************/ + +})(); diff --git a/src/js/code-viewer.js b/src/js/code-viewer.js new file mode 100644 index 0000000..f11289a --- /dev/null +++ b/src/js/code-viewer.js @@ -0,0 +1,311 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2023-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals CodeMirror, uBlockDashboard, beautifier */ + +'use strict'; + +/******************************************************************************/ + +import { dom, qs$ } from './dom.js'; +import { getActualTheme } from './theme.js'; + +/******************************************************************************/ + +const urlToDocMap = new Map(); +const params = new URLSearchParams(document.location.search); +let currentURL = ''; + +const cmEditor = new CodeMirror(qs$('#content'), { + autofocus: true, + gutters: [ 'CodeMirror-linenumbers' ], + lineNumbers: true, + lineWrapping: true, + matchBrackets: true, + styleActiveLine: { + nonEmpty: true, + }, +}); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +vAPI.messaging.send('dom', { what: 'uiStyles' }).then(response => { + if ( typeof response !== 'object' || response === null ) { return; } + if ( getActualTheme(response.uiTheme) === 'dark' ) { + dom.cl.add('#content .cm-s-default', 'cm-s-night'); + dom.cl.remove('#content .cm-s-default', 'cm-s-default'); + } +}); + +// Convert resource URLs into clickable links to code viewer +cmEditor.addOverlay({ + re: /\b(?:href|src)=["']([^"']+)["']/g, + match: null, + token: function(stream) { + if ( stream.sol() ) { + this.re.lastIndex = 0; + this.match = this.re.exec(stream.string); + } + if ( this.match === null ) { + stream.skipToEnd(); + return null; + } + const end = this.re.lastIndex - 1; + const beg = end - this.match[1].length; + if ( stream.pos < beg ) { + stream.pos = beg; + return null; + } + if ( stream.pos < end ) { + stream.pos = end; + return 'href'; + } + if ( stream.pos < this.re.lastIndex ) { + stream.pos = this.re.lastIndex; + this.match = this.re.exec(stream.string); + return null; + } + stream.skipToEnd(); + return null; + }, +}); + +urlToDocMap.set('', cmEditor.getDoc()); + +/******************************************************************************/ + +async function fetchResource(url) { + let response, text; + const fetchOptions = { + method: 'GET', + referrer: '', + }; + if ( urlToDocMap.has(url) ) { + fetchOptions.cache = 'reload'; + } + try { + response = await fetch(url, fetchOptions); + text = await response.text(); + } catch(reason) { + text = String(reason); + } + let mime = response && response.headers.get('Content-Type') || ''; + mime = mime.replace(/\s*;.*$/, '').trim(); + const beautifierOptions = { + end_with_newline: true, + indent_size: 3, + js: { + max_preserve_newlines: 3, + } + }; + switch ( mime ) { + case 'text/css': + text = beautifier.css(text, beautifierOptions); + break; + case 'text/html': + case 'application/xhtml+xml': + case 'application/xml': + case 'image/svg+xml': + text = beautifier.html(text, beautifierOptions); + break; + case 'text/javascript': + case 'application/javascript': + case 'application/x-javascript': + text = beautifier.js(text, beautifierOptions); + break; + case 'application/json': + text = beautifier.js(text, beautifierOptions); + break; + default: + break; + } + return { mime, text }; +} + +/******************************************************************************/ + +function addPastURLs(url) { + const list = qs$('#pastURLs'); + let current; + for ( let i = 0; i < list.children.length; i++ ) { + const span = list.children[i]; + dom.cl.remove(span, 'selected'); + if ( span.textContent !== url ) { continue; } + current = span; + } + if ( url === '' ) { return; } + if ( current === undefined ) { + current = document.createElement('span'); + current.textContent = url; + list.prepend(current); + } + dom.cl.add(current, 'selected'); +} + +/******************************************************************************/ + +function setInputURL(url) { + const input = qs$('#header input[type="url"]'); + if ( url === input.value ) { return; } + dom.attr(input, 'value', url); + input.value = url; +} + +/******************************************************************************/ + +async function setURL(resourceURL) { + // For convenience, remove potentially existing quotes around the URL + if ( /^(["']).+\1$/.test(resourceURL) ) { + resourceURL = resourceURL.slice(1, -1); + } + let afterURL; + if ( resourceURL !== '' ) { + try { + const url = new URL(resourceURL, currentURL || undefined); + url.hash = ''; + afterURL = url.href; + } catch(ex) { + } + if ( afterURL === undefined ) { return; } + } else { + afterURL = ''; + } + if ( afterURL !== '' && /^https?:\/\/./.test(afterURL) === false ) { + return; + } + if ( afterURL === currentURL ) { + if ( afterURL !== resourceURL ) { + setInputURL(afterURL); + } + return; + } + let afterDoc = urlToDocMap.get(afterURL); + if ( afterDoc === undefined ) { + const r = await fetchResource(afterURL) || { mime: '', text: '' }; + afterDoc = new CodeMirror.Doc(r.text, r.mime || ''); + urlToDocMap.set(afterURL, afterDoc); + } + swapDoc(afterDoc); + currentURL = afterURL; + setInputURL(afterURL); + const a = qs$('.cm-search-widget .sourceURL'); + dom.attr(a, 'href', afterURL); + dom.attr(a, 'title', afterURL); + addPastURLs(afterURL); + // For unknown reasons, calling focus() synchronously does not work... + vAPI.defer.once(1).then(( ) => { cmEditor.focus(); }); +} + +/******************************************************************************/ + +function removeURL(url) { + if ( url === '' ) { return; } + const list = qs$('#pastURLs'); + let foundAt = -1; + for ( let i = 0; i < list.children.length; i++ ) { + const span = list.children[i]; + if ( span.textContent !== url ) { continue; } + foundAt = i; + } + if ( foundAt === -1 ) { return; } + list.children[foundAt].remove(); + if ( foundAt >= list.children.length ) { + foundAt = list.children.length - 1; + } + const afterURL = foundAt !== -1 + ? list.children[foundAt].textContent + : ''; + setURL(afterURL); + urlToDocMap.delete(url); +} + +/******************************************************************************/ + +function swapDoc(doc) { + const r = cmEditor.swapDoc(doc); + if ( self.searchThread ) { + self.searchThread.setHaystack(cmEditor.getValue()); + } + const input = qs$('.cm-search-widget-input input[type="search"]'); + if ( input.value !== '' ) { + qs$('.cm-search-widget').dispatchEvent(new Event('input')); + } + return r; +} + +/******************************************************************************/ + +async function start() { + await setURL(params.get('url')); + + dom.on('#header input[type="url"]', 'change', ev => { + setURL(ev.target.value); + }); + + dom.on('#reloadURL', 'click', ( ) => { + const input = qs$('#header input[type="url"]'); + const url = input.value; + const beforeDoc = swapDoc(new CodeMirror.Doc('', '')); + fetchResource(url).then(r => { + if ( urlToDocMap.has(url) === false ) { return; } + const afterDoc = r !== undefined + ? new CodeMirror.Doc(r.text, r.mime || '') + : beforeDoc; + urlToDocMap.set(url, afterDoc); + if ( currentURL !== url ) { return; } + swapDoc(afterDoc); + }); + }); + + dom.on('#removeURL', 'click', ( ) => { + removeURL(qs$('#header input[type="url"]').value); + }); + + dom.on('#pastURLs', 'mousedown', 'span', ev => { + setURL(ev.target.textContent); + }); + + dom.on('#content', 'click', '.cm-href', ev => { + const target = ev.target; + const urlParts = [ target.textContent ]; + let previous = target; + for (;;) { + previous = previous.previousSibling; + if ( previous === null ) { break; } + if ( previous.nodeType !== 1 ) { break; } + if ( previous.classList.contains('cm-href') === false ) { break; } + urlParts.unshift(previous.textContent); + } + let next = target; + for (;;) { + next = next.nextSibling; + if ( next === null ) { break; } + if ( next.nodeType !== 1 ) { break; } + if ( next.classList.contains('cm-href') === false ) { break; } + urlParts.push(next.textContent); + } + setURL(urlParts.join('')); + }); +} + +start(); + +/******************************************************************************/ diff --git a/src/js/codemirror/search-thread.js b/src/js/codemirror/search-thread.js new file mode 100644 index 0000000..3a4416f --- /dev/null +++ b/src/js/codemirror/search-thread.js @@ -0,0 +1,199 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2020-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +(( ) => { +// >>>>> start of local scope + +/******************************************************************************/ + +// Worker context + +if ( + self.WorkerGlobalScope instanceof Object && + self instanceof self.WorkerGlobalScope +) { + let content = ''; + + const doSearch = function(details) { + const reEOLs = /\n\r|\r\n|\n|\r/g; + const t1 = Date.now() + 750; + + let reSearch; + try { + reSearch = new RegExp(details.pattern, details.flags); + } catch(ex) { + return; + } + + const response = []; + const maxOffset = content.length; + let iLine = 0; + let iOffset = 0; + let size = 0; + while ( iOffset < maxOffset ) { + // Find next match + const match = reSearch.exec(content); + if ( match === null ) { break; } + // Find number of line breaks between last and current match. + reEOLs.lastIndex = 0; + const eols = content.slice(iOffset, match.index).match(reEOLs); + if ( Array.isArray(eols) ) { + iLine += eols.length; + } + // Store line + response.push(iLine); + size += 1; + // Find next line break. + reEOLs.lastIndex = reSearch.lastIndex; + const eol = reEOLs.exec(content); + iOffset = eol !== null + ? reEOLs.lastIndex + : content.length; + reSearch.lastIndex = iOffset; + iLine += 1; + // Quit if this takes too long + if ( (size & 0x3FF) === 0 && Date.now() >= t1 ) { break; } + } + + return response; + }; + + self.onmessage = function(e) { + const msg = e.data; + + switch ( msg.what ) { + case 'setHaystack': + content = msg.content; + break; + + case 'doSearch': + const response = doSearch(msg); + self.postMessage({ id: msg.id, response }); + break; + } + }; + + return; +} + +/******************************************************************************/ + +// Main context + +{ + const workerTTL = { min: 5 }; + const pendingResponses = new Map(); + const workerTTLTimer = vAPI.defer.create(( ) => { + shutdown(); + }); + + let worker; + let messageId = 1; + + const onWorkerMessage = function(e) { + const msg = e.data; + const resolver = pendingResponses.get(msg.id); + if ( resolver === undefined ) { return; } + pendingResponses.delete(msg.id); + resolver(msg.response); + }; + + const cancelPendingTasks = function() { + for ( const resolver of pendingResponses.values() ) { + resolver(); + } + pendingResponses.clear(); + }; + + const destroy = function() { + shutdown(); + self.searchThread = undefined; + }; + + const shutdown = function() { + if ( worker === undefined ) { return; } + workerTTLTimer.off(); + worker.terminate(); + worker.onmessage = undefined; + worker = undefined; + cancelPendingTasks(); + }; + + const init = function() { + if ( self.searchThread instanceof Object === false ) { return; } + if ( worker === undefined ) { + worker = new Worker('js/codemirror/search-thread.js'); + worker.onmessage = onWorkerMessage; + } + workerTTLTimer.offon(workerTTL); + }; + + const needHaystack = function() { + return worker instanceof Object === false; + }; + + const setHaystack = function(content) { + init(); + worker.postMessage({ what: 'setHaystack', content }); + }; + + const search = function(query, overwrite = true) { + init(); + if ( worker instanceof Object === false ) { + return Promise.resolve(); + } + if ( overwrite ) { + cancelPendingTasks(); + } + const id = messageId++; + worker.postMessage({ + what: 'doSearch', + id, + pattern: query.source, + flags: query.flags, + isRE: query instanceof RegExp + }); + return new Promise(resolve => { + pendingResponses.set(id, resolve); + }); + }; + + self.addEventListener( + 'beforeunload', + ( ) => { destroy(); }, + { once: true } + ); + + self.searchThread = { needHaystack, setHaystack, search, shutdown }; +} + +/******************************************************************************/ + +// <<<<< end of local scope +})(); + +/******************************************************************************/ + +void 0; diff --git a/src/js/codemirror/search.js b/src/js/codemirror/search.js new file mode 100644 index 0000000..477e9cc --- /dev/null +++ b/src/js/codemirror/search.js @@ -0,0 +1,504 @@ +// The following code is heavily based on the standard CodeMirror +// search addon found at: https://codemirror.net/addon/search/search.js +// I added/removed and modified code in order to get a closer match to a +// browser's built-in find-in-page feature which are just enough for +// uBlock Origin. +// +// This file was originally wholly imported from: +// https://github.com/codemirror/CodeMirror/blob/3e1bb5fff682f8f6cbfaef0e56c61d62403d4798/addon/search/search.js +// +// And has been modified over time to better suit uBO's usage and coding style: +// https://github.com/gorhill/uBlock/commits/master/src/js/codemirror/search.js +// +// The original copyright notice is reproduced below: + +// ===== +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: http://codemirror.net/LICENSE + +// Define search commands. Depends on dialog.js or another +// implementation of the openDialog method. + +// Replace works a little oddly -- it will do the replace on the next +// Ctrl-G (or whatever is bound to findNext) press. You prevent a +// replace by making sure the match is no longer selected when hitting +// Ctrl-G. +// ===== + +'use strict'; + +import { dom, qs$ } from '../dom.js'; +import { i18n$ } from '../i18n.js'; + +{ + const CodeMirror = self.CodeMirror; + + const searchOverlay = function(query, caseInsensitive) { + if ( typeof query === 'string' ) + query = new RegExp( + query.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, '\\$&'), + caseInsensitive ? 'gi' : 'g' + ); + else if ( !query.global ) + query = new RegExp(query.source, query.ignoreCase ? 'gi' : 'g'); + + return { + token: function(stream) { + query.lastIndex = stream.pos; + const match = query.exec(stream.string); + if ( match && match.index === stream.pos ) { + stream.pos += match[0].length || 1; + return 'searching'; + } else if ( match ) { + stream.pos = match.index; + } else { + stream.skipToEnd(); + } + } + }; + }; + + const searchWidgetKeydownHandler = function(cm, ev) { + const keyName = CodeMirror.keyName(ev); + if ( !keyName ) { return; } + CodeMirror.lookupKey( + keyName, + cm.getOption('keyMap'), + function(command) { + if ( widgetCommandHandler(cm, command) ) { + ev.preventDefault(); + ev.stopPropagation(); + } + } + ); + }; + + const searchWidgetInputHandler = function(cm, ev) { + const state = getSearchState(cm); + if ( ev.isTrusted !== true ) { + if ( state.queryText === '' ) { + clearSearch(cm); + } else { + cm.operation(function() { + startSearch(cm, state); + }); + } + return; + } + if ( queryTextFromSearchWidget(cm) === state.queryText ) { return; } + state.queryTimer.offon(350); + }; + + const searchWidgetClickHandler = function(cm, ev) { + const tcl = ev.target.classList; + if ( tcl.contains('cm-search-widget-up') ) { + findNext(cm, -1); + } else if ( tcl.contains('cm-search-widget-down') ) { + findNext(cm, 1); + } else if ( tcl.contains('cm-linter-widget-up') ) { + findNextError(cm, -1); + } else if ( tcl.contains('cm-linter-widget-down') ) { + findNextError(cm, 1); + } + if ( ev.target.localName !== 'input' ) { + ev.preventDefault(); + } else { + ev.stopImmediatePropagation(); + } + }; + + const queryTextFromSearchWidget = function(cm) { + return getSearchState(cm).widget.querySelector('input[type="search"]').value; + }; + + const queryTextToSearchWidget = function(cm, q) { + const input = getSearchState(cm).widget.querySelector('input[type="search"]'); + if ( typeof q === 'string' && q !== input.value ) { + input.value = q; + } + input.setSelectionRange(0, input.value.length); + input.focus(); + }; + + const SearchState = function(cm) { + this.query = null; + this.panel = null; + const widgetParent = document.querySelector('.cm-search-widget-template').cloneNode(true); + this.widget = widgetParent.children[0]; + this.widget.addEventListener('keydown', searchWidgetKeydownHandler.bind(null, cm)); + this.widget.addEventListener('input', searchWidgetInputHandler.bind(null, cm)); + this.widget.addEventListener('mousedown', searchWidgetClickHandler.bind(null, cm)); + if ( typeof cm.addPanel === 'function' ) { + this.panel = cm.addPanel(this.widget); + } + this.queryText = ''; + this.dirty = true; + this.lines = []; + cm.on('changes', (cm, changes) => { + for ( const change of changes ) { + if ( change.text.length !== 0 || change.removed !== 0 ) { + this.dirty = true; + break; + } + } + }); + cm.on('cursorActivity', cm => { + updateCount(cm); + }); + this.queryTimer = vAPI.defer.create(( ) => { + findCommit(cm, 0); + }); + }; + + // We want the search widget to behave as if the focus was on the + // CodeMirror editor. + + const reSearchCommands = /^(?:find|findNext|findPrev|newlineAndIndent)$/; + + const widgetCommandHandler = function(cm, command) { + if ( reSearchCommands.test(command) === false ) { return false; } + const queryText = queryTextFromSearchWidget(cm); + if ( command === 'find' ) { + queryTextToSearchWidget(cm); + return true; + } + if ( queryText.length !== 0 ) { + findNext(cm, command === 'findPrev' ? -1 : 1); + } + return true; + }; + + const getSearchState = function(cm) { + return cm.state.search || (cm.state.search = new SearchState(cm)); + }; + + const queryCaseInsensitive = function(query) { + return typeof query === 'string' && query === query.toLowerCase(); + }; + + // Heuristic: if the query string is all lowercase, do a case insensitive search. + const getSearchCursor = function(cm, query, pos) { + return cm.getSearchCursor( + query, + pos, + { caseFold: queryCaseInsensitive(query), multiline: false } + ); + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/658 + // Modified to backslash-escape ONLY widely-used control characters. + const parseString = function(string) { + return string.replace(/\\[nrt\\]/g, match => { + if ( match === '\\n' ) { return '\n'; } + if ( match === '\\r' ) { return '\r'; } + if ( match === '\\t' ) { return '\t'; } + if ( match === '\\\\' ) { return '\\'; } + return match; + }); + }; + + const reEscape = /[.*+\-?^${}()|[\]\\]/g; + + // Must always return a RegExp object. + // + // Assume case-sensitivity if there is at least one uppercase in plain + // query text. + const parseQuery = function(query) { + let flags = 'i'; + let reParsed = query.match(/^\/(.+)\/([iu]*)$/); + if ( reParsed !== null ) { + try { + const re = new RegExp(reParsed[1], reParsed[2]); + query = re.source; + flags = re.flags; + } + catch (e) { + reParsed = null; + } + } + if ( reParsed === null ) { + if ( /[A-Z]/.test(query) ) { flags = ''; } + query = parseString(query).replace(reEscape, '\\$&'); + } + if ( typeof query === 'string' ? query === '' : query.test('') ) { + query = 'x^'; + } + return new RegExp(query, 'gm' + flags); + }; + + let intlNumberFormat; + + const formatNumber = function(n) { + if ( intlNumberFormat === undefined ) { + intlNumberFormat = null; + if ( Intl.NumberFormat instanceof Function ) { + const intl = new Intl.NumberFormat(undefined, { + notation: 'compact', + maximumSignificantDigits: 3 + }); + if ( + intl.resolvedOptions instanceof Function && + intl.resolvedOptions().hasOwnProperty('notation') + ) { + intlNumberFormat = intl; + } + } + } + return n > 10000 && intlNumberFormat instanceof Object + ? intlNumberFormat.format(n) + : n.toLocaleString(); + }; + + const updateCount = function(cm) { + const state = getSearchState(cm); + const lines = state.lines; + const current = cm.getCursor().line; + let l = 0; + let r = lines.length; + let i = -1; + while ( l < r ) { + i = l + r >>> 1; + const candidate = lines[i]; + if ( current === candidate ) { break; } + if ( current < candidate ) { + r = i; + } else /* if ( current > candidate ) */ { + l = i + 1; + } + } + let text = ''; + if ( i !== -1 ) { + text = formatNumber(i + 1); + if ( lines[i] !== current ) { + text = '~' + text; + } + text = text + '\xA0/\xA0'; + } + const count = lines.length; + text += formatNumber(count); + const span = state.widget.querySelector('.cm-search-widget-count'); + span.textContent = text; + span.title = count.toLocaleString(); + }; + + const startSearch = function(cm, state) { + state.query = parseQuery(state.queryText); + if ( state.overlay !== undefined ) { + cm.removeOverlay(state.overlay, queryCaseInsensitive(state.query)); + } + state.overlay = searchOverlay(state.query, queryCaseInsensitive(state.query)); + cm.addOverlay(state.overlay); + if ( state.dirty || self.searchThread.needHaystack() ) { + self.searchThread.setHaystack(cm.getValue()); + state.dirty = false; + } + self.searchThread.search(state.query).then(lines => { + if ( Array.isArray(lines) === false ) { return; } + state.lines = lines; + const count = lines.length; + updateCount(cm); + if ( state.annotate !== undefined ) { + state.annotate.clear(); + state.annotate = undefined; + } + if ( count === 0 ) { return; } + state.annotate = cm.annotateScrollbar('CodeMirror-search-match'); + const annotations = []; + let lineBeg = -1; + let lineEnd = -1; + for ( const line of lines ) { + if ( lineBeg === -1 ) { + lineBeg = line; + lineEnd = line + 1; + continue; + } else if ( line === lineEnd ) { + lineEnd = line + 1; + continue; + } + annotations.push({ + from: { line: lineBeg, ch: 0 }, + to: { line: lineEnd, ch: 0 } + }); + lineBeg = -1; + } + if ( lineBeg !== -1 ) { + annotations.push({ + from: { line: lineBeg, ch: 0 }, + to: { line: lineEnd, ch: 0 } + }); + } + state.annotate.update(annotations); + }); + state.widget.setAttribute('data-query', state.queryText); + // Ensure the caret is visible + const input = state.widget.querySelector('.cm-search-widget-input input'); + input.selectionStart = input.selectionStart; + }; + + const findNext = function(cm, dir, callback) { + cm.operation(function() { + const state = getSearchState(cm); + if ( !state.query ) { return; } + let cursor = getSearchCursor( + cm, + state.query, + dir <= 0 ? cm.getCursor('from') : cm.getCursor('to') + ); + const previous = dir < 0; + if (!cursor.find(previous)) { + cursor = getSearchCursor( + cm, + state.query, + previous + ? CodeMirror.Pos(cm.lastLine()) + : CodeMirror.Pos(cm.firstLine(), 0) + ); + if (!cursor.find(previous)) return; + } + cm.setSelection(cursor.from(), cursor.to()); + const { clientHeight } = cm.getScrollInfo(); + cm.scrollIntoView( + { from: cursor.from(), to: cursor.to() }, + clientHeight >>> 1 + ); + if (callback) callback(cursor.from(), cursor.to()); + }); + }; + + const findNextError = function(cm, dir) { + const doc = cm.getDoc(); + const cursor = cm.getCursor('from'); + const cursorLine = cursor.line; + const start = dir < 0 ? 0 : cursorLine + 1; + const end = dir < 0 ? cursorLine : doc.lineCount(); + let found = -1; + doc.eachLine(start, end, lineHandle => { + const markers = lineHandle.gutterMarkers || null; + if ( markers === null ) { return; } + const marker = markers['CodeMirror-lintgutter']; + if ( marker === undefined ) { return; } + if ( marker.dataset.error !== 'y' ) { return; } + const line = lineHandle.lineNo(); + if ( dir < 0 ) { + found = line; + return; + } + found = line; + return true; + }); + if ( found === -1 || found === cursorLine ) { return; } + cm.getDoc().setCursor(found); + const { clientHeight } = cm.getScrollInfo(); + cm.scrollIntoView({ line: found, ch: 0 }, clientHeight >>> 1); + }; + + const clearSearch = function(cm, hard) { + cm.operation(function() { + const state = getSearchState(cm); + if ( state.query ) { + state.query = state.queryText = null; + } + state.lines = []; + if ( state.overlay !== undefined ) { + cm.removeOverlay(state.overlay); + state.overlay = undefined; + } + if ( state.annotate ) { + state.annotate.clear(); + state.annotate = undefined; + } + state.widget.removeAttribute('data-query'); + if ( hard ) { + state.panel.clear(); + state.panel = null; + state.widget = null; + cm.state.search = null; + } + }); + }; + + const findCommit = function(cm, dir) { + const state = getSearchState(cm); + state.queryTimer.off(); + const queryText = queryTextFromSearchWidget(cm); + if ( queryText === state.queryText ) { return; } + state.queryText = queryText; + if ( state.queryText === '' ) { + clearSearch(cm); + } else { + cm.operation(function() { + startSearch(cm, state); + findNext(cm, dir); + }); + } + }; + + const findCommand = function(cm) { + let queryText = cm.getSelection() || undefined; + if ( !queryText ) { + const word = cm.findWordAt(cm.getCursor()); + queryText = cm.getRange(word.anchor, word.head); + if ( /^\W|\W$/.test(queryText) ) { + queryText = undefined; + } + cm.setCursor(word.anchor); + } + queryTextToSearchWidget(cm, queryText); + findCommit(cm, 1); + }; + + const findNextCommand = function(cm) { + const state = getSearchState(cm); + if ( state.query ) { return findNext(cm, 1); } + }; + + const findPrevCommand = function(cm) { + const state = getSearchState(cm); + if ( state.query ) { return findNext(cm, -1); } + }; + + { + const searchWidgetTemplate = + '<div class="cm-search-widget-template" style="display:none;">' + + '<div class="cm-search-widget">' + + '<span class="cm-search-widget-input">' + + '<span class="fa-icon fa-icon-ro">search</span> ' + + '<input type="search" spellcheck="false"> ' + + '<span class="cm-search-widget-up cm-search-widget-button fa-icon">angle-up</span> ' + + '<span class="cm-search-widget-down cm-search-widget-button fa-icon fa-icon-vflipped">angle-up</span> ' + + '<span class="cm-search-widget-count"></span>' + + '</span>' + + '<span class="cm-linter-widget" data-lint="0">' + + '<span class="cm-linter-widget-count"></span> ' + + '<span class="cm-linter-widget-up cm-search-widget-button fa-icon">angle-up</span> ' + + '<span class="cm-linter-widget-down cm-search-widget-button fa-icon fa-icon-vflipped">angle-up</span> ' + + '</span>' + + '<span>' + + '<a class="fa-icon sourceURL" href>external-link</a>' + + '</span>' + + '</div>' + + '</div>'; + const domParser = new DOMParser(); + const doc = domParser.parseFromString(searchWidgetTemplate, 'text/html'); + const widgetTemplate = document.adoptNode(doc.body.firstElementChild); + document.body.appendChild(widgetTemplate); + } + + CodeMirror.commands.find = findCommand; + CodeMirror.commands.findNext = findNextCommand; + CodeMirror.commands.findPrev = findPrevCommand; + + CodeMirror.defineInitHook(function(cm) { + getSearchState(cm); + cm.on('linterDone', details => { + const linterWidget = qs$('.cm-linter-widget'); + const count = details.errorCount; + if ( linterWidget.dataset.lint === `${count}` ) { return; } + linterWidget.dataset.lint = `${count}`; + dom.text( + qs$(linterWidget, '.cm-linter-widget-count'), + i18n$('linterMainReport').replace('{{count}}', count.toLocaleString()) + ); + }); + }); +} diff --git a/src/js/codemirror/ubo-dynamic-filtering.js b/src/js/codemirror/ubo-dynamic-filtering.js new file mode 100644 index 0000000..d0709a4 --- /dev/null +++ b/src/js/codemirror/ubo-dynamic-filtering.js @@ -0,0 +1,239 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2019-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror */ + +'use strict'; + +CodeMirror.defineMode('ubo-dynamic-filtering', ( ) => { + + const validSwitches = new Set([ + 'no-strict-blocking:', + 'no-popups:', + 'no-cosmetic-filtering:', + 'no-remote-fonts:', + 'no-large-media:', + 'no-csp-reports:', + 'no-scripting:', + ]); + const validSwitcheStates = new Set([ + 'true', + 'false', + ]); + const validHnRuleTypes = new Set([ + '*', + '3p', + 'image', + 'inline-script', + '1p-script', + '3p-script', + '3p-frame', + ]); + const invalidURLRuleTypes = new Set([ + 'doc', + 'main_frame', + ]); + const validActions = new Set([ + 'block', + 'allow', + 'noop', + ]); + const hnValidator = new URL(self.location.href); + const reBadHn = /[%]|^\.|\.$/; + const slices = []; + let sliceIndex = 0; + let sliceCount = 0; + let hostnameToDomainMap = new Map(); + let psl; + + const isValidHostname = hnin => { + if ( hnin === '*' ) { return true; } + hnValidator.hostname = '_'; + try { + hnValidator.hostname = hnin; + } catch(_) { + return false; + } + const hnout = hnValidator.hostname; + return hnout !== '_' && hnout !== '' && reBadHn.test(hnout) === false; + }; + + const addSlice = (len, style = null) => { + let i = sliceCount; + if ( i === slices.length ) { + slices[i] = { len: 0, style: null }; + } + const entry = slices[i]; + entry.len = len; + entry.style = style; + sliceCount += 1; + }; + + const addMatchSlice = (match, style = null) => { + const len = match !== null ? match[0].length : 0; + addSlice(len, style); + return match !== null ? match.input.slice(len) : ''; + }; + + const addMatchHnSlices = (match, style = null) => { + const hn = match[0]; + if ( hn === '*' ) { + return addMatchSlice(match, style); + } + let dn = hostnameToDomainMap.get(hn) || ''; + if ( dn === '' && psl !== undefined ) { + dn = /(\d|\])$/.test(hn) ? hn : (psl.getDomain(hn) || hn); + } + const entityBeg = hn.length - dn.length; + if ( entityBeg !== 0 ) { + addSlice(entityBeg, style); + } + let entityEnd = dn.indexOf('.'); + if ( entityEnd === -1 ) { entityEnd = dn.length; } + addSlice(entityEnd, style !== null ? `${style} strong` : 'strong'); + if ( entityEnd < dn.length ) { + addSlice(dn.length - entityEnd, style); + } + return match.input.slice(hn.length); + }; + + const makeSlices = (stream, opts) => { + sliceIndex = 0; + sliceCount = 0; + let { string } = stream; + if ( string === '...' ) { return; } + const { sortType } = opts; + const reNotToken = /^\s+/; + const reToken = /^\S+/; + const tokens = []; + // leading whitespaces + let match = reNotToken.exec(string); + if ( match !== null ) { + string = addMatchSlice(match); + } + // first token + match = reToken.exec(string); + if ( match === null ) { return; } + tokens.push(match[0]); + // hostname or switch + const isSwitchRule = validSwitches.has(match[0]); + if ( isSwitchRule ) { + string = addMatchSlice(match, sortType === 0 ? 'sortkey' : null); + } else if ( isValidHostname(match[0]) ) { + if ( sortType === 1 ) { + string = addMatchHnSlices(match, 'sortkey'); + } else { + string = addMatchHnSlices(match, null); + } + } else { + string = addMatchSlice(match, 'error'); + } + // whitespaces before second token + match = reNotToken.exec(string); + if ( match === null ) { return; } + string = addMatchSlice(match); + // second token + match = reToken.exec(string); + if ( match === null ) { return; } + tokens.push(match[0]); + // hostname or url + const isURLRule = isSwitchRule === false && match[0].indexOf('://') > 0; + if ( isURLRule ) { + string = addMatchSlice(match, sortType === 2 ? 'sortkey' : null); + } else if ( isValidHostname(match[0]) === false ) { + string = addMatchSlice(match, 'error'); + } else if ( sortType === 1 && isSwitchRule || sortType === 2 ) { + string = addMatchHnSlices(match, 'sortkey'); + } else { + string = addMatchHnSlices(match, null); + } + // whitespaces before third token + match = reNotToken.exec(string); + if ( match === null ) { return; } + string = addMatchSlice(match); + // third token + match = reToken.exec(string); + if ( match === null ) { return; } + tokens.push(match[0]); + // rule type or switch state + if ( isSwitchRule ) { + string = validSwitcheStates.has(match[0]) + ? addMatchSlice(match, match[0] === 'true' ? 'blockrule' : 'allowrule') + : addMatchSlice(match, 'error'); + } else if ( isURLRule ) { + string = invalidURLRuleTypes.has(match[0]) + ? addMatchSlice(match, 'error') + : addMatchSlice(match); + } else if ( tokens[1] === '*' ) { + string = validHnRuleTypes.has(match[0]) + ? addMatchSlice(match) + : addMatchSlice(match, 'error'); + } else { + string = match[0] === '*' + ? addMatchSlice(match) + : addMatchSlice(match, 'error'); + } + // whitespaces before fourth token + match = reNotToken.exec(string); + if ( match === null ) { return; } + string = addMatchSlice(match); + // fourth token + match = reToken.exec(string); + if ( match === null ) { return; } + tokens.push(match[0]); + string = isSwitchRule || validActions.has(match[0]) === false + ? addMatchSlice(match, 'error') + : addMatchSlice(match, `${match[0]}rule`); + // whitespaces before end of line + match = reNotToken.exec(string); + if ( match === null ) { return; } + string = addMatchSlice(match); + // any token beyond fourth token is invalid + match = reToken.exec(string); + if ( match !== null ) { + string = addMatchSlice(null, 'error'); + } + }; + + const token = function(stream) { + if ( stream.sol() ) { + makeSlices(stream, this); + } + if ( sliceIndex >= sliceCount ) { + stream.skipToEnd(stream); + return null; + } + const { len, style } = slices[sliceIndex++]; + if ( len === 0 ) { + stream.skipToEnd(); + } else { + stream.pos += len; + } + return style; + }; + + return { + token, + sortType: 1, + setHostnameToDomainMap: a => { hostnameToDomainMap = a; }, + setPSL: a => { psl = a; }, + }; +}); diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js new file mode 100644 index 0000000..ac1b048 --- /dev/null +++ b/src/js/codemirror/ubo-static-filtering.js @@ -0,0 +1,1200 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2018-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror */ + +'use strict'; + +/******************************************************************************/ + +import * as sfp from '../static-filtering-parser.js'; +import { dom, qs$ } from '../dom.js'; + +/******************************************************************************/ + +const redirectNames = new Map(); +const scriptletNames = new Map(); +const preparseDirectiveEnv = []; +const preparseDirectiveHints = []; +const originHints = []; +let hintHelperRegistered = false; + +/******************************************************************************/ + +CodeMirror.defineOption('trustedSource', false, (cm, state) => { + if ( typeof state !== 'boolean' ) { return; } + self.dispatchEvent(new CustomEvent('trustedSource', { + detail: state, + })); +}); + +CodeMirror.defineOption('trustedScriptletTokens', undefined, (cm, tokens) => { + if ( tokens === undefined || tokens === null ) { return; } + if ( typeof tokens[Symbol.iterator] !== 'function' ) { return; } + self.dispatchEvent(new CustomEvent('trustedScriptletTokens', { + detail: new Set(tokens), + })); +}); + +/******************************************************************************/ + +CodeMirror.defineMode('ubo-static-filtering', function() { + const astParser = new sfp.AstFilterParser({ + interactive: true, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); + const astWalker = astParser.getWalker(); + let currentWalkerNode = 0; + let lastNetOptionType = 0; + + const redirectTokenStyle = node => { + const rawToken = astParser.getNodeString(node || currentWalkerNode); + const { token } = sfp.parseRedirectValue(rawToken); + return redirectNames.has(token) ? 'value' : 'value warning'; + }; + + const nodeHasError = node => { + return astParser.getNodeFlags( + node || currentWalkerNode, sfp.NODE_FLAG_ERROR + ) !== 0; + }; + + const colorFromAstNode = function() { + if ( astParser.nodeIsEmptyString(currentWalkerNode) ) { return '+'; } + if ( nodeHasError() ) { return 'error'; } + const nodeType = astParser.getNodeType(currentWalkerNode); + switch ( nodeType ) { + case sfp.NODE_TYPE_WHITESPACE: + return ''; + case sfp.NODE_TYPE_COMMENT: + if ( astWalker.canGoDown() ) { break; } + return 'comment'; + case sfp.NODE_TYPE_COMMENT_URL: + return 'comment link'; + case sfp.NODE_TYPE_IGNORE: + return 'comment'; + case sfp.NODE_TYPE_PREPARSE_DIRECTIVE: + case sfp.NODE_TYPE_PREPARSE_DIRECTIVE_VALUE: + return 'directive'; + case sfp.NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE: { + const raw = astParser.getNodeString(currentWalkerNode); + const state = sfp.utils.preparser.evaluateExpr(raw, preparseDirectiveEnv); + return state ? 'positive strong' : 'negative strong'; + } + case sfp.NODE_TYPE_EXT_OPTIONS_ANCHOR: + return astParser.getFlags(sfp.AST_FLAG_IS_EXCEPTION) + ? 'tag strong' + : 'def strong'; + case sfp.NODE_TYPE_EXT_DECORATION: + return 'def'; + case sfp.NODE_TYPE_EXT_PATTERN_RAW: + if ( astWalker.canGoDown() ) { break; } + return 'variable'; + case sfp.NODE_TYPE_EXT_PATTERN_COSMETIC: + case sfp.NODE_TYPE_EXT_PATTERN_HTML: + return 'variable'; + case sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER: + case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET: + if ( astWalker.canGoDown() ) { break; } + return 'variable'; + case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN: { + const token = astParser.getNodeString(currentWalkerNode); + if ( scriptletNames.has(token) === false ) { + return 'warning'; + } + return 'variable'; + } + case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG: + return 'variable'; + case sfp.NODE_TYPE_NET_EXCEPTION: + return 'tag strong'; + case sfp.NODE_TYPE_NET_PATTERN: + if ( astWalker.canGoDown() ) { break; } + if ( astParser.isRegexPattern() ) { + if ( astParser.getNodeFlags(currentWalkerNode, sfp.NODE_FLAG_PATTERN_UNTOKENIZABLE) !== 0 ) { + return 'variable warning'; + } + return 'variable notice'; + } + return 'variable'; + case sfp.NODE_TYPE_NET_PATTERN_PART: + return 'variable'; + case sfp.NODE_TYPE_NET_PATTERN_PART_SPECIAL: + return 'keyword strong'; + case sfp.NODE_TYPE_NET_PATTERN_PART_UNICODE: + return 'variable unicode'; + case sfp.NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR: + case sfp.NODE_TYPE_NET_PATTERN_LEFT_ANCHOR: + case sfp.NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR: + case sfp.NODE_TYPE_NET_OPTION_NAME_NOT: + return 'keyword strong'; + case sfp.NODE_TYPE_NET_OPTIONS_ANCHOR: + case sfp.NODE_TYPE_NET_OPTION_SEPARATOR: + lastNetOptionType = 0; + return 'def strong'; + case sfp.NODE_TYPE_NET_OPTION_NAME_UNKNOWN: + lastNetOptionType = 0; + return 'error'; + case sfp.NODE_TYPE_NET_OPTION_NAME_1P: + case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P: + case sfp.NODE_TYPE_NET_OPTION_NAME_3P: + case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P: + case sfp.NODE_TYPE_NET_OPTION_NAME_ALL: + case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER: + case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME: + case sfp.NODE_TYPE_NET_OPTION_NAME_CSP: + case sfp.NODE_TYPE_NET_OPTION_NAME_CSS: + case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW: + case sfp.NODE_TYPE_NET_OPTION_NAME_DOC: + case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_EMPTY: + case sfp.NODE_TYPE_NET_OPTION_NAME_FONT: + case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME: + case sfp.NODE_TYPE_NET_OPTION_NAME_FROM: + case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK: + case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER: + case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE: + case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT: + case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT: + case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT: + case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE: + case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA: + case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD: + case sfp.NODE_TYPE_NET_OPTION_NAME_MP4: + case sfp.NODE_TYPE_NET_OPTION_NAME_NOOP: + case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT: + case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER: + case sfp.NODE_TYPE_NET_OPTION_NAME_PING: + case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER: + case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: + case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT: + case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_TO: + case sfp.NODE_TYPE_NET_OPTION_NAME_XHR: + case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC: + case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET: + lastNetOptionType = nodeType; + return 'def'; + case sfp.NODE_TYPE_NET_OPTION_ASSIGN: + return 'def'; + case sfp.NODE_TYPE_NET_OPTION_VALUE: + if ( astWalker.canGoDown() ) { break; } + switch ( lastNetOptionType ) { + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + return redirectTokenStyle(); + default: + break; + } + return 'value'; + case sfp.NODE_TYPE_OPTION_VALUE_NOT: + return 'keyword strong'; + case sfp.NODE_TYPE_OPTION_VALUE_DOMAIN: + return 'value'; + case sfp.NODE_TYPE_OPTION_VALUE_SEPARATOR: + return 'def'; + default: + break; + } + return '+'; + }; + + self.addEventListener('trustedSource', ev => { + astParser.options.trustedSource = ev.detail; + }); + + self.addEventListener('trustedScriptletTokens', ev => { + astParser.options.trustedScriptletTokens = ev.detail; + }); + + return { + lineComment: '!', + token: function(stream) { + if ( stream.sol() ) { + astParser.parse(stream.string); + if ( astParser.getFlags(sfp.AST_FLAG_UNSUPPORTED) !== 0 ) { + stream.skipToEnd(); + return 'error'; + } + if ( astParser.getType() === sfp.AST_TYPE_NONE ) { + stream.skipToEnd(); + return 'comment'; + } + currentWalkerNode = astWalker.reset(); + } else if ( nodeHasError() ) { + currentWalkerNode = astWalker.right(); + } else { + currentWalkerNode = astWalker.next(); + } + let style = ''; + while ( currentWalkerNode !== 0 ) { + style = colorFromAstNode(stream); + if ( style !== '+' ) { break; } + currentWalkerNode = astWalker.next(); + } + if ( style === '+' ) { + stream.skipToEnd(); + return null; + } + stream.pos = astParser.getNodeStringEnd(currentWalkerNode); + if ( astParser.isNetworkFilter() ) { + return style ? `line-cm-net ${style}` : 'line-cm-net'; + } + if ( astParser.isExtendedFilter() ) { + let flavor = ''; + if ( astParser.isCosmeticFilter() ) { + flavor = 'line-cm-ext-dom'; + } else if ( astParser.isScriptletFilter() ) { + flavor = 'line-cm-ext-js'; + } else if ( astParser.isHtmlFilter() ) { + flavor = 'line-cm-ext-html'; + } + if ( flavor !== '' ) { + style = `${flavor} ${style}`; + } + } + style = style.trim(); + return style !== '' ? style : null; + }, + parser: astParser, + }; +}); + +/******************************************************************************/ + +// Following code is for auto-completion. Reference: +// https://codemirror.net/demo/complete.html + +CodeMirror.defineOption('uboHints', null, (cm, hints) => { + if ( hints instanceof Object === false ) { return; } + if ( Array.isArray(hints.redirectResources) ) { + for ( const [ name, desc ] of hints.redirectResources ) { + const displayText = desc.aliasOf !== '' + ? `${name} (${desc.aliasOf})` + : ''; + if ( desc.canRedirect ) { + redirectNames.set(name, displayText); + } + if ( desc.canInject && name.endsWith('.js') ) { + scriptletNames.set(name.slice(0, -3), displayText); + } + } + } + if ( Array.isArray(hints.preparseDirectiveEnv)) { + preparseDirectiveEnv.length = 0; + preparseDirectiveEnv.push(...hints.preparseDirectiveEnv); + } + if ( Array.isArray(hints.preparseDirectiveHints)) { + preparseDirectiveHints.push(...hints.preparseDirectiveHints); + } + if ( Array.isArray(hints.originHints) ) { + originHints.length = 0; + for ( const hint of hints.originHints ) { + originHints.push(hint); + } + } + if ( hintHelperRegistered ) { return; } + hintHelperRegistered = true; + initHints(); +}); + +function initHints() { + const astParser = new sfp.AstFilterParser({ + interactive: true, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); + const proceduralOperatorNames = new Map( + Array.from(sfp.proceduralOperatorTokens) + .filter(item => (item[1] & 0b01) !== 0) + ); + const excludedHints = new Set([ + 'genericblock', + 'object-subrequest', + 'rewrite', + 'webrtc', + ]); + + const pickBestHints = function(cursor, seedLeft, seedRight, hints) { + const seed = (seedLeft + seedRight).trim(); + const out = []; + // First, compare against whole seed + for ( const hint of hints ) { + const text = hint instanceof Object + ? hint.displayText || hint.text + : hint; + if ( text.startsWith(seed) === false ) { continue; } + out.push(hint); + } + if ( out.length !== 0 ) { + return { + from: { line: cursor.line, ch: cursor.ch - seedLeft.length }, + to: { line: cursor.line, ch: cursor.ch + seedRight.length }, + list: out, + }; + } + // If no match, try again with a different heuristic: valid hints are + // those matching left seed, not matching right seed but right seed is + // found to be a valid hint. This is to take care of cases like: + // + // *$script,redomain=example.org + // ^ + // + cursor position + // + // In such case, [ redirect=, redirect-rule= ] should be returned + // as valid hints. + for ( const hint of hints ) { + const text = hint instanceof Object + ? hint.displayText || hint.text + : hint; + if ( seedLeft.length === 0 ) { continue; } + if ( text.startsWith(seedLeft) === false ) { continue; } + if ( hints.includes(seedRight) === false ) { continue; } + out.push(hint); + } + if ( out.length !== 0 ) { + return { + from: { line: cursor.line, ch: cursor.ch - seedLeft.length }, + to: { line: cursor.line, ch: cursor.ch }, + list: out, + }; + } + // If no match, try again with a different heuristic: valid hints are + // those containing seed as a substring. This is to take care of cases + // like: + // + // *$script,redirect=gif + // ^ + // + cursor position + // + // In such case, [ 1x1.gif, 1x1-transparent.gif ] should be returned + // as valid hints. + for ( const hint of hints ) { + const text = hint instanceof Object + ? hint.displayText || hint.text + : hint; + if ( seedLeft.length === 1 ) { + if ( text.startsWith(seedLeft) === false ) { continue; } + } else if ( text.includes(seed) === false ) { continue; } + out.push(hint); + } + if ( out.length !== 0 ) { + return { + from: { line: cursor.line, ch: cursor.ch - seedLeft.length }, + to: { line: cursor.line, ch: cursor.ch + seedRight.length }, + list: out, + }; + } + // If still no match, try again with a different heuristic: valid hints + // are those containing left seed as a substring. This is to take care + // of cases like: + // + // *$script,redirect=gifdomain=example.org + // ^ + // + cursor position + // + // In such case, [ 1x1.gif, 1x1-transparent.gif ] should be returned + // as valid hints. + for ( const hint of hints ) { + const text = hint instanceof Object + ? hint.displayText || hint.text + : hint; + if ( text.includes(seedLeft) === false ) { continue; } + out.push(hint); + } + if ( out.length !== 0 ) { + return { + from: { line: cursor.line, ch: cursor.ch - seedLeft.length }, + to: { line: cursor.line, ch: cursor.ch }, + list: out, + }; + } + }; + + const getOriginHints = function(cursor, line, suffix = '') { + const beg = cursor.ch; + const matchLeft = /[^,|=~]*$/.exec(line.slice(0, beg)); + const matchRight = /^[^#,|]*/.exec(line.slice(beg)); + if ( matchLeft === null || matchRight === null ) { return; } + const hints = []; + for ( const text of originHints ) { + hints.push(text + suffix); + } + return pickBestHints(cursor, matchLeft[0], matchRight[0], hints); + }; + + const getNetPatternHints = function(cursor, line) { + if ( /\|\|[\w.-]*$/.test(line.slice(0, cursor.ch)) ) { + return getOriginHints(cursor, line, '^'); + } + // Maybe a static extended filter is meant to be crafted. + if ( /[^\w\x80-\xF4#,.-]/.test(line) === false ) { + return getOriginHints(cursor, line); + } + }; + + const getNetOptionHints = function(cursor, seedLeft, seedRight) { + const isNegated = seedLeft.startsWith('~'); + if ( isNegated ) { + seedLeft = seedLeft.slice(1); + } + const assignPos = seedRight.indexOf('='); + if ( assignPos !== -1 ) { seedRight = seedRight.slice(0, assignPos); } + const isException = astParser.isException(); + const hints = []; + for ( let [ text, desc ] of sfp.netOptionTokenDescriptors ) { + if ( excludedHints.has(text) ) { continue; } + if ( isNegated && desc.canNegate !== true ) { continue; } + if ( isException ) { + if ( desc.blockOnly ) { continue; } + } else { + if ( desc.allowOnly ) { continue; } + if ( (assignPos === -1) && desc.mustAssign ) { + text += '='; + } + } + hints.push(text); + } + return pickBestHints(cursor, seedLeft, seedRight, hints); + }; + + const getNetRedirectHints = function(cursor, seedLeft, seedRight) { + const hints = []; + for ( const text of redirectNames.keys() ) { + if ( text.startsWith('abp-resource:') ) { continue; } + hints.push(text); + } + return pickBestHints(cursor, seedLeft, seedRight, hints); + }; + + const getNetHints = function(cursor, line) { + const patternNode = astParser.getBranchFromType(sfp.NODE_TYPE_NET_PATTERN_RAW); + if ( patternNode === 0 ) { return; } + const patternEnd = astParser.getNodeStringEnd(patternNode); + const beg = cursor.ch; + if ( beg <= patternEnd ) { + return getNetPatternHints(cursor, line); + } + const lineBefore = line.slice(0, beg); + const lineAfter = line.slice(beg); + let matchLeft = /[^$,]*$/.exec(lineBefore); + let matchRight = /^[^,]*/.exec(lineAfter); + if ( matchLeft === null || matchRight === null ) { return; } + const assignPos = matchLeft[0].indexOf('='); + if ( assignPos === -1 ) { + return getNetOptionHints(cursor, matchLeft[0], matchRight[0]); + } + if ( /^(redirect(-rule)?|rewrite)=/.test(matchLeft[0]) ) { + return getNetRedirectHints( + cursor, + matchLeft[0].slice(assignPos + 1), + matchRight[0] + ); + } + if ( /^(domain|from)=/.test(matchLeft[0]) ) { + return getOriginHints(cursor, line); + } + }; + + const getExtSelectorHints = function(cursor, line) { + const beg = cursor.ch; + // Special selector case: `^responseheader` + { + const match = /#\^([a-z]+)$/.exec(line.slice(0, beg)); + if ( + match !== null && + 'responseheader'.startsWith(match[1]) && + line.slice(beg) === '' + ) { + return pickBestHints( + cursor, + match[1], + '', + [ 'responseheader()' ] + ); + } + } + // Procedural operators + const matchLeft = /#\^?.*:([^:]*)$/.exec(line.slice(0, beg)); + const matchRight = /^([a-z-]*)\(?/.exec(line.slice(beg)); + if ( matchLeft === null || matchRight === null ) { return; } + const isStaticDOM = matchLeft[0].indexOf('^') !== -1; + const hints = []; + for ( let [ text, bits ] of proceduralOperatorNames ) { + if ( isStaticDOM && (bits & 0b10) !== 0 ) { continue; } + hints.push(text); + } + return pickBestHints(cursor, matchLeft[1], matchRight[1], hints); + }; + + const getExtHeaderHints = function(cursor, line) { + const beg = cursor.ch; + const matchLeft = /#\^responseheader\((.*)$/.exec(line.slice(0, beg)); + const matchRight = /^([^)]*)/.exec(line.slice(beg)); + if ( matchLeft === null || matchRight === null ) { return; } + const hints = []; + for ( const hint of sfp.removableHTTPHeaders ) { + hints.push(hint); + } + return pickBestHints(cursor, matchLeft[1], matchRight[1], hints); + }; + + const getExtScriptletHints = function(cursor, line) { + const beg = cursor.ch; + const matchLeft = /#\+\js\(([^,]*)$/.exec(line.slice(0, beg)); + const matchRight = /^([^,)]*)/.exec(line.slice(beg)); + if ( matchLeft === null || matchRight === null ) { return; } + const hints = []; + for ( const [ text, displayText ] of scriptletNames ) { + const hint = { text }; + if ( displayText !== '' ) { + hint.displayText = displayText; + } + hints.push(hint); + } + return pickBestHints(cursor, matchLeft[1], matchRight[1], hints); + }; + + const getCommentHints = function(cursor, line) { + const beg = cursor.ch; + if ( line.startsWith('!#if ') ) { + const matchLeft = /^!#if !?(\w*)$/.exec(line.slice(0, beg)); + const matchRight = /^\w*/.exec(line.slice(beg)); + if ( matchLeft === null || matchRight === null ) { return; } + return pickBestHints( + cursor, + matchLeft[1], + matchRight[0], + preparseDirectiveHints + ); + } + if ( line.startsWith('!#') && line !== '!#endif' ) { + const matchLeft = /^!#(\w*)$/.exec(line.slice(0, beg)); + const matchRight = /^\w*/.exec(line.slice(beg)); + if ( matchLeft === null || matchRight === null ) { return; } + const hints = [ 'if ', 'endif\n', 'include ' ]; + return pickBestHints(cursor, matchLeft[1], matchRight[0], hints); + } + }; + + CodeMirror.registerHelper('hint', 'ubo-static-filtering', function(cm) { + const cursor = cm.getCursor(); + const line = cm.getLine(cursor.line); + astParser.parse(line); + if ( astParser.isExtendedFilter() ) { + const anchorNode = astParser.getBranchFromType(sfp.NODE_TYPE_EXT_OPTIONS_ANCHOR); + if ( anchorNode === 0 ) { return; } + let hints; + if ( cursor.ch <= astParser.getNodeStringBeg(anchorNode) ) { + hints = getOriginHints(cursor, line); + } else if ( astParser.isScriptletFilter() ) { + hints = getExtScriptletHints(cursor, line); + } else if ( astParser.isResponseheaderFilter() ) { + hints = getExtHeaderHints(cursor, line); + } else { + hints = getExtSelectorHints(cursor, line); + } + return hints; + } + if ( astParser.isNetworkFilter() ) { + return getNetHints(cursor, line); + } + if ( astParser.isComment() ) { + return getCommentHints(cursor, line); + } + return getOriginHints(cursor, line); + }); +} + +/******************************************************************************/ + +CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => { + const foldIfEndif = function(startLineNo, startLine, cm) { + const lastLineNo = cm.lastLine(); + let endLineNo = startLineNo; + let depth = 1; + while ( endLineNo < lastLineNo ) { + endLineNo += 1; + const line = cm.getLine(endLineNo); + if ( line.startsWith('!#endif') ) { + depth -= 1; + if ( depth === 0 ) { + return { + from: CodeMirror.Pos(startLineNo, startLine.length), + to: CodeMirror.Pos(endLineNo, 0) + }; + } + } + if ( line.startsWith('!#if') ) { + depth += 1; + } + } + }; + + const foldInclude = function(startLineNo, startLine, cm) { + const lastLineNo = cm.lastLine(); + let endLineNo = startLineNo + 1; + if ( endLineNo >= lastLineNo ) { return; } + if ( cm.getLine(endLineNo).startsWith('! >>>>>>>> ') === false ) { + return; + } + while ( endLineNo < lastLineNo ) { + endLineNo += 1; + const line = cm.getLine(endLineNo); + if ( line.startsWith('! <<<<<<<< ') ) { + return { + from: CodeMirror.Pos(startLineNo, startLine.length), + to: CodeMirror.Pos(endLineNo, line.length) + }; + } + } + }; + + return function(cm, start) { + const startLineNo = start.line; + const startLine = cm.getLine(startLineNo); + if ( startLine.startsWith('!#if') ) { + return foldIfEndif(startLineNo, startLine, cm); + } + if ( startLine.startsWith('!#include ') ) { + return foldInclude(startLineNo, startLine, cm); + } + }; +})()); + +/******************************************************************************/ + +// Linter + +{ + const astParser = new sfp.AstFilterParser({ + interactive: true, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); + + const changeset = []; + let changesetTimer; + + const includeset = new Set(); + let errorCount = 0; + + const ifendifSet = new Set(); + let ifendifSetChanged = false; + + const extractMarkerDetails = (doc, lineHandle) => { + if ( astParser.isUnsupported() ) { + return { lint: 'error', msg: 'Unsupported filter syntax' }; + } + if ( astParser.hasError() ) { + let msg = 'Invalid filter'; + switch ( astParser.astError ) { + case sfp.AST_ERROR_UNSUPPORTED: + msg = `${msg}: Unsupported filter syntax`; + break; + case sfp.AST_ERROR_REGEX: + msg = `${msg}: Bad regular expression`; + break; + case sfp.AST_ERROR_PATTERN: + msg = `${msg}: Bad pattern`; + break; + case sfp.AST_ERROR_DOMAIN_NAME: + msg = `${msg}: Bad domain name`; + break; + case sfp.AST_ERROR_OPTION_BADVALUE: + msg = `${msg}: Bad value assigned to a valid option`; + break; + case sfp.AST_ERROR_OPTION_DUPLICATE: + msg = `${msg}: Duplicate filter option`; + break; + case sfp.AST_ERROR_OPTION_UNKNOWN: + msg = `${msg}: Unsupported filter option`; + break; + case sfp.AST_ERROR_IF_TOKEN_UNKNOWN: + msg = `${msg}: Unknown preparsing token`; + break; + case sfp.AST_ERROR_UNTRUSTED_SOURCE: + msg = `${msg}: Filter requires trusted source`; + break; + default: + if ( astParser.isCosmeticFilter() && astParser.result.error ) { + msg = `${msg}: ${astParser.result.error}`; + } + break; + } + return { lint: 'error', msg }; + } + if ( astParser.astType !== sfp.AST_TYPE_COMMENT ) { return; } + if ( astParser.astTypeFlavor !== sfp.AST_TYPE_COMMENT_PREPARSER ) { + if ( astParser.raw.startsWith('! <<<<<<<< ') === false ) { return; } + for ( const include of includeset ) { + if ( astParser.raw.endsWith(include) === false ) { continue; } + includeset.delete(include); + return { lint: 'include-end' }; + } + return; + } + if ( /^\s*!#if \S+/.test(astParser.raw) ) { + return { + lint: 'if-start', + data: { + state: sfp.utils.preparser.evaluateExpr( + astParser.getTypeString(sfp.NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE), + preparseDirectiveEnv + ) ? 'y' : 'n' + } + }; + } + if ( /^\s*!#endif\b/.test(astParser.raw) ) { + return { lint: 'if-end' }; + } + const match = /^\s*!#include\s*(\S+)/.exec(astParser.raw); + if ( match === null ) { return; } + const nextLineHandle = doc.getLineHandle(lineHandle.lineNo() + 1); + if ( nextLineHandle === undefined ) { return; } + if ( nextLineHandle.text.startsWith('! >>>>>>>> ') === false ) { return; } + const includeToken = `/${match[1]}`; + if ( nextLineHandle.text.endsWith(includeToken) === false ) { return; } + includeset.add(includeToken); + return { lint: 'include-start' }; + }; + + const extractMarker = lineHandle => { + const markers = lineHandle.gutterMarkers || null; + return markers !== null + ? markers['CodeMirror-lintgutter'] || null + : null; + }; + + const markerTemplates = { + 'error': { + node: null, + html: [ + '<div class="CodeMirror-lintmarker" data-lint="error" data-error="y"> ', + '<span class="msg"></span>', + '</div>', + ], + }, + 'if-start': { + node: null, + html: [ + '<div class="CodeMirror-lintmarker" data-lint="if" data-fold="start" data-state=""> ', + '<svg viewBox="0 0 100 100">', + '<polygon points="0,0 100,0 50,100" />', + '</svg>', + '<span class="msg">Mismatched if-endif directive</span>', + '</div>', + ], + }, + 'if-end': { + node: null, + html: [ + '<div class="CodeMirror-lintmarker" data-lint="if" data-fold="end"> ', + '<svg viewBox="0 0 100 100">', + '<polygon points="50,0 100,100 0,100" />', + '</svg>', + '<span class="msg">Mismatched if-endif directive</span>', + '</div>', + ], + }, + 'include-start': { + node: null, + html: [ + '<div class="CodeMirror-lintmarker" data-lint="include" data-fold="start"> ', + '<svg viewBox="0 0 100 100">', + '<polygon points="0,0 100,0 50,100" />', + '</svg>', + '</div>', + ], + }, + 'include-end': { + node: null, + html: [ + '<div class="CodeMirror-lintmarker" data-lint="include" data-fold="end"> ', + '<svg viewBox="0 0 100 100">', + '<polygon points="50,0 100,100 0,100" />', + '</svg>', + '</div>', + ], + }, + }; + + const markerFromTemplate = details => { + const template = markerTemplates[details.lint]; + if ( template.node === null ) { + const domParser = new DOMParser(); + const doc = domParser.parseFromString(template.html.join(''), 'text/html'); + template.node = document.adoptNode(qs$(doc, '.CodeMirror-lintmarker')); + } + const node = template.node.cloneNode(true); + if ( details.data instanceof Object ) { + for ( const [ k, v ] of Object.entries(details.data) ) { + node.dataset[k] = `${v}`; + } + } + return node; + }; + + const addMarker = (doc, lineHandle, marker, details) => { + if ( marker && marker.dataset.lint !== details.lint ) { + doc.setGutterMarker(lineHandle, 'CodeMirror-lintgutter', null); + if ( marker.dataset.error === 'y' ) { + errorCount -= 1; + } + if ( marker.dataset.lint === 'if' ) { + ifendifSet.delete(lineHandle); + ifendifSetChanged = true; + } + marker = null; + } + if ( marker === null ) { + marker = markerFromTemplate(details); + doc.setGutterMarker(lineHandle, 'CodeMirror-lintgutter', marker); + if ( marker.dataset.error === 'y' ) { + errorCount += 1; + } + if ( marker.dataset.lint === 'if' ) { + ifendifSet.add(lineHandle); + ifendifSetChanged = true; + } + } + if ( typeof details.msg !== 'string' || details.msg === '' ) { return; } + const msgElem = qs$(marker, '.msg'); + if ( msgElem === null ) { return; } + msgElem.textContent = details.msg; + }; + + const removeMarker = (doc, lineHandle, marker) => { + doc.setGutterMarker(lineHandle, 'CodeMirror-lintgutter', null); + if ( marker.dataset.error === 'y' ) { + errorCount -= 1; + } + if ( marker.dataset.lint === 'if' ) { + ifendifSet.delete(lineHandle); + ifendifSetChanged = true; + } + }; + + // Analyze whether all if-endif are properly paired + const processIfendifs = ( ) => { + if ( ifendifSet.size === 0 ) { return; } + if ( ifendifSetChanged !== true ) { return; } + const sortFn = (a, b) => a.lineNo() - b.lineNo(); + const sorted = Array.from(ifendifSet).sort(sortFn); + const bad = []; + const stack = []; + for ( const line of sorted ) { + const marker = extractMarker(line); + const fold = marker.dataset.fold; + if ( fold === 'start' ) { + stack.push(line); + } else if ( fold === 'end' ) { + if ( stack.length !== 0 ) { + if ( marker.dataset.error === 'y' ) { + marker.dataset.error = ''; + errorCount -= 1; + } + const ifstart = extractMarker(stack.pop()); + if ( ifstart.dataset.error === 'y' ) { + ifstart.dataset.error = ''; + errorCount -= 1; + } + } else { + bad.push(line); + } + } + } + bad.push(...stack); + for ( const line of bad ) { + const marker = extractMarker(line); + marker.dataset.error = 'y'; + errorCount += 1; + } + ifendifSetChanged = false; + }; + + const processDeletion = (doc, change) => { + let { from, to } = change; + doc.eachLine(from.line, to.line, lineHandle => { + const marker = extractMarker(lineHandle); + if ( marker === null ) { return; } + if ( marker.dataset.error === 'y' ) { + marker.dataset.error = ''; + errorCount -= 1; + } + ifendifSet.delete(lineHandle); + ifendifSetChanged = true; + }); + }; + + const processInsertion = (doc, deadline, change) => { + let { from, to } = change; + doc.eachLine(from, to, lineHandle => { + astParser.parse(lineHandle.text); + const markerDetails = extractMarkerDetails(doc, lineHandle); + const marker = extractMarker(lineHandle); + if ( markerDetails !== undefined ) { + addMarker(doc, lineHandle, marker, markerDetails); + } else if ( marker !== null ) { + removeMarker(doc, lineHandle, marker); + } + from += 1; + if ( (from & 0x0F) !== 0 ) { return; } + if ( deadline.timeRemaining() !== 0 ) { return; } + return true; + }); + if ( from !== to ) { + return { from, to }; + } + }; + + const processChangeset = (doc, deadline) => { + const cm = doc.getEditor(); + cm.startOperation(); + while ( changeset.length !== 0 ) { + const change = processInsertion(doc, deadline, changeset.shift()); + if ( change === undefined ) { continue; } + changeset.unshift(change); + break; + } + cm.endOperation(); + if ( changeset.length !== 0 ) { + return processChangesetAsync(doc); + } + includeset.clear(); + processIfendifs(doc); + CodeMirror.signal(doc.getEditor(), 'linterDone', { errorCount }); + }; + + const processChangesetAsync = doc => { + if ( changesetTimer !== undefined ) { return; } + changesetTimer = self.requestIdleCallback(deadline => { + changesetTimer = undefined; + processChangeset(doc, deadline); + }); + }; + + const onChanges = (cm, changes) => { + if ( changes.length === 0 ) { return; } + const doc = cm.getDoc(); + for ( const change of changes ) { + const from = change.from.line; + const to = from + change.text.length; + changeset.push({ from, to }); + } + processChangesetAsync(doc); + }; + + const onBeforeChanges = (cm, change) => { + const doc = cm.getDoc(); + processDeletion(doc, change); + }; + + const foldRangeFinder = (cm, from) => { + const lineNo = from.line; + const lineHandle = cm.getDoc().getLineHandle(lineNo); + const marker = extractMarker(lineHandle); + if ( marker === null ) { return; } + if ( marker.dataset.fold === undefined ) { return; } + const foldName = marker.dataset.lint; + from.ch = lineHandle.text.length; + const to = { line: 0, ch: 0 }; + const doc = cm.getDoc(); + let depth = 0; + doc.eachLine(from.line, doc.lineCount(), lineHandle => { + const marker = extractMarker(lineHandle); + if ( marker === null ) { return; } + if ( marker.dataset.lint === foldName && marker.dataset.fold === 'start' ) { + depth += 1; + return; + } + if ( marker.dataset.lint !== foldName ) { return; } + if ( marker.dataset.fold !== 'end' ) { return; } + depth -= 1; + if ( depth !== 0 ) { return; } + to.line = lineHandle.lineNo(); + return true; + }); + return { from, to }; + }; + + const onGutterClick = (cm, lineNo, gutterId, ev) => { + if ( ev.button !== 0 ) { return; } + if ( gutterId !== 'CodeMirror-lintgutter' ) { return; } + const doc = cm.getDoc(); + const lineHandle = doc.getLineHandle(lineNo); + const marker = extractMarker(lineHandle); + if ( marker === null ) { return; } + if ( marker.dataset.fold === 'start' ) { + if ( ev.ctrlKey ) { + if ( dom.cl.has(marker, 'folded') ) { + CodeMirror.commands.unfoldAll(cm); + } else { + CodeMirror.commands.foldAll(cm); + } + doc.setCursor(lineNo); + return; + } + cm.foldCode(lineNo, { + widget: '\u00A0\u22EF\u00A0', + rangeFinder: foldRangeFinder, + }); + return; + } + if ( marker.dataset.fold === 'end' ) { + let depth = 1; + let lineNo = lineHandle.lineNo(); + while ( lineNo-- ) { + const prevLineHandle = doc.getLineHandle(lineNo); + const markerFrom = extractMarker(prevLineHandle); + if ( markerFrom === null ) { continue; } + if ( markerFrom.dataset.fold === 'end' ) { + depth += 1; + } else if ( markerFrom.dataset.fold === 'start' ) { + depth -= 1; + if ( depth === 0 ) { + doc.setCursor(lineNo); + break; + } + } + } + return; + } + }; + + self.addEventListener('trustedSource', ev => { + astParser.options.trustedSource = ev.detail; + }); + + self.addEventListener('trustedScriptletTokens', ev => { + astParser.options.trustedScriptletTokens = ev.detail; + }); + + CodeMirror.defineInitHook(cm => { + cm.on('changes', onChanges); + cm.on('beforeChange', onBeforeChanges); + cm.on('gutterClick', onGutterClick); + cm.on('fold', function(cm, from) { + const doc = cm.getDoc(); + const lineHandle = doc.getLineHandle(from.line); + const marker = extractMarker(lineHandle); + dom.cl.add(marker, 'folded'); + }); + cm.on('unfold', function(cm, from) { + const doc = cm.getDoc(); + const lineHandle = doc.getLineHandle(from.line); + const marker = extractMarker(lineHandle); + dom.cl.remove(marker, 'folded'); + }); + }); +} + +/******************************************************************************/ + +// Enhanced word selection + +{ + const selectWordAt = function(cm, pos) { + const { line, ch } = pos; + const s = cm.getLine(line); + const { type: token } = cm.getTokenAt(pos); + let beg, end; + + // Select URL in comments + if ( /\bcomment\b/.test(token) && /\blink\b/.test(token) ) { + const l = /\S+$/.exec(s.slice(0, ch)); + if ( l && /^https?:\/\//.test(s.slice(l.index)) ) { + const r = /^\S+/.exec(s.slice(ch)); + if ( r ) { + beg = l.index; + end = ch + r[0].length; + } + } + } + + // Better word selection for extended filters: prefix + else if ( + /\bline-cm-ext-(?:dom|html|js)\b/.test(token) && + /\bvalue\b/.test(token) + ) { + const l = /[^,.]*$/i.exec(s.slice(0, ch)); + const r = /^[^#,]*/i.exec(s.slice(ch)); + if ( l && r ) { + beg = l.index; + end = ch + r[0].length; + } + } + + // Better word selection for cosmetic and HTML filters: suffix + else if ( /\bline-cm-ext-(?:dom|html)\b/.test(token) ) { + const l = /[#.]?[a-z0-9_-]+$/i.exec(s.slice(0, ch)); + const r = /^[a-z0-9_-]+/i.exec(s.slice(ch)); + if ( l && r ) { + beg = l.index; + end = ch + r[0].length; + if ( /\bdef\b/.test(cm.getTokenTypeAt({ line, ch: beg + 1 })) ) { + beg += 1; + } + } + } + + // Better word selection for network filters + else if ( /\bline-cm-net\b/.test(token) ) { + if ( /\bvalue\b/.test(token) ) { + const l = /[^ ,.=|]*$/i.exec(s.slice(0, ch)); + const r = /^[^ #,|]*/i.exec(s.slice(ch)); + if ( l && r ) { + beg = l.index; + end = ch + r[0].length; + } + } else if ( /\bdef\b/.test(token) ) { + const l = /[a-z0-9-]+$/i.exec(s.slice(0, ch)); + const r = /^[^,]*=[^,]+/i.exec(s.slice(ch)); + if ( l && r ) { + beg = l.index; + end = ch + r[0].length; + } + } + } + + if ( beg === undefined ) { + const { anchor, head } = cm.findWordAt(pos); + return { from: anchor, to: head }; + } + + return { + from: { line, ch: beg }, + to: { line, ch: end }, + }; + }; + + CodeMirror.defineInitHook(cm => { + cm.setOption('configureMouse', function(cm, repeat) { + return { + unit: repeat === 'double' ? selectWordAt : null, + }; + }); + }); +} + +/******************************************************************************/ diff --git a/src/js/commands.js b/src/js/commands.js new file mode 100644 index 0000000..8fd6341 --- /dev/null +++ b/src/js/commands.js @@ -0,0 +1,181 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; +import { hostnameFromURI } from './uri-utils.js'; + +/******************************************************************************/ + +(( ) => { + +// ***************************************************************************** +// start of local namespace + +if ( vAPI.commands instanceof Object === false ) { return; } + +const relaxBlockingMode = (( ) => { + const reloadTimers = new Map(); + + return function(tab) { + if ( tab instanceof Object === false || tab.id <= 0 ) { return; } + + const normalURL = µb.normalizeTabURL(tab.id, tab.url); + + if ( µb.getNetFilteringSwitch(normalURL) === false ) { return; } + + const hn = hostnameFromURI(normalURL); + const curProfileBits = µb.blockingModeFromHostname(hn); + let newProfileBits; + for ( const profile of µb.liveBlockingProfiles ) { + if ( (curProfileBits & profile.bits & ~1) !== curProfileBits ) { + newProfileBits = profile.bits; + break; + } + } + + // TODO: Reset to original blocking profile? + if ( newProfileBits === undefined ) { return; } + + const noReload = (newProfileBits & 0b00000001) === 0; + + if ( + (curProfileBits & 0b00000010) !== 0 && + (newProfileBits & 0b00000010) === 0 + ) { + µb.toggleHostnameSwitch({ + name: 'no-scripting', + hostname: hn, + state: false, + }); + } + if ( µb.userSettings.advancedUserEnabled ) { + if ( + (curProfileBits & 0b00000100) !== 0 && + (newProfileBits & 0b00000100) === 0 + ) { + µb.toggleFirewallRule({ + tabId: noReload ? tab.id : undefined, + srcHostname: hn, + desHostname: '*', + requestType: '3p', + action: 3, + }); + } + if ( + (curProfileBits & 0b00001000) !== 0 && + (newProfileBits & 0b00001000) === 0 + ) { + µb.toggleFirewallRule({ + srcHostname: hn, + desHostname: '*', + requestType: '3p-script', + action: 3, + }); + } + if ( + (curProfileBits & 0b00010000) !== 0 && + (newProfileBits & 0b00010000) === 0 + ) { + µb.toggleFirewallRule({ + srcHostname: hn, + desHostname: '*', + requestType: '3p-frame', + action: 3, + }); + } + } + + // Reload the target tab? + if ( noReload ) { return; } + + // Reload: use a timer to coalesce bursts of reload commands. + const timer = reloadTimers.get(tab.id) || (( ) => { + const t = vAPI.defer.create(tabId => { + reloadTimers.delete(tabId); + vAPI.tabs.reload(tabId); + }); + reloadTimers.set(tab.id, t); + return t; + })(); + timer.offon(547, tab.id); + }; +})(); + +vAPI.commands.onCommand.addListener(async command => { + // Generic commands + if ( command === 'open-dashboard' ) { + µb.openNewTab({ + url: 'dashboard.html', + select: true, + index: -1, + }); + return; + } + // Tab-specific commands + const tab = await vAPI.tabs.getCurrent(); + if ( tab instanceof Object === false ) { return; } + switch ( command ) { + case 'launch-element-picker': + case 'launch-element-zapper': { + µb.epickerArgs.mouse = false; + µb.elementPickerExec( + tab.id, + 0, + undefined, + command === 'launch-element-zapper' + ); + break; + } + case 'launch-logger': { + const hash = tab.url.startsWith(vAPI.getURL('')) + ? '' + : `#_+${tab.id}`; + µb.openNewTab({ + url: `logger-ui.html${hash}`, + select: true, + index: -1, + }); + break; + } + case 'relax-blocking-mode': + relaxBlockingMode(tab); + break; + case 'toggle-cosmetic-filtering': + µb.toggleHostnameSwitch({ + name: 'no-cosmetic-filtering', + hostname: hostnameFromURI(µb.normalizeTabURL(tab.id, tab.url)), + }); + break; + default: + break; + } +}); + +// end of local namespace +// ***************************************************************************** + +})(); + +/******************************************************************************/ diff --git a/src/js/console.js b/src/js/console.js new file mode 100644 index 0000000..410abbd --- /dev/null +++ b/src/js/console.js @@ -0,0 +1,59 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2019-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +function ubologSet(state = false) { + if ( state ) { + if ( ubolog.process instanceof Function ) { + ubolog.process(); + } + ubolog = ubologDo; + } else { + ubolog = ubologIgnore; + } +} + +function ubologDo(...args) { + console.info('[uBO]', ...args); +} + +function ubologIgnore() { +} + +let ubolog = (( ) => { + const pending = []; + const store = function(...args) { + pending.push(args); + }; + store.process = function() { + for ( const args of pending ) { + ubologDo(...args); + } + }; + return store; +})(); + +/******************************************************************************/ + +export { ubolog, ubologSet }; diff --git a/src/js/contentscript-extra.js b/src/js/contentscript-extra.js new file mode 100644 index 0000000..45c5262 --- /dev/null +++ b/src/js/contentscript-extra.js @@ -0,0 +1,662 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +if ( + typeof vAPI === 'object' && + typeof vAPI.DOMProceduralFilterer !== 'object' +) { +// >>>>>>>> start of local scope + +/******************************************************************************/ + +const nonVisualElements = { + script: true, + style: true, +}; + +const regexFromString = (s, exact = false) => { + if ( s === '' ) { return /^/; } + const match = /^\/(.+)\/([imu]*)$/.exec(s); + if ( match !== null ) { + return new RegExp(match[1], match[2] || undefined); + } + const reStr = s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(exact ? `^${reStr}$` : reStr); +}; + +// 'P' stands for 'Procedural' + +class PSelectorTask { + begin() { + } + end() { + } +} + +class PSelectorVoidTask extends PSelectorTask { + constructor(task) { + super(); + console.info(`uBO: :${task[0]}() operator does not exist`); + } + transpose() { + } +} + +class PSelectorHasTextTask extends PSelectorTask { + constructor(task) { + super(); + this.needle = regexFromString(task[1]); + } + transpose(node, output) { + if ( this.needle.test(node.textContent) ) { + output.push(node); + } + } +} + +class PSelectorIfTask extends PSelectorTask { + constructor(task) { + super(); + this.pselector = new PSelector(task[1]); + } + transpose(node, output) { + if ( this.pselector.test(node) === this.target ) { + output.push(node); + } + } +} +PSelectorIfTask.prototype.target = true; + +class PSelectorIfNotTask extends PSelectorIfTask { +} +PSelectorIfNotTask.prototype.target = false; + +class PSelectorMatchesAttrTask extends PSelectorTask { + constructor(task) { + super(); + this.reAttr = regexFromString(task[1].attr, true); + this.reValue = regexFromString(task[1].value, true); + } + transpose(node, output) { + const attrs = node.getAttributeNames(); + for ( const attr of attrs ) { + if ( this.reAttr.test(attr) === false ) { continue; } + if ( this.reValue.test(node.getAttribute(attr)) === false ) { continue; } + output.push(node); + } + } +} + +class PSelectorMatchesCSSTask extends PSelectorTask { + constructor(task) { + super(); + this.name = task[1].name; + this.pseudo = task[1].pseudo ? `::${task[1].pseudo}` : null; + let arg0 = task[1].value, arg1; + if ( Array.isArray(arg0) ) { + arg1 = arg0[1]; arg0 = arg0[0]; + } + this.value = new RegExp(arg0, arg1); + } + transpose(node, output) { + const style = window.getComputedStyle(node, this.pseudo); + if ( style !== null && this.value.test(style[this.name]) ) { + output.push(node); + } + } +} +class PSelectorMatchesCSSAfterTask extends PSelectorMatchesCSSTask { + constructor(task) { + super(task); + this.pseudo = '::after'; + } +} + +class PSelectorMatchesCSSBeforeTask extends PSelectorMatchesCSSTask { + constructor(task) { + super(task); + this.pseudo = '::before'; + } +} + +class PSelectorMatchesMediaTask extends PSelectorTask { + constructor(task) { + super(); + this.mql = window.matchMedia(task[1]); + if ( this.mql.media === 'not all' ) { return; } + this.mql.addEventListener('change', ( ) => { + if ( typeof vAPI !== 'object' ) { return; } + if ( vAPI === null ) { return; } + const filterer = vAPI.domFilterer && vAPI.domFilterer.proceduralFilterer; + if ( filterer instanceof Object === false ) { return; } + filterer.onDOMChanged([ null ]); + }); + } + transpose(node, output) { + if ( this.mql.matches === false ) { return; } + output.push(node); + } +} + +class PSelectorMatchesPathTask extends PSelectorTask { + constructor(task) { + super(); + this.needle = regexFromString( + task[1].replace(/\P{ASCII}/gu, s => encodeURIComponent(s)) + ); + } + transpose(node, output) { + if ( this.needle.test(self.location.pathname + self.location.search) ) { + output.push(node); + } + } +} + +class PSelectorMinTextLengthTask extends PSelectorTask { + constructor(task) { + super(); + this.min = task[1]; + } + transpose(node, output) { + if ( node.textContent.length >= this.min ) { + output.push(node); + } + } +} + +class PSelectorOthersTask extends PSelectorTask { + constructor() { + super(); + this.targets = new Set(); + } + begin() { + this.targets.clear(); + } + end(output) { + const toKeep = new Set(this.targets); + const toDiscard = new Set(); + const body = document.body; + let discard = null; + for ( let keep of this.targets ) { + while ( keep !== null && keep !== body ) { + toKeep.add(keep); + toDiscard.delete(keep); + discard = keep.previousElementSibling; + while ( discard !== null ) { + if ( + nonVisualElements[discard.localName] !== true && + toKeep.has(discard) === false + ) { + toDiscard.add(discard); + } + discard = discard.previousElementSibling; + } + discard = keep.nextElementSibling; + while ( discard !== null ) { + if ( + nonVisualElements[discard.localName] !== true && + toKeep.has(discard) === false + ) { + toDiscard.add(discard); + } + discard = discard.nextElementSibling; + } + keep = keep.parentElement; + } + } + for ( discard of toDiscard ) { + output.push(discard); + } + this.targets.clear(); + } + transpose(candidate) { + for ( const target of this.targets ) { + if ( target.contains(candidate) ) { return; } + if ( candidate.contains(target) ) { + this.targets.delete(target); + } + } + this.targets.add(candidate); + } +} + +// https://github.com/AdguardTeam/ExtendedCss/issues/31#issuecomment-302391277 +// Prepend `:scope ` if needed. +class PSelectorSpathTask extends PSelectorTask { + constructor(task) { + super(); + this.spath = task[1]; + this.nth = /^(?:\s*[+~]|:)/.test(this.spath); + if ( this.nth ) { return; } + if ( /^\s*>/.test(this.spath) ) { + this.spath = `:scope ${this.spath.trim()}`; + } + } + transpose(node, output) { + const nodes = this.nth + ? PSelectorSpathTask.qsa(node, this.spath) + : node.querySelectorAll(this.spath); + for ( const node of nodes ) { + output.push(node); + } + } + // Helper method for other operators. + static qsa(node, selector) { + const parent = node.parentElement; + if ( parent === null ) { return []; } + let pos = 1; + for (;;) { + node = node.previousElementSibling; + if ( node === null ) { break; } + pos += 1; + } + return parent.querySelectorAll( + `:scope > :nth-child(${pos})${selector}` + ); + } +} + +class PSelectorUpwardTask extends PSelectorTask { + constructor(task) { + super(); + const arg = task[1]; + if ( typeof arg === 'number' ) { + this.i = arg; + } else { + this.s = arg; + } + } + transpose(node, output) { + if ( this.s !== '' ) { + const parent = node.parentElement; + if ( parent === null ) { return; } + node = parent.closest(this.s); + if ( node === null ) { return; } + } else { + let nth = this.i; + for (;;) { + node = node.parentElement; + if ( node === null ) { return; } + nth -= 1; + if ( nth === 0 ) { break; } + } + } + output.push(node); + } +} +PSelectorUpwardTask.prototype.i = 0; +PSelectorUpwardTask.prototype.s = ''; + +class PSelectorWatchAttrs extends PSelectorTask { + constructor(task) { + super(); + this.observer = null; + this.observed = new WeakSet(); + this.observerOptions = { + attributes: true, + subtree: true, + }; + const attrs = task[1]; + if ( Array.isArray(attrs) && attrs.length !== 0 ) { + this.observerOptions.attributeFilter = task[1]; + } + } + // TODO: Is it worth trying to re-apply only the current selector? + handler() { + const filterer = + vAPI.domFilterer && vAPI.domFilterer.proceduralFilterer; + if ( filterer instanceof Object ) { + filterer.onDOMChanged([ null ]); + } + } + transpose(node, output) { + output.push(node); + if ( this.observed.has(node) ) { return; } + if ( this.observer === null ) { + this.observer = new MutationObserver(this.handler); + } + this.observer.observe(node, this.observerOptions); + this.observed.add(node); + } +} + +class PSelectorXpathTask extends PSelectorTask { + constructor(task) { + super(); + this.xpe = document.createExpression(task[1], null); + this.xpr = null; + } + transpose(node, output) { + this.xpr = this.xpe.evaluate( + node, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + this.xpr + ); + let j = this.xpr.snapshotLength; + while ( j-- ) { + const node = this.xpr.snapshotItem(j); + if ( node.nodeType === 1 ) { + output.push(node); + } + } + } +} + +class PSelector { + constructor(o) { + this.raw = o.raw; + this.selector = o.selector; + this.tasks = []; + const tasks = []; + if ( Array.isArray(o.tasks) === false ) { return; } + for ( const task of o.tasks ) { + const ctor = this.operatorToTaskMap.get(task[0]) || PSelectorVoidTask; + tasks.push(new ctor(task)); + } + this.tasks = tasks; + } + prime(input) { + const root = input || document; + if ( this.selector === '' ) { return [ root ]; } + if ( input !== document ) { + const c0 = this.selector.charCodeAt(0); + if ( c0 === 0x2B /* + */ || c0 === 0x7E /* ~ */ ) { + return Array.from(PSelectorSpathTask.qsa(input, this.selector)); + } else if ( c0 === 0x3E /* > */ ) { + return Array.from(input.querySelectorAll(`:scope ${this.selector}`)); + } + } + return Array.from(root.querySelectorAll(this.selector)); + } + exec(input) { + let nodes = this.prime(input); + for ( const task of this.tasks ) { + if ( nodes.length === 0 ) { break; } + const transposed = []; + task.begin(); + for ( const node of nodes ) { + task.transpose(node, transposed); + } + task.end(transposed); + nodes = transposed; + } + return nodes; + } + test(input) { + const nodes = this.prime(input); + for ( const node of nodes ) { + let output = [ node ]; + for ( const task of this.tasks ) { + const transposed = []; + task.begin(); + for ( const node of output ) { + task.transpose(node, transposed); + } + task.end(transposed); + output = transposed; + if ( output.length === 0 ) { break; } + } + if ( output.length !== 0 ) { return true; } + } + return false; + } +} +PSelector.prototype.operatorToTaskMap = new Map([ + [ 'has', PSelectorIfTask ], + [ 'has-text', PSelectorHasTextTask ], + [ 'if', PSelectorIfTask ], + [ 'if-not', PSelectorIfNotTask ], + [ 'matches-attr', PSelectorMatchesAttrTask ], + [ 'matches-css', PSelectorMatchesCSSTask ], + [ 'matches-css-after', PSelectorMatchesCSSAfterTask ], + [ 'matches-css-before', PSelectorMatchesCSSBeforeTask ], + [ 'matches-media', PSelectorMatchesMediaTask ], + [ 'matches-path', PSelectorMatchesPathTask ], + [ 'min-text-length', PSelectorMinTextLengthTask ], + [ 'not', PSelectorIfNotTask ], + [ 'others', PSelectorOthersTask ], + [ 'spath', PSelectorSpathTask ], + [ 'upward', PSelectorUpwardTask ], + [ 'watch-attr', PSelectorWatchAttrs ], + [ 'xpath', PSelectorXpathTask ], +]); + +class PSelectorRoot extends PSelector { + constructor(o) { + super(o); + this.budget = 200; // I arbitrary picked a 1/5 second + this.raw = o.raw; + this.cost = 0; + this.lastAllowanceTime = 0; + this.action = o.action; + } + prime(input) { + try { + return super.prime(input); + } catch (ex) { + } + return []; + } +} +PSelectorRoot.prototype.hit = false; + +class ProceduralFilterer { + constructor(domFilterer) { + this.domFilterer = domFilterer; + this.mustApplySelectors = false; + this.selectors = new Map(); + this.masterToken = vAPI.randomToken(); + this.styleTokenMap = new Map(); + this.styledNodes = new Set(); + if ( vAPI.domWatcher instanceof Object ) { + vAPI.domWatcher.addListener(this); + } + } + + addProceduralSelectors(selectors) { + const addedSelectors = []; + let mustCommit = false; + for ( const selector of selectors ) { + if ( this.selectors.has(selector.raw) ) { continue; } + const pselector = new PSelectorRoot(selector); + this.primeProceduralSelector(pselector); + this.selectors.set(selector.raw, pselector); + addedSelectors.push(pselector); + mustCommit = true; + } + if ( mustCommit === false ) { return; } + this.mustApplySelectors = this.selectors.size !== 0; + this.domFilterer.commit(); + if ( this.domFilterer.hasListeners() ) { + this.domFilterer.triggerListeners({ + procedural: addedSelectors + }); + } + } + + // This allows to perform potentially expensive initialization steps + // before the filters are ready to be applied. + primeProceduralSelector(pselector) { + if ( pselector.action === undefined ) { + this.styleTokenFromStyle(vAPI.hideStyle); + } else if ( pselector.action[0] === 'style' ) { + this.styleTokenFromStyle(pselector.action[1]); + } + return pselector; + } + + commitNow() { + if ( this.selectors.size === 0 ) { return; } + + this.mustApplySelectors = false; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/341 + // Be ready to unhide nodes which no longer matches any of + // the procedural selectors. + const toUnstyle = this.styledNodes; + this.styledNodes = new Set(); + + let t0 = Date.now(); + + for ( const pselector of this.selectors.values() ) { + const allowance = Math.floor((t0 - pselector.lastAllowanceTime) / 2000); + if ( allowance >= 1 ) { + pselector.budget += allowance * 50; + if ( pselector.budget > 200 ) { pselector.budget = 200; } + pselector.lastAllowanceTime = t0; + } + if ( pselector.budget <= 0 ) { continue; } + const nodes = pselector.exec(); + const t1 = Date.now(); + pselector.budget += t0 - t1; + if ( pselector.budget < -500 ) { + console.info('uBO: disabling %s', pselector.raw); + pselector.budget = -0x7FFFFFFF; + } + t0 = t1; + if ( nodes.length === 0 ) { continue; } + pselector.hit = true; + this.processNodes(nodes, pselector.action); + } + + this.unprocessNodes(toUnstyle); + } + + styleTokenFromStyle(style) { + if ( style === undefined ) { return; } + let styleToken = this.styleTokenMap.get(style); + if ( styleToken !== undefined ) { return styleToken; } + styleToken = vAPI.randomToken(); + this.styleTokenMap.set(style, styleToken); + this.domFilterer.addCSS( + `[${this.masterToken}][${styleToken}]\n{${style}}`, + { silent: true, mustInject: true } + ); + return styleToken; + } + + processNodes(nodes, action) { + const op = action && action[0] || ''; + const arg = op !== '' ? action[1] : ''; + switch ( op ) { + case '': + /* fall through */ + case 'style': { + const styleToken = this.styleTokenFromStyle( + arg === '' ? vAPI.hideStyle : arg + ); + for ( const node of nodes ) { + node.setAttribute(this.masterToken, ''); + node.setAttribute(styleToken, ''); + this.styledNodes.add(node); + } + break; + } + case 'remove': { + for ( const node of nodes ) { + node.remove(); + node.textContent = ''; + } + break; + } + case 'remove-attr': { + const reAttr = regexFromString(arg, true); + for ( const node of nodes ) { + for ( const name of node.getAttributeNames() ) { + if ( reAttr.test(name) === false ) { continue; } + node.removeAttribute(name); + } + } + break; + } + case 'remove-class': { + const reClass = regexFromString(arg, true); + for ( const node of nodes ) { + const cl = node.classList; + for ( const name of cl.values() ) { + if ( reClass.test(name) === false ) { continue; } + cl.remove(name); + } + } + break; + } + default: + break; + } + } + + // TODO: Current assumption is one style per hit element. Could be an + // issue if an element has multiple styling and one styling is + // brought back. Possibly too rare to care about this for now. + unprocessNodes(nodes) { + for ( const node of nodes ) { + if ( this.styledNodes.has(node) ) { continue; } + node.removeAttribute(this.masterToken); + } + } + + createProceduralFilter(o) { + return this.primeProceduralSelector( + new PSelectorRoot(typeof o === 'string' ? JSON.parse(o) : o) + ); + } + + onDOMCreated() { + } + + onDOMChanged(addedNodes, removedNodes) { + if ( this.selectors.size === 0 ) { return; } + this.mustApplySelectors = + this.mustApplySelectors || + addedNodes.length !== 0 || + removedNodes; + this.domFilterer.commit(); + } +} + +vAPI.DOMProceduralFilterer = ProceduralFilterer; + +/******************************************************************************/ + +// >>>>>>>> end of local scope +} + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/contentscript.js b/src/js/contentscript.js new file mode 100644 index 0000000..8f3a4cf --- /dev/null +++ b/src/js/contentscript.js @@ -0,0 +1,1364 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************* + + +--> domCollapser + | + | + domWatcher--+ + | +-- domSurveyor + | | + +--> domFilterer --+-- [domLogger] + | | + | +-- [domInspector] + | + [domProceduralFilterer] + + domWatcher: + Watches for changes in the DOM, and notify the other components about these + changes. + + domCollapser: + Enforces the collapsing of DOM elements for which a corresponding + resource was blocked through network filtering. + + domFilterer: + Enforces the filtering of DOM elements, by feeding it cosmetic filters. + + domProceduralFilterer: + Enforce the filtering of DOM elements through procedural cosmetic filters. + Loaded on demand, only when needed. + + domSurveyor: + Surveys the DOM to find new cosmetic filters to apply to the current page. + + domLogger: + Surveys the page to find and report the injected cosmetic filters blocking + actual elements on the current page. This component is dynamically loaded + IF AND ONLY IF uBO's logger is opened. + + If page is whitelisted: + - domWatcher: off + - domCollapser: off + - domFilterer: off + - domSurveyor: off + - domLogger: off + + I verified that the code in this file is completely flushed out of memory + when a page is whitelisted. + + If cosmetic filtering is disabled: + - domWatcher: on + - domCollapser: on + - domFilterer: off + - domSurveyor: off + - domLogger: off + + If generic cosmetic filtering is disabled: + - domWatcher: on + - domCollapser: on + - domFilterer: on + - domSurveyor: off + - domLogger: on if uBO logger is opened + + If generic cosmetic filtering is enabled: + - domWatcher: on + - domCollapser: on + - domFilterer: on + - domSurveyor: on + - domLogger: on if uBO logger is opened + + Additionally, the domSurveyor can turn itself off once it decides that + it has become pointless (repeatedly not finding new cosmetic filters). + + The domFilterer makes use of platform-dependent user stylesheets[1]. + + [1] "user stylesheets" refer to local CSS rules which have priority over, + and can't be overridden by a web page's own CSS rules. + +*/ + +// Abort execution if our global vAPI object does not exist. +// https://github.com/chrisaljoudi/uBlock/issues/456 +// https://github.com/gorhill/uBlock/issues/2029 + + // >>>>>>>> start of HUGE-IF-BLOCK +if ( typeof vAPI === 'object' && !vAPI.contentScript ) { + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +vAPI.contentScript = true; + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/688#issuecomment-663657508 +{ + let context = self; + try { + while ( + context !== self.top && + context.location.href.startsWith('about:blank') && + context.parent.location.href + ) { + context = context.parent; + } + } catch(ex) { + } + vAPI.effectiveSelf = context; +} + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +vAPI.userStylesheet = { + added: new Set(), + removed: new Set(), + apply: function(callback) { + if ( this.added.size === 0 && this.removed.size === 0 ) { return; } + vAPI.messaging.send('vapi', { + what: 'userCSS', + add: Array.from(this.added), + remove: Array.from(this.removed), + }).then(( ) => { + if ( callback instanceof Function === false ) { return; } + callback(); + }); + this.added.clear(); + this.removed.clear(); + }, + add: function(cssText, now) { + if ( cssText === '' ) { return; } + this.added.add(cssText); + if ( now ) { this.apply(); } + }, + remove: function(cssText, now) { + if ( cssText === '' ) { return; } + this.removed.add(cssText); + if ( now ) { this.apply(); } + } +}; + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************* + + The purpose of SafeAnimationFrame is to take advantage of the behavior of + window.requestAnimationFrame[1]. If we use an animation frame as a timer, + then this timer is described as follow: + + - time events are throttled by the browser when the viewport is not visible -- + there is no point for uBO to play with the DOM if the document is not + visible. + - time events are micro tasks[2]. + - time events are synchronized to monitor refresh, meaning that they can fire + at most 1/60 (typically). + + If a delay value is provided, a plain timer is first used. Plain timers are + macro-tasks, so this is good when uBO wants to yield to more important tasks + on a page. Once the plain timer elapse, an animation frame is used to trigger + the next time at which to execute the job. + + [1] https://developer.mozilla.org/en-US/docs/Web/API/window/requestAnimationFrame + [2] https://jakearchibald.com/2015/tasks-microtasks-queues-and-schedules/ + +*/ + +// https://github.com/gorhill/uBlock/issues/2147 + +vAPI.SafeAnimationFrame = class { + constructor(callback) { + this.fid = this.tid = undefined; + this.callback = callback; + } + start(delay) { + if ( self.vAPI instanceof Object === false ) { return; } + if ( delay === undefined ) { + if ( this.fid === undefined ) { + this.fid = requestAnimationFrame(( ) => { this.onRAF(); } ); + } + if ( this.tid === undefined ) { + this.tid = vAPI.setTimeout(( ) => { this.onSTO(); }, 20000); + } + return; + } + if ( this.fid === undefined && this.tid === undefined ) { + this.tid = vAPI.setTimeout(( ) => { this.macroToMicro(); }, delay); + } + } + clear() { + if ( this.fid !== undefined ) { + cancelAnimationFrame(this.fid); + this.fid = undefined; + } + if ( this.tid !== undefined ) { + clearTimeout(this.tid); + this.tid = undefined; + } + } + macroToMicro() { + this.tid = undefined; + this.start(); + } + onRAF() { + if ( this.tid !== undefined ) { + clearTimeout(this.tid); + this.tid = undefined; + } + this.fid = undefined; + this.callback(); + } + onSTO() { + if ( this.fid !== undefined ) { + cancelAnimationFrame(this.fid); + this.fid = undefined; + } + this.tid = undefined; + this.callback(); + } +}; + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/552 +// Listen and report CSP violations so that blocked resources through CSP +// are properly reported in the logger. + +{ + const newEvents = new Set(); + const allEvents = new Set(); + let timer; + + const send = function() { + if ( self.vAPI instanceof Object === false ) { return; } + vAPI.messaging.send('scriptlets', { + what: 'securityPolicyViolation', + type: 'net', + docURL: document.location.href, + violations: Array.from(newEvents), + }).then(response => { + if ( response === true ) { return; } + stop(); + }); + for ( const event of newEvents ) { + allEvents.add(event); + } + newEvents.clear(); + }; + + const sendAsync = function() { + if ( timer !== undefined ) { return; } + timer = self.requestIdleCallback( + ( ) => { timer = undefined; send(); }, + { timeout: 2063 } + ); + }; + + const listener = function(ev) { + if ( ev.isTrusted !== true ) { return; } + if ( ev.disposition !== 'enforce' ) { return; } + const json = JSON.stringify({ + url: ev.blockedURL || ev.blockedURI, + policy: ev.originalPolicy, + directive: ev.effectiveDirective || ev.violatedDirective, + }); + if ( allEvents.has(json) ) { return; } + newEvents.add(json); + sendAsync(); + }; + + const stop = function() { + newEvents.clear(); + allEvents.clear(); + if ( timer !== undefined ) { + self.cancelIdleCallback(timer); + timer = undefined; + } + document.removeEventListener('securitypolicyviolation', listener); + if ( vAPI ) { vAPI.shutdown.remove(stop); } + }; + + document.addEventListener('securitypolicyviolation', listener); + vAPI.shutdown.add(stop); + + // We need to call at least once to find out whether we really need to + // listen to CSP violations. + sendAsync(); +} + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +// vAPI.domWatcher + +{ + vAPI.domMutationTime = Date.now(); + + const addedNodeLists = []; + const removedNodeLists = []; + const addedNodes = []; + const ignoreTags = new Set([ 'br', 'head', 'link', 'meta', 'script', 'style' ]); + const listeners = []; + + let domLayoutObserver; + let listenerIterator = []; + let listenerIteratorDirty = false; + let removedNodes = false; + let safeObserverHandlerTimer; + + const safeObserverHandler = function() { + let i = addedNodeLists.length; + while ( i-- ) { + const nodeList = addedNodeLists[i]; + let iNode = nodeList.length; + while ( iNode-- ) { + const node = nodeList[iNode]; + if ( node.nodeType !== 1 ) { continue; } + if ( ignoreTags.has(node.localName) ) { continue; } + if ( node.parentElement === null ) { continue; } + addedNodes.push(node); + } + } + addedNodeLists.length = 0; + i = removedNodeLists.length; + while ( i-- && removedNodes === false ) { + const nodeList = removedNodeLists[i]; + let iNode = nodeList.length; + while ( iNode-- ) { + if ( nodeList[iNode].nodeType !== 1 ) { continue; } + removedNodes = true; + break; + } + } + removedNodeLists.length = 0; + if ( addedNodes.length === 0 && removedNodes === false ) { return; } + for ( const listener of getListenerIterator() ) { + try { listener.onDOMChanged(addedNodes, removedNodes); } + catch (ex) { } + } + addedNodes.length = 0; + removedNodes = false; + vAPI.domMutationTime = Date.now(); + }; + + // https://github.com/chrisaljoudi/uBlock/issues/205 + // Do not handle added node directly from within mutation observer. + const observerHandler = function(mutations) { + let i = mutations.length; + while ( i-- ) { + const mutation = mutations[i]; + let nodeList = mutation.addedNodes; + if ( nodeList.length !== 0 ) { + addedNodeLists.push(nodeList); + } + nodeList = mutation.removedNodes; + if ( nodeList.length !== 0 ) { + removedNodeLists.push(nodeList); + } + } + if ( addedNodeLists.length !== 0 || removedNodeLists.length !== 0 ) { + safeObserverHandlerTimer.start( + addedNodeLists.length < 100 ? 1 : undefined + ); + } + }; + + const startMutationObserver = function() { + if ( domLayoutObserver !== undefined ) { return; } + domLayoutObserver = new MutationObserver(observerHandler); + domLayoutObserver.observe(document, { + //attributeFilter: [ 'class', 'id' ], + //attributes: true, + childList: true, + subtree: true + }); + safeObserverHandlerTimer = new vAPI.SafeAnimationFrame(safeObserverHandler); + vAPI.shutdown.add(cleanup); + }; + + const stopMutationObserver = function() { + if ( domLayoutObserver === undefined ) { return; } + cleanup(); + vAPI.shutdown.remove(cleanup); + }; + + const getListenerIterator = function() { + if ( listenerIteratorDirty ) { + listenerIterator = listeners.slice(); + listenerIteratorDirty = false; + } + return listenerIterator; + }; + + const addListener = function(listener) { + if ( listeners.indexOf(listener) !== -1 ) { return; } + listeners.push(listener); + listenerIteratorDirty = true; + if ( domLayoutObserver === undefined ) { return; } + try { listener.onDOMCreated(); } + catch (ex) { } + startMutationObserver(); + }; + + const removeListener = function(listener) { + const pos = listeners.indexOf(listener); + if ( pos === -1 ) { return; } + listeners.splice(pos, 1); + listenerIteratorDirty = true; + if ( listeners.length === 0 ) { + stopMutationObserver(); + } + }; + + const cleanup = function() { + if ( domLayoutObserver !== undefined ) { + domLayoutObserver.disconnect(); + domLayoutObserver = undefined; + } + if ( safeObserverHandlerTimer !== undefined ) { + safeObserverHandlerTimer.clear(); + safeObserverHandlerTimer = undefined; + } + }; + + const start = function() { + for ( const listener of getListenerIterator() ) { + try { listener.onDOMCreated(); } + catch (ex) { } + } + startMutationObserver(); + }; + + vAPI.domWatcher = { start, addListener, removeListener }; +} + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +vAPI.injectScriptlet = function(doc, text) { + if ( !doc ) { return; } + let script, url; + try { + const blob = new self.Blob([ text ], { type: 'text/javascript; charset=utf-8' }); + url = self.URL.createObjectURL(blob); + script = doc.createElement('script'); + script.async = false; + script.src = url; + (doc.head || doc.documentElement || doc).appendChild(script); + } catch (ex) { + } + if ( url ) { + if ( script ) { script.remove(); } + self.URL.revokeObjectURL(url); + } +}; + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************* + + The DOM filterer is the heart of uBO's cosmetic filtering. + + DOMFilterer: adds procedural cosmetic filtering + +*/ + +vAPI.hideStyle = 'display:none!important;'; + +vAPI.DOMFilterer = class { + constructor() { + this.commitTimer = new vAPI.SafeAnimationFrame( + ( ) => { this.commitNow(); } + ); + this.disabled = false; + this.listeners = []; + this.stylesheets = []; + this.exceptedCSSRules = []; + this.exceptions = []; + this.convertedProceduralFilters = []; + this.proceduralFilterer = null; + } + + explodeCSS(css) { + const out = []; + const cssHide = `{${vAPI.hideStyle}}`; + const blocks = css.trim().split(/\n\n+/); + for ( const block of blocks ) { + if ( block.endsWith(cssHide) === false ) { continue; } + out.push(block.slice(0, -cssHide.length).trim()); + } + return out; + } + + addCSS(css, details = {}) { + if ( typeof css !== 'string' || css.length === 0 ) { return; } + if ( this.stylesheets.includes(css) ) { return; } + this.stylesheets.push(css); + if ( details.mustInject && this.disabled === false ) { + vAPI.userStylesheet.add(css); + } + if ( this.hasListeners() === false ) { return; } + if ( details.silent ) { return; } + this.triggerListeners({ declarative: this.explodeCSS(css) }); + } + + exceptCSSRules(exceptions) { + if ( exceptions.length === 0 ) { return; } + this.exceptedCSSRules.push(...exceptions); + if ( this.hasListeners() ) { + this.triggerListeners({ exceptions }); + } + } + + addListener(listener) { + if ( this.listeners.indexOf(listener) !== -1 ) { return; } + this.listeners.push(listener); + } + + removeListener(listener) { + const pos = this.listeners.indexOf(listener); + if ( pos === -1 ) { return; } + this.listeners.splice(pos, 1); + } + + hasListeners() { + return this.listeners.length !== 0; + } + + triggerListeners(changes) { + for ( const listener of this.listeners ) { + listener.onFiltersetChanged(changes); + } + } + + toggle(state, callback) { + if ( state === undefined ) { state = this.disabled; } + if ( state !== this.disabled ) { return; } + this.disabled = !state; + const uss = vAPI.userStylesheet; + for ( const css of this.stylesheets ) { + if ( this.disabled ) { + uss.remove(css); + } else { + uss.add(css); + } + } + uss.apply(callback); + } + + // Here we will deal with: + // - Injecting low priority user styles; + // - Notifying listeners about changed filterset. + // https://www.reddit.com/r/uBlockOrigin/comments/9jj0y1/no_longer_blocking_ads/ + // Ensure vAPI is still valid -- it can go away by the time we are + // called, since the port could be force-disconnected from the main + // process. Another approach would be to have vAPI.SafeAnimationFrame + // register a shutdown job: to evaluate. For now I will keep the fix + // trivial. + commitNow() { + this.commitTimer.clear(); + if ( vAPI instanceof Object === false ) { return; } + vAPI.userStylesheet.apply(); + if ( this.proceduralFilterer instanceof Object ) { + this.proceduralFilterer.commitNow(); + } + } + + commit(commitNow) { + if ( commitNow ) { + this.commitTimer.clear(); + this.commitNow(); + } else { + this.commitTimer.start(); + } + } + + proceduralFiltererInstance() { + if ( this.proceduralFilterer instanceof Object === false ) { + if ( vAPI.DOMProceduralFilterer instanceof Object === false ) { + return null; + } + this.proceduralFilterer = new vAPI.DOMProceduralFilterer(this); + } + return this.proceduralFilterer; + } + + addProceduralSelectors(selectors) { + const procedurals = []; + for ( const raw of selectors ) { + procedurals.push(JSON.parse(raw)); + } + if ( procedurals.length === 0 ) { return; } + const pfilterer = this.proceduralFiltererInstance(); + if ( pfilterer !== null ) { + pfilterer.addProceduralSelectors(procedurals); + } + } + + createProceduralFilter(o) { + const pfilterer = this.proceduralFiltererInstance(); + if ( pfilterer === null ) { return; } + return pfilterer.createProceduralFilter(o); + } + + getAllSelectors(bits = 0) { + const out = { + declarative: [], + exceptions: this.exceptedCSSRules, + }; + const hasProcedural = this.proceduralFilterer instanceof Object; + const includePrivateSelectors = (bits & 0b01) !== 0; + const masterToken = hasProcedural + ? `[${this.proceduralFilterer.masterToken}]` + : undefined; + for ( const css of this.stylesheets ) { + for ( const block of this.explodeCSS(css) ) { + if ( + includePrivateSelectors === false && + masterToken !== undefined && + block.startsWith(masterToken) + ) { + continue; + } + out.declarative.push(block); + } + } + const excludeProcedurals = (bits & 0b10) !== 0; + if ( excludeProcedurals === false ) { + out.procedural = []; + if ( hasProcedural ) { + out.procedural.push( + ...this.proceduralFilterer.selectors.values() + ); + } + const proceduralFilterer = this.proceduralFiltererInstance(); + if ( proceduralFilterer !== null ) { + for ( const json of this.convertedProceduralFilters ) { + const pfilter = proceduralFilterer.createProceduralFilter(json); + pfilter.converted = true; + out.procedural.push(pfilter); + } + } + } + return out; + } + + getAllExceptionSelectors() { + return this.exceptions.join(',\n'); + } +}; + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +// vAPI.domCollapser + +{ + const messaging = vAPI.messaging; + const toCollapse = new Map(); + const src1stProps = { + audio: 'currentSrc', + embed: 'src', + iframe: 'src', + img: 'currentSrc', + object: 'data', + video: 'currentSrc', + }; + const src2ndProps = { + audio: 'src', + img: 'src', + video: 'src', + }; + const tagToTypeMap = { + audio: 'media', + embed: 'object', + iframe: 'sub_frame', + img: 'image', + object: 'object', + video: 'media', + }; + let requestIdGenerator = 1, + processTimer, + cachedBlockedSet, + cachedBlockedSetHash, + cachedBlockedSetTimer, + toProcess = [], + toFilter = [], + netSelectorCacheCount = 0; + + const cachedBlockedSetClear = function() { + cachedBlockedSet = + cachedBlockedSetHash = + cachedBlockedSetTimer = undefined; + }; + + // https://github.com/chrisaljoudi/uBlock/issues/399 + // https://github.com/gorhill/uBlock/issues/2848 + // Use a user stylesheet to collapse placeholders. + const getCollapseToken = ( ) => { + if ( collapseToken === undefined ) { + collapseToken = vAPI.randomToken(); + vAPI.userStylesheet.add( + `[${collapseToken}]\n{display:none!important;}`, + true + ); + } + return collapseToken; + }; + let collapseToken; + + // https://github.com/chrisaljoudi/uBlock/issues/174 + // Do not remove fragment from src URL + const onProcessed = function(response) { + // This happens if uBO is disabled or restarted. + if ( response instanceof Object === false ) { + toCollapse.clear(); + return; + } + + const targets = toCollapse.get(response.id); + if ( targets === undefined ) { return; } + + toCollapse.delete(response.id); + if ( cachedBlockedSetHash !== response.hash ) { + cachedBlockedSet = new Set(response.blockedResources); + cachedBlockedSetHash = response.hash; + if ( cachedBlockedSetTimer !== undefined ) { + clearTimeout(cachedBlockedSetTimer); + } + cachedBlockedSetTimer = vAPI.setTimeout(cachedBlockedSetClear, 30000); + } + if ( cachedBlockedSet === undefined || cachedBlockedSet.size === 0 ) { + return; + } + + const selectors = []; + let netSelectorCacheCountMax = response.netSelectorCacheCountMax; + + for ( const target of targets ) { + const tag = target.localName; + let prop = src1stProps[tag]; + if ( prop === undefined ) { continue; } + let src = target[prop]; + if ( typeof src !== 'string' || src.length === 0 ) { + prop = src2ndProps[tag]; + if ( prop === undefined ) { continue; } + src = target[prop]; + if ( typeof src !== 'string' || src.length === 0 ) { continue; } + } + if ( cachedBlockedSet.has(tagToTypeMap[tag] + ' ' + src) === false ) { + continue; + } + target.setAttribute(getCollapseToken(), ''); + // https://github.com/chrisaljoudi/uBlock/issues/1048 + // Use attribute to construct CSS rule + if ( netSelectorCacheCount > netSelectorCacheCountMax ) { continue; } + const value = target.getAttribute(prop); + if ( value ) { + selectors.push(`${tag}[${prop}="${CSS.escape(value)}"]`); + netSelectorCacheCount += 1; + } + } + + if ( selectors.length === 0 ) { return; } + messaging.send('contentscript', { + what: 'cosmeticFiltersInjected', + type: 'net', + hostname: window.location.hostname, + selectors, + }); + }; + + const send = function() { + processTimer = undefined; + toCollapse.set(requestIdGenerator, toProcess); + messaging.send('contentscript', { + what: 'getCollapsibleBlockedRequests', + id: requestIdGenerator, + frameURL: window.location.href, + resources: toFilter, + hash: cachedBlockedSetHash, + }).then(response => { + onProcessed(response); + }); + toProcess = []; + toFilter = []; + requestIdGenerator += 1; + }; + + const process = function(delay) { + if ( toProcess.length === 0 ) { return; } + if ( delay === 0 ) { + if ( processTimer !== undefined ) { + clearTimeout(processTimer); + } + send(); + } else if ( processTimer === undefined ) { + processTimer = vAPI.setTimeout(send, delay || 20); + } + }; + + const add = function(target) { + toProcess[toProcess.length] = target; + }; + + const addMany = function(targets) { + for ( const target of targets ) { + add(target); + } + }; + + const iframeSourceModified = function(mutations) { + for ( const mutation of mutations ) { + addIFrame(mutation.target, true); + } + process(); + }; + const iframeSourceObserver = new MutationObserver(iframeSourceModified); + const iframeSourceObserverOptions = { + attributes: true, + attributeFilter: [ 'src' ] + }; + + // https://github.com/gorhill/uBlock/issues/162 + // Be prepared to deal with possible change of src attribute. + const addIFrame = function(iframe, dontObserve) { + if ( dontObserve !== true ) { + iframeSourceObserver.observe(iframe, iframeSourceObserverOptions); + } + const src = iframe.src; + if ( typeof src !== 'string' || src === '' ) { return; } + if ( src.startsWith('http') === false ) { return; } + toFilter.push({ type: 'sub_frame', url: iframe.src }); + add(iframe); + }; + + const addIFrames = function(iframes) { + for ( const iframe of iframes ) { + addIFrame(iframe); + } + }; + + const onResourceFailed = function(ev) { + if ( tagToTypeMap[ev.target.localName] !== undefined ) { + add(ev.target); + process(); + } + }; + + const stop = function() { + document.removeEventListener('error', onResourceFailed, true); + if ( processTimer !== undefined ) { + clearTimeout(processTimer); + } + if ( vAPI.domWatcher instanceof Object ) { + vAPI.domWatcher.removeListener(domWatcherInterface); + } + vAPI.shutdown.remove(stop); + vAPI.domCollapser = null; + }; + + const start = function() { + if ( vAPI.domWatcher instanceof Object ) { + vAPI.domWatcher.addListener(domWatcherInterface); + } + }; + + const domWatcherInterface = { + onDOMCreated: function() { + if ( self.vAPI instanceof Object === false ) { return; } + if ( vAPI.domCollapser instanceof Object === false ) { + if ( vAPI.domWatcher instanceof Object ) { + vAPI.domWatcher.removeListener(domWatcherInterface); + } + return; + } + // Listener to collapse blocked resources. + // - Future requests not blocked yet + // - Elements dynamically added to the page + // - Elements which resource URL changes + // https://github.com/chrisaljoudi/uBlock/issues/7 + // Preferring getElementsByTagName over querySelectorAll: + // http://jsperf.com/queryselectorall-vs-getelementsbytagname/145 + const elems = document.images || + document.getElementsByTagName('img'); + for ( const elem of elems ) { + if ( elem.complete ) { + add(elem); + } + } + addMany(document.embeds || document.getElementsByTagName('embed')); + addMany(document.getElementsByTagName('object')); + addIFrames(document.getElementsByTagName('iframe')); + process(0); + + document.addEventListener('error', onResourceFailed, true); + + vAPI.shutdown.add(stop); + }, + onDOMChanged: function(addedNodes) { + if ( addedNodes.length === 0 ) { return; } + for ( const node of addedNodes ) { + if ( node.localName === 'iframe' ) { + addIFrame(node); + } + if ( node.firstElementChild === null ) { continue; } + const iframes = node.getElementsByTagName('iframe'); + if ( iframes.length !== 0 ) { + addIFrames(iframes); + } + } + process(); + } + }; + + vAPI.domCollapser = { start }; +} + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +// vAPI.domSurveyor + +{ + // http://www.cse.yorku.ca/~oz/hash.html#djb2 + // Must mirror cosmetic filtering compiler's version + const hashFromStr = (type, s) => { + const len = s.length; + const step = len + 7 >>> 3; + let hash = (type << 5) + type ^ len; + for ( let i = 0; i < len; i += step ) { + hash = (hash << 5) + hash ^ s.charCodeAt(i); + } + return hash & 0xFFFFFF; + }; + + const addHashes = hashes => { + for ( const hash of hashes ) { + queriedHashes.add(hash); + } + }; + + const queriedHashes = new Set(); + const maxSurveyNodes = 65536; + const pendingLists = []; + const pendingNodes = []; + const processedSet = new Set(); + let domFilterer; + let hostname = ''; + let domChanged = false; + let scannedCount = 0; + let stopped = false; + + const addPendingList = list => { + if ( list.length === 0 ) { return; } + pendingLists.push(Array.from(list)); + }; + + const nextPendingNodes = ( ) => { + if ( pendingLists.length === 0 ) { return 0; } + const bufferSize = 256; + let j = 0; + do { + const nodeList = pendingLists[0]; + let n = bufferSize - j; + if ( n > nodeList.length ) { + n = nodeList.length; + } + for ( let i = 0; i < n; i++ ) { + pendingNodes[j+i] = nodeList[i]; + } + j += n; + if ( n !== nodeList.length ) { + pendingLists[0] = nodeList.slice(n); + break; + } + pendingLists.shift(); + } while ( j < bufferSize && pendingLists.length !== 0 ); + return j; + }; + + const hasPendingNodes = ( ) => { + return pendingLists.length !== 0; + }; + + // Extract all classes/ids: these will be passed to the cosmetic + // filtering engine, and in return we will obtain only the relevant + // CSS selectors. + + // https://github.com/gorhill/uBlock/issues/672 + // http://www.w3.org/TR/2014/REC-html5-20141028/infrastructure.html#space-separated-tokens + // http://jsperf.com/enumerate-classes/6 + + const idFromNode = (node, out) => { + const raw = node.id; + if ( typeof raw !== 'string' || raw.length === 0 ) { return; } + const hash = hashFromStr(0x23 /* '#' */, raw.trim()); + if ( queriedHashes.has(hash) ) { return; } + queriedHashes.add(hash); + out.push(hash); + }; + + // https://github.com/uBlockOrigin/uBlock-issues/discussions/2076 + // Performance: avoid using Element.classList + const classesFromNode = (node, out) => { + const s = node.getAttribute('class'); + if ( typeof s !== 'string' ) { return; } + const len = s.length; + for ( let beg = 0, end = 0; beg < len; beg += 1 ) { + end = s.indexOf(' ', beg); + if ( end === beg ) { continue; } + if ( end === -1 ) { end = len; } + const hash = hashFromStr(0x2E /* '.' */, s.slice(beg, end)); + beg = end; + if ( queriedHashes.has(hash) ) { continue; } + queriedHashes.add(hash); + out.push(hash); + } + }; + + const getSurveyResults = (hashes, safeOnly) => { + if ( self.vAPI.messaging instanceof Object === false ) { + stop(); return; + } + const promise = hashes.length === 0 + ? Promise.resolve(null) + : self.vAPI.messaging.send('contentscript', { + what: 'retrieveGenericCosmeticSelectors', + hostname, + hashes, + exceptions: domFilterer.exceptions, + safeOnly, + }); + promise.then(response => { + processSurveyResults(response); + }); + }; + + const doSurvey = ( ) => { + if ( self.vAPI instanceof Object === false ) { return; } + const t0 = performance.now(); + const hashes = []; + const nodes = pendingNodes; + const deadline = t0 + 4; + let processed = 0; + let scanned = 0; + for (;;) { + const n = nextPendingNodes(); + if ( n === 0 ) { break; } + for ( let i = 0; i < n; i++ ) { + const node = nodes[i]; nodes[i] = null; + if ( domChanged ) { + if ( processedSet.has(node) ) { continue; } + processedSet.add(node); + } + idFromNode(node, hashes); + classesFromNode(node, hashes); + scanned += 1; + } + processed += n; + if ( performance.now() >= deadline ) { break; } + } + //console.info(`[domSurveyor][${hostname}] Surveyed ${scanned}/${processed} nodes in ${(performance.now()-t0).toFixed(2)} ms: ${hashes.length} hashes`); + scannedCount += scanned; + if ( scannedCount >= maxSurveyNodes ) { + stop(); + } + processedSet.clear(); + getSurveyResults(hashes); + }; + + const surveyTimer = new vAPI.SafeAnimationFrame(doSurvey); + + // This is to shutdown the surveyor if result of surveying keeps being + // fruitless. This is useful on long-lived web page. I arbitrarily + // picked 5 minutes before the surveyor is allowed to shutdown. I also + // arbitrarily picked 256 misses before the surveyor is allowed to + // shutdown. + let canShutdownAfter = Date.now() + 300000; + let surveyResultMissCount = 0; + + // Handle main process' response. + + const processSurveyResults = response => { + if ( stopped ) { return; } + const result = response && response.result; + let mustCommit = false; + if ( result ) { + const css = result.injectedCSS; + if ( typeof css === 'string' && css.length !== 0 ) { + domFilterer.addCSS(css); + mustCommit = true; + } + const selectors = result.excepted; + if ( Array.isArray(selectors) && selectors.length !== 0 ) { + domFilterer.exceptCSSRules(selectors); + } + } + if ( hasPendingNodes() ) { + surveyTimer.start(1); + } + if ( mustCommit ) { + surveyResultMissCount = 0; + canShutdownAfter = Date.now() + 300000; + return; + } + surveyResultMissCount += 1; + if ( surveyResultMissCount < 256 || Date.now() < canShutdownAfter ) { + return; + } + //console.info(`[domSurveyor][${hostname}] Shutting down, too many misses`); + stop(); + self.vAPI.messaging.send('contentscript', { + what: 'disableGenericCosmeticFilteringSurveyor', + hostname, + }); + }; + + const domWatcherInterface = { + onDOMCreated: function() { + domFilterer = vAPI.domFilterer; + // https://github.com/uBlockOrigin/uBlock-issues/issues/1692 + // Look-up safe-only selectors to mitigate probability of + // html/body elements of erroneously being targeted. + const hashes = []; + if ( document.documentElement !== null ) { + idFromNode(document.documentElement, hashes); + classesFromNode(document.documentElement, hashes); + } + if ( document.body !== null ) { + idFromNode(document.body, hashes); + classesFromNode(document.body, hashes); + } + if ( hashes.length !== 0 ) { + getSurveyResults(hashes, true); + } + addPendingList(document.querySelectorAll( + '[id]:not(html):not(body),[class]:not(html):not(body)' + )); + if ( hasPendingNodes() ) { + surveyTimer.start(); + } + }, + onDOMChanged: function(addedNodes) { + if ( addedNodes.length === 0 ) { return; } + domChanged = true; + for ( const node of addedNodes ) { + addPendingList([ node ]); + if ( node.firstElementChild === null ) { continue; } + addPendingList( + node.querySelectorAll( + '[id]:not(html):not(body),[class]:not(html):not(body)' + ) + ); + } + if ( hasPendingNodes() ) { + surveyTimer.start(1); + } + } + }; + + const start = details => { + if ( self.vAPI instanceof Object === false ) { return; } + if ( self.vAPI.domFilterer instanceof Object === false ) { return; } + if ( self.vAPI.domWatcher instanceof Object === false ) { return; } + hostname = details.hostname; + self.vAPI.domWatcher.addListener(domWatcherInterface); + }; + + const stop = ( ) => { + stopped = true; + pendingLists.length = 0; + surveyTimer.clear(); + if ( self.vAPI instanceof Object === false ) { return; } + if ( self.vAPI.domWatcher instanceof Object ) { + self.vAPI.domWatcher.removeListener(domWatcherInterface); + } + self.vAPI.domSurveyor = null; + }; + + self.vAPI.domSurveyor = { start, addHashes }; +} + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +// vAPI.bootstrap: +// Bootstrapping allows all components of the content script +// to be launched if/when needed. + +{ + const onDomReady = ( ) => { + // This can happen on Firefox. For instance: + // https://github.com/gorhill/uBlock/issues/1893 + if ( window.location === null ) { return; } + if ( self.vAPI instanceof Object === false ) { return; } + + vAPI.messaging.send('contentscript', { + what: 'shouldRenderNoscriptTags', + }); + + if ( vAPI.domFilterer instanceof Object ) { + vAPI.domFilterer.commitNow(); + } + + if ( vAPI.domWatcher instanceof Object ) { + vAPI.domWatcher.start(); + } + + // Element picker works only in top window for now. + if ( + window !== window.top || + vAPI.domFilterer instanceof Object === false + ) { + return; + } + + // To be used by element picker/zapper. + vAPI.mouseClick = { x: -1, y: -1 }; + + const onMouseClick = function(ev) { + if ( ev.isTrusted === false ) { return; } + vAPI.mouseClick.x = ev.clientX; + vAPI.mouseClick.y = ev.clientY; + + // https://github.com/chrisaljoudi/uBlock/issues/1143 + // Find a link under the mouse, to try to avoid confusing new tabs + // as nuisance popups. + // https://github.com/uBlockOrigin/uBlock-issues/issues/777 + // Mind that href may not be a string. + const elem = ev.target.closest('a[href]'); + if ( elem === null || typeof elem.href !== 'string' ) { return; } + vAPI.messaging.send('contentscript', { + what: 'maybeGoodPopup', + url: elem.href || '', + }); + }; + + document.addEventListener('mousedown', onMouseClick, true); + + // https://github.com/gorhill/uMatrix/issues/144 + vAPI.shutdown.add(function() { + document.removeEventListener('mousedown', onMouseClick, true); + }); + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/403 + // If there was a spurious port disconnection -- in which case the + // response is expressly set to `null`, rather than undefined or + // an object -- let's stay around, we may be given the opportunity + // to try bootstrapping again later. + + const onResponseReady = response => { + if ( response instanceof Object === false ) { return; } + vAPI.bootstrap = undefined; + + // cosmetic filtering engine aka 'cfe' + const cfeDetails = response && response.specificCosmeticFilters; + if ( !cfeDetails || !cfeDetails.ready ) { + vAPI.domWatcher = vAPI.domCollapser = vAPI.domFilterer = + vAPI.domSurveyor = vAPI.domIsLoaded = null; + return; + } + + vAPI.domCollapser.start(); + + const { + noSpecificCosmeticFiltering, + noGenericCosmeticFiltering, + scriptletDetails, + } = response; + + vAPI.noSpecificCosmeticFiltering = noSpecificCosmeticFiltering; + vAPI.noGenericCosmeticFiltering = noGenericCosmeticFiltering; + + if ( noSpecificCosmeticFiltering && noGenericCosmeticFiltering ) { + vAPI.domFilterer = null; + vAPI.domSurveyor = null; + } else { + const domFilterer = vAPI.domFilterer = new vAPI.DOMFilterer(); + if ( noGenericCosmeticFiltering || cfeDetails.disableSurveyor ) { + vAPI.domSurveyor = null; + } + domFilterer.exceptions = cfeDetails.exceptionFilters; + domFilterer.addCSS(cfeDetails.injectedCSS); + domFilterer.addProceduralSelectors(cfeDetails.proceduralFilters); + domFilterer.exceptCSSRules(cfeDetails.exceptedFilters); + domFilterer.convertedProceduralFilters = cfeDetails.convertedProceduralFilters; + vAPI.userStylesheet.apply(); + } + + if ( scriptletDetails && typeof self.uBO_scriptletsInjected !== 'string' ) { + self.uBO_scriptletsInjected = scriptletDetails.filters; + if ( scriptletDetails.mainWorld ) { + vAPI.injectScriptlet(document, scriptletDetails.mainWorld); + vAPI.injectedScripts = scriptletDetails.mainWorld; + } + } + + if ( vAPI.domSurveyor ) { + if ( Array.isArray(cfeDetails.genericCosmeticHashes) ) { + vAPI.domSurveyor.addHashes(cfeDetails.genericCosmeticHashes); + } + vAPI.domSurveyor.start(cfeDetails); + } + + const readyState = document.readyState; + if ( readyState === 'interactive' || readyState === 'complete' ) { + return onDomReady(); + } + document.addEventListener('DOMContentLoaded', onDomReady, { once: true }); + }; + + vAPI.bootstrap = function() { + vAPI.messaging.send('contentscript', { + what: 'retrieveContentScriptParameters', + url: vAPI.effectiveSelf.location.href, + needScriptlets: typeof self.uBO_scriptletsInjected !== 'string', + }).then(response => { + onResponseReady(response); + }); + }; +} + +// This starts bootstrap process. +vAPI.bootstrap(); + +/******************************************************************************/ +/******************************************************************************/ +/******************************************************************************/ + +} +// <<<<<<<< end of HUGE-IF-BLOCK diff --git a/src/js/contextmenu.js b/src/js/contextmenu.js new file mode 100644 index 0000000..abf0582 --- /dev/null +++ b/src/js/contextmenu.js @@ -0,0 +1,270 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; +import { i18n$ } from './i18n.js'; + +/******************************************************************************/ + +const contextMenu = (( ) => { + +/******************************************************************************/ + +if ( vAPI.contextMenu === undefined ) { + return { + update: function() {} + }; +} + +/******************************************************************************/ + +const BLOCK_ELEMENT_BIT = 0b00001; +const BLOCK_RESOURCE_BIT = 0b00010; +const TEMP_ALLOW_LARGE_MEDIA_BIT = 0b00100; +const SUBSCRIBE_TO_LIST_BIT = 0b01000; +const VIEW_SOURCE_BIT = 0b10000; + +/******************************************************************************/ + +const onBlockElement = function(details, tab) { + if ( tab === undefined ) { return; } + if ( /^https?:\/\//.test(tab.url) === false ) { return; } + let tagName = details.tagName || ''; + let src = details.frameUrl || details.srcUrl || details.linkUrl || ''; + + if ( !tagName ) { + if ( typeof details.frameUrl === 'string' ) { + tagName = 'iframe'; + } else if ( typeof details.srcUrl === 'string' ) { + if ( details.mediaType === 'image' ) { + tagName = 'img'; + } else if ( details.mediaType === 'video' ) { + tagName = 'video'; + } else if ( details.mediaType === 'audio' ) { + tagName = 'audio'; + } + } else if ( typeof details.linkUrl === 'string' ) { + tagName = 'a'; + } + } + + µb.epickerArgs.mouse = true; + µb.elementPickerExec(tab.id, 0, `${tagName}\t${src}`); +}; + +/******************************************************************************/ + +const onBlockElementInFrame = function(details, tab) { + if ( tab === undefined ) { return; } + if ( /^https?:\/\//.test(details.frameUrl) === false ) { return; } + µb.epickerArgs.mouse = false; + µb.elementPickerExec(tab.id, details.frameId); +}; + +/******************************************************************************/ + +const onSubscribeToList = function(details) { + let parsedURL; + try { + parsedURL = new URL(details.linkUrl); + } + catch(ex) { + } + if ( parsedURL instanceof URL === false ) { return; } + const url = parsedURL.searchParams.get('location'); + if ( url === null ) { return; } + const title = parsedURL.searchParams.get('title') || '?'; + const hash = µb.selectedFilterLists.indexOf(parsedURL) !== -1 + ? '#subscribed' + : ''; + vAPI.tabs.open({ + url: + `/asset-viewer.html` + + `?url=${encodeURIComponent(url)}` + + `&title=${encodeURIComponent(title)}` + + `&subscribe=1${hash}`, + select: true, + }); +}; + +/******************************************************************************/ + +const onTemporarilyAllowLargeMediaElements = function(details, tab) { + if ( tab === undefined ) { return; } + const pageStore = µb.pageStoreFromTabId(tab.id); + if ( pageStore === null ) { return; } + pageStore.temporarilyAllowLargeMediaElements(true); +}; + +/******************************************************************************/ + +const onViewSource = function(details, tab) { + if ( tab === undefined ) { return; } + const url = details.linkUrl || details.frameUrl || details.pageUrl || ''; + if ( /^https?:\/\//.test(url) === false ) { return; } + µb.openNewTab({ + url: `code-viewer.html?url=${self.encodeURIComponent(url)}`, + select: true, + }); +}; + +/******************************************************************************/ + +const onEntryClicked = function(details, tab) { + if ( details.menuItemId === 'uBlock0-blockElement' ) { + return onBlockElement(details, tab); + } + if ( details.menuItemId === 'uBlock0-blockElementInFrame' ) { + return onBlockElementInFrame(details, tab); + } + if ( details.menuItemId === 'uBlock0-blockResource' ) { + return onBlockElement(details, tab); + } + if ( details.menuItemId === 'uBlock0-subscribeToList' ) { + return onSubscribeToList(details); + } + if ( details.menuItemId === 'uBlock0-temporarilyAllowLargeMediaElements' ) { + return onTemporarilyAllowLargeMediaElements(details, tab); + } + if ( details.menuItemId === 'uBlock0-viewSource' ) { + return onViewSource(details, tab); + } +}; + +/******************************************************************************/ + +const menuEntries = { + blockElement: { + id: 'uBlock0-blockElement', + title: i18n$('pickerContextMenuEntry'), + contexts: [ 'all' ], + documentUrlPatterns: [ 'http://*/*', 'https://*/*' ], + }, + blockElementInFrame: { + id: 'uBlock0-blockElementInFrame', + title: i18n$('contextMenuBlockElementInFrame'), + contexts: [ 'frame' ], + documentUrlPatterns: [ 'http://*/*', 'https://*/*' ], + }, + blockResource: { + id: 'uBlock0-blockResource', + title: i18n$('pickerContextMenuEntry'), + contexts: [ 'audio', 'frame', 'image', 'video' ], + documentUrlPatterns: [ 'http://*/*', 'https://*/*' ], + }, + subscribeToList: { + id: 'uBlock0-subscribeToList', + title: i18n$('contextMenuSubscribeToList'), + contexts: [ 'link' ], + targetUrlPatterns: [ 'abp:*', 'https://subscribe.adblockplus.org/*' ], + }, + temporarilyAllowLargeMediaElements: { + id: 'uBlock0-temporarilyAllowLargeMediaElements', + title: i18n$('contextMenuTemporarilyAllowLargeMediaElements'), + contexts: [ 'all' ], + documentUrlPatterns: [ 'http://*/*', 'https://*/*' ], + }, + viewSource: { + id: 'uBlock0-viewSource', + title: i18n$('contextMenuViewSource'), + contexts: [ 'page', 'frame', 'link' ], + documentUrlPatterns: [ 'http://*/*', 'https://*/*' ], + }, +}; + +/******************************************************************************/ + +let currentBits = 0; + +const update = function(tabId = undefined) { + let newBits = 0; + if ( µb.userSettings.contextMenuEnabled && tabId !== undefined ) { + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore && pageStore.getNetFilteringSwitch() ) { + if ( pageStore.shouldApplySpecificCosmeticFilters(0) ) { + newBits |= BLOCK_ELEMENT_BIT; + } else { + newBits |= BLOCK_RESOURCE_BIT; + } + if ( pageStore.largeMediaCount !== 0 ) { + newBits |= TEMP_ALLOW_LARGE_MEDIA_BIT; + } + } + newBits |= SUBSCRIBE_TO_LIST_BIT; + } + if ( µb.hiddenSettings.filterAuthorMode ) { + newBits |= VIEW_SOURCE_BIT; + } + if ( newBits === currentBits ) { return; } + currentBits = newBits; + const usedEntries = []; + if ( (newBits & BLOCK_ELEMENT_BIT) !== 0 ) { + usedEntries.push(menuEntries.blockElement); + usedEntries.push(menuEntries.blockElementInFrame); + } + if ( (newBits & BLOCK_RESOURCE_BIT) !== 0 ) { + usedEntries.push(menuEntries.blockResource); + } + if ( (newBits & TEMP_ALLOW_LARGE_MEDIA_BIT) !== 0 ) { + usedEntries.push(menuEntries.temporarilyAllowLargeMediaElements); + } + if ( (newBits & SUBSCRIBE_TO_LIST_BIT) !== 0 ) { + usedEntries.push(menuEntries.subscribeToList); + } + if ( (newBits & VIEW_SOURCE_BIT) !== 0 ) { + usedEntries.push(menuEntries.viewSource); + } + vAPI.contextMenu.setEntries(usedEntries, onEntryClicked); +}; + +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/151 +// For unknown reasons, the currently active tab will not be successfully +// looked up after closing a window. + +vAPI.contextMenu.onMustUpdate = async function(tabId = undefined) { + if ( µb.userSettings.contextMenuEnabled === false ) { + return update(); + } + if ( tabId !== undefined ) { + return update(tabId); + } + const tab = await vAPI.tabs.getCurrent(); + if ( tab instanceof Object === false ) { return; } + update(tab.id); +}; + +return { update: vAPI.contextMenu.onMustUpdate }; + +/******************************************************************************/ + +})(); + +/******************************************************************************/ + +export default contextMenu; + +/******************************************************************************/ diff --git a/src/js/cosmetic-filtering.js b/src/js/cosmetic-filtering.js new file mode 100644 index 0000000..f4782bc --- /dev/null +++ b/src/js/cosmetic-filtering.js @@ -0,0 +1,983 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import logger from './logger.js'; +import µb from './background.js'; + +import { MRUCache } from './mrucache.js'; +import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js'; + +/******************************************************************************/ +/******************************************************************************/ + +const SelectorCacheEntry = class { + constructor() { + this.reset(); + } + + reset() { + this.cosmetic = new Set(); + this.cosmeticHashes = new Set(); + this.disableSurveyor = false; + this.net = new Map(); + this.accessId = SelectorCacheEntry.accessId++; + return this; + } + + dispose() { + this.cosmetic = this.cosmeticHashes = this.net = null; + if ( SelectorCacheEntry.junkyard.length < 25 ) { + SelectorCacheEntry.junkyard.push(this); + } + } + + addCosmetic(details) { + const selectors = details.selectors.join(',\n'); + if ( selectors.length !== 0 ) { + this.cosmetic.add(selectors); + } + for ( const hash of details.hashes ) { + this.cosmeticHashes.add(hash); + } + } + + addNet(selectors) { + if ( typeof selectors === 'string' ) { + this.net.set(selectors, this.accessId); + } else { + this.net.set(selectors.join(',\n'), this.accessId); + } + // Net request-derived selectors: I limit the number of cached + // selectors, as I expect cases where the blocked network requests + // are never the exact same URL. + if ( this.net.size < SelectorCacheEntry.netHighWaterMark ) { return; } + const keys = Array.from(this.net) + .sort((a, b) => b[1] - a[1]) + .slice(SelectorCacheEntry.netLowWaterMark) + .map(a => a[0]); + for ( const key of keys ) { + this.net.delete(key); + } + } + + addNetOne(selector, token) { + this.net.set(selector, token); + } + + add(details) { + this.accessId = SelectorCacheEntry.accessId++; + if ( details.type === 'cosmetic' ) { + this.addCosmetic(details); + } else { + this.addNet(details.selectors); + } + } + + // https://github.com/chrisaljoudi/uBlock/issues/420 + remove(type) { + this.accessId = SelectorCacheEntry.accessId++; + if ( type === undefined || type === 'cosmetic' ) { + this.cosmetic.clear(); + } + if ( type === undefined || type === 'net' ) { + this.net.clear(); + } + } + + retrieveToArray(iterator, out) { + for ( const selector of iterator ) { + out.push(selector); + } + } + + retrieveToSet(iterator, out) { + for ( const selector of iterator ) { + out.add(selector); + } + } + + retrieveNet(out) { + this.accessId = SelectorCacheEntry.accessId++; + if ( this.net.size === 0 ) { return false; } + this.retrieveToArray(this.net.keys(), out); + return true; + } + + retrieveCosmetic(selectors, hashes) { + this.accessId = SelectorCacheEntry.accessId++; + if ( this.cosmetic.size === 0 ) { return false; } + this.retrieveToSet(this.cosmetic, selectors); + this.retrieveToArray(this.cosmeticHashes, hashes); + return true; + } + + static factory() { + const entry = SelectorCacheEntry.junkyard.pop(); + return entry + ? entry.reset() + : new SelectorCacheEntry(); + } +}; + +SelectorCacheEntry.accessId = 1; +SelectorCacheEntry.netLowWaterMark = 20; +SelectorCacheEntry.netHighWaterMark = 30; +SelectorCacheEntry.junkyard = []; + +/******************************************************************************/ +/******************************************************************************/ + +// http://www.cse.yorku.ca/~oz/hash.html#djb2 +// Must mirror content script surveyor's version + +const hashFromStr = (type, s) => { + const len = s.length; + const step = len + 7 >>> 3; + let hash = (type << 5) + type ^ len; + for ( let i = 0; i < len; i += step ) { + hash = (hash << 5) + hash ^ s.charCodeAt(i); + } + return hash & 0xFFFFFF; +}; + +// https://github.com/gorhill/uBlock/issues/1668 +// The key must be literal: unescape escaped CSS before extracting key. +// It's an uncommon case, so it's best to unescape only when needed. + +const keyFromSelector = selector => { + let key = ''; + let matches = rePlainSelector.exec(selector); + if ( matches !== null ) { + key = matches[0]; + } else { + matches = rePlainSelectorEx.exec(selector); + if ( matches === null ) { return; } + key = matches[1] || matches[2]; + } + if ( key.includes('\\') === false ) { return key; } + matches = rePlainSelectorEscaped.exec(selector); + if ( matches === null ) { return; } + key = ''; + const escaped = matches[0]; + let beg = 0; + reEscapeSequence.lastIndex = 0; + for (;;) { + matches = reEscapeSequence.exec(escaped); + if ( matches === null ) { + return key + escaped.slice(beg); + } + key += escaped.slice(beg, matches.index); + beg = reEscapeSequence.lastIndex; + if ( matches[1].length === 1 ) { + key += matches[1]; + } else { + key += String.fromCharCode(parseInt(matches[1], 16)); + } + } +}; + +const rePlainSelector = /^[#.][\w\\-]+/; +const rePlainSelectorEx = /^[^#.\[(]+([#.][\w-]+)|([#.][\w-]+)$/; +const rePlainSelectorEscaped = /^[#.](?:\\[0-9A-Fa-f]+ |\\.|\w|-)+/; +const reEscapeSequence = /\\([0-9A-Fa-f]+ |.)/g; + +/******************************************************************************/ +/******************************************************************************/ + +// Cosmetic filter family tree: +// +// Generic +// Low generic simple: class or id only +// Low generic complex: class or id + extra stuff after +// High generic: +// High-low generic: [alt="..."],[title="..."] +// High-medium generic: [href^="..."] +// High-high generic: everything else +// Specific +// Specific hostname +// Specific entity +// Generic filters can only be enforced once the main document is loaded. +// Specific filers can be enforced before the main document is loaded. + +const FilterContainer = function() { + this.reSimpleHighGeneric = /^(?:[a-z]*\[[^\]]+\]|\S+)$/; + + this.selectorCache = new Map(); + this.selectorCachePruneDelay = 10; // 10 minutes + this.selectorCacheCountMin = 40; + this.selectorCacheCountMax = 50; + this.selectorCacheTimer = vAPI.defer.create(( ) => { + this.pruneSelectorCacheAsync(); + }); + + // specific filters + this.specificFilters = new StaticExtFilteringHostnameDB(2); + + // low generic cosmetic filters: map of hash => stringified selector list + this.lowlyGeneric = new Map(); + + // highly generic selectors sets + this.highlyGeneric = Object.create(null); + this.highlyGeneric.simple = { + canonical: 'highGenericHideSimple', + dict: new Set(), + str: '', + mru: new MRUCache(16) + }; + this.highlyGeneric.complex = { + canonical: 'highGenericHideComplex', + dict: new Set(), + str: '', + mru: new MRUCache(16) + }; + + // Short-lived: content is valid only during one function call. These + // is to prevent repeated allocation/deallocation overheads -- the + // constructors/destructors of javascript Set/Map is assumed to be costlier + // than just calling clear() on these. + this.$specificSet = new Set(); + this.$exceptionSet = new Set(); + this.$proceduralSet = new Set(); + this.$dummySet = new Set(); + + this.reset(); +}; + +/******************************************************************************/ + +// Reset all, thus reducing to a minimum memory footprint of the context. + +FilterContainer.prototype.reset = function() { + this.frozen = false; + this.acceptedCount = 0; + this.discardedCount = 0; + this.duplicateBuster = new Set(); + + this.selectorCache.clear(); + this.selectorCacheTimer.off(); + + // hostname, entity-based filters + this.specificFilters.clear(); + + // low generic cosmetic filters + this.lowlyGeneric.clear(); + + // highly generic selectors sets + this.highlyGeneric.simple.dict.clear(); + this.highlyGeneric.simple.str = ''; + this.highlyGeneric.simple.mru.reset(); + this.highlyGeneric.complex.dict.clear(); + this.highlyGeneric.complex.str = ''; + this.highlyGeneric.complex.mru.reset(); + + this.selfieVersion = 1; +}; + +/******************************************************************************/ + +FilterContainer.prototype.freeze = function() { + this.duplicateBuster.clear(); + this.specificFilters.collectGarbage(); + + this.highlyGeneric.simple.str = Array.from(this.highlyGeneric.simple.dict).join(',\n'); + this.highlyGeneric.simple.mru.reset(); + this.highlyGeneric.complex.str = Array.from(this.highlyGeneric.complex.dict).join(',\n'); + this.highlyGeneric.complex.mru.reset(); + + this.frozen = true; +}; + +/******************************************************************************/ + +FilterContainer.prototype.compile = function(parser, writer) { + if ( parser.hasOptions() === false ) { + this.compileGenericSelector(parser, writer); + return true; + } + + // https://github.com/chrisaljoudi/uBlock/issues/151 + // Negated hostname means the filter applies to all non-negated hostnames + // of same filter OR globally if there is no non-negated hostnames. + let applyGlobally = true; + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + if ( not === false ) { + applyGlobally = false; + } + this.compileSpecificSelector(parser, hn, not, writer); + } + if ( applyGlobally ) { + this.compileGenericSelector(parser, writer); + } + + return true; +}; + +/******************************************************************************/ + +FilterContainer.prototype.compileGenericSelector = function(parser, writer) { + if ( parser.isException() ) { + this.compileGenericUnhideSelector(parser, writer); + } else { + this.compileGenericHideSelector(parser, writer); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.compileGenericHideSelector = function( + parser, + writer +) { + const { raw, compiled } = parser.result; + if ( compiled === undefined ) { + const who = writer.properties.get('name') || '?'; + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid generic cosmetic filter in ${who}: ${raw}` + }); + return; + } + + writer.select('COSMETIC_FILTERS:GENERIC'); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/131 + // Support generic procedural filters as per advanced settings. + if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { + if ( µb.hiddenSettings.allowGenericProceduralFilters === true ) { + return this.compileSpecificSelector(parser, '', false, writer); + } + const who = writer.properties.get('name') || '?'; + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid generic cosmetic filter in ${who}: ##${raw}` + }); + return; + } + + const key = keyFromSelector(compiled); + if ( key !== undefined ) { + writer.push([ + 0, + hashFromStr(key.charCodeAt(0), key.slice(1)), + compiled, + ]); + return; + } + + // Pass this point, we are dealing with highly-generic cosmetic filters. + // + // For efficiency purpose, we will distinguish between simple and complex + // selectors. + + if ( this.reSimpleHighGeneric.test(compiled) ) { + writer.push([ 4 /* simple */, compiled ]); + } else { + writer.push([ 5 /* complex */, compiled ]); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.compileGenericUnhideSelector = function( + parser, + writer +) { + // Procedural cosmetic filters are acceptable as generic exception filters. + const { raw, compiled } = parser.result; + if ( compiled === undefined ) { + const who = writer.properties.get('name') || '?'; + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid cosmetic filter in ${who}: #@#${raw}` + }); + return; + } + + writer.select('COSMETIC_FILTERS:SPECIFIC'); + + // https://github.com/chrisaljoudi/uBlock/issues/497 + // All generic exception filters are stored as hostname-based filter + // whereas the hostname is the empty string (which matches all + // hostnames). No distinction is made between declarative and + // procedural selectors, since they really exist only to cancel + // out other cosmetic filters. + writer.push([ 8, '', 0b001, compiled ]); +}; + +/******************************************************************************/ + +FilterContainer.prototype.compileSpecificSelector = function( + parser, + hostname, + not, + writer +) { + const { raw, compiled, exception } = parser.result; + if ( compiled === undefined ) { + const who = writer.properties.get('name') || '?'; + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid cosmetic filter in ${who}: ##${raw}` + }); + return; + } + + writer.select('COSMETIC_FILTERS:SPECIFIC'); + + // https://github.com/chrisaljoudi/uBlock/issues/145 + let unhide = exception ? 1 : 0; + if ( not ) { unhide ^= 1; } + + let kind = 0; + if ( unhide === 1 ) { + kind |= 0b001; // Exception + } + if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { + kind |= 0b010; // Procedural + } + if ( hostname === '*' ) { + kind |= 0b100; // Applies everywhere + } + + writer.push([ 8, hostname, kind, compiled ]); +}; + +/******************************************************************************/ + +FilterContainer.prototype.fromCompiledContent = function(reader, options) { + if ( options.skipCosmetic ) { + this.skipCompiledContent(reader, 'SPECIFIC'); + this.skipCompiledContent(reader, 'GENERIC'); + return; + } + + // Specific cosmetic filter section + reader.select('COSMETIC_FILTERS:SPECIFIC'); + while ( reader.next() ) { + this.acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( this.duplicateBuster.has(fingerprint) ) { + this.discardedCount += 1; + continue; + } + this.duplicateBuster.add(fingerprint); + const args = reader.args(); + switch ( args[0] ) { + // hash, example.com, .promoted-tweet + // hash, example.*, .promoted-tweet + // + // https://github.com/uBlockOrigin/uBlock-issues/issues/803 + // Handle specific filters meant to apply everywhere, i.e. selectors + // not to be injected conditionally through the DOM surveyor. + // hash, *, .promoted-tweet + case 8: + if ( args[2] === 0b100 ) { + if ( this.reSimpleHighGeneric.test(args[3]) ) + this.highlyGeneric.simple.dict.add(args[3]); + else { + this.highlyGeneric.complex.dict.add(args[3]); + } + break; + } + this.specificFilters.store(args[1], args[2] & 0b011, args[3]); + break; + default: + this.discardedCount += 1; + break; + } + } + + if ( options.skipGenericCosmetic ) { + this.skipCompiledContent(reader, 'GENERIC'); + return; + } + + // Generic cosmetic filter section + reader.select('COSMETIC_FILTERS:GENERIC'); + while ( reader.next() ) { + this.acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( this.duplicateBuster.has(fingerprint) ) { + this.discardedCount += 1; + continue; + } + this.duplicateBuster.add(fingerprint); + const args = reader.args(); + switch ( args[0] ) { + // low generic + case 0: { + if ( this.lowlyGeneric.has(args[1]) ) { + const selector = this.lowlyGeneric.get(args[1]); + this.lowlyGeneric.set(args[1], `${selector},\n${args[2]}`); + } else { + this.lowlyGeneric.set(args[1], args[2]); + } + break; + } + // High-high generic hide/simple selectors + // div[id^="allo"] + case 4: + this.highlyGeneric.simple.dict.add(args[1]); + break; + // High-high generic hide/complex selectors + // div[id^="allo"] > span + case 5: + this.highlyGeneric.complex.dict.add(args[1]); + break; + default: + this.discardedCount += 1; + break; + } + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.skipCompiledContent = function(reader, sectionId) { + reader.select(`COSMETIC_FILTERS:${sectionId}`); + while ( reader.next() ) { + this.acceptedCount += 1; + this.discardedCount += 1; + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.toSelfie = function() { + return { + version: this.selfieVersion, + acceptedCount: this.acceptedCount, + discardedCount: this.discardedCount, + specificFilters: this.specificFilters.toSelfie(), + lowlyGeneric: Array.from(this.lowlyGeneric), + highSimpleGenericHideArray: Array.from(this.highlyGeneric.simple.dict), + highComplexGenericHideArray: Array.from(this.highlyGeneric.complex.dict), + }; +}; + +/******************************************************************************/ + +FilterContainer.prototype.fromSelfie = function(selfie) { + if ( selfie.version !== this.selfieVersion ) { + throw new Error( + `cosmeticFilteringEngine: mismatched selfie version, ${selfie.version}, expected ${this.selfieVersion}` + ); + } + this.acceptedCount = selfie.acceptedCount; + this.discardedCount = selfie.discardedCount; + this.specificFilters.fromSelfie(selfie.specificFilters); + this.lowlyGeneric = new Map(selfie.lowlyGeneric); + this.highlyGeneric.simple.dict = new Set(selfie.highSimpleGenericHideArray); + this.highlyGeneric.simple.str = selfie.highSimpleGenericHideArray.join(',\n'); + this.highlyGeneric.complex.dict = new Set(selfie.highComplexGenericHideArray); + this.highlyGeneric.complex.str = selfie.highComplexGenericHideArray.join(',\n'); + this.frozen = true; +}; + +/******************************************************************************/ + +FilterContainer.prototype.addToSelectorCache = function(details) { + const hostname = details.hostname; + if ( typeof hostname !== 'string' || hostname === '' ) { return; } + const selectors = details.selectors; + if ( Array.isArray(selectors) === false ) { return; } + let entry = this.selectorCache.get(hostname); + if ( entry === undefined ) { + entry = SelectorCacheEntry.factory(); + this.selectorCache.set(hostname, entry); + if ( this.selectorCache.size > this.selectorCacheCountMax ) { + this.selectorCacheTimer.on({ min: this.selectorCachePruneDelay }); + } + } + entry.add(details); +}; + +/******************************************************************************/ + +FilterContainer.prototype.removeFromSelectorCache = function( + targetHostname = '*', + type = undefined +) { + const targetHostnameLength = targetHostname.length; + for ( let entry of this.selectorCache ) { + let hostname = entry[0]; + let item = entry[1]; + if ( targetHostname !== '*' ) { + if ( hostname.endsWith(targetHostname) === false ) { continue; } + if ( + hostname.length !== targetHostnameLength && + hostname.charAt(hostname.length - targetHostnameLength - 1) !== '.' + ) { + continue; + } + } + item.remove(type); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.pruneSelectorCacheAsync = function() { + if ( this.selectorCache.size <= this.selectorCacheCountMax ) { return; } + const cache = this.selectorCache; + const hostnames = Array.from(cache.keys()) + .sort((a, b) => cache.get(b).accessId - cache.get(a).accessId) + .slice(this.selectorCacheCountMin); + for ( const hn of hostnames ) { + cache.get(hn).dispose(); + cache.delete(hn); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.disableSurveyor = function(details) { + const hostname = details.hostname; + if ( typeof hostname !== 'string' || hostname === '' ) { return; } + const cacheEntry = this.selectorCache.get(hostname); + if ( cacheEntry === undefined ) { return; } + cacheEntry.disableSurveyor = true; +}; + +/******************************************************************************/ + +FilterContainer.prototype.cssRuleFromProcedural = function(pfilter) { + if ( pfilter.cssable !== true ) { return; } + const { tasks, action } = pfilter; + let mq, selector; + if ( Array.isArray(tasks) ) { + if ( tasks[0][0] !== 'matches-media' ) { return; } + mq = tasks[0][1]; + if ( tasks.length > 2 ) { return; } + if ( tasks.length === 2 ) { + if ( tasks[1][0] !== 'spath' ) { return; } + selector = tasks[1][1]; + } + } + let style; + if ( Array.isArray(action) ) { + if ( action[0] !== 'style' ) { return; } + selector = selector || pfilter.selector; + style = action[1]; + } + if ( mq === undefined && style === undefined && selector === undefined ) { return; } + if ( mq === undefined ) { + return `${selector}\n{${style}}`; + } + if ( style === undefined ) { + return `@media ${mq} {\n${selector}\n{display:none!important;}\n}`; + } + return `@media ${mq} {\n${selector}\n{${style}}\n}`; +}; + +/******************************************************************************/ + +FilterContainer.prototype.retrieveGenericSelectors = function(request) { + if ( this.lowlyGeneric.size === 0 ) { return; } + if ( Array.isArray(request.hashes) === false ) { return; } + if ( request.hashes.length === 0 ) { return; } + + const selectorsSet = new Set(); + const hashes = []; + const safeOnly = request.safeOnly === true; + for ( const hash of request.hashes ) { + const bucket = this.lowlyGeneric.get(hash); + if ( bucket === undefined ) { continue; } + for ( const selector of bucket.split(',\n') ) { + if ( safeOnly && selector === keyFromSelector(selector) ) { continue; } + selectorsSet.add(selector); + } + hashes.push(hash); + } + + // Apply exceptions: it is the responsibility of the caller to provide + // the exceptions to be applied. + const excepted = []; + if ( selectorsSet.size !== 0 && Array.isArray(request.exceptions) ) { + for ( const exception of request.exceptions ) { + if ( selectorsSet.delete(exception) ) { + excepted.push(exception); + } + } + } + + if ( selectorsSet.size === 0 && excepted.length === 0 ) { return; } + + const out = { injectedCSS: '', excepted, }; + const selectors = Array.from(selectorsSet); + + if ( typeof request.hostname === 'string' && request.hostname !== '' ) { + this.addToSelectorCache({ + hostname: request.hostname, + selectors, + hashes, + type: 'cosmetic', + }); + } + + if ( selectors.length === 0 ) { return out; } + + out.injectedCSS = `${selectors.join(',\n')}\n{display:none!important;}`; + vAPI.tabs.insertCSS(request.tabId, { + code: out.injectedCSS, + frameId: request.frameId, + matchAboutBlank: true, + runAt: 'document_start', + }); + + return out; +}; + +/******************************************************************************/ + +FilterContainer.prototype.retrieveSpecificSelectors = function( + request, + options +) { + const hostname = request.hostname; + const cacheEntry = this.selectorCache.get(hostname); + + // https://github.com/chrisaljoudi/uBlock/issues/587 + // out.ready will tell the content script the cosmetic filtering engine is + // up and ready. + + // https://github.com/chrisaljoudi/uBlock/issues/497 + // Generic exception filters are to be applied on all pages. + + const out = { + ready: this.frozen, + hostname: hostname, + domain: request.domain, + exceptionFilters: [], + exceptedFilters: [], + proceduralFilters: [], + convertedProceduralFilters: [], + disableSurveyor: this.lowlyGeneric.size === 0, + }; + const injectedCSS = []; + + if ( + options.noSpecificCosmeticFiltering !== true || + options.noGenericCosmeticFiltering !== true + ) { + const specificSet = this.$specificSet; + const proceduralSet = this.$proceduralSet; + const exceptionSet = this.$exceptionSet; + const dummySet = this.$dummySet; + + // Cached cosmetic filters: these are always declarative. + if ( cacheEntry !== undefined ) { + cacheEntry.retrieveCosmetic(specificSet, out.genericCosmeticHashes = []); + if ( cacheEntry.disableSurveyor ) { + out.disableSurveyor = true; + } + } + + // Retrieve filters with a non-empty hostname + const retrieveSets = [ specificSet, exceptionSet, proceduralSet, exceptionSet ]; + const discardSets = [ dummySet, exceptionSet ]; + this.specificFilters.retrieve( + hostname, + options.noSpecificCosmeticFiltering ? discardSets : retrieveSets, + 1 + ); + // Retrieve filters with a regex-based hostname value + this.specificFilters.retrieve( + hostname, + options.noSpecificCosmeticFiltering ? discardSets : retrieveSets, + 3 + ); + // Retrieve filters with a entity-based hostname value + if ( request.entity !== '' ) { + this.specificFilters.retrieve( + `${hostname.slice(0, -request.domain.length)}${request.entity}`, + options.noSpecificCosmeticFiltering ? discardSets : retrieveSets, + 1 + ); + } + // Retrieve filters with an empty hostname + this.specificFilters.retrieve( + hostname, + options.noGenericCosmeticFiltering ? discardSets : retrieveSets, + 2 + ); + + // Apply exceptions to specific filterset + if ( exceptionSet.size !== 0 ) { + out.exceptionFilters = Array.from(exceptionSet); + for ( const selector of specificSet ) { + if ( exceptionSet.has(selector) === false ) { continue; } + specificSet.delete(selector); + out.exceptedFilters.push(selector); + } + } + + if ( specificSet.size !== 0 ) { + injectedCSS.push( + `${Array.from(specificSet).join(',\n')}\n{display:none!important;}` + ); + } + + // Apply exceptions to procedural filterset. + // Also, some procedural filters are really declarative cosmetic + // filters, so we extract and inject them immediately. + if ( proceduralSet.size !== 0 ) { + for ( const json of proceduralSet ) { + const pfilter = JSON.parse(json); + if ( exceptionSet.has(json) ) { + proceduralSet.delete(json); + out.exceptedFilters.push(json); + continue; + } + if ( exceptionSet.has(pfilter.raw) ) { + proceduralSet.delete(json); + out.exceptedFilters.push(pfilter.raw); + continue; + } + const cssRule = this.cssRuleFromProcedural(pfilter); + if ( cssRule === undefined ) { continue; } + injectedCSS.push(cssRule); + proceduralSet.delete(json); + out.convertedProceduralFilters.push(json); + } + out.proceduralFilters.push(...proceduralSet); + } + + // Highly generic cosmetic filters: sent once along with specific ones. + // A most-recent-used cache is used to skip computing the resulting set + // of high generics for a given set of exceptions. + // The resulting set of high generics is stored as a string, ready to + // be used as-is by the content script. The string is stored + // indirectly in the mru cache: this is to prevent duplication of the + // string in memory, which I have observed occurs when the string is + // stored directly as a value in a Map. + if ( options.noGenericCosmeticFiltering !== true ) { + const exceptionSetHash = out.exceptionFilters.join(); + for ( const key in this.highlyGeneric ) { + const entry = this.highlyGeneric[key]; + let str = entry.mru.lookup(exceptionSetHash); + if ( str === undefined ) { + str = { s: entry.str, excepted: [] }; + let genericSet = entry.dict; + let hit = false; + for ( const exception of exceptionSet ) { + if ( (hit = genericSet.has(exception)) ) { break; } + } + if ( hit ) { + genericSet = new Set(entry.dict); + for ( const exception of exceptionSet ) { + if ( genericSet.delete(exception) ) { + str.excepted.push(exception); + } + } + str.s = Array.from(genericSet).join(',\n'); + } + entry.mru.add(exceptionSetHash, str); + } + if ( str.excepted.length !== 0 ) { + out.exceptedFilters.push(...str.excepted); + } + if ( str.s.length !== 0 ) { + injectedCSS.push(`${str.s}\n{display:none!important;}`); + } + } + } + + // Important: always clear used registers before leaving. + specificSet.clear(); + proceduralSet.clear(); + exceptionSet.clear(); + dummySet.clear(); + } + + const details = { + code: '', + frameId: request.frameId, + matchAboutBlank: true, + runAt: 'document_start', + }; + + // Inject all declarative-based filters as a single stylesheet. + if ( injectedCSS.length !== 0 ) { + out.injectedCSS = injectedCSS.join('\n\n'); + details.code = out.injectedCSS; + if ( request.tabId !== undefined ) { + vAPI.tabs.insertCSS(request.tabId, details); + } + } + + // CSS selectors for collapsible blocked elements + if ( cacheEntry ) { + const networkFilters = []; + if ( cacheEntry.retrieveNet(networkFilters) ) { + details.code = `${networkFilters.join('\n')}\n{display:none!important;}`; + if ( request.tabId !== undefined ) { + vAPI.tabs.insertCSS(request.tabId, details); + } + } + } + + return out; +}; + +/******************************************************************************/ + +FilterContainer.prototype.getFilterCount = function() { + return this.acceptedCount - this.discardedCount; +}; + +/******************************************************************************/ + +FilterContainer.prototype.dump = function() { + const lowlyGenerics = []; + for ( const selectors of this.lowlyGeneric.values() ) { + lowlyGenerics.push(...selectors.split(',\n')); + } + lowlyGenerics.sort(); + const highlyGenerics = Array.from(this.highlyGeneric.simple.dict).sort(); + highlyGenerics.push(...Array.from(this.highlyGeneric.complex.dict).sort()); + return [ + 'Cosmetic Filtering Engine internals:', + `specific: ${this.specificFilters.size}`, + `generic: ${lowlyGenerics.length + highlyGenerics.length}`, + `+ lowly generic: ${lowlyGenerics.length}`, + ...lowlyGenerics.map(a => ` ${a}`), + `+ highly generic: ${highlyGenerics.length}`, + ...highlyGenerics.map(a => ` ${a}`), + ].join('\n'); +}; + +/******************************************************************************/ + +const cosmeticFilteringEngine = new FilterContainer(); + +export default cosmeticFilteringEngine; + +/******************************************************************************/ diff --git a/src/js/dashboard-common.js b/src/js/dashboard-common.js new file mode 100644 index 0000000..feceb1f --- /dev/null +++ b/src/js/dashboard-common.js @@ -0,0 +1,215 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { dom } from './dom.js'; + +/******************************************************************************/ + +self.uBlockDashboard = self.uBlockDashboard || {}; + +/******************************************************************************/ + +// Helper for client panes: +// Remove literal duplicate lines from a set based on another set. + +self.uBlockDashboard.mergeNewLines = function(text, newText) { + // Step 1: build dictionary for existing lines. + const fromDict = new Map(); + let lineBeg = 0; + let textEnd = text.length; + while ( lineBeg < textEnd ) { + let lineEnd = text.indexOf('\n', lineBeg); + if ( lineEnd === -1 ) { + lineEnd = text.indexOf('\r', lineBeg); + if ( lineEnd === -1 ) { + lineEnd = textEnd; + } + } + const line = text.slice(lineBeg, lineEnd).trim(); + lineBeg = lineEnd + 1; + if ( line.length === 0 ) { continue; } + const hash = line.slice(0, 8); + const bucket = fromDict.get(hash); + if ( bucket === undefined ) { + fromDict.set(hash, line); + } else if ( typeof bucket === 'string' ) { + fromDict.set(hash, [ bucket, line ]); + } else /* if ( Array.isArray(bucket) ) */ { + bucket.push(line); + } + } + + // Step 2: use above dictionary to filter out duplicate lines. + const out = [ '' ]; + lineBeg = 0; + textEnd = newText.length; + while ( lineBeg < textEnd ) { + let lineEnd = newText.indexOf('\n', lineBeg); + if ( lineEnd === -1 ) { + lineEnd = newText.indexOf('\r', lineBeg); + if ( lineEnd === -1 ) { + lineEnd = textEnd; + } + } + const line = newText.slice(lineBeg, lineEnd).trim(); + lineBeg = lineEnd + 1; + if ( line.length === 0 ) { + if ( out[out.length - 1] !== '' ) { + out.push(''); + } + continue; + } + const bucket = fromDict.get(line.slice(0, 8)); + if ( bucket === undefined ) { + out.push(line); + continue; + } + if ( typeof bucket === 'string' && line !== bucket ) { + out.push(line); + continue; + } + if ( bucket.indexOf(line) === -1 ) { + out.push(line); + /* continue; */ + } + } + + const append = out.join('\n').trim(); + if ( text !== '' && append !== '' ) { + text += '\n\n'; + } + return text + append; +}; + +/******************************************************************************/ + +self.uBlockDashboard.dateNowToSensibleString = function() { + const now = new Date(Date.now() - (new Date()).getTimezoneOffset() * 60000); + return now.toISOString().replace(/\.\d+Z$/, '') + .replace(/:/g, '.') + .replace('T', '_'); +}; + +/******************************************************************************/ + +self.uBlockDashboard.patchCodeMirrorEditor = (function() { + let grabFocusTarget; + + const grabFocus = function() { + grabFocusTarget.focus(); + grabFocusTarget = undefined; + }; + + const grabFocusTimer = vAPI.defer.create(grabFocus); + + const grabFocusAsync = function(cm) { + grabFocusTarget = cm; + grabFocusTimer.on(1); + }; + + // https://github.com/gorhill/uBlock/issues/3646 + const patchSelectAll = function(cm, details) { + var vp = cm.getViewport(); + if ( details.ranges.length !== 1 ) { return; } + var range = details.ranges[0], + lineFrom = range.anchor.line, + lineTo = range.head.line; + if ( lineTo === lineFrom ) { return; } + if ( range.head.ch !== 0 ) { lineTo += 1; } + if ( lineFrom !== vp.from || lineTo !== vp.to ) { return; } + details.update([ + { + anchor: { line: 0, ch: 0 }, + head: { line: cm.lineCount(), ch: 0 } + } + ]); + grabFocusAsync(cm); + }; + + let lastGutterClick = 0; + let lastGutterLine = 0; + + const onGutterClicked = function(cm, line, gutter) { + if ( gutter !== 'CodeMirror-linenumbers' ) { return; } + grabFocusAsync(cm); + const delta = Date.now() - lastGutterClick; + // Single click + if ( delta >= 500 || line !== lastGutterLine ) { + cm.setSelection( + { line, ch: 0 }, + { line: line + 1, ch: 0 } + ); + lastGutterClick = Date.now(); + lastGutterLine = line; + return; + } + // Double click: select fold-able block or all + let lineFrom = 0; + let lineTo = cm.lineCount(); + const foldFn = cm.getHelper({ line, ch: 0 }, 'fold'); + if ( foldFn instanceof Function ) { + const range = foldFn(cm, { line, ch: 0 }); + if ( range !== undefined ) { + lineFrom = range.from.line; + lineTo = range.to.line + 1; + } + } + cm.setSelection( + { line: lineFrom, ch: 0 }, + { line: lineTo, ch: 0 }, + { scroll: false } + ); + lastGutterClick = 0; + }; + + return function(cm) { + if ( cm.options.inputStyle === 'contenteditable' ) { + cm.on('beforeSelectionChange', patchSelectAll); + } + cm.on('gutterClick', onGutterClicked); + }; +})(); + +/******************************************************************************/ + +self.uBlockDashboard.openOrSelectPage = function(url, options = {}) { + let ev; + if ( url instanceof MouseEvent ) { + ev = url; + url = dom.attr(ev.target, 'href'); + } + const details = Object.assign({ url, select: true, index: -1 }, options); + vAPI.messaging.send('default', { + what: 'gotoURL', + details, + }); + if ( ev ) { + ev.preventDefault(); + } +}; + +/******************************************************************************/ + +// Open links in the proper window +dom.attr('a', 'target', '_blank'); +dom.attr('a[href*="dashboard.html"]', 'target', '_parent'); diff --git a/src/js/dashboard.js b/src/js/dashboard.js new file mode 100644 index 0000000..e82ec28 --- /dev/null +++ b/src/js/dashboard.js @@ -0,0 +1,166 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { dom, qs$ } from './dom.js'; + +/******************************************************************************/ + +const discardUnsavedData = function(synchronous = false) { + const paneFrame = qs$('#iframe'); + const paneWindow = paneFrame.contentWindow; + if ( + typeof paneWindow.hasUnsavedData !== 'function' || + paneWindow.hasUnsavedData() === false + ) { + return true; + } + + if ( synchronous ) { + return false; + } + + return new Promise(resolve => { + const modal = qs$('#unsavedWarning'); + dom.cl.add(modal, 'on'); + modal.focus(); + + const onDone = status => { + dom.cl.remove(modal, 'on'); + dom.off(document, 'click', onClick, true); + resolve(status); + }; + + const onClick = ev => { + const target = ev.target; + if ( target.matches('[data-i18n="dashboardUnsavedWarningStay"]') ) { + return onDone(false); + } + if ( target.matches('[data-i18n="dashboardUnsavedWarningIgnore"]') ) { + return onDone(true); + } + if ( qs$(modal, '[data-i18n="dashboardUnsavedWarning"]').contains(target) ) { + return; + } + onDone(false); + }; + + dom.on(document, 'click', onClick, true); + }); +}; + +const loadDashboardPanel = function(pane, first) { + const tabButton = qs$(`[data-pane="${pane}"]`); + if ( tabButton === null || dom.cl.has(tabButton, 'selected') ) { return; } + const loadPane = ( ) => { + self.location.replace(`#${pane}`); + dom.cl.remove('.tabButton.selected', 'selected'); + dom.cl.add(tabButton, 'selected'); + tabButton.scrollIntoView(); + qs$('#iframe').contentWindow.location.replace(pane); + if ( pane !== 'no-dashboard.html' ) { + vAPI.localStorage.setItem('dashboardLastVisitedPane', pane); + } + }; + if ( first ) { + return loadPane(); + } + const r = discardUnsavedData(); + if ( r === false ) { return; } + if ( r === true ) { return loadPane(); } + r.then(status => { + if ( status === false ) { return; } + loadPane(); + }); +}; + +const onTabClickHandler = function(ev) { + loadDashboardPanel(dom.attr(ev.target, 'data-pane')); +}; + +if ( self.location.hash.slice(1) === 'no-dashboard.html' ) { + dom.cl.add(dom.body, 'noDashboard'); +} + +(async ( ) => { + // Wait for uBO's main process to be ready + await new Promise(resolve => { + const check = async ( ) => { + try { + const response = await vAPI.messaging.send('dashboard', { + what: 'readyToFilter' + }); + if ( response ) { return resolve(true); } + const iframe = qs$('#iframe'); + if ( iframe.src !== '' ) { + iframe.src = ''; + } + } catch(ex) { + } + vAPI.defer.once(250).then(( ) => check()); + }; + check(); + }); + + dom.cl.remove(dom.body, 'notReady'); + + const results = await Promise.all([ + // https://github.com/uBlockOrigin/uBlock-issues/issues/106 + vAPI.messaging.send('dashboard', { what: 'dashboardConfig' }), + vAPI.localStorage.getItemAsync('dashboardLastVisitedPane'), + ]); + + { + const details = results[0] || {}; + if ( details.noDashboard ) { + self.location.hash = '#no-dashboard.html'; + dom.cl.add(dom.body, 'noDashboard'); + } else if ( self.location.hash === '#no-dashboard.html' ) { + self.location.hash = ''; + } + } + + { + let pane = results[1] || null; + if ( self.location.hash !== '' ) { + pane = self.location.hash.slice(1) || null; + } + loadDashboardPanel(pane !== null ? pane : 'settings.html', true); + + dom.on('.tabButton', 'click', onTabClickHandler); + + // https://developer.mozilla.org/en-US/docs/Web/API/Window/beforeunload_event + dom.on(self, 'beforeunload', ( ) => { + if ( discardUnsavedData(true) ) { return; } + event.preventDefault(); + event.returnValue = ''; + }); + + // https://developer.mozilla.org/en-US/docs/Web/API/Window/beforeunload_event + dom.on(self, 'hashchange', ( ) => { + const pane = self.location.hash.slice(1); + if ( pane === '' ) { return; } + loadDashboardPanel(pane); + }); + + } +})(); diff --git a/src/js/devtools.js b/src/js/devtools.js new file mode 100644 index 0000000..93b2697 --- /dev/null +++ b/src/js/devtools.js @@ -0,0 +1,192 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uBlockDashboard */ + +'use strict'; + +import { dom, qs$ } from './dom.js'; + +/******************************************************************************/ + +const reFoldable = /^ *(?=\+ \S)/; + +/******************************************************************************/ + +CodeMirror.registerGlobalHelper( + 'fold', + 'ubo-dump', + ( ) => true, + (cm, start) => { + const startLineNo = start.line; + const startLine = cm.getLine(startLineNo); + let endLineNo = startLineNo; + let endLine = startLine; + const match = reFoldable.exec(startLine); + if ( match === null ) { return; } + const foldCandidate = ' ' + match[0]; + const lastLineNo = cm.lastLine(); + let nextLineNo = startLineNo + 1; + while ( nextLineNo < lastLineNo ) { + const nextLine = cm.getLine(nextLineNo); + // TODO: use regex to find folding end + if ( nextLine.startsWith(foldCandidate) === false && nextLine !== ']' ) { + if ( startLineNo >= endLineNo ) { return; } + return { + from: CodeMirror.Pos(startLineNo, startLine.length), + to: CodeMirror.Pos(endLineNo, endLine.length) + }; + } + endLine = nextLine; + endLineNo = nextLineNo; + nextLineNo += 1; + } + } +); + +const cmEditor = new CodeMirror(qs$('#console'), { + autofocus: true, + foldGutter: true, + gutters: [ 'CodeMirror-linenumbers', 'CodeMirror-foldgutter' ], + lineNumbers: true, + lineWrapping: true, + mode: 'ubo-dump', + styleActiveLine: true, + undoDepth: 5, +}); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +/******************************************************************************/ + +function log(text) { + cmEditor.replaceRange(text.trim() + '\n\n', { line: 0, ch: 0 }); +} + +/******************************************************************************/ + +dom.on('#console-clear', 'click', ( ) => { + cmEditor.setValue(''); +}); + +dom.on('#console-fold', 'click', ( ) => { + const unfolded = []; + let maxUnfolded = -1; + cmEditor.eachLine(handle => { + const match = reFoldable.exec(handle.text); + if ( match === null ) { return; } + const depth = match[0].length; + const line = handle.lineNo(); + const isFolded = cmEditor.isFolded({ line, ch: handle.text.length }); + if ( isFolded === true ) { return; } + unfolded.push({ line, depth }); + maxUnfolded = Math.max(maxUnfolded, depth); + }); + if ( maxUnfolded === -1 ) { return; } + cmEditor.startOperation(); + for ( const details of unfolded ) { + if ( details.depth !== maxUnfolded ) { continue; } + cmEditor.foldCode(details.line, null, 'fold'); + } + cmEditor.endOperation(); +}); + +dom.on('#console-unfold', 'click', ( ) => { + const folded = []; + let minFolded = Number.MAX_SAFE_INTEGER; + cmEditor.eachLine(handle => { + const match = reFoldable.exec(handle.text); + if ( match === null ) { return; } + const depth = match[0].length; + const line = handle.lineNo(); + const isFolded = cmEditor.isFolded({ line, ch: handle.text.length }); + if ( isFolded !== true ) { return; } + folded.push({ line, depth }); + minFolded = Math.min(minFolded, depth); + }); + if ( minFolded === Number.MAX_SAFE_INTEGER ) { return; } + cmEditor.startOperation(); + for ( const details of folded ) { + if ( details.depth !== minFolded ) { continue; } + cmEditor.foldCode(details.line, null, 'unfold'); + } + cmEditor.endOperation(); +}); + +dom.on('#snfe-dump', 'click', ev => { + const button = ev.target; + dom.attr(button, 'disabled', ''); + vAPI.messaging.send('devTools', { + what: 'snfeDump', + }).then(result => { + log(result); + dom.attr(button, 'disabled', null); + }); +}); + +dom.on('#snfe-todnr', 'click', ev => { + const button = ev.target; + dom.attr(button, 'disabled', ''); + vAPI.messaging.send('devTools', { + what: 'snfeToDNR', + }).then(result => { + log(result); + dom.attr(button, 'disabled', null); + }); +}); + +dom.on('#cfe-dump', 'click', ev => { + const button = ev.target; + dom.attr(button, 'disabled', ''); + vAPI.messaging.send('devTools', { + what: 'cfeDump', + }).then(result => { + log(result); + dom.attr(button, 'disabled', null); + }); +}); + +dom.on('#purge-all-caches', 'click', ( ) => { + vAPI.messaging.send('devTools', { + what: 'purgeAllCaches' + }).then(result => { + log(result); + }); +}); + +vAPI.messaging.send('dashboard', { + what: 'getAppData', +}).then(appData => { + if ( appData.canBenchmark !== true ) { return; } + dom.attr('#snfe-benchmark', 'disabled', null); + dom.on('#snfe-benchmark', 'click', ev => { + const button = ev.target; + dom.attr(button, 'disabled', ''); + vAPI.messaging.send('devTools', { + what: 'snfeBenchmark', + }).then(result => { + log(result); + dom.attr(button, 'disabled', null); + }); + }); +}); + +/******************************************************************************/ diff --git a/src/js/diff-updater.js b/src/js/diff-updater.js new file mode 100644 index 0000000..4e6ece1 --- /dev/null +++ b/src/js/diff-updater.js @@ -0,0 +1,288 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +// This module can be dynamically loaded or spun off as a worker. + +/******************************************************************************/ + +const patches = new Map(); +const encoder = new TextEncoder(); +const reFileName = /([^\/]+?)(?:#.+)?$/; +const EMPTYLINE = ''; + +/******************************************************************************/ + +const suffleArray = arr => { + const out = arr.slice(); + for ( let i = 0, n = out.length; i < n; i++ ) { + const j = Math.floor(Math.random() * n); + if ( j === i ) { continue; } + [ out[j], out[i] ] = [ out[i], out[j] ]; + } + return out; +}; + +const basename = url => { + const match = reFileName.exec(url); + return match && match[1] || ''; +}; + +const resolveURL = (path, url) => { + try { + return new URL(path, url); + } + catch(_) { + } +}; + +const expectedTimeFromPatch = assetDetails => { + const match = /(\d+)\.(\d+)\.(\d+)\.(\d+)/.exec(assetDetails.patchPath); + if ( match === null ) { return 0; } + const date = new Date(); + date.setUTCFullYear( + parseInt(match[1], 10), + parseInt(match[2], 10) - 1, + parseInt(match[3], 10) + ); + date.setUTCHours(0, parseInt(match[4], 10), 0, 0); + return date.getTime() + assetDetails.diffExpires; +}; + +function parsePatch(patch) { + const patchDetails = new Map(); + const diffLines = patch.split('\n'); + let i = 0, n = diffLines.length; + while ( i < n ) { + const line = diffLines[i++]; + if ( line.startsWith('diff ') === false ) { continue; } + const fields = line.split(/\s+/); + const diffBlock = {}; + for ( let j = 0; j < fields.length; j++ ) { + const field = fields[j]; + const pos = field.indexOf(':'); + if ( pos === -1 ) { continue; } + const name = field.slice(0, pos); + if ( name === '' ) { continue; } + const value = field.slice(pos+1); + switch ( name ) { + case 'name': + case 'checksum': + diffBlock[name] = value; + break; + case 'lines': + diffBlock.lines = parseInt(value, 10); + break; + default: + break; + } + } + if ( diffBlock.name === undefined ) { return; } + if ( isNaN(diffBlock.lines) || diffBlock.lines <= 0 ) { return; } + if ( diffBlock.checksum === undefined ) { return; } + patchDetails.set(diffBlock.name, diffBlock); + diffBlock.diff = diffLines.slice(i, i + diffBlock.lines).join('\n'); + i += diffBlock.lines; + } + if ( patchDetails.size === 0 ) { return; } + return patchDetails; +} + +function applyPatch(text, diff) { + // Inspired from (Perl) "sub _patch" at: + // https://twiki.org/p/pub/Codev/RcsLite/RcsLite.pm + // Apparently authored by John Talintyre in Jan. 2002 + // https://twiki.org/cgi-bin/view/Codev/RcsLite + const lines = text.split('\n'); + const diffLines = diff.split('\n'); + let iAdjust = 0; + let iDiff = 0, nDiff = diffLines.length; + while ( iDiff < nDiff ) { + const diffLine = diffLines[iDiff++]; + if ( diffLine === '' ) { break; } + const diffParsed = /^([ad])(\d+) (\d+)$/.exec(diffLine); + if ( diffParsed === null ) { return; } + const op = diffParsed[1]; + const iOp = parseInt(diffParsed[2], 10); + const nOp = parseInt(diffParsed[3], 10); + const iOpAdj = iOp + iAdjust; + if ( iOpAdj > lines.length ) { return; } + // Delete lines + if ( op === 'd' ) { + lines.splice(iOpAdj-1, nOp); + iAdjust -= nOp; + continue; + } + // Add lines: Don't use splice() to avoid stack limit issues + for ( let i = 0; i < nOp; i++ ) { + lines.push(EMPTYLINE); + } + lines.copyWithin(iOpAdj+nOp, iOpAdj); + for ( let i = 0; i < nOp; i++ ) { + lines[iOpAdj+i] = diffLines[iDiff+i]; + } + iAdjust += nOp; + iDiff += nOp; + } + return lines.join('\n'); +} + +function hasPatchDetails(assetDetails) { + const { patchPath } = assetDetails; + const patchFile = basename(patchPath); + return patchFile !== '' && patches.has(patchFile); +} + +/******************************************************************************/ + +// Async + +async function applyPatchAndValidate(assetDetails, diffDetails) { + const { text } = assetDetails; + const { diff, checksum } = diffDetails; + const textAfter = applyPatch(text, diff); + if ( typeof textAfter !== 'string' ) { + assetDetails.error = 'baddiff'; + return false; + } + const crypto = globalThis.crypto; + if ( typeof crypto !== 'object' ) { + assetDetails.error = 'nocrypto'; + return false; + } + const arrayin = encoder.encode(textAfter); + const arraybuffer = await crypto.subtle.digest('SHA-1', arrayin); + const arrayout = new Uint8Array(arraybuffer); + const sha1Full = Array.from(arrayout).map(i => + i.toString(16).padStart(2, '0') + ).join(''); + if ( sha1Full.startsWith(checksum) === false ) { + assetDetails.error = `badchecksum: expected ${checksum}, computed ${sha1Full.slice(0, checksum.length)}`; + return false; + } + assetDetails.text = textAfter; + return true; +} + +async function fetchPatchDetailsFromCDNs(assetDetails) { + const { patchPath, cdnURLs } = assetDetails; + if ( Array.isArray(cdnURLs) === false ) { return null; } + if ( cdnURLs.length === 0 ) { return null; } + for ( const cdnURL of suffleArray(cdnURLs) ) { + const patchURL = resolveURL(patchPath, cdnURL); + if ( patchURL === undefined ) { continue; } + const response = await fetch(patchURL).catch(reason => { + console.error(reason, patchURL); + }); + if ( response === undefined ) { continue; } + if ( response.status === 404 ) { break; } + if ( response.ok !== true ) { continue; } + const patchText = await response.text(); + const patchDetails = parsePatch(patchText); + if ( patchURL.hash.length > 1 ) { + assetDetails.diffName = patchURL.hash.slice(1); + patchURL.hash = ''; + } + return { + patchURL: patchURL.href, + patchSize: `${(patchText.length / 1000).toFixed(1)} KB`, + patchDetails, + }; + } + return null; +} + +async function fetchPatchDetails(assetDetails) { + const { patchPath } = assetDetails; + const patchFile = basename(patchPath); + if ( patchFile === '' ) { return null; } + if ( patches.has(patchFile) ) { + return patches.get(patchFile); + } + const patchDetailsPromise = fetchPatchDetailsFromCDNs(assetDetails); + patches.set(patchFile, patchDetailsPromise); + return patchDetailsPromise; +} + +async function fetchAndApplyAllPatches(assetDetails) { + if ( assetDetails.fetch === false ) { + if ( hasPatchDetails(assetDetails) === false ) { + assetDetails.status = 'nodiff'; + return assetDetails; + } + } + // uBO-specific, to avoid pointless fetches which are likely to fail + // because the patch has not yet been created + const patchTime = expectedTimeFromPatch(assetDetails); + if ( patchTime > Date.now() ) { + assetDetails.status = 'nopatch-yet'; + return assetDetails; + } + const patchData = await fetchPatchDetails(assetDetails); + if ( patchData === null ) { + assetDetails.status = (Date.now() - patchTime) < (4 * assetDetails.diffExpires) + ? 'nopatch-yet' + : 'nopatch'; + return assetDetails; + } + const { patchDetails } = patchData; + if ( patchDetails instanceof Map === false ) { + assetDetails.status = 'nodiff'; + return assetDetails; + } + const diffDetails = patchDetails.get(assetDetails.diffName); + if ( diffDetails === undefined ) { + assetDetails.status = 'nodiff'; + return assetDetails; + } + if ( assetDetails.text === undefined ) { + assetDetails.status = 'needtext'; + return assetDetails; + } + const outcome = await applyPatchAndValidate(assetDetails, diffDetails); + if ( outcome !== true ) { return assetDetails; } + assetDetails.status = 'updated'; + assetDetails.patchURL = patchData.patchURL; + assetDetails.patchSize = patchData.patchSize; + return assetDetails; +} + +/******************************************************************************/ + +const bc = new globalThis.BroadcastChannel('diffUpdater'); + +bc.onmessage = ev => { + const message = ev.data || {}; + switch ( message.what ) { + case 'update': + fetchAndApplyAllPatches(message).then(response => { + bc.postMessage(response); + }).catch(error => { + bc.postMessage({ what: 'broken', error }); + }); + break; + } +}; + +bc.postMessage({ what: 'ready' }); + +/******************************************************************************/ diff --git a/src/js/document-blocked.js b/src/js/document-blocked.js new file mode 100644 index 0000000..59a6bc8 --- /dev/null +++ b/src/js/document-blocked.js @@ -0,0 +1,230 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { i18n, i18n$ } from './i18n.js'; +import { dom, qs$ } from './dom.js'; + +/******************************************************************************/ + +const messaging = vAPI.messaging; +let details = {}; + +{ + const matches = /details=([^&]+)/.exec(window.location.search); + if ( matches !== null ) { + details = JSON.parse(decodeURIComponent(matches[1])); + } +} + +/******************************************************************************/ + +(async ( ) => { + const response = await messaging.send('documentBlocked', { + what: 'listsFromNetFilter', + rawFilter: details.fs, + }); + if ( response instanceof Object === false ) { return; } + + let lists; + for ( const rawFilter in response ) { + if ( response.hasOwnProperty(rawFilter) ) { + lists = response[rawFilter]; + break; + } + } + + if ( Array.isArray(lists) === false || lists.length === 0 ) { + qs$('#whyex').style.setProperty('visibility', 'collapse'); + return; + } + + const parent = qs$('#whyex > ul'); + parent.firstElementChild.remove(); // remove placeholder element + for ( const list of lists ) { + const listElem = dom.clone('#templates .filterList'); + const sourceElem = qs$(listElem, '.filterListSource'); + sourceElem.href += encodeURIComponent(list.assetKey); + sourceElem.append(i18n.patchUnicodeFlags(list.title)); + if ( typeof list.supportURL === 'string' && list.supportURL !== '' ) { + const supportElem = qs$(listElem, '.filterListSupport'); + dom.attr(supportElem, 'href', list.supportURL); + dom.cl.remove(supportElem, 'hidden'); + } + parent.appendChild(listElem); + } + qs$('#whyex').style.removeProperty('visibility'); +})(); + +/******************************************************************************/ + +dom.text('#theURL > p > span:first-of-type', details.url); +dom.text('#why', details.fs); + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/691 +// Parse URL to extract as much useful information as possible. This is +// useful to assist the user in deciding whether to navigate to the web page. +(( ) => { + if ( typeof URL !== 'function' ) { return; } + + const reURL = /^https?:\/\//; + + const liFromParam = function(name, value) { + if ( value === '' ) { + value = name; + name = ''; + } + const li = dom.create('li'); + let span = dom.create('span'); + dom.text(span, name); + li.appendChild(span); + if ( name !== '' && value !== '' ) { + li.appendChild(document.createTextNode(' = ')); + } + span = dom.create('span'); + if ( reURL.test(value) ) { + const a = dom.create('a'); + dom.attr(a, 'href', value); + dom.text(a, value); + span.appendChild(a); + } else { + dom.text(span, value); + } + li.appendChild(span); + return li; + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1649 + // Limit recursion. + const renderParams = function(parentNode, rawURL, depth = 0) { + let url; + try { + url = new URL(rawURL); + } catch(ex) { + return false; + } + + const search = url.search.slice(1); + if ( search === '' ) { return false; } + + url.search = ''; + const li = liFromParam(i18n$('docblockedNoParamsPrompt'), url.href); + parentNode.appendChild(li); + + const params = new self.URLSearchParams(search); + for ( const [ name, value ] of params ) { + const li = liFromParam(name, value); + if ( depth < 2 && reURL.test(value) ) { + const ul = dom.create('ul'); + renderParams(ul, value, depth + 1); + li.appendChild(ul); + } + parentNode.appendChild(li); + } + + return true; + }; + + if ( renderParams(qs$('#parsed'), details.url) === false ) { + return; + } + + dom.cl.remove('#toggleParse', 'hidden'); + + dom.on('#toggleParse', 'click', ( ) => { + dom.cl.toggle('#theURL', 'collapsed'); + vAPI.localStorage.setItem( + 'document-blocked-expand-url', + (dom.cl.has('#theURL', 'collapsed') === false).toString() + ); + }); + + vAPI.localStorage.getItemAsync('document-blocked-expand-url').then(value => { + dom.cl.toggle('#theURL', 'collapsed', value !== 'true' && value !== true); + }); +})(); + +/******************************************************************************/ + +// https://www.reddit.com/r/uBlockOrigin/comments/breeux/close_this_window_doesnt_work_on_firefox/ + +if ( window.history.length > 1 ) { + dom.on('#back', 'click', ( ) => { + window.history.back(); + }); + qs$('#bye').style.display = 'none'; +} else { + dom.on('#bye', 'click', ( ) => { + messaging.send('documentBlocked', { + what: 'closeThisTab', + }); + }); + qs$('#back').style.display = 'none'; +} + +/******************************************************************************/ + +const getTargetHostname = function() { + return details.hn; +}; + +const proceedToURL = function() { + window.location.replace(details.url); +}; + +const proceedTemporary = async function() { + await messaging.send('documentBlocked', { + what: 'temporarilyWhitelistDocument', + hostname: getTargetHostname(), + }); + proceedToURL(); +}; + +const proceedPermanent = async function() { + await messaging.send('documentBlocked', { + what: 'toggleHostnameSwitch', + name: 'no-strict-blocking', + hostname: getTargetHostname(), + deep: true, + state: true, + persist: true, + }); + proceedToURL(); +}; + +dom.on('#disableWarning', 'change', ev => { + const checked = ev.target.checked; + dom.cl.toggle('[data-i18n="docblockedBack"]', 'disabled', checked); + dom.cl.toggle('[data-i18n="docblockedClose"]', 'disabled', checked); +}); + +dom.on('#proceed', 'click', ( ) => { + if ( qs$('#disableWarning').checked ) { + proceedPermanent(); + } else { + proceedTemporary(); + } +}); + +/******************************************************************************/ diff --git a/src/js/dom-inspector.js b/src/js/dom-inspector.js new file mode 100644 index 0000000..a0d334b --- /dev/null +++ b/src/js/dom-inspector.js @@ -0,0 +1,68 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ +/******************************************************************************/ + +const svgRoot = document.querySelector('svg'); +let inspectorContentPort; + +const shutdown = ( ) => { + inspectorContentPort.close(); + inspectorContentPort.onmessage = inspectorContentPort.onmessageerror = null; + inspectorContentPort = undefined; +}; + +const contentInspectorChannel = ev => { + const msg = ev.data || {}; + switch ( msg.what ) { + case 'quitInspector': { + shutdown(); + break; + } + case 'svgPaths': { + const paths = svgRoot.children; + paths[0].setAttribute('d', msg.paths[0]); + paths[1].setAttribute('d', msg.paths[1]); + paths[2].setAttribute('d', msg.paths[2]); + paths[3].setAttribute('d', msg.paths[3]); + break; + } + default: + break; + } +}; + +// Wait for the content script to establish communication +globalThis.addEventListener('message', ev => { + const msg = ev.data || {}; + if ( msg.what !== 'startInspector' ) { return; } + if ( Array.isArray(ev.ports) === false ) { return; } + if ( ev.ports.length === 0 ) { return; } + inspectorContentPort = ev.ports[0]; + inspectorContentPort.onmessage = contentInspectorChannel; + inspectorContentPort.onmessageerror = shutdown; + inspectorContentPort.postMessage({ what: 'startInspector' }); +}, { once: true }); + +/******************************************************************************/ diff --git a/src/js/dom.js b/src/js/dom.js new file mode 100644 index 0000000..3d2f517 --- /dev/null +++ b/src/js/dom.js @@ -0,0 +1,213 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* jshint esversion:11 */ + +'use strict'; + +/******************************************************************************/ + +const normalizeTarget = target => { + if ( typeof target === 'string' ) { return Array.from(qsa$(target)); } + if ( target instanceof Element ) { return [ target ]; } + if ( target === null ) { return []; } + if ( Array.isArray(target) ) { return target; } + return Array.from(target); +}; + +const makeEventHandler = (selector, callback) => { + return function(event) { + const dispatcher = event.currentTarget; + if ( + dispatcher instanceof HTMLElement === false || + typeof dispatcher.querySelectorAll !== 'function' + ) { + return; + } + const receiver = event.target; + const ancestor = receiver.closest(selector); + if ( + ancestor === receiver && + ancestor !== dispatcher && + dispatcher.contains(ancestor) + ) { + callback.call(receiver, event); + } + }; +}; + +/******************************************************************************/ + +class dom { + static attr(target, attr, value = undefined) { + for ( const elem of normalizeTarget(target) ) { + if ( value === undefined ) { + return elem.getAttribute(attr); + } + if ( value === null ) { + elem.removeAttribute(attr); + } else { + elem.setAttribute(attr, value); + } + } + } + + static clear(target) { + for ( const elem of normalizeTarget(target) ) { + while ( elem.firstChild !== null ) { + elem.removeChild(elem.firstChild); + } + } + } + + static clone(target) { + const elements = normalizeTarget(target); + if ( elements.length === 0 ) { return null; } + return elements[0].cloneNode(true); + } + + static create(a) { + if ( typeof a === 'string' ) { + return document.createElement(a); + } + } + + static prop(target, prop, value = undefined) { + for ( const elem of normalizeTarget(target) ) { + if ( value === undefined ) { return elem[prop]; } + elem[prop] = value; + } + } + + static text(target, text) { + const targets = normalizeTarget(target); + if ( text === undefined ) { + return targets.length !== 0 ? targets[0].textContent : undefined; + } + for ( const elem of targets ) { + elem.textContent = text; + } + } + + static remove(target) { + for ( const elem of normalizeTarget(target) ) { + elem.remove(); + } + } + + // target, type, callback, [options] + // target, type, subtarget, callback, [options] + + static on(target, type, subtarget, callback, options) { + if ( typeof subtarget === 'function' ) { + options = callback; + callback = subtarget; + subtarget = undefined; + if ( typeof options === 'boolean' ) { + options = { capture: true }; + } + } else { + callback = makeEventHandler(subtarget, callback); + if ( options === undefined || typeof options === 'boolean' ) { + options = { capture: true }; + } else { + options.capture = true; + } + } + const targets = target instanceof Window || target instanceof Document + ? [ target ] + : normalizeTarget(target); + for ( const elem of targets ) { + elem.addEventListener(type, callback, options); + } + } + + static off(target, type, callback, options) { + if ( typeof callback !== 'function' ) { return; } + if ( typeof options === 'boolean' ) { + options = { capture: true }; + } + const targets = target instanceof Window || target instanceof Document + ? [ target ] + : normalizeTarget(target); + for ( const elem of targets ) { + elem.removeEventListener(type, callback, options); + } + } +} + +dom.cl = class { + static add(target, name) { + for ( const elem of normalizeTarget(target) ) { + elem.classList.add(name); + } + } + + static remove(target, name) { + for ( const elem of normalizeTarget(target) ) { + elem.classList.remove(name); + } + } + + static toggle(target, name, state) { + let r; + for ( const elem of normalizeTarget(target) ) { + r = elem.classList.toggle(name, state); + } + return r; + } + + static has(target, name) { + for ( const elem of normalizeTarget(target) ) { + if ( elem.classList.contains(name) ) { + return true; + } + } + return false; + } +}; + +/******************************************************************************/ + +function qs$(a, b) { + if ( typeof a === 'string') { + return document.querySelector(a); + } + if ( a === null ) { return null; } + return a.querySelector(b); +} + +function qsa$(a, b) { + if ( typeof a === 'string') { + return document.querySelectorAll(a); + } + if ( a === null ) { return []; } + return a.querySelectorAll(b); +} + +dom.root = qs$(':root'); +dom.html = document.documentElement; +dom.head = document.head; +dom.body = document.body; + +/******************************************************************************/ + +export { dom, qs$, qsa$ }; diff --git a/src/js/dyna-rules.js b/src/js/dyna-rules.js new file mode 100644 index 0000000..ea79742 --- /dev/null +++ b/src/js/dyna-rules.js @@ -0,0 +1,678 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uMatrix +*/ + +/* global CodeMirror, diff_match_patch, uBlockDashboard */ + +'use strict'; + +import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; + +import { hostnameFromURI } from './uri-utils.js'; +import { i18n$ } from './i18n.js'; +import { dom, qs$, qsa$ } from './dom.js'; + +import './codemirror/ubo-dynamic-filtering.js'; + +/******************************************************************************/ + +const hostnameToDomainMap = new Map(); + +const mergeView = new CodeMirror.MergeView( + qs$('.codeMirrorMergeContainer'), + { + allowEditingOriginals: true, + connect: 'align', + inputStyle: 'contenteditable', + lineNumbers: true, + lineWrapping: false, + origLeft: '', + revertButtons: true, + value: '', + } +); +mergeView.editor().setOption('styleActiveLine', true); +mergeView.editor().setOption('lineNumbers', false); +mergeView.leftOriginal().setOption('readOnly', 'nocursor'); + +uBlockDashboard.patchCodeMirrorEditor(mergeView.editor()); + +const thePanes = { + orig: { + doc: mergeView.leftOriginal(), + original: [], + modified: [], + }, + edit: { + doc: mergeView.editor(), + original: [], + modified: [], + }, +}; + +let cleanEditToken = 0; +let cleanEditText = ''; +let isCollapsed = false; + +/******************************************************************************/ + +// The following code is to take care of properly internationalizing +// the tooltips of the arrows used by the CodeMirror merge view. These +// are hard-coded by CodeMirror ("Push to left", "Push to right"). An +// observer is necessary because there is no hook for uBO to overwrite +// reliably the default title attribute assigned by CodeMirror. + +{ + const i18nCommitStr = i18n$('rulesCommit'); + const i18nRevertStr = i18n$('rulesRevert'); + const commitArrowSelector = '.CodeMirror-merge-copybuttons-left .CodeMirror-merge-copy-reverse:not([title="' + i18nCommitStr + '"])'; + const revertArrowSelector = '.CodeMirror-merge-copybuttons-left .CodeMirror-merge-copy:not([title="' + i18nRevertStr + '"])'; + + dom.attr('.CodeMirror-merge-scrolllock', 'title', i18n$('genericMergeViewScrollLock')); + + const translate = function() { + let elems = qsa$(commitArrowSelector); + for ( const elem of elems ) { + dom.attr(elem, 'title', i18nCommitStr); + } + elems = qsa$(revertArrowSelector); + for ( const elem of elems ) { + dom.attr(elem, 'title', i18nRevertStr); + } + }; + + const mergeGapObserver = new MutationObserver(translate); + + mergeGapObserver.observe( + qs$('.CodeMirror-merge-copybuttons-left'), + { attributes: true, attributeFilter: [ 'title' ], subtree: true } + ); + +} + +/******************************************************************************/ + +const getDiffer = (( ) => { + let differ; + return ( ) => { + if ( differ === undefined ) { differ = new diff_match_patch(); } + return differ; + }; +})(); + +/******************************************************************************/ + +// Borrowed from... +// https://github.com/codemirror/CodeMirror/blob/3e1bb5fff682f8f6cbfaef0e56c61d62403d4798/addon/search/search.js#L22 +// ... and modified as needed. + +const updateOverlay = (( ) => { + let reFilter; + const mode = { + token: function(stream) { + if ( reFilter !== undefined ) { + reFilter.lastIndex = stream.pos; + let match = reFilter.exec(stream.string); + if ( match !== null ) { + if ( match.index === stream.pos ) { + stream.pos += match[0].length || 1; + return 'searching'; + } + stream.pos = match.index; + return; + } + } + stream.skipToEnd(); + } + }; + return function(filter) { + reFilter = typeof filter === 'string' && filter !== '' ? + new RegExp(filter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi') : + undefined; + return mode; + }; +})(); + +/******************************************************************************/ + +// Incrementally update text in a CodeMirror editor for best user experience: +// - Scroll position preserved +// - Minimum amount of text updated + +const rulesToDoc = function(clearHistory) { + const orig = thePanes.orig.doc; + const edit = thePanes.edit.doc; + orig.startOperation(); + edit.startOperation(); + + for ( const key in thePanes ) { + if ( thePanes.hasOwnProperty(key) === false ) { continue; } + const doc = thePanes[key].doc; + const rules = filterRules(key); + if ( + clearHistory || + doc.lineCount() === 1 && doc.getValue() === '' || + rules.length === 0 + ) { + doc.setValue(rules.length !== 0 ? rules.join('\n') + '\n' : ''); + continue; + } + // https://github.com/uBlockOrigin/uBlock-issues/issues/593 + // Ensure the text content always ends with an empty line to avoid + // spurious diff entries. + // https://github.com/uBlockOrigin/uBlock-issues/issues/657 + // Diff against unmodified beforeText so that the last newline can + // be reported in the diff and thus appended if needed. + let beforeText = doc.getValue(); + let afterText = rules.join('\n').trim(); + if ( afterText !== '' ) { afterText += '\n'; } + const diffs = getDiffer().diff_main(beforeText, afterText); + let i = diffs.length; + let iedit = beforeText.length; + while ( i-- ) { + const diff = diffs[i]; + if ( diff[0] === 0 ) { + iedit -= diff[1].length; + continue; + } + const end = doc.posFromIndex(iedit); + if ( diff[0] === 1 ) { + doc.replaceRange(diff[1], end, end); + continue; + } + /* diff[0] === -1 */ + iedit -= diff[1].length; + const beg = doc.posFromIndex(iedit); + doc.replaceRange('', beg, end); + } + } + + // Mark ellipses as read-only + const marks = edit.getAllMarks(); + for ( const mark of marks ) { + if ( mark.uboEllipsis !== true ) { continue; } + mark.clear(); + } + if ( isCollapsed ) { + for ( let iline = 0, n = edit.lineCount(); iline < n; iline++ ) { + if ( edit.getLine(iline) !== '...' ) { continue; } + const mark = edit.markText( + { line: iline, ch: 0 }, + { line: iline + 1, ch: 0 }, + { atomic: true, readOnly: true } + ); + mark.uboEllipsis = true; + } + } + + orig.endOperation(); + edit.endOperation(); + cleanEditText = mergeView.editor().getValue().trim(); + cleanEditToken = mergeView.editor().changeGeneration(); + + if ( clearHistory !== true ) { return; } + + mergeView.editor().clearHistory(); + const chunks = mergeView.leftChunks(); + if ( chunks.length === 0 ) { return; } + const ldoc = thePanes.orig.doc; + const { clientHeight } = ldoc.getScrollInfo(); + const line = Math.min(chunks[0].editFrom, chunks[0].origFrom); + ldoc.setCursor(line, 0); + ldoc.scrollIntoView( + { line, ch: 0 }, + (clientHeight - ldoc.defaultTextHeight()) / 2 + ); +}; + +/******************************************************************************/ + +const filterRules = function(key) { + const filter = qs$('#ruleFilter input').value; + const rules = thePanes[key].modified; + if ( filter === '' ) { return rules; } + const out = []; + for ( const rule of rules ) { + if ( rule.indexOf(filter) === -1 ) { continue; } + out.push(rule); + } + return out; +}; + +/******************************************************************************/ + +const applyDiff = async function(permanent, toAdd, toRemove) { + const details = await vAPI.messaging.send('dashboard', { + what: 'modifyRuleset', + permanent: permanent, + toAdd: toAdd, + toRemove: toRemove, + }); + thePanes.orig.original = details.permanentRules; + thePanes.edit.original = details.sessionRules; + onPresentationChanged(); +}; + +/******************************************************************************/ + +// CodeMirror quirk: sometimes fromStart.ch and/or toStart.ch is undefined. +// When this happens, use 0. + +mergeView.options.revertChunk = function( + mv, + from, fromStart, fromEnd, + to, toStart, toEnd +) { + // https://github.com/gorhill/uBlock/issues/3611 + if ( dom.attr(dom.body, 'dir') === 'rtl' ) { + let tmp = from; from = to; to = tmp; + tmp = fromStart; fromStart = toStart; toStart = tmp; + tmp = fromEnd; fromEnd = toEnd; toEnd = tmp; + } + if ( typeof fromStart.ch !== 'number' ) { fromStart.ch = 0; } + if ( fromEnd.ch !== 0 ) { fromEnd.line += 1; } + const toAdd = from.getRange( + { line: fromStart.line, ch: 0 }, + { line: fromEnd.line, ch: 0 } + ); + if ( typeof toStart.ch !== 'number' ) { toStart.ch = 0; } + if ( toEnd.ch !== 0 ) { toEnd.line += 1; } + const toRemove = to.getRange( + { line: toStart.line, ch: 0 }, + { line: toEnd.line, ch: 0 } + ); + applyDiff(from === mv.editor(), toAdd, toRemove); +}; + +/******************************************************************************/ + +function handleImportFilePicker() { + const fileReaderOnLoadHandler = function() { + if ( typeof this.result !== 'string' || this.result === '' ) { return; } + // https://github.com/chrisaljoudi/uBlock/issues/757 + // Support RequestPolicy rule syntax + let result = this.result; + let matches = /\[origins-to-destinations\]([^\[]+)/.exec(result); + if ( matches && matches.length === 2 ) { + result = matches[1].trim() + .replace(/\|/g, ' ') + .replace(/\n/g, ' * noop\n'); + } + applyDiff(false, result, ''); + }; + const file = this.files[0]; + if ( file === undefined || file.name === '' ) { return; } + if ( file.type.indexOf('text') !== 0 ) { return; } + const fr = new FileReader(); + fr.onload = fileReaderOnLoadHandler; + fr.readAsText(file); +} + +/******************************************************************************/ + +const startImportFilePicker = function() { + const input = qs$('#importFilePicker'); + // Reset to empty string, this will ensure an change event is properly + // triggered if the user pick a file, even if it is the same as the last + // one picked. + input.value = ''; + input.click(); +}; + +/******************************************************************************/ + +function exportUserRulesToFile() { + const filename = i18n$('rulesDefaultFileName') + .replace('{{datetime}}', uBlockDashboard.dateNowToSensibleString()) + .replace(/ +/g, '_'); + vAPI.download({ + url: 'data:text/plain,' + encodeURIComponent( + mergeView.leftOriginal().getValue().trim() + '\n' + ), + filename: filename, + saveAs: true + }); +} + +/******************************************************************************/ + +const onFilterChanged = (( ) => { + let timer; + let overlay = null; + let last = ''; + + const process = function() { + timer = undefined; + if ( mergeView.editor().isClean(cleanEditToken) === false ) { return; } + const filter = qs$('#ruleFilter input').value; + if ( filter === last ) { return; } + last = filter; + if ( overlay !== null ) { + mergeView.leftOriginal().removeOverlay(overlay); + mergeView.editor().removeOverlay(overlay); + overlay = null; + } + if ( filter !== '' ) { + overlay = updateOverlay(filter); + mergeView.leftOriginal().addOverlay(overlay); + mergeView.editor().addOverlay(overlay); + } + rulesToDoc(true); + }; + + return function() { + if ( timer !== undefined ) { self.cancelIdleCallback(timer); } + timer = self.requestIdleCallback(process, { timeout: 773 }); + }; +})(); + +/******************************************************************************/ + +const onPresentationChanged = (( ) => { + let sortType = 1; + + const reSwRule = /^([^/]+): ([^/ ]+) ([^ ]+)/; + const reRule = /^([^ ]+) ([^/ ]+) ([^ ]+ [^ ]+)/; + const reUrlRule = /^([^ ]+) ([^ ]+) ([^ ]+ [^ ]+)/; + + const sortNormalizeHn = function(hn) { + let domain = hostnameToDomainMap.get(hn); + if ( domain === undefined ) { + domain = /(\d|\])$/.test(hn) + ? hn + : publicSuffixList.getDomain(hn); + hostnameToDomainMap.set(hn, domain); + } + let normalized = domain || hn; + if ( hn.length !== domain.length ) { + const subdomains = hn.slice(0, hn.length - domain.length - 1); + normalized += '.' + ( + subdomains.includes('.') + ? subdomains.split('.').reverse().join('.') + : subdomains + ); + } + return normalized; + }; + + const slotFromRule = rule => { + let type, srcHn, desHn, extra; + let match = reSwRule.exec(rule); + if ( match !== null ) { + type = ' ' + match[1]; + srcHn = sortNormalizeHn(match[2]); + desHn = srcHn; + extra = match[3]; + } else if ( (match = reRule.exec(rule)) !== null ) { + type = '\x10FFFE'; + srcHn = sortNormalizeHn(match[1]); + desHn = sortNormalizeHn(match[2]); + extra = match[3]; + } else if ( (match = reUrlRule.exec(rule)) !== null ) { + type = '\x10FFFF'; + srcHn = sortNormalizeHn(match[1]); + desHn = sortNormalizeHn(hostnameFromURI(match[2])); + extra = match[3]; + } + if ( sortType === 0 ) { + return { rule, token: `${type} ${srcHn} ${desHn} ${extra}` }; + } + if ( sortType === 1 ) { + return { rule, token: `${srcHn} ${type} ${desHn} ${extra}` }; + } + return { rule, token: `${desHn} ${type} ${srcHn} ${extra}` }; + }; + + const sort = rules => { + const slots = []; + for ( let i = 0; i < rules.length; i++ ) { + slots.push(slotFromRule(rules[i], 1)); + } + slots.sort((a, b) => a.token.localeCompare(b.token)); + for ( let i = 0; i < rules.length; i++ ) { + rules[i] = slots[i].rule; + } + }; + + const collapse = ( ) => { + if ( isCollapsed !== true ) { return; } + const diffs = getDiffer().diff_main( + thePanes.orig.modified.join('\n'), + thePanes.edit.modified.join('\n') + ); + const ll = []; let il = 0, lellipsis = false; + const rr = []; let ir = 0, rellipsis = false; + for ( let i = 0; i < diffs.length; i++ ) { + const diff = diffs[i]; + if ( diff[0] === 0 ) { + lellipsis = rellipsis = true; + il += 1; ir += 1; + continue; + } + if ( diff[0] < 0 ) { + if ( lellipsis ) { + ll.push('...'); + if ( rellipsis ) { rr.push('...'); } + lellipsis = rellipsis = false; + } + ll.push(diff[1].trim()); + il += 1; + continue; + } + /* diff[0] > 0 */ + if ( rellipsis ) { + rr.push('...'); + if ( lellipsis ) { ll.push('...'); } + lellipsis = rellipsis = false; + } + rr.push(diff[1].trim()); + ir += 1; + } + if ( lellipsis ) { ll.push('...'); } + if ( rellipsis ) { rr.push('...'); } + thePanes.orig.modified = ll; + thePanes.edit.modified = rr; + }; + + return function(clearHistory) { + const origPane = thePanes.orig; + const editPane = thePanes.edit; + origPane.modified = origPane.original.slice(); + editPane.modified = editPane.original.slice(); + const select = qs$('#ruleFilter select'); + sortType = parseInt(select.value, 10); + if ( isNaN(sortType) ) { sortType = 1; } + { + const mode = origPane.doc.getMode(); + mode.sortType = sortType; + mode.setHostnameToDomainMap(hostnameToDomainMap); + mode.setPSL(publicSuffixList); + } + { + const mode = editPane.doc.getMode(); + mode.sortType = sortType; + mode.setHostnameToDomainMap(hostnameToDomainMap); + mode.setPSL(publicSuffixList); + } + sort(origPane.modified); + sort(editPane.modified); + collapse(); + rulesToDoc(clearHistory); + onTextChanged(clearHistory); + }; +})(); + +/******************************************************************************/ + +const onTextChanged = (( ) => { + let timer; + + const process = details => { + timer = undefined; + const diff = qs$('#diff'); + let isClean = mergeView.editor().isClean(cleanEditToken); + if ( + details === undefined && + isClean === false && + mergeView.editor().getValue().trim() === cleanEditText + ) { + cleanEditToken = mergeView.editor().changeGeneration(); + isClean = true; + } + const isDirty = mergeView.leftChunks().length !== 0; + dom.cl.toggle(dom.body, 'editing', isClean === false); + dom.cl.toggle(diff, 'dirty', isDirty); + dom.cl.toggle('#editSaveButton', 'disabled', isClean); + dom.cl.toggle('#exportButton,#importButton', 'disabled', isClean === false); + dom.cl.toggle('#revertButton,#commitButton', 'disabled', isClean === false || isDirty === false); + const input = qs$('#ruleFilter input'); + if ( isClean ) { + dom.attr(input, 'disabled', null); + CodeMirror.commands.save = undefined; + } else { + dom.attr(input, 'disabled', ''); + CodeMirror.commands.save = editSaveHandler; + } + }; + + return function(now) { + if ( timer !== undefined ) { self.cancelIdleCallback(timer); } + timer = now ? process() : self.requestIdleCallback(process, { timeout: 57 }); + }; +})(); + +/******************************************************************************/ + +const revertAllHandler = function() { + const toAdd = [], toRemove = []; + const left = mergeView.leftOriginal(); + const edit = mergeView.editor(); + for ( const chunk of mergeView.leftChunks() ) { + const addedLines = left.getRange( + { line: chunk.origFrom, ch: 0 }, + { line: chunk.origTo, ch: 0 } + ); + const removedLines = edit.getRange( + { line: chunk.editFrom, ch: 0 }, + { line: chunk.editTo, ch: 0 } + ); + toAdd.push(addedLines.trim()); + toRemove.push(removedLines.trim()); + } + applyDiff(false, toAdd.join('\n'), toRemove.join('\n')); +}; + +/******************************************************************************/ + +const commitAllHandler = function() { + const toAdd = [], toRemove = []; + const left = mergeView.leftOriginal(); + const edit = mergeView.editor(); + for ( const chunk of mergeView.leftChunks() ) { + const addedLines = edit.getRange( + { line: chunk.editFrom, ch: 0 }, + { line: chunk.editTo, ch: 0 } + ); + const removedLines = left.getRange( + { line: chunk.origFrom, ch: 0 }, + { line: chunk.origTo, ch: 0 } + ); + toAdd.push(addedLines.trim()); + toRemove.push(removedLines.trim()); + } + applyDiff(true, toAdd.join('\n'), toRemove.join('\n')); +}; + +/******************************************************************************/ + +const editSaveHandler = function() { + const editor = mergeView.editor(); + const editText = editor.getValue().trim(); + if ( editText === cleanEditText ) { + onTextChanged(true); + return; + } + const toAdd = [], toRemove = []; + const diffs = getDiffer().diff_main(cleanEditText, editText); + for ( const diff of diffs ) { + if ( diff[0] === 1 ) { + toAdd.push(diff[1]); + } else if ( diff[0] === -1 ) { + toRemove.push(diff[1]); + } + } + applyDiff(false, toAdd.join(''), toRemove.join('')); +}; + +/******************************************************************************/ + +self.cloud.onPush = function() { + return thePanes.orig.original.join('\n'); +}; + +self.cloud.onPull = function(data, append) { + if ( typeof data !== 'string' ) { return; } + applyDiff( + false, + data, + append ? '' : mergeView.editor().getValue().trim() + ); +}; + +/******************************************************************************/ + +self.hasUnsavedData = function() { + return mergeView.editor().isClean(cleanEditToken) === false; +}; + +/******************************************************************************/ + +vAPI.messaging.send('dashboard', { + what: 'getRules', +}).then(details => { + thePanes.orig.original = details.permanentRules; + thePanes.edit.original = details.sessionRules; + publicSuffixList.fromSelfie(details.pslSelfie); + onPresentationChanged(true); +}); + +// Handle user interaction +dom.on('#importButton', 'click', startImportFilePicker); +dom.on('#importFilePicker', 'change', handleImportFilePicker); +dom.on('#exportButton', 'click', exportUserRulesToFile); +dom.on('#revertButton', 'click', revertAllHandler); +dom.on('#commitButton', 'click', commitAllHandler); +dom.on('#editSaveButton', 'click', editSaveHandler); +dom.on('#ruleFilter input', 'input', onFilterChanged); +dom.on('#ruleFilter select', 'input', ( ) => { + onPresentationChanged(true); +}); +dom.on('#ruleFilter #diffCollapse', 'click', ev => { + isCollapsed = dom.cl.toggle(ev.target, 'active'); + onPresentationChanged(true); +}); + +// https://groups.google.com/forum/#!topic/codemirror/UQkTrt078Vs +mergeView.editor().on('updateDiff', ( ) => { + onTextChanged(); +}); + +/******************************************************************************/ + diff --git a/src/js/dynamic-net-filtering.js b/src/js/dynamic-net-filtering.js new file mode 100644 index 0000000..ec7a7c9 --- /dev/null +++ b/src/js/dynamic-net-filtering.js @@ -0,0 +1,488 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import punycode from '../lib/punycode.js'; + +import { LineIterator } from './text-utils.js'; + +import { + decomposeHostname, + domainFromHostname, +} from './uri-utils.js'; + +/******************************************************************************/ + +// Object.create(null) is used below to eliminate worries about unexpected +// property names in prototype chain -- and this way we don't have to use +// hasOwnProperty() to avoid this. + +const supportedDynamicTypes = Object.create(null); +Object.assign(supportedDynamicTypes, { + '3p': true, + 'image': true, +'inline-script': true, + '1p-script': true, + '3p-script': true, + '3p-frame': true +}); + +const typeBitOffsets = Object.create(null); +Object.assign(typeBitOffsets, { + '*': 0, +'inline-script': 2, + '1p-script': 4, + '3p-script': 6, + '3p-frame': 8, + 'image': 10, + '3p': 12 +}); + +const nameToActionMap = Object.create(null); +Object.assign(nameToActionMap, { + 'block': 1, + 'allow': 2, + 'noop': 3 +}); + +const intToActionMap = new Map([ + [ 1, 'block' ], + [ 2, 'allow' ], + [ 3, 'noop' ] +]); + +// For performance purpose, as simple tests as possible +const reBadHostname = /[^0-9a-z_.\[\]:%-]/; +const reNotASCII = /[^\x20-\x7F]/; +const decomposedSource = []; +const decomposedDestination = []; + +/******************************************************************************/ + +function is3rdParty(srcHostname, desHostname) { + // If at least one is party-less, the relation can't be labelled + // "3rd-party" + if ( desHostname === '*' || srcHostname === '*' || srcHostname === '' ) { + return false; + } + + // No domain can very well occurs, for examples: + // - localhost + // - file-scheme + // etc. + const srcDomain = domainFromHostname(srcHostname) || srcHostname; + + if ( desHostname.endsWith(srcDomain) === false ) { + return true; + } + // Do not confuse 'example.com' with 'anotherexample.com' + return desHostname.length !== srcDomain.length && + desHostname.charAt(desHostname.length - srcDomain.length - 1) !== '.'; +} + +/******************************************************************************/ + +class DynamicHostRuleFiltering { + + constructor() { + this.reset(); + } + + reset() { + this.r = 0; + this.type = ''; + this.y = ''; + this.z = ''; + this.rules = new Map(); + this.changed = false; + } + + assign(other) { + // Remove rules not in other + for ( const k of this.rules.keys() ) { + if ( other.rules.has(k) === false ) { + this.rules.delete(k); + this.changed = true; + } + } + // Add/change rules in other + for ( const entry of other.rules ) { + if ( this.rules.get(entry[0]) !== entry[1] ) { + this.rules.set(entry[0], entry[1]); + this.changed = true; + } + } + } + + copyRules(from, srcHostname, desHostnames) { + // Specific types + let thisBits = this.rules.get('* *'); + let fromBits = from.rules.get('* *'); + if ( fromBits !== thisBits ) { + if ( fromBits !== undefined ) { + this.rules.set('* *', fromBits); + } else { + this.rules.delete('* *'); + } + this.changed = true; + } + + let key = `${srcHostname} *`; + thisBits = this.rules.get(key); + fromBits = from.rules.get(key); + if ( fromBits !== thisBits ) { + if ( fromBits !== undefined ) { + this.rules.set(key, fromBits); + } else { + this.rules.delete(key); + } + this.changed = true; + } + + // Specific destinations + for ( const desHostname in desHostnames ) { + key = `* ${desHostname}`; + thisBits = this.rules.get(key); + fromBits = from.rules.get(key); + if ( fromBits !== thisBits ) { + if ( fromBits !== undefined ) { + this.rules.set(key, fromBits); + } else { + this.rules.delete(key); + } + this.changed = true; + } + key = `${srcHostname} ${desHostname}` ; + thisBits = this.rules.get(key); + fromBits = from.rules.get(key); + if ( fromBits !== thisBits ) { + if ( fromBits !== undefined ) { + this.rules.set(key, fromBits); + } else { + this.rules.delete(key); + } + this.changed = true; + } + } + + return this.changed; + } + + // - * * type + // - from * type + // - * to * + // - from to * + + hasSameRules(other, srcHostname, desHostnames) { + // Specific types + let key = '* *'; + if ( this.rules.get(key) !== other.rules.get(key) ) { return false; } + key = `${srcHostname} *`; + if ( this.rules.get(key) !== other.rules.get(key) ) { return false; } + // Specific destinations + for ( const desHostname in desHostnames ) { + key = `* ${desHostname}`; + if ( this.rules.get(key) !== other.rules.get(key) ) { + return false; + } + key = `${srcHostname} ${desHostname}`; + if ( this.rules.get(key) !== other.rules.get(key) ) { + return false; + } + } + return true; + } + + setCell(srcHostname, desHostname, type, state) { + const bitOffset = typeBitOffsets[type]; + const k = `${srcHostname} ${desHostname}`; + const oldBitmap = this.rules.get(k) || 0; + const newBitmap = oldBitmap & ~(3 << bitOffset) | (state << bitOffset); + if ( newBitmap === oldBitmap ) { return false; } + if ( newBitmap === 0 ) { + this.rules.delete(k); + } else { + this.rules.set(k, newBitmap); + } + this.changed = true; + return true; + } + + unsetCell(srcHostname, desHostname, type) { + this.evaluateCellZY(srcHostname, desHostname, type); + if ( this.r === 0 ) { return false; } + this.setCell(srcHostname, desHostname, type, 0); + this.changed = true; + return true; + } + + evaluateCell(srcHostname, desHostname, type) { + const key = `${srcHostname} ${desHostname}`; + const bitmap = this.rules.get(key); + if ( bitmap === undefined ) { return 0; } + return bitmap >> typeBitOffsets[type] & 3; + } + + clearRegisters() { + this.r = 0; + this.type = this.y = this.z = ''; + return this; + } + + evaluateCellZ(srcHostname, desHostname, type) { + decomposeHostname(srcHostname, decomposedSource); + this.type = type; + const bitOffset = typeBitOffsets[type]; + for ( const srchn of decomposedSource ) { + this.z = srchn; + let v = this.rules.get(`${srchn} ${desHostname}`); + if ( v === undefined ) { continue; } + v = v >>> bitOffset & 3; + if ( v === 0 ) { continue; } + return (this.r = v); + } + // srcHostname is '*' at this point + this.r = 0; + return 0; + } + + evaluateCellZY(srcHostname, desHostname, type) { + // Pathological cases. + if ( desHostname === '' ) { + this.r = 0; + return 0; + } + + // Precedence: from most specific to least specific + + // Specific-destination, any party, any type + decomposeHostname(desHostname, decomposedDestination); + for ( const deshn of decomposedDestination ) { + if ( deshn === '*' ) { break; } + this.y = deshn; + if ( this.evaluateCellZ(srcHostname, deshn, '*') !== 0 ) { + return this.r; + } + } + + const thirdParty = is3rdParty(srcHostname, desHostname); + + // Any destination + this.y = '*'; + + // Specific party + if ( thirdParty ) { + // 3rd-party, specific type + if ( type === 'script' ) { + if ( this.evaluateCellZ(srcHostname, '*', '3p-script') !== 0 ) { + return this.r; + } + } else if ( type === 'sub_frame' || type === 'object' ) { + if ( this.evaluateCellZ(srcHostname, '*', '3p-frame') !== 0 ) { + return this.r; + } + } + // 3rd-party, any type + if ( this.evaluateCellZ(srcHostname, '*', '3p') !== 0 ) { + return this.r; + } + } else if ( type === 'script' ) { + // 1st party, specific type + if ( this.evaluateCellZ(srcHostname, '*', '1p-script') !== 0 ) { + return this.r; + } + } + + // Any destination, any party, specific type + if ( supportedDynamicTypes[type] !== undefined ) { + if ( this.evaluateCellZ(srcHostname, '*', type) !== 0 ) { + return this.r; + } + if ( type.startsWith('3p-') ) { + if ( this.evaluateCellZ(srcHostname, '*', '3p') !== 0 ) { + return this.r; + } + } + } + + // Any destination, any party, any type + if ( this.evaluateCellZ(srcHostname, '*', '*') !== 0 ) { + return this.r; + } + + this.type = ''; + return 0; + } + + mustAllowCellZY(srcHostname, desHostname, type) { + return this.evaluateCellZY(srcHostname, desHostname, type) === 2; + } + + mustBlockOrAllow() { + return this.r === 1 || this.r === 2; + } + + mustBlock() { + return this.r === 1; + } + + mustAbort() { + return this.r === 3; + } + + lookupRuleData(src, des, type) { + const r = this.evaluateCellZY(src, des, type); + if ( r === 0 ) { return; } + return `${this.z} ${this.y} ${this.type} ${r}`; + } + + toLogData() { + if ( this.r === 0 || this.type === '' ) { return; } + return { + source: 'dynamicHost', + result: this.r, + raw: `${this.z} ${this.y} ${this.type} ${intToActionMap.get(this.r)}` + }; + } + + srcHostnameFromRule(rule) { + return rule.slice(0, rule.indexOf(' ')); + } + + desHostnameFromRule(rule) { + return rule.slice(rule.indexOf(' ') + 1); + } + + toArray() { + const out = []; + for ( const key of this.rules.keys() ) { + const srchn = this.srcHostnameFromRule(key); + const deshn = this.desHostnameFromRule(key); + const srchnPretty = srchn.includes('xn--') && punycode + ? punycode.toUnicode(srchn) + : srchn; + const deshnPretty = deshn.includes('xn--') && punycode + ? punycode.toUnicode(deshn) + : deshn; + for ( const type in typeBitOffsets ) { + if ( typeBitOffsets[type] === undefined ) { continue; } + const val = this.evaluateCell(srchn, deshn, type); + if ( val === 0 ) { continue; } + const action = intToActionMap.get(val); + if ( action === undefined ) { continue; } + out.push(`${srchnPretty} ${deshnPretty} ${type} ${action}`); + } + } + return out; + } + + toString() { + return this.toArray().join('\n'); + } + + fromString(text, append) { + const lineIter = new LineIterator(text); + if ( append !== true ) { this.reset(); } + while ( lineIter.eot() === false ) { + this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); + } + } + + validateRuleParts(parts) { + if ( parts.length < 4 ) { return; } + + // Ignore hostname-based switch rules + if ( parts[0].endsWith(':') ) { return; } + + // Ignore URL-based rules + if ( parts[1].includes('/') ) { return; } + + if ( typeBitOffsets[parts[2]] === undefined ) { return; } + + if ( nameToActionMap[parts[3]] === undefined ) { return; } + + // https://github.com/chrisaljoudi/uBlock/issues/840 + // Discard invalid rules + if ( parts[1] !== '*' && parts[2] !== '*' ) { return; } + + // Performance: avoid punycoding when only ASCII chars + if ( punycode !== undefined ) { + if ( reNotASCII.test(parts[0]) ) { + parts[0] = punycode.toASCII(parts[0]); + } + if ( reNotASCII.test(parts[1]) ) { + parts[1] = punycode.toASCII(parts[1]); + } + } + + // https://github.com/chrisaljoudi/uBlock/issues/1082 + // Discard rules with invalid hostnames + if ( + (parts[0] !== '*' && reBadHostname.test(parts[0])) || + (parts[1] !== '*' && reBadHostname.test(parts[1])) + ) { + return; + } + + return parts; + } + + addFromRuleParts(parts) { + if ( this.validateRuleParts(parts) !== undefined ) { + this.setCell(parts[0], parts[1], parts[2], nameToActionMap[parts[3]]); + return true; + } + return false; + } + + removeFromRuleParts(parts) { + if ( this.validateRuleParts(parts) !== undefined ) { + this.setCell(parts[0], parts[1], parts[2], 0); + return true; + } + return false; + } + + toSelfie() { + return { + magicId: this.magicId, + rules: Array.from(this.rules) + }; + } + + fromSelfie(selfie) { + if ( selfie.magicId !== this.magicId ) { return false; } + this.rules = new Map(selfie.rules); + this.changed = true; + return true; + } +} + +DynamicHostRuleFiltering.prototype.magicId = 1; + +/******************************************************************************/ + +export default DynamicHostRuleFiltering; + +/******************************************************************************/ diff --git a/src/js/epicker-ui.js b/src/js/epicker-ui.js new file mode 100644 index 0000000..49fc116 --- /dev/null +++ b/src/js/epicker-ui.js @@ -0,0 +1,900 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror */ + +'use strict'; + +import './codemirror/ubo-static-filtering.js'; + +import { hostnameFromURI } from './uri-utils.js'; +import punycode from '../lib/punycode.js'; +import * as sfp from './static-filtering-parser.js'; + +/******************************************************************************/ +/******************************************************************************/ + +(( ) => { + +/******************************************************************************/ + +if ( typeof vAPI !== 'object' ) { return; } + +const $id = id => document.getElementById(id); +const $stor = selector => document.querySelector(selector); +const $storAll = selector => document.querySelectorAll(selector); + +const pickerRoot = document.documentElement; +const dialog = $stor('aside'); +let staticFilteringParser; + +const svgRoot = $stor('svg'); +const svgOcean = svgRoot.children[0]; +const svgIslands = svgRoot.children[1]; +const NoPaths = 'M0 0'; + +const reCosmeticAnchor = /^#(\$|\?|\$\?)?#/; + +{ + const url = new URL(self.location.href); + if ( url.searchParams.has('zap') ) { + pickerRoot.classList.add('zap'); + } +} + +const docURL = new URL(vAPI.getURL('')); + +let resultsetOpt; + +let netFilterCandidates = []; +let cosmeticFilterCandidates = []; +let computedCandidateSlot = 0; +let computedCandidate = ''; +const computedSpecificityCandidates = new Map(); +let needBody = false; + +/******************************************************************************/ + +const cmEditor = new CodeMirror(document.querySelector('.codeMirrorContainer'), { + autoCloseBrackets: true, + autofocus: true, + extraKeys: { + 'Ctrl-Space': 'autocomplete', + }, + lineWrapping: true, + matchBrackets: true, + maxScanLines: 1, +}); + +vAPI.messaging.send('dashboard', { + what: 'getAutoCompleteDetails' +}).then(hints => { + // For unknown reasons, `instanceof Object` does not work here in Firefox. + if ( hints instanceof Object === false ) { return; } + cmEditor.setOption('uboHints', hints); +}); + +/******************************************************************************/ + +const rawFilterFromTextarea = function() { + const text = cmEditor.getValue(); + const pos = text.indexOf('\n'); + return pos === -1 ? text : text.slice(0, pos); +}; + +/******************************************************************************/ + +const filterFromTextarea = function() { + const filter = rawFilterFromTextarea(); + if ( filter === '' ) { return ''; } + const parser = staticFilteringParser; + parser.parse(filter); + if ( parser.isFilter() === false ) { return '!'; } + if ( parser.isExtendedFilter() ) { + if ( parser.isCosmeticFilter() === false ) { return '!'; } + } else if ( parser.isNetworkFilter() === false ) { + return '!'; + } + return filter; +}; + +/******************************************************************************/ + +const renderRange = function(id, value, invert = false) { + const input = $stor(`#${id} input`); + const max = parseInt(input.max, 10); + if ( typeof value !== 'number' ) { + value = parseInt(input.value, 10); + } + if ( invert ) { + value = max - value; + } + input.value = value; + const slider = $stor(`#${id} > span`); + const lside = slider.children[0]; + const thumb = slider.children[1]; + const sliderWidth = slider.offsetWidth; + const maxPercent = (sliderWidth - thumb.offsetWidth) / sliderWidth * 100; + const widthPercent = value / max * maxPercent; + lside.style.width = `${widthPercent}%`; +}; + +/******************************************************************************/ + +const userFilterFromCandidate = function(filter) { + if ( filter === '' || filter === '!' ) { return; } + + let hn = hostnameFromURI(docURL.href); + if ( hn.startsWith('xn--') ) { + hn = punycode.toUnicode(hn); + } + + // Cosmetic filter? + if ( reCosmeticAnchor.test(filter) ) { + return hn + filter; + } + + // Assume net filter + const opts = []; + + // If no domain included in filter, we need domain option + if ( filter.startsWith('||') === false ) { + opts.push(`domain=${hn}`); + } + + if ( resultsetOpt !== undefined ) { + opts.push(resultsetOpt); + } + + if ( opts.length ) { + filter += '$' + opts.join(','); + } + + return filter; +}; + +/******************************************************************************/ + +const candidateFromFilterChoice = function(filterChoice) { + let { slot, filters } = filterChoice; + let filter = filters[slot]; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/47 + for ( const elem of $storAll('#candidateFilters li') ) { + elem.classList.remove('active'); + } + + computedCandidateSlot = slot; + computedCandidate = ''; + + if ( filter === undefined ) { return ''; } + + // For net filters there no such thing as a path + if ( filter.startsWith('##') === false ) { + $stor(`#netFilters li:nth-of-type(${slot+1})`) + .classList.add('active'); + return filter; + } + + // At this point, we have a cosmetic filter + + $stor(`#cosmeticFilters li:nth-of-type(${slot+1})`) + .classList.add('active'); + + return cosmeticCandidatesFromFilterChoice(filterChoice); +}; + +/******************************************************************************/ + +const cosmeticCandidatesFromFilterChoice = function(filterChoice) { + let { slot, filters } = filterChoice; + + renderRange('resultsetDepth', slot, true); + renderRange('resultsetSpecificity'); + + if ( computedSpecificityCandidates.has(slot) ) { + onCandidatesOptimized({ slot }); + return; + } + + const specificities = [ + 0b0000, // remove hierarchy; remove id, nth-of-type, attribute values + 0b0010, // remove hierarchy; remove id, nth-of-type + 0b0011, // remove hierarchy + 0b1000, // trim hierarchy; remove id, nth-of-type, attribute values + 0b1010, // trim hierarchy; remove id, nth-of-type + 0b1100, // remove id, nth-of-type, attribute values + 0b1110, // remove id, nth-of-type + 0b1111, // keep all = most specific + ]; + + const candidates = []; + + let filter = filters[slot]; + + for ( const specificity of specificities ) { + // Return path: the target element, then all siblings prepended + const paths = []; + for ( let i = slot; i < filters.length; i++ ) { + filter = filters[i].slice(2); + // Remove id, nth-of-type + // https://github.com/uBlockOrigin/uBlock-issues/issues/162 + // Mind escaped periods: they do not denote a class identifier. + if ( (specificity & 0b0001) === 0 ) { + filter = filter.replace(/:nth-of-type\(\d+\)/, ''); + if ( + filter.charAt(0) === '#' && ( + (specificity & 0b1000) === 0 || i === slot + ) + ) { + const pos = filter.search(/[^\\]\./); + if ( pos !== -1 ) { + filter = filter.slice(pos + 1); + } + } + } + // Remove attribute values. + if ( (specificity & 0b0010) === 0 ) { + const match = /^\[([^^*$=]+)[\^*$]?=.+\]$/.exec(filter); + if ( match !== null ) { + filter = `[${match[1]}]`; + } + } + // Remove all classes when an id exists. + // https://github.com/uBlockOrigin/uBlock-issues/issues/162 + // Mind escaped periods: they do not denote a class identifier. + if ( filter.charAt(0) === '#' ) { + filter = filter.replace(/([^\\])\..+$/, '$1'); + } + if ( paths.length !== 0 ) { + filter += ' > '; + } + paths.unshift(filter); + // Stop at any element with an id: these are unique in a web page + if ( (specificity & 0b1000) === 0 || filter.startsWith('#') ) { + break; + } + } + + // Trim hierarchy: remove generic elements from path + if ( (specificity & 0b1100) === 0b1000 ) { + let i = 0; + while ( i < paths.length - 1 ) { + if ( /^[a-z0-9]+ > $/.test(paths[i+1]) ) { + if ( paths[i].endsWith(' > ') ) { + paths[i] = paths[i].slice(0, -2); + } + paths.splice(i + 1, 1); + } else { + i += 1; + } + } + } + + if ( + needBody && + paths.length !== 0 && + paths[0].startsWith('#') === false && + paths[0].startsWith('body ') === false && + (specificity & 0b1100) !== 0 + ) { + paths.unshift('body > '); + } + + candidates.push(paths); + } + + pickerContentPort.postMessage({ + what: 'optimizeCandidates', + candidates, + slot, + }); +}; + +/******************************************************************************/ + +const onCandidatesOptimized = function(details) { + $id('resultsetModifiers').classList.remove('hide'); + const i = parseInt($stor('#resultsetSpecificity input').value, 10); + if ( Array.isArray(details.candidates) ) { + computedSpecificityCandidates.set(details.slot, details.candidates); + } + const candidates = computedSpecificityCandidates.get(details.slot); + computedCandidate = candidates[i]; + cmEditor.setValue(computedCandidate); + cmEditor.clearHistory(); + onCandidateChanged(); +}; + +/******************************************************************************/ + +const onSvgClicked = function(ev) { + // If zap mode, highlight element under mouse, this makes the zapper usable + // on touch screens. + if ( pickerRoot.classList.contains('zap') ) { + pickerContentPort.postMessage({ + what: 'zapElementAtPoint', + mx: ev.clientX, + my: ev.clientY, + options: { + stay: ev.shiftKey || ev.type === 'touch', + highlight: ev.target !== svgIslands, + }, + }); + return; + } + // https://github.com/chrisaljoudi/uBlock/issues/810#issuecomment-74600694 + // Unpause picker if: + // - click outside dialog AND + // - not in preview mode + if ( pickerRoot.classList.contains('paused') ) { + if ( pickerRoot.classList.contains('preview') === false ) { + unpausePicker(); + } + return; + } + // Force dialog to always be visible when using a touch-driven device. + if ( ev.type === 'touch' ) { + pickerRoot.classList.add('show'); + } + pickerContentPort.postMessage({ + what: 'filterElementAtPoint', + mx: ev.clientX, + my: ev.clientY, + broad: ev.ctrlKey, + }); +}; + +/******************************************************************************* + + Swipe right: + If picker not paused: quit picker + If picker paused and dialog visible: hide dialog + If picker paused and dialog not visible: quit picker + + Swipe left: + If picker paused and dialog not visible: show dialog + +*/ + +const onSvgTouch = (( ) => { + let startX = 0, startY = 0; + let t0 = 0; + return ev => { + if ( ev.type === 'touchstart' ) { + startX = ev.touches[0].screenX; + startY = ev.touches[0].screenY; + t0 = ev.timeStamp; + return; + } + if ( startX === undefined ) { return; } + const stopX = ev.changedTouches[0].screenX; + const stopY = ev.changedTouches[0].screenY; + const angle = Math.abs(Math.atan2(stopY - startY, stopX - startX)); + const distance = Math.sqrt( + Math.pow(stopX - startX, 2), + Math.pow(stopY - startY, 2) + ); + // Interpret touch events as a tap if: + // - Swipe is not valid; and + // - The time between start and stop was less than 200ms. + const duration = ev.timeStamp - t0; + if ( distance < 32 && duration < 200 ) { + onSvgClicked({ + type: 'touch', + target: ev.target, + clientX: ev.changedTouches[0].pageX, + clientY: ev.changedTouches[0].pageY, + }); + ev.preventDefault(); + return; + } + if ( distance < 64 ) { return; } + const angleUpperBound = Math.PI * 0.25 * 0.5; + const swipeRight = angle < angleUpperBound; + if ( swipeRight === false && angle < Math.PI - angleUpperBound ) { + return; + } + if ( ev.cancelable ) { + ev.preventDefault(); + } + // Swipe left. + if ( swipeRight === false ) { + if ( pickerRoot.classList.contains('paused') ) { + pickerRoot.classList.remove('hide'); + pickerRoot.classList.add('show'); + } + return; + } + // Swipe right. + if ( + pickerRoot.classList.contains('zap') && + svgIslands.getAttribute('d') !== NoPaths + ) { + pickerContentPort.postMessage({ + what: 'unhighlight' + }); + return; + } + else if ( + pickerRoot.classList.contains('paused') && + pickerRoot.classList.contains('show') + ) { + pickerRoot.classList.remove('show'); + pickerRoot.classList.add('hide'); + return; + } + quitPicker(); + }; +})(); + +/******************************************************************************/ + +const onCandidateChanged = function() { + const filter = filterFromTextarea(); + const bad = filter === '!'; + $stor('section').classList.toggle('invalidFilter', bad); + if ( bad ) { + $id('resultsetCount').textContent = 'E'; + $id('create').setAttribute('disabled', ''); + } + const text = rawFilterFromTextarea(); + $id('resultsetModifiers').classList.toggle( + 'hide', text === '' || text !== computedCandidate + ); + pickerContentPort.postMessage({ + what: 'dialogSetFilter', + filter, + compiled: reCosmeticAnchor.test(filter) + ? staticFilteringParser.result.compiled + : undefined, + }); +}; + +/******************************************************************************/ + +const onPreviewClicked = function() { + const state = pickerRoot.classList.toggle('preview'); + pickerContentPort.postMessage({ + what: 'togglePreview', + state, + }); +}; + +/******************************************************************************/ + +const onCreateClicked = function() { + const candidate = filterFromTextarea(); + const filter = userFilterFromCandidate(candidate); + if ( filter !== undefined ) { + vAPI.messaging.send('elementPicker', { + what: 'createUserFilter', + autoComment: true, + filters: filter, + docURL: docURL.href, + killCache: reCosmeticAnchor.test(candidate) === false, + }); + } + pickerContentPort.postMessage({ + what: 'dialogCreate', + filter: candidate, + compiled: reCosmeticAnchor.test(candidate) + ? staticFilteringParser.result.compiled + : undefined, + }); +}; + +/******************************************************************************/ + +const onPickClicked = function() { + unpausePicker(); +}; + +/******************************************************************************/ + +const onQuitClicked = function() { + quitPicker(); +}; + +/******************************************************************************/ + +const onDepthChanged = function() { + const input = $stor('#resultsetDepth input'); + const max = parseInt(input.max, 10); + const value = parseInt(input.value, 10); + const text = candidateFromFilterChoice({ + filters: cosmeticFilterCandidates, + slot: max - value, + }); + if ( text === undefined ) { return; } + cmEditor.setValue(text); + cmEditor.clearHistory(); + onCandidateChanged(); +}; + +/******************************************************************************/ + +const onSpecificityChanged = function() { + renderRange('resultsetSpecificity'); + if ( rawFilterFromTextarea() !== computedCandidate ) { return; } + const depthInput = $stor('#resultsetDepth input'); + const slot = parseInt(depthInput.max, 10) - parseInt(depthInput.value, 10); + const i = parseInt($stor('#resultsetSpecificity input').value, 10); + const candidates = computedSpecificityCandidates.get(slot); + computedCandidate = candidates[i]; + cmEditor.setValue(computedCandidate); + cmEditor.clearHistory(); + onCandidateChanged(); +}; + +/******************************************************************************/ + +const onCandidateClicked = function(ev) { + let li = ev.target.closest('li'); + if ( li === null ) { return; } + const ul = li.closest('.changeFilter'); + if ( ul === null ) { return; } + const choice = { + filters: Array.from(ul.querySelectorAll('li')).map(a => a.textContent), + slot: 0, + }; + while ( li.previousElementSibling !== null ) { + li = li.previousElementSibling; + choice.slot += 1; + } + const text = candidateFromFilterChoice(choice); + if ( text === undefined ) { return; } + cmEditor.setValue(text); + cmEditor.clearHistory(); + onCandidateChanged(); +}; + +/******************************************************************************/ + +const onKeyPressed = function(ev) { + // Delete + if ( + (ev.key === 'Delete' || ev.key === 'Backspace') && + pickerRoot.classList.contains('zap') + ) { + pickerContentPort.postMessage({ + what: 'zapElementAtPoint', + options: { stay: true }, + }); + return; + } + // Esc + if ( ev.key === 'Escape' || ev.which === 27 ) { + onQuitClicked(); + return; + } +}; + +/******************************************************************************/ + +const onStartMoving = (( ) => { + let isTouch = false; + let mx0 = 0, my0 = 0; + let mx1 = 0, my1 = 0; + let r0 = 0, b0 = 0; + let rMax = 0, bMax = 0; + let timer; + + const eatEvent = function(ev) { + ev.stopPropagation(); + ev.preventDefault(); + }; + + const move = ( ) => { + timer = undefined; + const r1 = Math.min(Math.max(r0 - mx1 + mx0, 2), rMax); + const b1 = Math.min(Math.max(b0 - my1 + my0, 2), bMax); + dialog.style.setProperty('right', `${r1}px`); + dialog.style.setProperty('bottom', `${b1}px`); + }; + + const moveAsync = ev => { + if ( timer !== undefined ) { return; } + if ( isTouch ) { + const touch = ev.touches[0]; + mx1 = touch.pageX; + my1 = touch.pageY; + } else { + mx1 = ev.pageX; + my1 = ev.pageY; + } + timer = self.requestAnimationFrame(move); + }; + + const stop = ev => { + if ( dialog.classList.contains('moving') === false ) { return; } + dialog.classList.remove('moving'); + if ( isTouch ) { + self.removeEventListener('touchmove', moveAsync, { capture: true }); + } else { + self.removeEventListener('mousemove', moveAsync, { capture: true }); + } + eatEvent(ev); + }; + + return function(ev) { + const target = dialog.querySelector('#move'); + if ( ev.target !== target ) { return; } + if ( dialog.classList.contains('moving') ) { return; } + isTouch = ev.type.startsWith('touch'); + if ( isTouch ) { + const touch = ev.touches[0]; + mx0 = touch.pageX; + my0 = touch.pageY; + } else { + mx0 = ev.pageX; + my0 = ev.pageY; + } + const style = self.getComputedStyle(dialog); + r0 = parseInt(style.right, 10); + b0 = parseInt(style.bottom, 10); + const rect = dialog.getBoundingClientRect(); + rMax = pickerRoot.clientWidth - 2 - rect.width ; + bMax = pickerRoot.clientHeight - 2 - rect.height; + dialog.classList.add('moving'); + if ( isTouch ) { + self.addEventListener('touchmove', moveAsync, { capture: true }); + self.addEventListener('touchend', stop, { capture: true, once: true }); + } else { + self.addEventListener('mousemove', moveAsync, { capture: true }); + self.addEventListener('mouseup', stop, { capture: true, once: true }); + } + eatEvent(ev); + }; +})(); + +/******************************************************************************/ + +const svgListening = (( ) => { + let on = false; + let timer; + let mx = 0, my = 0; + + const onTimer = ( ) => { + timer = undefined; + pickerContentPort.postMessage({ + what: 'highlightElementAtPoint', + mx, + my, + }); + }; + + const onHover = ev => { + mx = ev.clientX; + my = ev.clientY; + if ( timer === undefined ) { + timer = self.requestAnimationFrame(onTimer); + } + }; + + return state => { + if ( state === on ) { return; } + on = state; + if ( on ) { + document.addEventListener('mousemove', onHover, { passive: true }); + return; + } + document.removeEventListener('mousemove', onHover, { passive: true }); + if ( timer !== undefined ) { + self.cancelAnimationFrame(timer); + timer = undefined; + } + }; +})(); + +/******************************************************************************/ + +// Create lists of candidate filters. This takes into account whether the +// current mode is narrow or broad. + +const populateCandidates = function(candidates, selector) { + + const root = dialog.querySelector(selector); + const ul = root.querySelector('ul'); + while ( ul.firstChild !== null ) { + ul.firstChild.remove(); + } + for ( let i = 0; i < candidates.length; i++ ) { + const li = document.createElement('li'); + li.textContent = candidates[i]; + ul.appendChild(li); + } + if ( candidates.length !== 0 ) { + root.style.removeProperty('display'); + } else { + root.style.setProperty('display', 'none'); + } +}; + +/******************************************************************************/ + +const showDialog = function(details) { + pausePicker(); + + const { netFilters, cosmeticFilters, filter } = details; + + netFilterCandidates = netFilters; + + needBody = + cosmeticFilters.length !== 0 && + cosmeticFilters[cosmeticFilters.length - 1] === '##body'; + if ( needBody ) { + cosmeticFilters.pop(); + } + cosmeticFilterCandidates = cosmeticFilters; + + docURL.href = details.url; + + populateCandidates(netFilters, '#netFilters'); + populateCandidates(cosmeticFilters, '#cosmeticFilters'); + computedSpecificityCandidates.clear(); + + const depthInput = $stor('#resultsetDepth input'); + depthInput.max = cosmeticFilters.length - 1; + depthInput.value = depthInput.max; + + dialog.querySelector('ul').style.display = + netFilters.length || cosmeticFilters.length ? '' : 'none'; + $id('create').setAttribute('disabled', ''); + + // Auto-select a candidate filter + + // 2020-09-01: + // In Firefox, `details instanceof Object` resolves to `false` despite + // `details` being a valid object. Consequently, falling back to use + // `typeof details`. + // This is an issue which surfaced when the element picker code was + // revisited to isolate the picker dialog DOM from the page DOM. + if ( typeof filter !== 'object' || filter === null ) { + cmEditor.setValue(''); + return; + } + + const filterChoice = { + filters: filter.filters, + slot: filter.slot, + }; + + const text = candidateFromFilterChoice(filterChoice); + if ( text === undefined ) { return; } + cmEditor.setValue(text); + onCandidateChanged(); +}; + +/******************************************************************************/ + +const pausePicker = function() { + pickerRoot.classList.add('paused'); + svgListening(false); +}; + +/******************************************************************************/ + +const unpausePicker = function() { + pickerRoot.classList.remove('paused', 'preview'); + pickerContentPort.postMessage({ + what: 'togglePreview', + state: false, + }); + svgListening(true); +}; + +/******************************************************************************/ + +const startPicker = function() { + self.addEventListener('keydown', onKeyPressed, true); + const svg = $stor('svg'); + svg.addEventListener('click', onSvgClicked); + svg.addEventListener('touchstart', onSvgTouch); + svg.addEventListener('touchend', onSvgTouch); + + unpausePicker(); + + if ( pickerRoot.classList.contains('zap') ) { return; } + + cmEditor.on('changes', onCandidateChanged); + + $id('preview').addEventListener('click', onPreviewClicked); + $id('create').addEventListener('click', onCreateClicked); + $id('pick').addEventListener('click', onPickClicked); + $id('quit').addEventListener('click', onQuitClicked); + $id('move').addEventListener('mousedown', onStartMoving); + $id('move').addEventListener('touchstart', onStartMoving); + $id('candidateFilters').addEventListener('click', onCandidateClicked); + $stor('#resultsetDepth input').addEventListener('input', onDepthChanged); + $stor('#resultsetSpecificity input').addEventListener('input', onSpecificityChanged); + staticFilteringParser = new sfp.AstFilterParser({ + interactive: true, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); +}; + +/******************************************************************************/ + +const quitPicker = function() { + pickerContentPort.postMessage({ what: 'quitPicker' }); + pickerContentPort.close(); + pickerContentPort = undefined; +}; + +/******************************************************************************/ + +const onPickerMessage = function(msg) { + switch ( msg.what ) { + case 'candidatesOptimized': + onCandidatesOptimized(msg); + break; + case 'showDialog': + showDialog(msg); + break; + case 'resultsetDetails': { + resultsetOpt = msg.opt; + $id('resultsetCount').textContent = msg.count; + if ( msg.count !== 0 ) { + $id('create').removeAttribute('disabled'); + } else { + $id('create').setAttribute('disabled', ''); + } + break; + } + case 'svgPaths': { + let { ocean, islands } = msg; + ocean += islands; + svgOcean.setAttribute('d', ocean); + svgIslands.setAttribute('d', islands || NoPaths); + break; + } + default: + break; + } +}; + +/******************************************************************************/ + +// Wait for the content script to establish communication + +let pickerContentPort; + +globalThis.addEventListener('message', ev => { + const msg = ev.data || {}; + if ( msg.what !== 'epickerStart' ) { return; } + if ( Array.isArray(ev.ports) === false ) { return; } + if ( ev.ports.length === 0 ) { return; } + pickerContentPort = ev.ports[0]; + pickerContentPort.onmessage = ev => { + const msg = ev.data || {}; + onPickerMessage(msg); + }; + pickerContentPort.onmessageerror = ( ) => { + quitPicker(); + }; + startPicker(); + pickerContentPort.postMessage({ what: 'start' }); +}, { once: true }); + +/******************************************************************************/ + +})(); diff --git a/src/js/fa-icons.js b/src/js/fa-icons.js new file mode 100644 index 0000000..79968d0 --- /dev/null +++ b/src/js/fa-icons.js @@ -0,0 +1,129 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2018-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uMatrix +*/ + +'use strict'; + +/******************************************************************************/ + +export const faIconsInit = (( ) => { + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1196 + const svgIcons = new Map([ + // See /img/fontawesome/fontawesome-defs.svg + [ 'angle-up', { viewBox: '0 0 998 582', path: 'm 998,499 q 0,13 -10,23 l -50,50 q -10,10 -23,10 -13,0 -23,-10 L 499,179 106,572 Q 96,582 83,582 70,582 60,572 L 10,522 Q 0,512 0,499 0,486 10,476 L 476,10 q 10,-10 23,-10 13,0 23,10 l 466,466 q 10,10 10,23 z' } ], + [ 'arrow-right', { viewBox: '0 0 1472 1558', path: 'm 1472,779 q 0,54 -37,91 l -651,651 q -39,37 -91,37 -51,0 -90,-37 l -75,-75 q -38,-38 -38,-91 0,-53 38,-91 L 821,971 H 117 Q 65,971 32.5,933.5 0,896 0,843 V 715 Q 0,662 32.5,624.5 65,587 117,587 H 821 L 528,293 q -38,-36 -38,-90 0,-54 38,-90 l 75,-75 q 38,-38 90,-38 53,0 91,38 l 651,651 q 37,35 37,90 z' } ], + [ 'bar-chart', { viewBox: '0 0 2048 1536', path: 'm 640,768 0,512 -256,0 0,-512 256,0 z m 384,-512 0,1024 -256,0 0,-1024 256,0 z m 1024,1152 0,128 L 0,1536 0,0 l 128,0 0,1408 1920,0 z m -640,-896 0,768 -256,0 0,-768 256,0 z m 384,-384 0,1152 -256,0 0,-1152 256,0 z' } ], + [ 'bolt', { viewBox: '0 0 896 1664', path: 'm 885.08696,438 q 18,20 7,44 l -540,1157 q -13,25 -42,25 -4,0 -14,-2 -17,-5 -25.5,-19 -8.5,-14 -4.5,-30 l 197,-808 -406,101 q -4,1 -12,1 -18,0 -31,-11 Q -3.9130435,881 1.0869565,857 L 202.08696,32 q 4,-14 16,-23 12,-9 28,-9 l 328,0 q 19,0 32,12.5 13,12.5 13,29.5 0,8 -5,18 l -171,463 396,-98 q 8,-2 12,-2 19,0 34,15 z' } ], + [ 'clipboard', { viewBox: '0 0 1792 1792', path: 'm 768,1664 896,0 0,-640 -416,0 q -40,0 -68,-28 -28,-28 -28,-68 l 0,-416 -384,0 0,1152 z m 256,-1440 0,-64 q 0,-13 -9.5,-22.5 Q 1005,128 992,128 l -704,0 q -13,0 -22.5,9.5 Q 256,147 256,160 l 0,64 q 0,13 9.5,22.5 9.5,9.5 22.5,9.5 l 704,0 q 13,0 22.5,-9.5 9.5,-9.5 9.5,-22.5 z m 256,672 299,0 -299,-299 0,299 z m 512,128 0,672 q 0,40 -28,68 -28,28 -68,28 l -960,0 q -40,0 -68,-28 -28,-28 -28,-68 l 0,-160 -544,0 Q 56,1536 28,1508 0,1480 0,1440 L 0,96 Q 0,56 28,28 56,0 96,0 l 1088,0 q 40,0 68,28 28,28 28,68 l 0,328 q 21,13 36,28 l 408,408 q 28,28 48,76 20,48 20,88 z' } ], + [ 'clock-o', { viewBox: '0 0 1536 1536', path: 'm 896,416 v 448 q 0,14 -9,23 -9,9 -23,9 H 544 q -14,0 -23,-9 -9,-9 -9,-23 v -64 q 0,-14 9,-23 9,-9 23,-9 H 768 V 416 q 0,-14 9,-23 9,-9 23,-9 h 64 q 14,0 23,9 9,9 9,23 z m 416,352 q 0,-148 -73,-273 -73,-125 -198,-198 -125,-73 -273,-73 -148,0 -273,73 -125,73 -198,198 -73,125 -73,273 0,148 73,273 73,125 198,198 125,73 273,73 148,0 273,-73 125,-73 198,-198 73,-125 73,-273 z m 224,0 q 0,209 -103,385.5 Q 1330,1330 1153.5,1433 977,1536 768,1536 559,1536 382.5,1433 206,1330 103,1153.5 0,977 0,768 0,559 103,382.5 206,206 382.5,103 559,0 768,0 977,0 1153.5,103 1330,206 1433,382.5 1536,559 1536,768 Z' } ], + [ 'cloud-download', { viewBox: '0 0 1920 1408', path: 'm 1280,800 q 0,-14 -9,-23 -9,-9 -23,-9 l -224,0 0,-352 q 0,-13 -9.5,-22.5 Q 1005,384 992,384 l -192,0 q -13,0 -22.5,9.5 Q 768,403 768,416 l 0,352 -224,0 q -13,0 -22.5,9.5 -9.5,9.5 -9.5,22.5 0,14 9,23 l 352,352 q 9,9 23,9 14,0 23,-9 l 351,-351 q 10,-12 10,-24 z m 640,224 q 0,159 -112.5,271.5 Q 1695,1408 1536,1408 l -1088,0 Q 263,1408 131.5,1276.5 0,1145 0,960 0,830 70,720 140,610 258,555 256,525 256,512 256,300 406,150 556,0 768,0 q 156,0 285.5,87 129.5,87 188.5,231 71,-62 166,-62 106,0 181,75 75,75 75,181 0,76 -41,138 130,31 213.5,135.5 Q 1920,890 1920,1024 Z' } ], + [ 'cloud-upload', { viewBox: '0 0 1920 1408', path: 'm 1280,736 q 0,-14 -9,-23 L 919,361 q -9,-9 -23,-9 -14,0 -23,9 L 522,712 q -10,12 -10,24 0,14 9,23 9,9 23,9 l 224,0 0,352 q 0,13 9.5,22.5 9.5,9.5 22.5,9.5 l 192,0 q 13,0 22.5,-9.5 9.5,-9.5 9.5,-22.5 l 0,-352 224,0 q 13,0 22.5,-9.5 9.5,-9.5 9.5,-22.5 z m 640,288 q 0,159 -112.5,271.5 Q 1695,1408 1536,1408 l -1088,0 Q 263,1408 131.5,1276.5 0,1145 0,960 0,830 70,720 140,610 258,555 256,525 256,512 256,300 406,150 556,0 768,0 q 156,0 285.5,87 129.5,87 188.5,231 71,-62 166,-62 106,0 181,75 75,75 75,181 0,76 -41,138 130,31 213.5,135.5 Q 1920,890 1920,1024 Z' } ], + [ 'check', { viewBox: '0 0 1550 1188', path: 'm 1550,232 q 0,40 -28,68 l -724,724 -136,136 q -28,28 -68,28 -40,0 -68,-28 L 390,1024 28,662 Q 0,634 0,594 0,554 28,526 L 164,390 q 28,-28 68,-28 40,0 68,28 L 594,685 1250,28 q 28,-28 68,-28 40,0 68,28 l 136,136 q 28,28 28,68 z' } ], + [ 'code', { viewBox: '0 0 1830 1373', path: 'm 572,1125.5 -50,50 q -10,10 -23,10 -13,0 -23,-10 l -466,-466 q -10,-10 -10,-23 0,-13 10,-23 l 466,-466 q 10,-10 23,-10 13,0 23,10 l 50,50 q 10,10 10,23 0,13 -10,23 l -393,393 393,393 q 10,10 10,23 0,13 -10,23 z M 1163,58.476203 790,1349.4762 q -4,13 -15.5,19.5 -11.5,6.5 -23.5,2.5 l -62,-17 q -13,-4 -19.5,-15.5 -6.5,-11.5 -2.5,-24.5 L 1040,23.5 q 4,-13 15.5,-19.5 11.5,-6.5 23.5,-2.5 l 62,17 q 13,4 19.5,15.5 6.5,11.5 2.5,24.5 z m 657,651 -466,466 q -10,10 -23,10 -13,0 -23,-10 l -50,-50 q -10,-10 -10,-23 0,-13 10,-23 l 393,-393 -393,-393 q -10,-10 -10,-23 0,-13 10,-23 l 50,-50 q 10,-10 23,-10 13,0 23,10 l 466,466 q 10,10 10,23 0,13 -10,23 z' } ], + [ 'cog', { viewBox: '0 0 1536 1536', path: 'm 1024,768 q 0,-106 -75,-181 -75,-75 -181,-75 -106,0 -181,75 -75,75 -75,181 0,106 75,181 75,75 181,75 106,0 181,-75 75,-75 75,-181 z m 512,-109 0,222 q 0,12 -8,23 -8,11 -20,13 l -185,28 q -19,54 -39,91 35,50 107,138 10,12 10,25 0,13 -9,23 -27,37 -99,108 -72,71 -94,71 -12,0 -26,-9 l -138,-108 q -44,23 -91,38 -16,136 -29,186 -7,28 -36,28 l -222,0 q -14,0 -24.5,-8.5 Q 622,1519 621,1506 l -28,-184 q -49,-16 -90,-37 l -141,107 q -10,9 -25,9 -14,0 -25,-11 -126,-114 -165,-168 -7,-10 -7,-23 0,-12 8,-23 15,-21 51,-66.5 36,-45.5 54,-70.5 -27,-50 -41,-99 L 29,913 Q 16,911 8,900.5 0,890 0,877 L 0,655 q 0,-12 8,-23 8,-11 19,-13 l 186,-28 q 14,-46 39,-92 -40,-57 -107,-138 -10,-12 -10,-24 0,-10 9,-23 26,-36 98.5,-107.5 Q 315,135 337,135 q 13,0 26,10 L 501,252 Q 545,229 592,214 608,78 621,28 628,0 657,0 L 879,0 Q 893,0 903.5,8.5 914,17 915,30 l 28,184 q 49,16 90,37 l 142,-107 q 9,-9 24,-9 13,0 25,10 129,119 165,170 7,8 7,22 0,12 -8,23 -15,21 -51,66.5 -36,45.5 -54,70.5 26,50 41,98 l 183,28 q 13,2 21,12.5 8,10.5 8,23.5 z' } ], + [ 'cogs', { viewBox: '0 0 1920 1761', path: 'm 896,880 q 0,-106 -75,-181 -75,-75 -181,-75 -106,0 -181,75 -75,75 -75,181 0,106 75,181 75,75 181,75 106,0 181,-75 75,-75 75,-181 z m 768,512 q 0,-52 -38,-90 -38,-38 -90,-38 -52,0 -90,38 -38,38 -38,90 0,53 37.5,90.5 37.5,37.5 90.5,37.5 53,0 90.5,-37.5 37.5,-37.5 37.5,-90.5 z m 0,-1024 q 0,-52 -38,-90 -38,-38 -90,-38 -52,0 -90,38 -38,38 -38,90 0,53 37.5,90.5 37.5,37.5 90.5,37.5 53,0 90.5,-37.5 Q 1664,421 1664,368 Z m -384,421 v 185 q 0,10 -7,19.5 -7,9.5 -16,10.5 l -155,24 q -11,35 -32,76 34,48 90,115 7,11 7,20 0,12 -7,19 -23,30 -82.5,89.5 -59.5,59.5 -78.5,59.5 -11,0 -21,-7 l -115,-90 q -37,19 -77,31 -11,108 -23,155 -7,24 -30,24 H 547 q -11,0 -20,-7.5 -9,-7.5 -10,-17.5 l -23,-153 q -34,-10 -75,-31 l -118,89 q -7,7 -20,7 -11,0 -21,-8 -144,-133 -144,-160 0,-9 7,-19 10,-14 41,-53 31,-39 47,-61 -23,-44 -35,-82 L 24,1000 Q 14,999 7,990.5 0,982 0,971 V 786 Q 0,776 7,766.5 14,757 23,756 l 155,-24 q 11,-35 32,-76 -34,-48 -90,-115 -7,-11 -7,-20 0,-12 7,-20 22,-30 82,-89 60,-59 79,-59 11,0 21,7 l 115,90 q 34,-18 77,-32 11,-108 23,-154 7,-24 30,-24 h 186 q 11,0 20,7.5 9,7.5 10,17.5 l 23,153 q 34,10 75,31 l 118,-89 q 8,-7 20,-7 11,0 21,8 144,133 144,160 0,8 -7,19 -12,16 -42,54 -30,38 -45,60 23,48 34,82 l 152,23 q 10,2 17,10.5 7,8.5 7,19.5 z m 640,533 v 140 q 0,16 -149,31 -12,27 -30,52 51,113 51,138 0,4 -4,7 -122,71 -124,71 -8,0 -46,-47 -38,-47 -52,-68 -20,2 -30,2 -10,0 -30,-2 -14,21 -52,68 -38,47 -46,47 -2,0 -124,-71 -4,-3 -4,-7 0,-25 51,-138 -18,-25 -30,-52 -149,-15 -149,-31 v -140 q 0,-16 149,-31 13,-29 30,-52 -51,-113 -51,-138 0,-4 4,-7 4,-2 35,-20 31,-18 59,-34 28,-16 30,-16 8,0 46,46.5 38,46.5 52,67.5 20,-2 30,-2 10,0 30,2 51,-71 92,-112 l 6,-2 q 4,0 124,70 4,3 4,7 0,25 -51,138 17,23 30,52 149,15 149,31 z m 0,-1024 v 140 q 0,16 -149,31 -12,27 -30,52 51,113 51,138 0,4 -4,7 -122,71 -124,71 -8,0 -46,-47 -38,-47 -52,-68 -20,2 -30,2 -10,0 -30,-2 -14,21 -52,68 -38,47 -46,47 -2,0 -124,-71 -4,-3 -4,-7 0,-25 51,-138 -18,-25 -30,-52 -149,-15 -149,-31 V 298 q 0,-16 149,-31 13,-29 30,-52 -51,-113 -51,-138 0,-4 4,-7 4,-2 35,-20 31,-18 59,-34 28,-16 30,-16 8,0 46,46.5 38,46.5 52,67.5 20,-2 30,-2 10,0 30,2 51,-71 92,-112 l 6,-2 q 4,0 124,70 4,3 4,7 0,25 -51,138 17,23 30,52 149,15 149,31 z' } ], + [ 'comment-alt', { viewBox: '0 0 1792 1536', path: 'M 896,128 Q 692,128 514.5,197.5 337,267 232.5,385 128,503 128,640 128,752 199.5,853.5 271,955 401,1029 l 87,50 -27,96 q -24,91 -70,172 152,-63 275,-171 l 43,-38 57,6 q 69,8 130,8 204,0 381.5,-69.5 Q 1455,1013 1559.5,895 1664,777 1664,640 1664,503 1559.5,385 1455,267 1277.5,197.5 1100,128 896,128 Z m 896,512 q 0,174 -120,321.5 -120,147.5 -326,233 -206,85.5 -450,85.5 -70,0 -145,-8 -198,175 -460,242 -49,14 -114,22 h -5 q -15,0 -27,-10.5 -12,-10.5 -16,-27.5 v -1 q -3,-4 -0.5,-12 2.5,-8 2,-10 -0.5,-2 4.5,-9.5 l 6,-9 q 0,0 7,-8.5 7,-8.5 8,-9 7,-8 31,-34.5 24,-26.5 34.5,-38 10.5,-11.5 31,-39.5 20.5,-28 32.5,-51 12,-23 27,-59 15,-36 26,-76 Q 181,1052 90.5,921 0,790 0,640 0,466 120,318.5 240,171 446,85.5 652,0 896,0 q 244,0 450,85.5 206,85.5 326,233 120,147.5 120,321.5 z' } ], + [ 'double-angle-left', { viewBox: '0 0 966 998', path: 'm 582,915 q 0,13 -10,23 l -50,50 q -10,10 -23,10 -13,0 -23,-10 L 10,522 Q 0,512 0,499 0,486 10,476 L 476,10 q 10,-10 23,-10 13,0 23,10 l 50,50 q 10,10 10,23 0,13 -10,23 L 179,499 572,892 q 10,10 10,23 z m 384,0 q 0,13 -10,23 l -50,50 q -10,10 -23,10 -13,0 -23,-10 L 394,522 q -10,-10 -10,-23 0,-13 10,-23 L 860,10 q 10,-10 23,-10 13,0 23,10 l 50,50 q 10,10 10,23 0,13 -10,23 L 563,499 956,892 q 10,10 10,23 z' } ], + [ 'double-angle-up', { viewBox: '0 0 998 966', path: 'm 998,883 q 0,13 -10,23 l -50,50 q -10,10 -23,10 -13,0 -23,-10 L 499,563 106,956 Q 96,966 83,966 70,966 60,956 L 10,906 Q 0,896 0,883 0,870 10,860 L 476,394 q 10,-10 23,-10 13,0 23,10 l 466,466 q 10,10 10,23 z m 0,-384 q 0,13 -10,23 l -50,50 q -10,10 -23,10 -13,0 -23,-10 L 499,179 106,572 Q 96,582 83,582 70,582 60,572 L 10,522 Q 0,512 0,499 0,486 10,476 L 476,10 q 10,-10 23,-10 13,0 23,10 l 466,466 q 10,10 10,23 z' } ], + [ 'download-alt', { viewBox: '0 0 1664 1536', path: 'm 1280,1344 q 0,-26 -19,-45 -19,-19 -45,-19 -26,0 -45,19 -19,19 -19,45 0,26 19,45 19,19 45,19 26,0 45,-19 19,-19 19,-45 z m 256,0 q 0,-26 -19,-45 -19,-19 -45,-19 -26,0 -45,19 -19,19 -19,45 0,26 19,45 19,19 45,19 26,0 45,-19 19,-19 19,-45 z m 128,-224 v 320 q 0,40 -28,68 -28,28 -68,28 H 96 q -40,0 -68,-28 -28,-28 -28,-68 v -320 q 0,-40 28,-68 28,-28 68,-28 h 465 l 135,136 q 58,56 136,56 78,0 136,-56 l 136,-136 h 464 q 40,0 68,28 28,28 28,68 z M 1339,551 q 17,41 -14,70 l -448,448 q -18,19 -45,19 -27,0 -45,-19 L 339,621 q -31,-29 -14,-70 17,-39 59,-39 H 640 V 64 Q 640,38 659,19 678,0 704,0 h 256 q 26,0 45,19 19,19 19,45 v 448 h 256 q 42,0 59,39 z' } ], + [ 'eraser', { viewBox: '0 0 1920 1280', path: 'M 896,1152 1232,768 l -768,0 -336,384 768,0 z M 1909,75 q 15,34 9.5,71.5 Q 1913,184 1888,212 L 992,1236 q -38,44 -96,44 l -768,0 q -38,0 -69.5,-20.5 -31.5,-20.5 -47.5,-54.5 -15,-34 -9.5,-71.5 5.5,-37.5 30.5,-65.5 L 928,44 Q 966,0 1024,0 l 768,0 q 38,0 69.5,20.5 Q 1893,41 1909,75 Z' } ], + [ 'exclamation-triangle', { viewBox: '0 0 1794 1664', path: 'm 1025.0139,1375 0,-190 q 0,-14 -9.5,-23.5 -9.5,-9.5 -22.5,-9.5 l -192,0 q -13,0 -22.5,9.5 -9.5,9.5 -9.5,23.5 l 0,190 q 0,14 9.5,23.5 9.5,9.5 22.5,9.5 l 192,0 q 13,0 22.5,-9.5 9.5,-9.5 9.5,-23.5 z m -2,-374 18,-459 q 0,-12 -10,-19 -13,-11 -24,-11 l -220,0 q -11,0 -24,11 -10,7 -10,21 l 17,457 q 0,10 10,16.5 10,6.5 24,6.5 l 185,0 q 14,0 23.5,-6.5 9.5,-6.5 10.5,-16.5 z m -14,-934 768,1408 q 35,63 -2,126 -17,29 -46.5,46 -29.5,17 -63.5,17 l -1536,0 q -34,0 -63.5,-17 -29.5,-17 -46.5,-46 -37,-63 -2,-126 L 785.01389,67 q 17,-31 47,-49 30,-18 65,-18 35,0 65,18 30,18 47,49 z' } ], + [ 'external-link', { viewBox: '0 0 1792 1536', path: 'm 1408,928 0,320 q 0,119 -84.5,203.5 Q 1239,1536 1120,1536 l -832,0 Q 169,1536 84.5,1451.5 0,1367 0,1248 L 0,416 Q 0,297 84.5,212.5 169,128 288,128 l 704,0 q 14,0 23,9 9,9 9,23 l 0,64 q 0,14 -9,23 -9,9 -23,9 l -704,0 q -66,0 -113,47 -47,47 -47,113 l 0,832 q 0,66 47,113 47,47 113,47 l 832,0 q 66,0 113,-47 47,-47 47,-113 l 0,-320 q 0,-14 9,-23 9,-9 23,-9 l 64,0 q 14,0 23,9 9,9 9,23 z m 384,-864 0,512 q 0,26 -19,45 -19,19 -45,19 -26,0 -45,-19 L 1507,445 855,1097 q -10,10 -23,10 -13,0 -23,-10 L 695,983 q -10,-10 -10,-23 0,-13 10,-23 L 1347,285 1171,109 q -19,-19 -19,-45 0,-26 19,-45 19,-19 45,-19 l 512,0 q 26,0 45,19 19,19 19,45 z' } ], + [ 'eye-dropper', { viewBox: '0 0 1792 1792', path: 'm 1698,94 q 94,94 94,226.5 0,132.5 -94,225.5 l -225,223 104,104 q 10,10 10,23 0,13 -10,23 l -210,210 q -10,10 -23,10 -13,0 -23,-10 l -105,-105 -603,603 q -37,37 -90,37 l -203,0 -256,128 -64,-64 128,-256 0,-203 q 0,-53 37,-90 L 768,576 663,471 q -10,-10 -10,-23 0,-13 10,-23 L 873,215 q 10,-10 23,-10 13,0 23,10 L 1023,319 1246,94 Q 1339,0 1471.5,0 1604,0 1698,94 Z M 512,1472 1088,896 896,704 l -576,576 0,192 192,0 z' } ], + [ 'eye-open', { viewBox: '0 0 1792 1152', path: 'm 1664,576 q -152,-236 -381,-353 61,104 61,225 0,185 -131.5,316.5 Q 1081,896 896,896 711,896 579.5,764.5 448,633 448,448 448,327 509,223 280,340 128,576 261,781 461.5,902.5 662,1024 896,1024 1130,1024 1330.5,902.5 1531,781 1664,576 Z M 944,192 q 0,-20 -14,-34 -14,-14 -34,-14 -125,0 -214.5,89.5 Q 592,323 592,448 q 0,20 14,34 14,14 34,14 20,0 34,-14 14,-14 14,-34 0,-86 61,-147 61,-61 147,-61 20,0 34,-14 14,-14 14,-34 z m 848,384 q 0,34 -20,69 -140,230 -376.5,368.5 Q 1159,1152 896,1152 633,1152 396.5,1013 160,874 20,645 0,610 0,576 0,542 20,507 160,278 396.5,139 633,0 896,0 q 263,0 499.5,139 236.5,139 376.5,368 20,35 20,69 z' } ], + [ 'eye-slash', { viewBox: '0 0 1792 1344', path: 'M 555,1047 633,906 Q 546,843 497,747 448,651 448,544 448,423 509,319 280,436 128,672 295,930 555,1047 Z M 944,288 q 0,-20 -14,-34 -14,-14 -34,-14 -125,0 -214.5,89.5 Q 592,419 592,544 q 0,20 14,34 14,14 34,14 20,0 34,-14 14,-14 14,-34 0,-86 61,-147 61,-61 147,-61 20,0 34,-14 14,-14 14,-34 z M 1307,97 q 0,7 -1,9 -106,189 -316,567 -210,378 -315,566 l -49,89 q -10,16 -28,16 -12,0 -134,-70 -16,-10 -16,-28 0,-12 44,-87 Q 349,1094 228.5,986 108,878 20,741 0,710 0,672 0,634 20,603 173,368 400,232 627,96 896,96 q 89,0 180,17 l 54,-97 q 10,-16 28,-16 5,0 18,6 13,6 31,15.5 18,9.5 33,18.5 15,9 31.5,18.5 16.5,9.5 19.5,11.5 16,10 16,27 z m 37,447 q 0,139 -79,253.5 Q 1186,912 1056,962 l 280,-502 q 8,45 8,84 z m 448,128 q 0,35 -20,69 -39,64 -109,145 -150,172 -347.5,267 -197.5,95 -419.5,95 l 74,-132 Q 1182,1098 1362.5,979 1543,860 1664,672 1549,493 1382,378 l 63,-112 q 95,64 182.5,153 87.5,89 144.5,184 20,34 20,69 z' } ], + [ 'files-o', { viewBox: '0 0 1792 1792', path: 'm 1696,384 q 40,0 68,28 28,28 28,68 l 0,1216 q 0,40 -28,68 -28,28 -68,28 l -960,0 q -40,0 -68,-28 -28,-28 -28,-68 l 0,-288 -544,0 Q 56,1408 28,1380 0,1352 0,1312 L 0,640 Q 0,600 20,552 40,504 68,476 L 476,68 Q 504,40 552,20 600,0 640,0 l 416,0 q 40,0 68,28 28,28 28,68 l 0,328 q 68,-40 128,-40 l 416,0 z m -544,213 -299,299 299,0 0,-299 z M 512,213 213,512 l 299,0 0,-299 z m 196,647 316,-316 0,-416 -384,0 0,416 q 0,40 -28,68 -28,28 -68,28 l -416,0 0,640 512,0 0,-256 q 0,-40 20,-88 20,-48 48,-76 z m 956,804 0,-1152 -384,0 0,416 q 0,40 -28,68 -28,28 -68,28 l -416,0 0,640 896,0 z' } ], + [ 'film', { viewBox: '0 0 1920 1664', path: 'm 384,1472 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 0,-384 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 0,-384 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 1024,768 0,-512 q 0,-26 -19,-45 -19,-19 -45,-19 l -768,0 q -26,0 -45,19 -19,19 -19,45 l 0,512 q 0,26 19,45 19,19 45,19 l 768,0 q 26,0 45,-19 19,-19 19,-45 z M 384,320 384,192 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 1408,1152 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m -384,-768 0,-512 q 0,-26 -19,-45 -19,-19 -45,-19 l -768,0 q -26,0 -45,19 -19,19 -19,45 l 0,512 q 0,26 19,45 19,19 45,19 l 768,0 q 26,0 45,-19 19,-19 19,-45 z m 384,384 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 0,-384 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 0,-384 0,-128 q 0,-26 -19,-45 -19,-19 -45,-19 l -128,0 q -26,0 -45,19 -19,19 -19,45 l 0,128 q 0,26 19,45 19,19 45,19 l 128,0 q 26,0 45,-19 19,-19 19,-45 z m 128,-160 0,1344 q 0,66 -47,113 -47,47 -113,47 l -1600,0 Q 94,1664 47,1617 0,1570 0,1504 L 0,160 Q 0,94 47,47 94,0 160,0 l 1600,0 q 66,0 113,47 47,47 47,113 z' } ], + [ 'filter', { viewBox: '0 0 1410 1408', path: 'm 1404.0208,39 q 17,41 -14,70 l -493,493 0,742 q 0,42 -39,59 -13,5 -25,5 -27,0 -45,-19 l -256,-256 q -19,-19 -19,-45 l 0,-486 L 20.020833,109 q -31,-29 -14,-70 Q 23.020833,0 65.020833,0 L 1345.0208,0 q 42,0 59,39 z' } ], + [ 'floppy-o', { viewBox: '0 0 1536 1536', path: 'm 384,1408 768,0 0,-384 -768,0 0,384 z m 896,0 128,0 0,-896 q 0,-14 -10,-38.5 Q 1388,449 1378,439 L 1097,158 q -10,-10 -34,-20 -24,-10 -39,-10 l 0,416 q 0,40 -28,68 -28,28 -68,28 l -576,0 q -40,0 -68,-28 -28,-28 -28,-68 l 0,-416 -128,0 0,1280 128,0 0,-416 q 0,-40 28,-68 28,-28 68,-28 l 832,0 q 40,0 68,28 28,28 28,68 l 0,416 z M 896,480 896,160 q 0,-13 -9.5,-22.5 Q 877,128 864,128 l -192,0 q -13,0 -22.5,9.5 Q 640,147 640,160 l 0,320 q 0,13 9.5,22.5 9.5,9.5 22.5,9.5 l 192,0 q 13,0 22.5,-9.5 Q 896,493 896,480 Z m 640,32 0,928 q 0,40 -28,68 -28,28 -68,28 L 96,1536 Q 56,1536 28,1508 0,1480 0,1440 L 0,96 Q 0,56 28,28 56,0 96,0 l 928,0 q 40,0 88,20 48,20 76,48 l 280,280 q 28,28 48,76 20,48 20,88 z' } ], + [ 'font', { viewBox: '0 0 1664 1536', path: 'M 725,431 555,881 q 33,0 136.5,2 103.5,2 160.5,2 19,0 57,-2 Q 822,630 725,431 Z M 0,1536 2,1457 q 23,-7 56,-12.5 33,-5.5 57,-10.5 24,-5 49.5,-14.5 25.5,-9.5 44.5,-29 19,-19.5 31,-50.5 L 477,724 757,0 l 75,0 53,0 q 8,14 11,21 l 205,480 q 33,78 106,257.5 73,179.5 114,274.5 15,34 58,144.5 43,110.5 72,168.5 20,45 35,57 19,15 88,29.5 69,14.5 84,20.5 6,38 6,57 0,5 -0.5,13.5 -0.5,8.5 -0.5,12.5 -63,0 -190,-8 -127,-8 -191,-8 -76,0 -215,7 -139,7 -178,8 0,-43 4,-78 l 131,-28 q 1,0 12.5,-2.5 11.5,-2.5 15.5,-3.5 4,-1 14.5,-4.5 10.5,-3.5 15,-6.5 4.5,-3 11,-8 6.5,-5 9,-11 2.5,-6 2.5,-14 0,-16 -31,-96.5 -31,-80.5 -72,-177.5 -41,-97 -42,-100 l -450,-2 q -26,58 -76.5,195.5 Q 382,1336 382,1361 q 0,22 14,37.5 14,15.5 43.5,24.5 29.5,9 48.5,13.5 19,4.5 57,8.5 38,4 41,4 1,19 1,58 0,9 -2,27 -58,0 -174.5,-10 -116.5,-10 -174.5,-10 -8,0 -26.5,4 -18.5,4 -21.5,4 -80,14 -188,14 z' } ], + [ 'home', { viewBox: '0 0 1612 1283', path: 'm 1382.1111,739 v 480 q 0,26 -19,45 -19,19 -45,19 H 934.11111 V 899 h -256 v 384 h -384 q -26,0 -45,-19 -19,-19 -19,-45 V 739 q 0,-1 0.5,-3 0.5,-2 0.5,-3 l 575,-474 574.99999,474 q 1,2 1,6 z m 223,-69 -62,74 q -8,9 -21,11 h -3 q -13,0 -21,-7 l -691.99999,-577 -692,577 q -12,8 -23.999999,7 -13,-2 -21,-11 L 7.1111111,670 Q -0.88888889,660 0.11111111,646.5 1.1111111,633 11.111111,625 L 730.11111,26 q 32,-26 76,-26 44,0 76,26 L 1126.1111,230 V 35 q 0,-14 9,-23 9,-9 23,-9 h 192 q 14,0 23,9 9,9 9,23 v 408 l 219,182 q 10,8 11,21.5 1,13.5 -7,23.5 z' } ], + [ 'info-circle', { viewBox: '0 0 1536 1536', path: 'm 1024,1248 0,-160 q 0,-14 -9,-23 -9,-9 -23,-9 l -96,0 0,-512 q 0,-14 -9,-23 -9,-9 -23,-9 l -320,0 q -14,0 -23,9 -9,9 -9,23 l 0,160 q 0,14 9,23 9,9 23,9 l 96,0 0,320 -96,0 q -14,0 -23,9 -9,9 -9,23 l 0,160 q 0,14 9,23 9,9 23,9 l 448,0 q 14,0 23,-9 9,-9 9,-23 z M 896,352 896,192 q 0,-14 -9,-23 -9,-9 -23,-9 l -192,0 q -14,0 -23,9 -9,9 -9,23 l 0,160 q 0,14 9,23 9,9 23,9 l 192,0 q 14,0 23,-9 9,-9 9,-23 z m 640,416 q 0,209 -103,385.5 Q 1330,1330 1153.5,1433 977,1536 768,1536 559,1536 382.5,1433 206,1330 103,1153.5 0,977 0,768 0,559 103,382.5 206,206 382.5,103 559,0 768,0 977,0 1153.5,103 1330,206 1433,382.5 1536,559 1536,768 Z' } ], + [ 'list-alt', { viewBox: '0 0 1792 1408', path: 'm 384,1056 0,64 q 0,13 -9.5,22.5 -9.5,9.5 -22.5,9.5 l -64,0 q -13,0 -22.5,-9.5 Q 256,1133 256,1120 l 0,-64 q 0,-13 9.5,-22.5 9.5,-9.5 22.5,-9.5 l 64,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 0,-256 0,64 q 0,13 -9.5,22.5 Q 365,896 352,896 l -64,0 q -13,0 -22.5,-9.5 Q 256,877 256,864 l 0,-64 q 0,-13 9.5,-22.5 Q 275,768 288,768 l 64,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 0,-256 0,64 q 0,13 -9.5,22.5 Q 365,640 352,640 l -64,0 q -13,0 -22.5,-9.5 Q 256,621 256,608 l 0,-64 q 0,-13 9.5,-22.5 Q 275,512 288,512 l 64,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 1152,512 0,64 q 0,13 -9.5,22.5 -9.5,9.5 -22.5,9.5 l -960,0 q -13,0 -22.5,-9.5 Q 512,1133 512,1120 l 0,-64 q 0,-13 9.5,-22.5 9.5,-9.5 22.5,-9.5 l 960,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 0,-256 0,64 q 0,13 -9.5,22.5 -9.5,9.5 -22.5,9.5 l -960,0 q -13,0 -22.5,-9.5 Q 512,877 512,864 l 0,-64 q 0,-13 9.5,-22.5 Q 531,768 544,768 l 960,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 0,-256 0,64 q 0,13 -9.5,22.5 -9.5,9.5 -22.5,9.5 l -960,0 q -13,0 -22.5,-9.5 Q 512,621 512,608 l 0,-64 q 0,-13 9.5,-22.5 Q 531,512 544,512 l 960,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 128,704 0,-832 q 0,-13 -9.5,-22.5 Q 1645,384 1632,384 l -1472,0 q -13,0 -22.5,9.5 Q 128,403 128,416 l 0,832 q 0,13 9.5,22.5 9.5,9.5 22.5,9.5 l 1472,0 q 13,0 22.5,-9.5 9.5,-9.5 9.5,-22.5 z m 128,-1088 0,1088 q 0,66 -47,113 -47,47 -113,47 l -1472,0 Q 94,1408 47,1361 0,1314 0,1248 L 0,160 Q 0,94 47,47 94,0 160,0 l 1472,0 q 66,0 113,47 47,47 47,113 z' } ], + [ 'lock', { viewBox: '0 0 1152 1408', path: 'm 320,640 512,0 0,-192 q 0,-106 -75,-181 -75,-75 -181,-75 -106,0 -181,75 -75,75 -75,181 l 0,192 z m 832,96 0,576 q 0,40 -28,68 -28,28 -68,28 l -960,0 Q 56,1408 28,1380 0,1352 0,1312 L 0,736 q 0,-40 28,-68 28,-28 68,-28 l 32,0 0,-192 Q 128,264 260,132 392,0 576,0 q 184,0 316,132 132,132 132,316 l 0,192 32,0 q 40,0 68,28 28,28 28,68 z' } ], + [ 'magic', { viewBox: '0 0 1637 1637', path: 'M 1163,581 1456,288 1349,181 1056,474 Z m 447,-293 q 0,27 -18,45 L 306,1619 q -18,18 -45,18 -27,0 -45,-18 L 18,1421 Q 0,1403 0,1376 0,1349 18,1331 L 1304,45 q 18,-18 45,-18 27,0 45,18 l 198,198 q 18,18 18,45 z M 259,98 l 98,30 -98,30 -30,98 -30,-98 -98,-30 98,-30 30,-98 z M 609,260 805,320 609,380 549,576 489,380 293,320 489,260 549,64 Z m 930,478 98,30 -98,30 -30,98 -30,-98 -98,-30 98,-30 30,-98 z M 899,98 l 98,30 -98,30 -30,98 -30,-98 -98,-30 98,-30 30,-98 z' } ], + [ 'pause-circle-o', { viewBox: '0 0 1536 1536', path: 'M 768,0 Q 977,0 1153.5,103 1330,206 1433,382.5 1536,559 1536,768 1536,977 1433,1153.5 1330,1330 1153.5,1433 977,1536 768,1536 559,1536 382.5,1433 206,1330 103,1153.5 0,977 0,768 0,559 103,382.5 206,206 382.5,103 559,0 768,0 Z m 0,1312 q 148,0 273,-73 125,-73 198,-198 73,-125 73,-273 0,-148 -73,-273 -73,-125 -198,-198 -125,-73 -273,-73 -148,0 -273,73 -125,73 -198,198 -73,125 -73,273 0,148 73,273 73,125 198,198 125,73 273,73 z m 96,-224 q -14,0 -23,-9 -9,-9 -9,-23 l 0,-576 q 0,-14 9,-23 9,-9 23,-9 l 192,0 q 14,0 23,9 9,9 9,23 l 0,576 q 0,14 -9,23 -9,9 -23,9 l -192,0 z m -384,0 q -14,0 -23,-9 -9,-9 -9,-23 l 0,-576 q 0,-14 9,-23 9,-9 23,-9 l 192,0 q 14,0 23,9 9,9 9,23 l 0,576 q 0,14 -9,23 -9,9 -23,9 l -192,0 z' } ], + [ 'play-circle-o', { viewBox: '0 0 1536 1536', path: 'm 1184,768 q 0,37 -32,55 l -544,320 q -15,9 -32,9 -16,0 -32,-8 -32,-19 -32,-56 l 0,-640 q 0,-37 32,-56 33,-18 64,1 l 544,320 q 32,18 32,55 z m 128,0 q 0,-148 -73,-273 -73,-125 -198,-198 -125,-73 -273,-73 -148,0 -273,73 -125,73 -198,198 -73,125 -73,273 0,148 73,273 73,125 198,198 125,73 273,73 148,0 273,-73 125,-73 198,-198 73,-125 73,-273 z m 224,0 q 0,209 -103,385.5 Q 1330,1330 1153.5,1433 977,1536 768,1536 559,1536 382.5,1433 206,1330 103,1153.5 0,977 0,768 0,559 103,382.5 206,206 382.5,103 559,0 768,0 977,0 1153.5,103 1330,206 1433,382.5 1536,559 1536,768 Z' } ], + [ 'plus', { viewBox: '0 0 1408 1408', path: 'm 1408,608 0,192 q 0,40 -28,68 -28,28 -68,28 l -416,0 0,416 q 0,40 -28,68 -28,28 -68,28 l -192,0 q -40,0 -68,-28 -28,-28 -28,-68 l 0,-416 -416,0 Q 56,896 28,868 0,840 0,800 L 0,608 q 0,-40 28,-68 28,-28 68,-28 l 416,0 0,-416 Q 512,56 540,28 568,0 608,0 l 192,0 q 40,0 68,28 28,28 28,68 l 0,416 416,0 q 40,0 68,28 28,28 28,68 z' } ], + [ 'power-off', { viewBox: '0 0 1536 1664', path: 'm 1536,896 q 0,156 -61,298 -61,142 -164,245 -103,103 -245,164 -142,61 -298,61 -156,0 -298,-61 Q 328,1542 225,1439 122,1336 61,1194 0,1052 0,896 0,714 80.5,553 161,392 307,283 q 43,-32 95.5,-25 52.5,7 83.5,50 32,42 24.5,94.5 Q 503,455 461,487 363,561 309.5,668 256,775 256,896 q 0,104 40.5,198.5 40.5,94.5 109.5,163.5 69,69 163.5,109.5 94.5,40.5 198.5,40.5 104,0 198.5,-40.5 Q 1061,1327 1130,1258 1199,1189 1239.5,1094.5 1280,1000 1280,896 1280,775 1226.5,668 1173,561 1075,487 1033,455 1025.5,402.5 1018,350 1050,308 q 31,-43 84,-50 53,-7 95,25 146,109 226.5,270 80.5,161 80.5,343 z m -640,-768 0,640 q 0,52 -38,90 -38,38 -90,38 -52,0 -90,-38 -38,-38 -38,-90 l 0,-640 q 0,-52 38,-90 38,-38 90,-38 52,0 90,38 38,38 38,90 z' } ], + [ 'question-circle', { viewBox: '0 0 1536 1536', path: 'm 896,1248 v -192 q 0,-14 -9,-23 -9,-9 -23,-9 H 672 q -14,0 -23,9 -9,9 -9,23 v 192 q 0,14 9,23 9,9 23,9 h 192 q 14,0 23,-9 9,-9 9,-23 z m 256,-672 q 0,-88 -55.5,-163 Q 1041,338 958,297 875,256 788,256 q -243,0 -371,213 -15,24 8,42 l 132,100 q 7,6 19,6 16,0 25,-12 53,-68 86,-92 34,-24 86,-24 48,0 85.5,26 37.5,26 37.5,59 0,38 -20,61 -20,23 -68,45 -63,28 -115.5,86.5 Q 640,825 640,892 v 36 q 0,14 9,23 9,9 23,9 h 192 q 14,0 23,-9 9,-9 9,-23 0,-19 21.5,-49.5 Q 939,848 972,829 q 32,-18 49,-28.5 17,-10.5 46,-35 29,-24.5 44.5,-48 15.5,-23.5 28,-60.5 12.5,-37 12.5,-81 z m 384,192 q 0,209 -103,385.5 Q 1330,1330 1153.5,1433 977,1536 768,1536 559,1536 382.5,1433 206,1330 103,1153.5 0,977 0,768 0,559 103,382.5 206,206 382.5,103 559,0 768,0 977,0 1153.5,103 1330,206 1433,382.5 1536,559 1536,768 Z' } ], + [ 'refresh', { viewBox: '0 0 1536 1536', path: 'm 1511,928 q 0,5 -1,7 -64,268 -268,434.5 Q 1038,1536 764,1536 618,1536 481.5,1481 345,1426 238,1324 l -129,129 q -19,19 -45,19 -26,0 -45,-19 Q 0,1434 0,1408 L 0,960 q 0,-26 19,-45 19,-19 45,-19 l 448,0 q 26,0 45,19 19,19 19,45 0,26 -19,45 l -137,137 q 71,66 161,102 90,36 187,36 134,0 250,-65 116,-65 186,-179 11,-17 53,-117 8,-23 30,-23 l 192,0 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 25,-800 0,448 q 0,26 -19,45 -19,19 -45,19 l -448,0 q -26,0 -45,-19 -19,-19 -19,-45 0,-26 19,-45 L 1117,393 Q 969,256 768,256 q -134,0 -250,65 -116,65 -186,179 -11,17 -53,117 -8,23 -30,23 L 50,640 Q 37,640 27.5,630.5 18,621 18,608 l 0,-7 Q 83,333 288,166.5 493,0 768,0 914,0 1052,55.5 1190,111 1297,212 L 1427,83 q 19,-19 45,-19 26,0 45,19 19,19 19,45 z' } ], + [ 'save', { viewBox: '0 0 1536 1536', path: 'm 384,1408 h 768 V 1024 H 384 Z m 896,0 h 128 V 512 q 0,-14 -10,-38.5 Q 1388,449 1378,439 L 1097,158 q -10,-10 -34,-20 -24,-10 -39,-10 v 416 q 0,40 -28,68 -28,28 -68,28 H 352 q -40,0 -68,-28 -28,-28 -28,-68 V 128 H 128 V 1408 H 256 V 992 q 0,-40 28,-68 28,-28 68,-28 h 832 q 40,0 68,28 28,28 28,68 z M 896,480 V 160 q 0,-13 -9.5,-22.5 Q 877,128 864,128 H 672 q -13,0 -22.5,9.5 Q 640,147 640,160 v 320 q 0,13 9.5,22.5 9.5,9.5 22.5,9.5 h 192 q 13,0 22.5,-9.5 Q 896,493 896,480 Z m 640,32 v 928 q 0,40 -28,68 -28,28 -68,28 H 96 Q 56,1536 28,1508 0,1480 0,1440 V 96 Q 0,56 28,28 56,0 96,0 h 928 q 40,0 88,20 48,20 76,48 l 280,280 q 28,28 48,76 20,48 20,88 z' } ], + [ 'search', { viewBox: '0 0 1664 1664', path: 'M 1152,704 Q 1152,519 1020.5,387.5 889,256 704,256 519,256 387.5,387.5 256,519 256,704 256,889 387.5,1020.5 519,1152 704,1152 889,1152 1020.5,1020.5 1152,889 1152,704 Z m 512,832 q 0,52 -38,90 -38,38 -90,38 -54,0 -90,-38 L 1103,1284 Q 924,1408 704,1408 561,1408 430.5,1352.5 300,1297 205.5,1202.5 111,1108 55.5,977.5 0,847 0,704 0,561 55.5,430.5 111,300 205.5,205.5 300,111 430.5,55.5 561,0 704,0 q 143,0 273.5,55.5 130.5,55.5 225,150 94.5,94.5 150,225 55.5,130.5 55.5,273.5 0,220 -124,399 l 343,343 q 37,37 37,90 z' } ], + [ 'sliders', { viewBox: '0 0 1536 1408', path: 'm 352,1152 0,128 -352,0 0,-128 352,0 z m 352,-128 q 26,0 45,19 19,19 19,45 l 0,256 q 0,26 -19,45 -19,19 -45,19 l -256,0 q -26,0 -45,-19 -19,-19 -19,-45 l 0,-256 q 0,-26 19,-45 19,-19 45,-19 l 256,0 z m 160,-384 0,128 -864,0 0,-128 864,0 z m -640,-512 0,128 -224,0 0,-128 224,0 z m 1312,1024 0,128 -736,0 0,-128 736,0 z M 576,0 q 26,0 45,19 19,19 19,45 l 0,256 q 0,26 -19,45 -19,19 -45,19 l -256,0 q -26,0 -45,-19 -19,-19 -19,-45 L 256,64 Q 256,38 275,19 294,0 320,0 l 256,0 z m 640,512 q 26,0 45,19 19,19 19,45 l 0,256 q 0,26 -19,45 -19,19 -45,19 l -256,0 q -26,0 -45,-19 -19,-19 -19,-45 l 0,-256 q 0,-26 19,-45 19,-19 45,-19 l 256,0 z m 320,128 0,128 -224,0 0,-128 224,0 z m 0,-512 0,128 -864,0 0,-128 864,0 z' } ], + [ 'spinner', { viewBox: '0 0 1664 1728', path: 'm 462,1394 q 0,53 -37.5,90.5 -37.5,37.5 -90.5,37.5 -52,0 -90,-38 -38,-38 -38,-90 0,-53 37.5,-90.5 37.5,-37.5 90.5,-37.5 53,0 90.5,37.5 37.5,37.5 37.5,90.5 z m 498,206 q 0,53 -37.5,90.5 Q 885,1728 832,1728 779,1728 741.5,1690.5 704,1653 704,1600 q 0,-53 37.5,-90.5 37.5,-37.5 90.5,-37.5 53,0 90.5,37.5 Q 960,1547 960,1600 Z M 256,896 q 0,53 -37.5,90.5 Q 181,1024 128,1024 75,1024 37.5,986.5 0,949 0,896 0,843 37.5,805.5 75,768 128,768 q 53,0 90.5,37.5 Q 256,843 256,896 Z m 1202,498 q 0,52 -38,90 -38,38 -90,38 -53,0 -90.5,-37.5 -37.5,-37.5 -37.5,-90.5 0,-53 37.5,-90.5 37.5,-37.5 90.5,-37.5 53,0 90.5,37.5 37.5,37.5 37.5,90.5 z M 494,398 q 0,66 -47,113 -47,47 -113,47 -66,0 -113,-47 -47,-47 -47,-113 0,-66 47,-113 47,-47 113,-47 66,0 113,47 47,47 47,113 z m 1170,498 q 0,53 -37.5,90.5 -37.5,37.5 -90.5,37.5 -53,0 -90.5,-37.5 Q 1408,949 1408,896 q 0,-53 37.5,-90.5 37.5,-37.5 90.5,-37.5 53,0 90.5,37.5 Q 1664,843 1664,896 Z M 1024,192 q 0,80 -56,136 -56,56 -136,56 -80,0 -136,-56 -56,-56 -56,-136 0,-80 56,-136 56,-56 136,-56 80,0 136,56 56,56 56,136 z m 530,206 q 0,93 -66,158.5 -66,65.5 -158,65.5 -93,0 -158.5,-65.5 Q 1106,491 1106,398 q 0,-92 65.5,-158 65.5,-66 158.5,-66 92,0 158,66 66,66 66,158 z' } ], + [ 'sun', { viewBox: '0 0 1708 1792', path: 'm 1706,1172.5 c -3,10 -11,17 -20,20 l -292,96 v 306 c 0,10 -5,20 -13,26 -9,6 -19,8 -29,4 l -292,-94 -180,248 c -6,8 -16,13 -26,13 -10,0 -20,-5 -26,-13 l -180,-248 -292,94 c -10,4 -20,2 -29,-4 -8,-6 -13,-16 -13,-26 v -306 l -292,-96 c -9,-3 -17,-10 -20,-20 -3,-10 -2,-21 4,-29 l 180,-248 -180,-248 c -6,-9 -7,-19 -4,-29 3,-10 11,-17 20,-20 l 292,-96 v -306 c 0,-10 5,-20 13,-26 9,-6 19,-8 29,-4 l 292,94 180,-248 c 12,-16 40,-16 52,0 L 1060,260.5 l 292,-94 c 10,-4 20,-2 29,4 8,6 13,16 13,26 v 306 l 292,96 c 9,3 17,10 20,20 3,10 2,20 -4,29 l -180,248 180,248 c 6,8 7,19 4,29 z' } ], + [ 'sun-o', { viewBox: '0 0 1708 1792', path: 'm 1430,895.5 c 0,-318 -258,-576 -576,-576 -318,0 -576,258 -576,576 0,318 258,576 576,576 C 1172,1471.5 1430,1213.5 1430,895.5 Z m 276,277 c -3,10 -11,17 -20,20 l -292,96 v 306 c 0,10 -5,20 -13,26 -9,6 -19,8 -29,4 l -292,-94 -180,248 c -6,8 -16,13 -26,13 -10,0 -20,-5 -26,-13 l -180,-248 -292,94 c -10,4 -20,2 -29,-4 -8,-6 -13,-16 -13,-26 v -306 l -292,-96 c -9,-3 -17,-10 -20,-20 -3,-10 -2,-21 4,-29 l 180,-248 -180,-248 c -6,-9 -7,-19 -4,-29 3,-10 11,-17 20,-20 l 292,-96 v -306 c 0,-10 5,-20 13,-26 9,-6 19,-8 29,-4 l 292,94 180,-248 c 12,-16 40,-16 52,0 L 1060,260.5 l 292,-94 c 10,-4 20,-2 29,4 8,6 13,16 13,26 v 306 l 292,96 c 9,3 17,10 20,20 3,10 2,20 -4,29 l -180,248 180,248 c 6,8 7,19 4,29 z' } ], + [ 'times', { viewBox: '0 0 1188 1188', path: 'm 1188,956 q 0,40 -28,68 l -136,136 q -28,28 -68,28 -40,0 -68,-28 L 594,866 300,1160 q -28,28 -68,28 -40,0 -68,-28 L 28,1024 Q 0,996 0,956 0,916 28,888 L 322,594 28,300 Q 0,272 0,232 0,192 28,164 L 164,28 Q 192,0 232,0 272,0 300,28 L 594,322 888,28 q 28,-28 68,-28 40,0 68,28 l 136,136 q 28,28 28,68 0,40 -28,68 l -294,294 294,294 q 28,28 28,68 z' } ], + [ 'trash-o', { viewBox: '0 0 1408 1536', path: 'm 512,608 v 576 q 0,14 -9,23 -9,9 -23,9 h -64 q -14,0 -23,-9 -9,-9 -9,-23 V 608 q 0,-14 9,-23 9,-9 23,-9 h 64 q 14,0 23,9 9,9 9,23 z m 256,0 v 576 q 0,14 -9,23 -9,9 -23,9 h -64 q -14,0 -23,-9 -9,-9 -9,-23 V 608 q 0,-14 9,-23 9,-9 23,-9 h 64 q 14,0 23,9 9,9 9,23 z m 256,0 v 576 q 0,14 -9,23 -9,9 -23,9 h -64 q -14,0 -23,-9 -9,-9 -9,-23 V 608 q 0,-14 9,-23 9,-9 23,-9 h 64 q 14,0 23,9 9,9 9,23 z m 128,724 V 384 H 256 v 948 q 0,22 7,40.5 7,18.5 14.5,27 7.5,8.5 10.5,8.5 h 832 q 3,0 10.5,-8.5 7.5,-8.5 14.5,-27 7,-18.5 7,-40.5 z M 480,256 H 928 L 880,139 q -7,-9 -17,-11 H 546 q -10,2 -17,11 z m 928,32 v 64 q 0,14 -9,23 -9,9 -23,9 h -96 v 948 q 0,83 -47,143.5 -47,60.5 -113,60.5 H 288 q -66,0 -113,-58.5 Q 128,1419 128,1336 V 384 H 32 Q 18,384 9,375 0,366 0,352 v -64 q 0,-14 9,-23 9,-9 23,-9 H 341 L 411,89 Q 426,52 465,26 504,0 544,0 h 320 q 40,0 79,26 39,26 54,63 l 70,167 h 309 q 14,0 23,9 9,9 9,23 z' } ], + [ 'undo', { viewBox: '0 0 1536 1536', path: 'm 1536,768 q 0,156 -61,298 -61,142 -164,245 -103,103 -245,164 -142,61 -298,61 -172,0 -327,-72.5 Q 286,1391 177,1259 q -7,-10 -6.5,-22.5 0.5,-12.5 8.5,-20.5 l 137,-138 q 10,-9 25,-9 16,2 23,12 73,95 179,147 106,52 225,52 104,0 198.5,-40.5 Q 1061,1199 1130,1130 1199,1061 1239.5,966.5 1280,872 1280,768 1280,664 1239.5,569.5 1199,475 1130,406 1061,337 966.5,296.5 872,256 768,256 670,256 580,291.5 490,327 420,393 l 137,138 q 31,30 14,69 -17,40 -59,40 H 64 Q 38,640 19,621 0,602 0,576 V 128 Q 0,86 40,69 79,52 109,83 L 239,212 Q 346,111 483.5,55.5 621,0 768,0 q 156,0 298,61 142,61 245,164 103,103 164,245 61,142 61,298 z' } ], + [ 'unlink', { viewBox: '0 0 1664 1664', path: 'm 439,1271 -256,256 q -11,9 -23,9 -12,0 -23,-9 -9,-10 -9,-23 0,-13 9,-23 l 256,-256 q 10,-9 23,-9 13,0 23,9 9,10 9,23 0,13 -9,23 z m 169,41 v 320 q 0,14 -9,23 -9,9 -23,9 -14,0 -23,-9 -9,-9 -9,-23 v -320 q 0,-14 9,-23 9,-9 23,-9 14,0 23,9 9,9 9,23 z M 384,1088 q 0,14 -9,23 -9,9 -23,9 H 32 q -14,0 -23,-9 -9,-9 -9,-23 0,-14 9,-23 9,-9 23,-9 h 320 q 14,0 23,9 9,9 9,23 z m 1264,128 q 0,120 -85,203 l -147,146 q -83,83 -203,83 -121,0 -204,-85 L 675,1228 q -21,-21 -42,-56 l 239,-18 273,274 q 27,27 68,27.5 41,0.5 68,-26.5 l 147,-146 q 28,-28 28,-67 0,-40 -28,-68 l -274,-275 18,-239 q 35,21 56,42 l 336,336 q 84,86 84,204 z M 1031,492 792,510 519,236 q -28,-28 -68,-28 -39,0 -68,27 L 236,381 q -28,28 -28,67 0,40 28,68 l 274,274 -18,240 q -35,-21 -56,-42 L 100,652 Q 16,566 16,448 16,328 101,245 L 248,99 q 83,-83 203,-83 121,0 204,85 l 334,335 q 21,21 42,56 z m 633,84 q 0,14 -9,23 -9,9 -23,9 h -320 q -14,0 -23,-9 -9,-9 -9,-23 0,-14 9,-23 9,-9 23,-9 h 320 q 14,0 23,9 9,9 9,23 z M 1120,32 v 320 q 0,14 -9,23 -9,9 -23,9 -14,0 -23,-9 -9,-9 -9,-23 V 32 q 0,-14 9,-23 9,-9 23,-9 14,0 23,9 9,9 9,23 z m 407,151 -256,256 q -11,9 -23,9 -12,0 -23,-9 -9,-10 -9,-23 0,-13 9,-23 l 256,-256 q 10,-9 23,-9 13,0 23,9 9,10 9,23 0,13 -9,23 z' } ], + [ 'unlock-alt', { viewBox: '0 0 1152 1536', path: 'm 1056,768 q 40,0 68,28 28,28 28,68 v 576 q 0,40 -28,68 -28,28 -68,28 H 96 Q 56,1536 28,1508 0,1480 0,1440 V 864 q 0,-40 28,-68 28,-28 68,-28 h 32 V 448 Q 128,263 259.5,131.5 391,0 576,0 761,0 892.5,131.5 1024,263 1024,448 q 0,26 -19,45 -19,19 -45,19 h -64 q -26,0 -45,-19 -19,-19 -19,-45 0,-106 -75,-181 -75,-75 -181,-75 -106,0 -181,75 -75,75 -75,181 v 320 z' } ], + [ 'upload-alt', { viewBox: '0 0 1664 1600', path: 'm 1280,1408 q 0,-26 -19,-45 -19,-19 -45,-19 -26,0 -45,19 -19,19 -19,45 0,26 19,45 19,19 45,19 26,0 45,-19 19,-19 19,-45 z m 256,0 q 0,-26 -19,-45 -19,-19 -45,-19 -26,0 -45,19 -19,19 -19,45 0,26 19,45 19,19 45,19 26,0 45,-19 19,-19 19,-45 z m 128,-224 v 320 q 0,40 -28,68 -28,28 -68,28 H 96 q -40,0 -68,-28 -28,-28 -28,-68 v -320 q 0,-40 28,-68 28,-28 68,-28 h 427 q 21,56 70.5,92 49.5,36 110.5,36 h 256 q 61,0 110.5,-36 49.5,-36 70.5,-92 h 427 q 40,0 68,28 28,28 28,68 z M 1339,536 q -17,40 -59,40 h -256 v 448 q 0,26 -19,45 -19,19 -45,19 H 704 q -26,0 -45,-19 -19,-19 -19,-45 V 576 H 384 q -42,0 -59,-40 -17,-39 14,-69 L 787,19 q 18,-19 45,-19 27,0 45,19 l 448,448 q 31,30 14,69 z' } ], + [ 'zoom-in', { viewBox: '0 0 1664 1664', path: 'm 1024,672 v 64 q 0,13 -9.5,22.5 Q 1005,768 992,768 H 768 v 224 q 0,13 -9.5,22.5 -9.5,9.5 -22.5,9.5 h -64 q -13,0 -22.5,-9.5 Q 640,1005 640,992 V 768 H 416 q -13,0 -22.5,-9.5 Q 384,749 384,736 v -64 q 0,-13 9.5,-22.5 Q 403,640 416,640 H 640 V 416 q 0,-13 9.5,-22.5 Q 659,384 672,384 h 64 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 v 224 h 224 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 128,32 Q 1152,519 1020.5,387.5 889,256 704,256 519,256 387.5,387.5 256,519 256,704 256,889 387.5,1020.5 519,1152 704,1152 889,1152 1020.5,1020.5 1152,889 1152,704 Z m 512,832 q 0,53 -37.5,90.5 -37.5,37.5 -90.5,37.5 -54,0 -90,-38 L 1103,1284 Q 924,1408 704,1408 561,1408 430.5,1352.5 300,1297 205.5,1202.5 111,1108 55.5,977.5 0,847 0,704 0,561 55.5,430.5 111,300 205.5,205.5 300,111 430.5,55.5 561,0 704,0 q 143,0 273.5,55.5 130.5,55.5 225,150 94.5,94.5 150,225 55.5,130.5 55.5,273.5 0,220 -124,399 l 343,343 q 37,37 37,90 z' } ], + [ 'zoom-out', { viewBox: '0 0 1664 1664', path: 'm 1024,672 v 64 q 0,13 -9.5,22.5 Q 1005,768 992,768 H 416 q -13,0 -22.5,-9.5 Q 384,749 384,736 v -64 q 0,-13 9.5,-22.5 Q 403,640 416,640 h 576 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 128,32 Q 1152,519 1020.5,387.5 889,256 704,256 519,256 387.5,387.5 256,519 256,704 256,889 387.5,1020.5 519,1152 704,1152 889,1152 1020.5,1020.5 1152,889 1152,704 Z m 512,832 q 0,53 -37.5,90.5 -37.5,37.5 -90.5,37.5 -54,0 -90,-38 L 1103,1284 Q 924,1408 704,1408 561,1408 430.5,1352.5 300,1297 205.5,1202.5 111,1108 55.5,977.5 0,847 0,704 0,561 55.5,430.5 111,300 205.5,205.5 300,111 430.5,55.5 561,0 704,0 q 143,0 273.5,55.5 130.5,55.5 225,150 94.5,94.5 150,225 55.5,130.5 55.5,273.5 0,220 -124,399 l 343,343 q 37,37 37,90 z' } ], + // See /img/photon.svg + [ 'ph-popups', { viewBox: '0 0 20 20', path: 'm 3.146,1.8546316 a 0.5006316,0.5006316 0 0 0 0.708,-0.708 l -1,-1 a 0.5006316,0.5006316 0 0 0 -0.708,0.708 z m -0.836,2.106 a 0.406,0.406 0 0 0 0.19,0.04 0.5,0.5 0 0 0 0.35,-0.851 0.493,0.493 0 0 0 -0.54,-0.109 0.361,0.361 0 0 0 -0.16,0.109 0.485,0.485 0 0 0 0,0.7 0.372,0.372 0 0 0 0.16,0.111 z m 3,-3 a 0.406,0.406 0 0 0 0.19,0.04 0.513,0.513 0 0 0 0.5,-0.5 0.473,0.473 0 0 0 -0.15,-0.351 0.5,0.5 0 0 0 -0.7,0 0.485,0.485 0 0 0 0,0.7 0.372,0.372 0 0 0 0.16,0.111 z m 13.19,1.04 a 0.5,0.5 0 0 0 0.354,-0.146 l 1,-1 a 0.5006316,0.5006316 0 0 0 -0.708,-0.708 l -1,1 a 0.5,0.5 0 0 0 0.354,0.854 z m 1.35,1.149 a 0.361,0.361 0 0 0 -0.16,-0.109 0.5,0.5 0 0 0 -0.38,0 0.361,0.361 0 0 0 -0.16,0.109 0.485,0.485 0 0 0 0,0.7 0.372,0.372 0 0 0 0.16,0.11 0.471,0.471 0 0 0 0.38,0 0.372,0.372 0 0 0 0.16,-0.11 0.469,0.469 0 0 0 0.15,-0.349 0.43,0.43 0 0 0 -0.04,-0.19 0.358,0.358 0 0 0 -0.11,-0.161 z m -3.54,-2.189 a 0.406,0.406 0 0 0 0.19,0.04 0.469,0.469 0 0 0 0.35,-0.15 0.353,0.353 0 0 0 0.11,-0.161 0.469,0.469 0 0 0 0,-0.379 0.358,0.358 0 0 0 -0.11,-0.161 0.361,0.361 0 0 0 -0.16,-0.109 0.493,0.493 0 0 0 -0.54,0.109 0.358,0.358 0 0 0 -0.11,0.161 0.43,0.43 0 0 0 -0.04,0.19 0.469,0.469 0 0 0 0.15,0.35 0.372,0.372 0 0 0 0.16,0.11 z m 2.544,15.1860004 a 0.5006316,0.5006316 0 0 0 -0.708,0.708 l 1,1 a 0.5006316,0.5006316 0 0 0 0.708,-0.708 z m 0.3,-2 a 0.473,0.473 0 0 0 -0.154,0.354 0.4,0.4 0 0 0 0.04,0.189 0.353,0.353 0 0 0 0.11,0.161 0.469,0.469 0 0 0 0.35,0.15 0.406,0.406 0 0 0 0.19,-0.04 0.372,0.372 0 0 0 0.16,-0.11 0.454,0.454 0 0 0 0.15,-0.35 0.473,0.473 0 0 0 -0.15,-0.351 0.5,0.5 0 0 0 -0.7,0 z m -3,3 a 0.473,0.473 0 0 0 -0.154,0.354 0.454,0.454 0 0 0 0.15,0.35 0.372,0.372 0 0 0 0.16,0.11 0.406,0.406 0 0 0 0.19,0.04 0.469,0.469 0 0 0 0.35,-0.15 0.353,0.353 0 0 0 0.11,-0.161 0.4,0.4 0 0 0 0.04,-0.189 0.473,0.473 0 0 0 -0.15,-0.351 0.5,0.5 0 0 0 -0.7,0 z M 18,5.0006316 a 3,3 0 0 0 -3,-3 H 7 a 3,3 0 0 0 -3,3 v 8.0000004 a 3,3 0 0 0 3,3 h 8 a 3,3 0 0 0 3,-3 z m -2,8.0000004 a 1,1 0 0 1 -1,1 H 7 a 1,1 0 0 1 -1,-1 V 7.0006316 H 16 Z M 16,6.0006316 H 6 v -1 a 1,1 0 0 1 1,-1 h 8 a 1,1 0 0 1 1,1 z M 11,18.000632 H 3 a 1,1 0 0 1 -1,-1 v -6 h 1 v -1 H 2 V 9.0006316 a 1,1 0 0 1 1,-1 v -2 a 3,3 0 0 0 -3,3 v 8.0000004 a 3,3 0 0 0 3,3 h 8 a 3,3 0 0 0 3,-3 h -2 a 1,1 0 0 1 -1,1 z' } ], + [ 'ph-readermode-text-size', { viewBox: '0 0 20 12.5', path: 'M 10.422,11.223 A 0.712,0.712 0 0 1 10.295,11.007 L 6.581,0 H 4.68 L 0.933,11.309 0,11.447 V 12.5 H 3.594 V 11.447 L 2.655,11.325 A 0.3,0.3 0 0 1 2.468,11.211 0.214,0.214 0 0 1 2.419,10.974 L 3.341,8.387 h 3.575 l 0.906,2.652 a 0.18,0.18 0 0 1 -0.016,0.18 0.217,0.217 0 0 1 -0.139,0.106 L 6.679,11.447 V 12.5 h 4.62 V 11.447 L 10.663,11.325 A 0.512,0.512 0 0 1 10.422,11.223 Z M 3.659,7.399 5.063,2.57 6.5,7.399 Z M 19.27,11.464 A 0.406,0.406 0 0 1 19.009,11.337 0.368,0.368 0 0 1 18.902,11.072 V 6.779 A 3.838,3.838 0 0 0 18.67,5.318 1.957,1.957 0 0 0 18.01,4.457 2.48,2.48 0 0 0 16.987,4.044 7.582,7.582 0 0 0 15.67,3.938 a 6.505,6.505 0 0 0 -1.325,0.139 5.2,5.2 0 0 0 -1.2,0.4 2.732,2.732 0 0 0 -0.864,0.624 1.215,1.215 0 0 0 -0.331,0.833 0.532,0.532 0 0 0 0.119,0.383 0.665,0.665 0 0 0 0.257,0.172 0.916,0.916 0 0 0 0.375,0.041 h 1.723 V 4.942 A 4.429,4.429 0 0 1 14.611,4.91 2.045,2.045 0 0 1 14.836,4.885 c 0.09,0 0.192,-0.008 0.306,-0.008 a 1.849,1.849 0 0 1 0.808,0.151 1.247,1.247 0 0 1 0.71,0.89 2.164,2.164 0 0 1 0.049,0.51 c 0,0.076 -0.008,0.152 -0.008,0.228 0,0.076 -0.008,0.139 -0.008,0.221 v 0.2 q -1.152,0.252 -1.976,0.489 a 12.973,12.973 0 0 0 -1.391,0.474 4.514,4.514 0 0 0 -0.91,0.485 2.143,2.143 0 0 0 -0.527,0.523 1.594,1.594 0 0 0 -0.245,0.592 3.739,3.739 0 0 0 -0.061,0.693 2.261,2.261 0 0 0 0.171,0.9 2.024,2.024 0 0 0 0.469,0.682 2.084,2.084 0 0 0 0.693,0.432 2.364,2.364 0 0 0 0.852,0.151 3.587,3.587 0 0 0 1.068,-0.159 6.441,6.441 0 0 0 1.835,-0.877 l 0.22,0.832 H 20 v -0.783 z m -2.588,-0.719 a 4.314,4.314 0 0 1 -0.5,0.188 5.909,5.909 0 0 1 -0.493,0.123 2.665,2.665 0 0 1 -0.543,0.057 1.173,1.173 0 0 1 -0.861,-0.363 1.166,1.166 0 0 1 -0.245,-0.392 1.357,1.357 0 0 1 -0.086,-0.486 1.632,1.632 0 0 1 0.123,-0.657 1.215,1.215 0 0 1 0.432,-0.5 3.151,3.151 0 0 1 0.837,-0.392 12.429,12.429 0 0 1 1.334,-0.334 z' } ], + ]); + + return function(root) { + const icons = (root || document).querySelectorAll('.fa-icon'); + if ( icons.length === 0 ) { return; } + const svgNS = 'http://www.w3.org/2000/svg'; + for ( const icon of icons ) { + if ( icon.firstChild === null || icon.firstChild.nodeType !== 3 ) { + continue; + } + const name = icon.firstChild.nodeValue.trim(); + if ( name === '' ) { continue; } + const svg = document.createElementNS(svgNS, 'svg'); + svg.classList.add('fa-icon_' + name); + const details = svgIcons.get(name); + if ( details === undefined ) { + let file; + if ( name.startsWith('ph-') ) { + file = 'photon'; + } else if ( name.startsWith('md-') ) { + file = 'material-design'; + } else { + continue; + } + const use = document.createElementNS(svgNS, 'use'); + use.setAttribute('href', `/img/${file}.svg#${name}`); + svg.appendChild(use); + } else { + svg.setAttribute('viewBox', details.viewBox); + const path = document.createElementNS(svgNS, 'path'); + path.setAttribute('d', details.path); + svg.appendChild(path); + } + icon.replaceChild(svg, icon.firstChild); + if ( icon.classList.contains('fa-icon-badged') ) { + const badge = document.createElement('span'); + badge.className = 'fa-icon-badge'; + icon.insertBefore(badge, icon.firstChild.nextSibling); + } + } + }; +})(); + +faIconsInit(); diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js new file mode 100644 index 0000000..5bc9aa1 --- /dev/null +++ b/src/js/filtering-context.js @@ -0,0 +1,461 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2018-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import { + hostnameFromURI, + domainFromHostname, + originFromURI, +} from './uri-utils.js'; + +/******************************************************************************/ + +// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webRequest/ResourceType + +// Long term, convert code wherever possible to work with integer-based type +// values -- the assumption being that integer operations are faster than +// string operations. + +export const NO_TYPE = 0; +export const BEACON = 1 << 0; +export const CSP_REPORT = 1 << 1; +export const FONT = 1 << 2; +export const IMAGE = 1 << 4; +export const IMAGESET = 1 << 4; +export const MAIN_FRAME = 1 << 5; +export const MEDIA = 1 << 6; +export const OBJECT = 1 << 7; +export const OBJECT_SUBREQUEST = 1 << 7; +export const PING = 1 << 8; +export const SCRIPT = 1 << 9; +export const STYLESHEET = 1 << 10; +export const SUB_FRAME = 1 << 11; +export const WEBSOCKET = 1 << 12; +export const XMLHTTPREQUEST = 1 << 13; +export const INLINE_FONT = 1 << 14; +export const INLINE_SCRIPT = 1 << 15; +export const OTHER = 1 << 16; +export const FRAME_ANY = MAIN_FRAME | SUB_FRAME; +export const FONT_ANY = FONT | INLINE_FONT; +export const INLINE_ANY = INLINE_FONT | INLINE_SCRIPT; +export const PING_ANY = BEACON | CSP_REPORT | PING; +export const SCRIPT_ANY = SCRIPT | INLINE_SCRIPT; + +const typeStrToIntMap = { + 'no_type': NO_TYPE, + 'beacon': BEACON, + 'csp_report': CSP_REPORT, + 'font': FONT, + 'image': IMAGE, + 'imageset': IMAGESET, + 'main_frame': MAIN_FRAME, + 'media': MEDIA, + 'object': OBJECT, + 'object_subrequest': OBJECT_SUBREQUEST, + 'ping': PING, + 'script': SCRIPT, + 'stylesheet': STYLESHEET, + 'sub_frame': SUB_FRAME, + 'websocket': WEBSOCKET, + 'xmlhttprequest': XMLHTTPREQUEST, + 'inline-font': INLINE_FONT, + 'inline-script': INLINE_SCRIPT, + 'other': OTHER, +}; + +export const METHOD_NONE = 0; +export const METHOD_CONNECT = 1 << 1; +export const METHOD_DELETE = 1 << 2; +export const METHOD_GET = 1 << 3; +export const METHOD_HEAD = 1 << 4; +export const METHOD_OPTIONS = 1 << 5; +export const METHOD_PATCH = 1 << 6; +export const METHOD_POST = 1 << 7; +export const METHOD_PUT = 1 << 8; + +const methodStrToBitMap = { + '': METHOD_NONE, + 'connect': METHOD_CONNECT, + 'delete': METHOD_DELETE, + 'get': METHOD_GET, + 'head': METHOD_HEAD, + 'options': METHOD_OPTIONS, + 'patch': METHOD_PATCH, + 'post': METHOD_POST, + 'put': METHOD_PUT, + 'CONNECT': METHOD_CONNECT, + 'DELETE': METHOD_DELETE, + 'GET': METHOD_GET, + 'HEAD': METHOD_HEAD, + 'OPTIONS': METHOD_OPTIONS, + 'PATCH': METHOD_PATCH, + 'POST': METHOD_POST, + 'PUT': METHOD_PUT, +}; + +const methodBitToStrMap = new Map([ + [ METHOD_NONE, '' ], + [ METHOD_CONNECT, 'connect' ], + [ METHOD_DELETE, 'delete' ], + [ METHOD_GET, 'get' ], + [ METHOD_HEAD, 'head' ], + [ METHOD_OPTIONS, 'options' ], + [ METHOD_PATCH, 'patch' ], + [ METHOD_POST, 'post' ], + [ METHOD_PUT, 'put' ], +]); + +/******************************************************************************/ + +export const FilteringContext = class { + constructor(other) { + if ( other instanceof FilteringContext ) { + return this.fromFilteringContext(other); + } + this.tstamp = 0; + this.realm = ''; + this.id = undefined; + this.method = 0; + this.itype = NO_TYPE; + this.stype = undefined; + this.url = undefined; + this.aliasURL = undefined; + this.hostname = undefined; + this.domain = undefined; + this.docId = -1; + this.frameId = -1; + this.docOrigin = undefined; + this.docHostname = undefined; + this.docDomain = undefined; + this.tabId = undefined; + this.tabOrigin = undefined; + this.tabHostname = undefined; + this.tabDomain = undefined; + this.redirectURL = undefined; + this.filter = undefined; + } + + get type() { + return this.stype; + } + + set type(a) { + this.itype = typeStrToIntMap[a] || NO_TYPE; + this.stype = a; + } + + isDocument() { + return (this.itype & FRAME_ANY) !== 0; + } + + isFont() { + return (this.itype & FONT_ANY) !== 0; + } + + fromFilteringContext(other) { + this.realm = other.realm; + this.id = other.id; + this.type = other.type; + this.method = other.method; + this.url = other.url; + this.hostname = other.hostname; + this.domain = other.domain; + this.docId = other.docId; + this.frameId = other.frameId; + this.docOrigin = other.docOrigin; + this.docHostname = other.docHostname; + this.docDomain = other.docDomain; + this.tabId = other.tabId; + this.tabOrigin = other.tabOrigin; + this.tabHostname = other.tabHostname; + this.tabDomain = other.tabDomain; + this.redirectURL = other.redirectURL; + this.filter = undefined; + return this; + } + + fromDetails({ originURL, url, type }) { + this.setDocOriginFromURL(originURL) + .setURL(url) + .setType(type); + return this; + } + + duplicate() { + return (new FilteringContext(this)); + } + + setRealm(a) { + this.realm = a; + return this; + } + + setType(a) { + this.type = a; + return this; + } + + setURL(a) { + if ( a !== this.url ) { + this.hostname = this.domain = undefined; + this.url = a; + } + return this; + } + + getHostname() { + if ( this.hostname === undefined ) { + this.hostname = hostnameFromURI(this.url); + } + return this.hostname; + } + + setHostname(a) { + if ( a !== this.hostname ) { + this.domain = undefined; + this.hostname = a; + } + return this; + } + + getDomain() { + if ( this.domain === undefined ) { + this.domain = domainFromHostname(this.getHostname()); + } + return this.domain; + } + + setDomain(a) { + this.domain = a; + return this; + } + + getDocOrigin() { + if ( this.docOrigin === undefined ) { + this.docOrigin = this.tabOrigin; + } + return this.docOrigin; + } + + setDocOrigin(a) { + if ( a !== this.docOrigin ) { + this.docHostname = this.docDomain = undefined; + this.docOrigin = a; + } + return this; + } + + setDocOriginFromURL(a) { + return this.setDocOrigin(originFromURI(a)); + } + + getDocHostname() { + if ( this.docHostname === undefined ) { + this.docHostname = hostnameFromURI(this.getDocOrigin()); + } + return this.docHostname; + } + + setDocHostname(a) { + if ( a !== this.docHostname ) { + this.docDomain = undefined; + this.docHostname = a; + } + return this; + } + + getDocDomain() { + if ( this.docDomain === undefined ) { + this.docDomain = domainFromHostname(this.getDocHostname()); + } + return this.docDomain; + } + + setDocDomain(a) { + this.docDomain = a; + return this; + } + + // The idea is to minimize the amount of work done to figure out whether + // the resource is 3rd-party to the document. + is3rdPartyToDoc() { + let docDomain = this.getDocDomain(); + if ( docDomain === '' ) { docDomain = this.docHostname; } + if ( this.domain !== undefined && this.domain !== '' ) { + return this.domain !== docDomain; + } + const hostname = this.getHostname(); + if ( hostname.endsWith(docDomain) === false ) { return true; } + const i = hostname.length - docDomain.length; + if ( i === 0 ) { return false; } + return hostname.charCodeAt(i - 1) !== 0x2E /* '.' */; + } + + setTabId(a) { + this.tabId = a; + return this; + } + + getTabOrigin() { + return this.tabOrigin; + } + + setTabOrigin(a) { + if ( a !== this.tabOrigin ) { + this.tabHostname = this.tabDomain = undefined; + this.tabOrigin = a; + } + return this; + } + + setTabOriginFromURL(a) { + return this.setTabOrigin(originFromURI(a)); + } + + getTabHostname() { + if ( this.tabHostname === undefined ) { + this.tabHostname = hostnameFromURI(this.getTabOrigin()); + } + return this.tabHostname; + } + + setTabHostname(a) { + if ( a !== this.tabHostname ) { + this.tabDomain = undefined; + this.tabHostname = a; + } + return this; + } + + getTabDomain() { + if ( this.tabDomain === undefined ) { + this.tabDomain = domainFromHostname(this.getTabHostname()); + } + return this.tabDomain; + } + + setTabDomain(a) { + this.docDomain = a; + return this; + } + + // The idea is to minimize the amount of work done to figure out whether + // the resource is 3rd-party to the top document. + is3rdPartyToTab() { + let tabDomain = this.getTabDomain(); + if ( tabDomain === '' ) { tabDomain = this.tabHostname; } + if ( this.domain !== undefined && this.domain !== '' ) { + return this.domain !== tabDomain; + } + const hostname = this.getHostname(); + if ( hostname.endsWith(tabDomain) === false ) { return true; } + const i = hostname.length - tabDomain.length; + if ( i === 0 ) { return false; } + return hostname.charCodeAt(i - 1) !== 0x2E /* '.' */; + } + + setFilter(a) { + this.filter = a; + return this; + } + + pushFilter(a) { + if ( this.filter === undefined ) { + return this.setFilter(a); + } + if ( Array.isArray(this.filter) ) { + this.filter.push(a); + } else { + this.filter = [ this.filter, a ]; + } + return this; + } + + pushFilters(a) { + if ( this.filter === undefined ) { + return this.setFilter(a); + } + if ( Array.isArray(this.filter) ) { + this.filter.push(...a); + } else { + this.filter = [ this.filter, ...a ]; + } + return this; + } + + setMethod(a) { + this.method = methodStrToBitMap[a] || 0; + return this; + } + + getMethodName() { + return FilteringContext.getMethodName(this.method); + } + + static getMethod(a) { + return methodStrToBitMap[a] || 0; + } + + static getMethodName(a) { + return methodBitToStrMap.get(a) || ''; + } +}; + +/******************************************************************************/ + +FilteringContext.prototype.BEACON = FilteringContext.BEACON = BEACON; +FilteringContext.prototype.CSP_REPORT = FilteringContext.CSP_REPORT = CSP_REPORT; +FilteringContext.prototype.FONT = FilteringContext.FONT = FONT; +FilteringContext.prototype.IMAGE = FilteringContext.IMAGE = IMAGE; +FilteringContext.prototype.IMAGESET = FilteringContext.IMAGESET = IMAGESET; +FilteringContext.prototype.MAIN_FRAME = FilteringContext.MAIN_FRAME = MAIN_FRAME; +FilteringContext.prototype.MEDIA = FilteringContext.MEDIA = MEDIA; +FilteringContext.prototype.OBJECT = FilteringContext.OBJECT = OBJECT; +FilteringContext.prototype.OBJECT_SUBREQUEST = FilteringContext.OBJECT_SUBREQUEST = OBJECT_SUBREQUEST; +FilteringContext.prototype.PING = FilteringContext.PING = PING; +FilteringContext.prototype.SCRIPT = FilteringContext.SCRIPT = SCRIPT; +FilteringContext.prototype.STYLESHEET = FilteringContext.STYLESHEET = STYLESHEET; +FilteringContext.prototype.SUB_FRAME = FilteringContext.SUB_FRAME = SUB_FRAME; +FilteringContext.prototype.WEBSOCKET = FilteringContext.WEBSOCKET = WEBSOCKET; +FilteringContext.prototype.XMLHTTPREQUEST = FilteringContext.XMLHTTPREQUEST = XMLHTTPREQUEST; +FilteringContext.prototype.INLINE_FONT = FilteringContext.INLINE_FONT = INLINE_FONT; +FilteringContext.prototype.INLINE_SCRIPT = FilteringContext.INLINE_SCRIPT = INLINE_SCRIPT; +FilteringContext.prototype.OTHER = FilteringContext.OTHER = OTHER; +FilteringContext.prototype.FRAME_ANY = FilteringContext.FRAME_ANY = FRAME_ANY; +FilteringContext.prototype.FONT_ANY = FilteringContext.FONT_ANY = FONT_ANY; +FilteringContext.prototype.INLINE_ANY = FilteringContext.INLINE_ANY = INLINE_ANY; +FilteringContext.prototype.PING_ANY = FilteringContext.PING_ANY = PING_ANY; +FilteringContext.prototype.SCRIPT_ANY = FilteringContext.SCRIPT_ANY = SCRIPT_ANY; + +FilteringContext.prototype.METHOD_NONE = FilteringContext.METHOD_NONE = METHOD_NONE; +FilteringContext.prototype.METHOD_CONNECT = FilteringContext.METHOD_CONNECT = METHOD_CONNECT; +FilteringContext.prototype.METHOD_DELETE = FilteringContext.METHOD_DELETE = METHOD_DELETE; +FilteringContext.prototype.METHOD_GET = FilteringContext.METHOD_GET = METHOD_GET; +FilteringContext.prototype.METHOD_HEAD = FilteringContext.METHOD_HEAD = METHOD_HEAD; +FilteringContext.prototype.METHOD_OPTIONS = FilteringContext.METHOD_OPTIONS = METHOD_OPTIONS; +FilteringContext.prototype.METHOD_PATCH = FilteringContext.METHOD_PATCH = METHOD_PATCH; +FilteringContext.prototype.METHOD_POST = FilteringContext.METHOD_POST = METHOD_POST; +FilteringContext.prototype.METHOD_PUT = FilteringContext.METHOD_PUT = METHOD_PUT; + +/******************************************************************************/ diff --git a/src/js/filtering-engines.js b/src/js/filtering-engines.js new file mode 100644 index 0000000..d72ff9d --- /dev/null +++ b/src/js/filtering-engines.js @@ -0,0 +1,50 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import DynamicHostRuleFiltering from './dynamic-net-filtering.js'; +import DynamicSwitchRuleFiltering from './hnswitches.js'; +import DynamicURLRuleFiltering from './url-net-filtering.js'; + +/******************************************************************************/ + +const permanentFirewall = new DynamicHostRuleFiltering(); +const sessionFirewall = new DynamicHostRuleFiltering(); + +const permanentURLFiltering = new DynamicURLRuleFiltering(); +const sessionURLFiltering = new DynamicURLRuleFiltering(); + +const permanentSwitches = new DynamicSwitchRuleFiltering(); +const sessionSwitches = new DynamicSwitchRuleFiltering(); + +/******************************************************************************/ + +export { + permanentFirewall, + sessionFirewall, + permanentURLFiltering, + sessionURLFiltering, + permanentSwitches, + sessionSwitches, +}; diff --git a/src/js/hnswitches.js b/src/js/hnswitches.js new file mode 100644 index 0000000..9e94a8e --- /dev/null +++ b/src/js/hnswitches.js @@ -0,0 +1,289 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* jshint bitwise: false */ + +'use strict'; + +/******************************************************************************/ + +import punycode from '../lib/punycode.js'; + +import { decomposeHostname } from './uri-utils.js'; +import { LineIterator } from './text-utils.js'; + +/******************************************************************************/ + +const decomposedSource = []; + +// Object.create(null) is used below to eliminate worries about unexpected +// property names in prototype chain -- and this way we don't have to use +// hasOwnProperty() to avoid this. + +const switchBitOffsets = Object.create(null); +Object.assign(switchBitOffsets, { + 'no-strict-blocking': 0, + 'no-popups': 2, + 'no-cosmetic-filtering': 4, + 'no-remote-fonts': 6, + 'no-large-media': 8, + 'no-csp-reports': 10, + 'no-scripting': 12, +}); + +const switchStateToNameMap = Object.create(null); +Object.assign(switchStateToNameMap, { + '1': 'true', + '2': 'false', +}); + +const nameToSwitchStateMap = Object.create(null); +Object.assign(nameToSwitchStateMap, { + 'true': 1, + 'false': 2, + 'on': 1, + 'off': 2, +}); + +/******************************************************************************/ + +// For performance purpose, as simple test as possible +const reNotASCII = /[^\x20-\x7F]/; + +// http://tools.ietf.org/html/rfc5952 +// 4.3: "MUST be represented in lowercase" +// Also: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers + +/******************************************************************************/ + +class DynamicSwitchRuleFiltering { + constructor() { + this.reset(); + } + + reset() { + this.switches = new Map(); + this.n = ''; + this.z = ''; + this.r = 0; + this.changed = true; + } + + assign(from) { + // Remove rules not in other + for ( const hn of this.switches.keys() ) { + if ( from.switches.has(hn) === false ) { + this.switches.delete(hn); + this.changed = true; + } + } + // Add/change rules in other + for ( const [hn, bits] of from.switches ) { + if ( this.switches.get(hn) !== bits ) { + this.switches.set(hn, bits); + this.changed = true; + } + } + } + + copyRules(from, srcHostname) { + const thisBits = this.switches.get(srcHostname); + const fromBits = from.switches.get(srcHostname); + if ( fromBits !== thisBits ) { + if ( fromBits !== undefined ) { + this.switches.set(srcHostname, fromBits); + } else { + this.switches.delete(srcHostname); + } + this.changed = true; + } + return this.changed; + } + + hasSameRules(other, srcHostname) { + return this.switches.get(srcHostname) === other.switches.get(srcHostname); + } + + toggle(switchName, hostname, newVal) { + const bitOffset = switchBitOffsets[switchName]; + if ( bitOffset === undefined ) { return false; } + if ( newVal === this.evaluate(switchName, hostname) ) { return false; } + let bits = this.switches.get(hostname) || 0; + bits &= ~(3 << bitOffset); + bits |= newVal << bitOffset; + if ( bits === 0 ) { + this.switches.delete(hostname); + } else { + this.switches.set(hostname, bits); + } + this.changed = true; + return true; + } + + toggleOneZ(switchName, hostname, newState) { + const bitOffset = switchBitOffsets[switchName]; + if ( bitOffset === undefined ) { return false; } + let state = this.evaluateZ(switchName, hostname); + if ( newState === state ) { return false; } + if ( newState === undefined ) { + newState = !state; + } + let bits = this.switches.get(hostname) || 0; + bits &= ~(3 << bitOffset); + if ( bits === 0 ) { + this.switches.delete(hostname); + } else { + this.switches.set(hostname, bits); + } + state = this.evaluateZ(switchName, hostname); + if ( state !== newState ) { + this.switches.set(hostname, bits | ((newState ? 1 : 2) << bitOffset)); + } + this.changed = true; + return true; + } + + toggleBranchZ(switchName, targetHostname, newState) { + this.toggleOneZ(switchName, targetHostname, newState); + + // Turn off all descendant switches, they will inherit the state of the + // branch's origin. + const targetLen = targetHostname.length; + for ( const hostname of this.switches.keys() ) { + if ( hostname === targetHostname ) { continue; } + if ( hostname.length <= targetLen ) { continue; } + if ( hostname.endsWith(targetHostname) === false ) { continue; } + if ( hostname.charAt(hostname.length - targetLen - 1) !== '.' ) { + continue; + } + this.toggle(switchName, hostname, 0); + } + + return this.changed; + } + + toggleZ(switchName, hostname, deep, newState) { + if ( deep === true ) { + return this.toggleBranchZ(switchName, hostname, newState); + } + return this.toggleOneZ(switchName, hostname, newState); + } + + // 0 = inherit from broader scope, up to default state + // 1 = non-default state + // 2 = forced default state (to override a broader non-default state) + + evaluate(switchName, hostname) { + const bits = this.switches.get(hostname); + if ( bits === undefined ) { return 0; } + let bitOffset = switchBitOffsets[switchName]; + if ( bitOffset === undefined ) { return 0; } + return (bits >>> bitOffset) & 3; + } + + evaluateZ(switchName, hostname) { + const bitOffset = switchBitOffsets[switchName]; + if ( bitOffset === undefined ) { + this.r = 0; + return false; + } + this.n = switchName; + for ( const shn of decomposeHostname(hostname, decomposedSource) ) { + let bits = this.switches.get(shn); + if ( bits === undefined ) { continue; } + bits = bits >>> bitOffset & 3; + if ( bits === 0 ) { continue; } + this.z = shn; + this.r = bits; + return bits === 1; + } + this.r = 0; + return false; + } + + toLogData() { + return { + source: 'switch', + result: this.r, + raw: `${this.n}: ${this.z} true` + }; + } + + toArray() { + const out = []; + for ( const hostname of this.switches.keys() ) { + const prettyHn = hostname.includes('xn--') && punycode + ? punycode.toUnicode(hostname) + : hostname; + for ( const switchName in switchBitOffsets ) { + if ( switchBitOffsets[switchName] === undefined ) { continue; } + const val = this.evaluate(switchName, hostname); + if ( val === 0 ) { continue; } + out.push(`${switchName}: ${prettyHn} ${switchStateToNameMap[val]}`); + } + } + return out; + } + + toString() { + return this.toArray().join('\n'); + } + + fromString(text, append) { + const lineIter = new LineIterator(text); + if ( append !== true ) { this.reset(); } + while ( lineIter.eot() === false ) { + this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); + } + } + + validateRuleParts(parts) { + if ( parts.length < 3 ) { return; } + if ( parts[0].endsWith(':') === false ) { return; } + if ( nameToSwitchStateMap[parts[2]] === undefined ) { return; } + if ( reNotASCII.test(parts[1]) && punycode !== undefined ) { + parts[1] = punycode.toASCII(parts[1]); + } + return parts; + } + + addFromRuleParts(parts) { + if ( this.validateRuleParts(parts) === undefined ) { return false; } + const switchName = parts[0].slice(0, -1); + if ( switchBitOffsets[switchName] === undefined ) { return false; } + this.toggle(switchName, parts[1], nameToSwitchStateMap[parts[2]]); + return true; + } + + removeFromRuleParts(parts) { + if ( this.validateRuleParts(parts) !== undefined ) { + this.toggle(parts[0].slice(0, -1), parts[1], 0); + return true; + } + return false; + } +} + +/******************************************************************************/ + +export default DynamicSwitchRuleFiltering; + +/******************************************************************************/ diff --git a/src/js/hntrie.js b/src/js/hntrie.js new file mode 100644 index 0000000..e8031a6 --- /dev/null +++ b/src/js/hntrie.js @@ -0,0 +1,780 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals WebAssembly */ + +'use strict'; + +/******************************************************************************* + + The original prototype was to develop an idea I had about using jump indices + in a TypedArray for quickly matching hostnames (or more generally strings)[1]. + Once I had a working, un-optimized prototype, I realized I had ended up + with something formally named a "trie": <https://en.wikipedia.org/wiki/Trie>, + hence the name. I have no idea whether the implementation here or one + resembling it has been done elsewhere. + + "HN" in HNTrieContainer stands for "HostName", because the trie is + specialized to deal with matching hostnames -- which is a bit more + complicated than matching plain strings. + + For example, `www.abc.com` is deemed matching `abc.com`, because the former + is a subdomain of the latter. The opposite is of course not true. + + The resulting read-only tries created as a result of using HNTrieContainer + are simply just typed arrays filled with integers. The matching algorithm is + just a matter of reading/comparing these integers, and further using them as + indices in the array as a way to move around in the trie. + + [1] To solve <https://github.com/gorhill/uBlock/issues/3193> + + Since this trie is specialized for matching hostnames, the stored + strings are reversed internally, because of hostname comparison logic: + + Correct matching: + index 0123456 + abc.com + | + www.abc.com + index 01234567890 + + Incorrect matching (typically used for plain strings): + index 0123456 + abc.com + | + www.abc.com + index 01234567890 + + ------------------------------------------------------------------------------ + + 1st iteration: + - https://github.com/gorhill/uBlock/blob/ff58107dac3a32607f8113e39ed5015584506813/src/js/hntrie.js + - Suitable for small to medium set of hostnames + - One buffer per trie + + 2nd iteration: goal was to make matches() method wasm-able + - https://github.com/gorhill/uBlock/blob/c3b0fd31f64bd7ffecdd282fb1208fe07aac3eb0/src/js/hntrie.js + - Suitable for small to medium set of hostnames + - Distinct tries all share same buffer: + - Reduced memory footprint + - https://stackoverflow.com/questions/45803829/memory-overhead-of-typed-arrays-vs-strings/45808835#45808835 + - Reusing needle character lookups for all tries + - This significantly reduce the number of String.charCodeAt() calls + - Slightly improved creation time + + This is the 3rd iteration: goal was to make add() method wasm-able and + further improve memory/CPU efficiency. + + This 3rd iteration has the following new traits: + - Suitable for small to large set of hostnames + - Support multiple trie containers (instanciable) + - Designed to hold large number of hostnames + - Hostnames can be added at any time (instead of all at once) + - This means pre-sorting is no longer a requirement + - The trie is always compact + - There is no longer a need for a `vacuum` method + - This makes the add() method wasm-able + - It can return the exact hostname which caused the match + - serializable/unserializable available for fast loading + - Distinct trie reference support the iteration protocol, thus allowing + to extract all the hostnames in the trie + + Its primary purpose is to replace the use of Set() as a mean to hold + large number of hostnames (ex. FilterHostnameDict in static filtering + engine). + + A HNTrieContainer is mostly a large buffer in which distinct but related + tries are stored. The memory layout of the buffer is as follow: + + 0-254: needle being processed + 255: length of needle + 256-259: offset to start of trie data section (=> trie0) + 260-263: offset to end of trie data section (=> trie1) + 264-267: offset to start of character data section (=> char0) + 268-271: offset to end of character data section (=> char1) + 272: start of trie data section + +*/ + +const PAGE_SIZE = 65536; + // i32 / i8 +const TRIE0_SLOT = 256 >>> 2; // 64 / 256 +const TRIE1_SLOT = TRIE0_SLOT + 1; // 65 / 260 +const CHAR0_SLOT = TRIE0_SLOT + 2; // 66 / 264 +const CHAR1_SLOT = TRIE0_SLOT + 3; // 67 / 268 +const TRIE0_START = TRIE0_SLOT + 4 << 2; // 272 + +const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); + +class HNTrieContainer { + + constructor() { + const len = PAGE_SIZE * 2; + this.buf = new Uint8Array(len); + this.buf32 = new Uint32Array(this.buf.buffer); + this.needle = ''; + this.buf32[TRIE0_SLOT] = TRIE0_START; + this.buf32[TRIE1_SLOT] = this.buf32[TRIE0_SLOT]; + this.buf32[CHAR0_SLOT] = len >>> 1; + this.buf32[CHAR1_SLOT] = this.buf32[CHAR0_SLOT]; + this.wasmMemory = null; + + this.lastStored = ''; + this.lastStoredLen = this.lastStoredIndex = 0; + } + + //-------------------------------------------------------------------------- + // Public methods + //-------------------------------------------------------------------------- + + reset(details) { + if ( + details instanceof Object && + typeof details.byteLength === 'number' && + typeof details.char0 === 'number' + ) { + if ( details.byteLength > this.buf.byteLength ) { + this.reallocateBuf(details.byteLength); + } + this.buf32[CHAR0_SLOT] = details.char0; + } + this.buf32[TRIE1_SLOT] = this.buf32[TRIE0_SLOT]; + this.buf32[CHAR1_SLOT] = this.buf32[CHAR0_SLOT]; + + this.lastStored = ''; + this.lastStoredLen = this.lastStoredIndex = 0; + } + + setNeedle(needle) { + if ( needle !== this.needle ) { + const buf = this.buf; + let i = needle.length; + if ( i > 255 ) { i = 255; } + buf[255] = i; + while ( i-- ) { + buf[i] = needle.charCodeAt(i); + } + this.needle = needle; + } + return this; + } + + matchesJS(iroot) { + const buf32 = this.buf32; + const buf8 = this.buf; + const char0 = buf32[CHAR0_SLOT]; + let ineedle = buf8[255]; + let icell = buf32[iroot+0]; + if ( icell === 0 ) { return -1; } + let c = 0, v = 0, i0 = 0, n = 0; + for (;;) { + if ( ineedle === 0 ) { return -1; } + ineedle -= 1; + c = buf8[ineedle]; + // find first segment with a first-character match + for (;;) { + v = buf32[icell+2]; + i0 = char0 + (v >>> 8); + if ( buf8[i0] === c ) { break; } + icell = buf32[icell+0]; + if ( icell === 0 ) { return -1; } + } + // all characters in segment must match + n = v & 0x7F; + if ( n > 1 ) { + n -= 1; + if ( n > ineedle ) { return -1; } + i0 += 1; + const i1 = i0 + n; + do { + ineedle -= 1; + if ( buf8[i0] !== buf8[ineedle] ) { return -1; } + i0 += 1; + } while ( i0 < i1 ); + } + // boundary at end of segment? + if ( (v & 0x80) !== 0 ) { + if ( ineedle === 0 || buf8[ineedle-1] === 0x2E /* '.' */ ) { + return ineedle; + } + } + // next segment + icell = buf32[icell+1]; + if ( icell === 0 ) { break; } + } + return -1; + } + + createTrie() { + // grow buffer if needed + if ( (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < 12 ) { + this.growBuf(12, 0); + } + const iroot = this.buf32[TRIE1_SLOT] >>> 2; + this.buf32[TRIE1_SLOT] += 12; + this.buf32[iroot+0] = 0; + this.buf32[iroot+1] = 0; + this.buf32[iroot+2] = 0; + return iroot; + } + + createTrieFromIterable(hostnames) { + const itrie = this.createTrie(); + for ( const hn of hostnames ) { + if ( hn === '' ) { continue; } + this.setNeedle(hn).add(itrie); + } + return itrie; + } + + createTrieFromStoredDomainOpt(i, n) { + const itrie = this.createTrie(); + const jend = i + n; + let j = i, offset = 0, k = 0, c = 0; + while ( j !== jend ) { + offset = this.buf32[CHAR0_SLOT]; // Important + k = 0; + for (;;) { + if ( j === jend ) { break; } + c = this.buf[offset+j]; + j += 1; + if ( c === 0x7C /* '|' */ ) { break; } + if ( k === 255 ) { continue; } + this.buf[k] = c; + k += 1; + } + if ( k !== 0 ) { + this.buf[255] = k; + this.add(itrie); + } + } + this.needle = ''; // Important + this.buf[255] = 0; // Important + return itrie; + } + + dumpTrie(iroot) { + let hostnames = Array.from(this.trieIterator(iroot)); + if ( String.prototype.padStart instanceof Function ) { + const maxlen = Math.min( + hostnames.reduce((maxlen, hn) => Math.max(maxlen, hn.length), 0), + 64 + ); + hostnames = hostnames.map(hn => hn.padStart(maxlen)); + } + for ( const hn of hostnames ) { + console.log(hn); + } + } + + trieIterator(iroot) { + return { + value: undefined, + done: false, + next() { + if ( this.icell === 0 ) { + if ( this.forks.length === 0 ) { + this.value = undefined; + this.done = true; + return this; + } + this.charPtr = this.forks.pop(); + this.icell = this.forks.pop(); + } + for (;;) { + const idown = this.container.buf32[this.icell+0]; + if ( idown !== 0 ) { + this.forks.push(idown, this.charPtr); + } + const v = this.container.buf32[this.icell+2]; + let i0 = this.container.buf32[CHAR0_SLOT] + (v >>> 8); + const i1 = i0 + (v & 0x7F); + while ( i0 < i1 ) { + this.charPtr -= 1; + this.charBuf[this.charPtr] = this.container.buf[i0]; + i0 += 1; + } + this.icell = this.container.buf32[this.icell+1]; + if ( (v & 0x80) !== 0 ) { + return this.toHostname(); + } + } + }, + toHostname() { + this.value = this.textDecoder.decode( + new Uint8Array(this.charBuf.buffer, this.charPtr) + ); + return this; + }, + container: this, + icell: this.buf32[iroot], + charBuf: new Uint8Array(256), + charPtr: 256, + forks: [], + textDecoder: new TextDecoder(), + [Symbol.iterator]() { return this; }, + }; + } + + // TODO: + // Rework code to add from a string already present in the character + // buffer, i.e. not having to go through setNeedle() when adding a new + // hostname to a trie. This will require much work though, and probably + // changing the order in which string segments are stored in the + // character buffer. + addJS(iroot) { + let lhnchar = this.buf[255]; + if ( lhnchar === 0 ) { return 0; } + // grow buffer if needed + if ( + (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < 24 || + (this.buf.length - this.buf32[CHAR1_SLOT]) < 256 + ) { + this.growBuf(24, 256); + } + let icell = this.buf32[iroot+0]; + // special case: first node in trie + if ( icell === 0 ) { + this.buf32[iroot+0] = this.addLeafCell(lhnchar); + return 1; + } + // + const char0 = this.buf32[CHAR0_SLOT]; + let isegchar, lsegchar, boundaryBit, inext; + // find a matching cell: move down + for (;;) { + const v = this.buf32[icell+2]; + let isegchar0 = char0 + (v >>> 8); + // if first character is no match, move to next descendant + if ( this.buf[isegchar0] !== this.buf[lhnchar-1] ) { + inext = this.buf32[icell+0]; + if ( inext === 0 ) { + this.buf32[icell+0] = this.addLeafCell(lhnchar); + return 1; + } + icell = inext; + continue; + } + // 1st character was tested + isegchar = 1; + lhnchar -= 1; + // find 1st mismatch in rest of segment + lsegchar = v & 0x7F; + if ( lsegchar !== 1 ) { + for (;;) { + if ( isegchar === lsegchar ) { break; } + if ( lhnchar === 0 ) { break; } + if ( this.buf[isegchar0+isegchar] !== this.buf[lhnchar-1] ) { break; } + isegchar += 1; + lhnchar -= 1; + } + } + boundaryBit = v & 0x80; + // all segment characters matched + if ( isegchar === lsegchar ) { + // needle remainder: no + if ( lhnchar === 0 ) { + // boundary: yes, already present + if ( boundaryBit !== 0 ) { return 0; } + // boundary: no, mark as boundary + this.buf32[icell+2] = v | 0x80; + } + // needle remainder: yes + else { + // remainder is at label boundary? if yes, no need to add + // the rest since the shortest match is always reported + if ( boundaryBit !== 0 ) { + if ( this.buf[lhnchar-1] === 0x2E /* '.' */ ) { return -1; } + } + inext = this.buf32[icell+1]; + if ( inext !== 0 ) { + icell = inext; + continue; + } + // add needle remainder + this.buf32[icell+1] = this.addLeafCell(lhnchar); + } + } + // some segment characters matched + else { + // split current cell + isegchar0 -= char0; + this.buf32[icell+2] = isegchar0 << 8 | isegchar; + inext = this.addCell( + 0, + this.buf32[icell+1], + isegchar0 + isegchar << 8 | boundaryBit | lsegchar - isegchar + ); + this.buf32[icell+1] = inext; + // needle remainder: yes, need new cell for remaining characters + if ( lhnchar !== 0 ) { + this.buf32[inext+0] = this.addLeafCell(lhnchar); + } + // needle remainder: no, need boundary cell + else { + this.buf32[icell+2] |= 0x80; + } + } + return 1; + } + } + + optimize() { + this.shrinkBuf(); + return { + byteLength: this.buf.byteLength, + char0: this.buf32[CHAR0_SLOT], + }; + } + + serialize(encoder) { + if ( encoder instanceof Object ) { + return encoder.encode( + this.buf32.buffer, + this.buf32[CHAR1_SLOT] + ); + } + return Array.from( + new Uint32Array( + this.buf32.buffer, + 0, + this.buf32[CHAR1_SLOT] + 3 >>> 2 + ) + ); + } + + unserialize(selfie, decoder) { + this.needle = ''; + const shouldDecode = typeof selfie === 'string'; + let byteLength = shouldDecode + ? decoder.decodeSize(selfie) + : selfie.length << 2; + if ( byteLength === 0 ) { return false; } + byteLength = roundToPageSize(byteLength); + if ( this.wasmMemory !== null ) { + const pageCountBefore = this.buf.length >>> 16; + const pageCountAfter = byteLength >>> 16; + if ( pageCountAfter > pageCountBefore ) { + this.wasmMemory.grow(pageCountAfter - pageCountBefore); + this.buf = new Uint8Array(this.wasmMemory.buffer); + this.buf32 = new Uint32Array(this.buf.buffer); + } + } else if ( byteLength > this.buf.length ) { + this.buf = new Uint8Array(byteLength); + this.buf32 = new Uint32Array(this.buf.buffer); + } + if ( shouldDecode ) { + decoder.decode(selfie, this.buf.buffer); + } else { + this.buf32.set(selfie); + } + // https://github.com/uBlockOrigin/uBlock-issues/issues/2925 + this.buf[255] = 0; + return true; + } + + // The following *Hostname() methods can be used to store hostname strings + // outside the trie. This is useful to store/match hostnames which are + // not part of a collection, and yet still benefit from storing the strings + // into a trie container's character buffer. + // TODO: WASM version of matchesHostname() + + storeHostname(hn) { + let n = hn.length; + if ( n > 255 ) { + hn = hn.slice(-255); + n = 255; + } + if ( n === this.lastStoredLen && hn === this.lastStored ) { + return this.lastStoredIndex; + } + this.lastStored = hn; + this.lastStoredLen = n; + if ( (this.buf.length - this.buf32[CHAR1_SLOT]) < n ) { + this.growBuf(0, n); + } + const offset = this.buf32[CHAR1_SLOT]; + this.buf32[CHAR1_SLOT] = offset + n; + const buf8 = this.buf; + for ( let i = 0; i < n; i++ ) { + buf8[offset+i] = hn.charCodeAt(i); + } + return (this.lastStoredIndex = offset - this.buf32[CHAR0_SLOT]); + } + + extractHostname(i, n) { + const textDecoder = new TextDecoder(); + const offset = this.buf32[CHAR0_SLOT] + i; + return textDecoder.decode(this.buf.subarray(offset, offset + n)); + } + + storeDomainOpt(s) { + let n = s.length; + if ( n === this.lastStoredLen && s === this.lastStored ) { + return this.lastStoredIndex; + } + this.lastStored = s; + this.lastStoredLen = n; + if ( (this.buf.length - this.buf32[CHAR1_SLOT]) < n ) { + this.growBuf(0, n); + } + const offset = this.buf32[CHAR1_SLOT]; + this.buf32[CHAR1_SLOT] = offset + n; + const buf8 = this.buf; + for ( let i = 0; i < n; i++ ) { + buf8[offset+i] = s.charCodeAt(i); + } + return (this.lastStoredIndex = offset - this.buf32[CHAR0_SLOT]); + } + + extractDomainOpt(i, n) { + const textDecoder = new TextDecoder(); + const offset = this.buf32[CHAR0_SLOT] + i; + return textDecoder.decode(this.buf.subarray(offset, offset + n)); + } + + matchesHostname(hn, i, n) { + this.setNeedle(hn); + const buf8 = this.buf; + const hr = buf8[255]; + if ( n > hr ) { return false; } + const hl = hr - n; + const nl = this.buf32[CHAR0_SLOT] + i; + for ( let j = 0; j < n; j++ ) { + if ( buf8[nl+j] !== buf8[hl+j] ) { return false; } + } + return n === hr || hn.charCodeAt(hl-1) === 0x2E /* '.' */; + } + + async enableWASM(wasmModuleFetcher, path) { + if ( typeof WebAssembly === 'undefined' ) { return false; } + if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; } + const module = await getWasmModule(wasmModuleFetcher, path); + if ( module instanceof WebAssembly.Module === false ) { return false; } + const memory = new WebAssembly.Memory({ initial: 2 }); + const instance = await WebAssembly.instantiate(module, { + imports: { + memory, + growBuf: this.growBuf.bind(this, 24, 256) + } + }); + if ( instance instanceof WebAssembly.Instance === false ) { return false; } + this.wasmMemory = memory; + const curPageCount = memory.buffer.byteLength >>> 16; + const newPageCount = roundToPageSize(this.buf.byteLength) >>> 16; + if ( newPageCount > curPageCount ) { + memory.grow(newPageCount - curPageCount); + } + const buf = new Uint8Array(memory.buffer); + buf.set(this.buf); + this.buf = buf; + this.buf32 = new Uint32Array(this.buf.buffer); + this.matches = this.matchesWASM = instance.exports.matches; + this.add = this.addWASM = instance.exports.add; + return true; + } + + dumpInfo() { + return [ + `Buffer size (Uint8Array): ${this.buf32[CHAR1_SLOT].toLocaleString('en')}`, + `WASM: ${this.wasmMemory === null ? 'disabled' : 'enabled'}`, + ].join('\n'); + } + + //-------------------------------------------------------------------------- + // Private methods + //-------------------------------------------------------------------------- + + addCell(idown, iright, v) { + let icell = this.buf32[TRIE1_SLOT]; + this.buf32[TRIE1_SLOT] = icell + 12; + icell >>>= 2; + this.buf32[icell+0] = idown; + this.buf32[icell+1] = iright; + this.buf32[icell+2] = v; + return icell; + } + + addLeafCell(lsegchar) { + const r = this.buf32[TRIE1_SLOT] >>> 2; + let i = r; + while ( lsegchar > 127 ) { + this.buf32[i+0] = 0; + this.buf32[i+1] = i + 3; + this.buf32[i+2] = this.addSegment(lsegchar, lsegchar - 127); + lsegchar -= 127; + i += 3; + } + this.buf32[i+0] = 0; + this.buf32[i+1] = 0; + this.buf32[i+2] = this.addSegment(lsegchar, 0) | 0x80; + this.buf32[TRIE1_SLOT] = i + 3 << 2; + return r; + } + + addSegment(lsegchar, lsegend) { + if ( lsegchar === 0 ) { return 0; } + let char1 = this.buf32[CHAR1_SLOT]; + const isegchar = char1 - this.buf32[CHAR0_SLOT]; + let i = lsegchar; + do { + this.buf[char1++] = this.buf[--i]; + } while ( i !== lsegend ); + this.buf32[CHAR1_SLOT] = char1; + return isegchar << 8 | lsegchar - lsegend; + } + + growBuf(trieGrow, charGrow) { + const char0 = Math.max( + roundToPageSize(this.buf32[TRIE1_SLOT] + trieGrow), + this.buf32[CHAR0_SLOT] + ); + const char1 = char0 + this.buf32[CHAR1_SLOT] - this.buf32[CHAR0_SLOT]; + const bufLen = Math.max( + roundToPageSize(char1 + charGrow), + this.buf.length + ); + this.resizeBuf(bufLen, char0); + } + + shrinkBuf() { + // Can't shrink WebAssembly.Memory + if ( this.wasmMemory !== null ) { return; } + const char0 = this.buf32[TRIE1_SLOT] + 24; + const char1 = char0 + this.buf32[CHAR1_SLOT] - this.buf32[CHAR0_SLOT]; + const bufLen = char1 + 256; + this.resizeBuf(bufLen, char0); + } + + resizeBuf(bufLen, char0) { + bufLen = roundToPageSize(bufLen); + if ( bufLen === this.buf.length && char0 === this.buf32[CHAR0_SLOT] ) { + return; + } + const charDataLen = this.buf32[CHAR1_SLOT] - this.buf32[CHAR0_SLOT]; + if ( this.wasmMemory !== null ) { + const pageCount = (bufLen >>> 16) - (this.buf.byteLength >>> 16); + if ( pageCount > 0 ) { + this.wasmMemory.grow(pageCount); + this.buf = new Uint8Array(this.wasmMemory.buffer); + this.buf32 = new Uint32Array(this.wasmMemory.buffer); + } + } else if ( bufLen !== this.buf.length ) { + const newBuf = new Uint8Array(bufLen); + newBuf.set( + new Uint8Array( + this.buf.buffer, + 0, + this.buf32[TRIE1_SLOT] + ), + 0 + ); + newBuf.set( + new Uint8Array( + this.buf.buffer, + this.buf32[CHAR0_SLOT], + charDataLen + ), + char0 + ); + this.buf = newBuf; + this.buf32 = new Uint32Array(this.buf.buffer); + this.buf32[CHAR0_SLOT] = char0; + this.buf32[CHAR1_SLOT] = char0 + charDataLen; + } + if ( char0 !== this.buf32[CHAR0_SLOT] ) { + this.buf.set( + new Uint8Array( + this.buf.buffer, + this.buf32[CHAR0_SLOT], + charDataLen + ), + char0 + ); + this.buf32[CHAR0_SLOT] = char0; + this.buf32[CHAR1_SLOT] = char0 + charDataLen; + } + } + + reallocateBuf(newSize) { + newSize = roundToPageSize(newSize); + if ( newSize === this.buf.length ) { return; } + if ( this.wasmMemory === null ) { + const newBuf = new Uint8Array(newSize); + newBuf.set( + newBuf.length < this.buf.length + ? this.buf.subarray(0, newBuf.length) + : this.buf + ); + this.buf = newBuf; + } else { + const growBy = + ((newSize + 0xFFFF) >>> 16) - (this.buf.length >>> 16); + if ( growBy <= 0 ) { return; } + this.wasmMemory.grow(growBy); + this.buf = new Uint8Array(this.wasmMemory.buffer); + } + this.buf32 = new Uint32Array(this.buf.buffer); + } +} + +HNTrieContainer.prototype.matches = HNTrieContainer.prototype.matchesJS; +HNTrieContainer.prototype.matchesWASM = null; + +HNTrieContainer.prototype.add = HNTrieContainer.prototype.addJS; +HNTrieContainer.prototype.addWASM = null; + +/******************************************************************************/ + +// Code below is to attempt to load a WASM module which implements: +// +// - HNTrieContainer.add() +// - HNTrieContainer.matches() +// +// The WASM module is entirely optional, the JS implementations will be +// used should the WASM module be unavailable for whatever reason. + +const getWasmModule = (( ) => { + let wasmModulePromise; + + return async function(wasmModuleFetcher, path) { + if ( wasmModulePromise instanceof Promise ) { + return wasmModulePromise; + } + + // The wasm module will work only if CPU is natively little-endian, + // as we use native uint32 array in our js code. + const uint32s = new Uint32Array(1); + const uint8s = new Uint8Array(uint32s.buffer); + uint32s[0] = 1; + if ( uint8s[0] !== 1 ) { return; } + + wasmModulePromise = wasmModuleFetcher(`${path}hntrie`).catch(reason => { + console.info(reason); + }); + + return wasmModulePromise; + }; +})(); + +/******************************************************************************/ + +export default HNTrieContainer; diff --git a/src/js/html-filtering.js b/src/js/html-filtering.js new file mode 100644 index 0000000..81d66ee --- /dev/null +++ b/src/js/html-filtering.js @@ -0,0 +1,465 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import logger from './logger.js'; +import µb from './background.js'; +import { sessionFirewall } from './filtering-engines.js'; +import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js'; +import { entityFromDomain } from './uri-utils.js'; + +/******************************************************************************/ + +const pselectors = new Map(); +const duplicates = new Set(); + +const filterDB = new StaticExtFilteringHostnameDB(2); + +let acceptedCount = 0; +let discardedCount = 0; +let docRegister; + +const htmlFilteringEngine = { + get acceptedCount() { + return acceptedCount; + }, + get discardedCount() { + return discardedCount; + }, + getFilterCount() { + return filterDB.size; + }, +}; + +const regexFromString = (s, exact = false) => { + if ( s === '' ) { return /^/; } + const match = /^\/(.+)\/([i]?)$/.exec(s); + if ( match !== null ) { + return new RegExp(match[1], match[2] || undefined); + } + const reStr = s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(exact ? `^${reStr}$` : reStr, 'i'); +}; + +class PSelectorVoidTask { + constructor(task) { + console.info(`[uBO] HTML filtering: :${task[0]}() operator is not supported`); + } + transpose() { + } +} +class PSelectorHasTextTask { + constructor(task) { + this.needle = regexFromString(task[1]); + } + transpose(node, output) { + if ( this.needle.test(node.textContent) ) { + output.push(node); + } + } +} + +const PSelectorIfTask = class { + constructor(task) { + this.pselector = new PSelector(task[1]); + } + transpose(node, output) { + if ( this.pselector.test(node) === this.target ) { + output.push(node); + } + } +}; +PSelectorIfTask.prototype.target = true; + +class PSelectorIfNotTask extends PSelectorIfTask { +} +PSelectorIfNotTask.prototype.target = false; + +class PSelectorMinTextLengthTask { + constructor(task) { + this.min = task[1]; + } + transpose(node, output) { + if ( node.textContent.length >= this.min ) { + output.push(node); + } + } +} + +class PSelectorSpathTask { + constructor(task) { + this.spath = task[1]; + this.nth = /^(?:\s*[+~]|:)/.test(this.spath); + if ( this.nth ) { return; } + if ( /^\s*>/.test(this.spath) ) { + this.spath = `:scope ${this.spath.trim()}`; + } + } + transpose(node, output) { + const nodes = this.nth + ? PSelectorSpathTask.qsa(node, this.spath) + : node.querySelectorAll(this.spath); + for ( const node of nodes ) { + output.push(node); + } + } + // Helper method for other operators. + static qsa(node, selector) { + const parent = node.parentElement; + if ( parent === null ) { return []; } + let pos = 1; + for (;;) { + node = node.previousElementSibling; + if ( node === null ) { break; } + pos += 1; + } + return parent.querySelectorAll( + `:scope > :nth-child(${pos})${selector}` + ); + } +} + +class PSelectorUpwardTask { + constructor(task) { + const arg = task[1]; + if ( typeof arg === 'number' ) { + this.i = arg; + } else { + this.s = arg; + } + } + transpose(node, output) { + if ( this.s !== '' ) { + const parent = node.parentElement; + if ( parent === null ) { return; } + node = parent.closest(this.s); + if ( node === null ) { return; } + } else { + let nth = this.i; + for (;;) { + node = node.parentElement; + if ( node === null ) { return; } + nth -= 1; + if ( nth === 0 ) { break; } + } + } + output.push(node); + } +} +PSelectorUpwardTask.prototype.i = 0; +PSelectorUpwardTask.prototype.s = ''; + +class PSelectorXpathTask { + constructor(task) { + this.xpe = task[1]; + } + transpose(node, output) { + const xpr = docRegister.evaluate( + this.xpe, + node, + null, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null + ); + let j = xpr.snapshotLength; + while ( j-- ) { + const node = xpr.snapshotItem(j); + if ( node.nodeType === 1 ) { + output.push(node); + } + } + } +} + +class PSelector { + constructor(o) { + this.raw = o.raw; + this.selector = o.selector; + this.tasks = []; + if ( !o.tasks ) { return; } + for ( const task of o.tasks ) { + const ctor = this.operatorToTaskMap.get(task[0]) || PSelectorVoidTask; + const pselector = new ctor(task); + this.tasks.push(pselector); + } + } + prime(input) { + const root = input || docRegister; + if ( this.selector === '' ) { return [ root ]; } + if ( input !== docRegister && /^ ?[>+~]/.test(this.selector) ) { + return Array.from(PSelectorSpathTask.qsa(input, this.selector)); + } + return Array.from(root.querySelectorAll(this.selector)); + } + exec(input) { + let nodes = this.prime(input); + for ( const task of this.tasks ) { + if ( nodes.length === 0 ) { break; } + const transposed = []; + for ( const node of nodes ) { + task.transpose(node, transposed); + } + nodes = transposed; + } + return nodes; + } + test(input) { + const nodes = this.prime(input); + for ( const node of nodes ) { + let output = [ node ]; + for ( const task of this.tasks ) { + const transposed = []; + for ( const node of output ) { + task.transpose(node, transposed); + } + output = transposed; + if ( output.length === 0 ) { break; } + } + if ( output.length !== 0 ) { return true; } + } + return false; + } +} +PSelector.prototype.operatorToTaskMap = new Map([ + [ 'has', PSelectorIfTask ], + [ 'has-text', PSelectorHasTextTask ], + [ 'if', PSelectorIfTask ], + [ 'if-not', PSelectorIfNotTask ], + [ 'min-text-length', PSelectorMinTextLengthTask ], + [ 'not', PSelectorIfNotTask ], + [ 'nth-ancestor', PSelectorUpwardTask ], + [ 'spath', PSelectorSpathTask ], + [ 'upward', PSelectorUpwardTask ], + [ 'xpath', PSelectorXpathTask ], +]); + +function logOne(details, exception, selector) { + µb.filteringContext + .duplicate() + .fromTabId(details.tabId) + .setRealm('extended') + .setType('dom') + .setURL(details.url) + .setDocOriginFromURL(details.url) + .setFilter({ + source: 'extended', + raw: `${exception === 0 ? '##' : '#@#'}^${selector}` + }) + .toLogger(); +} + +function applyProceduralSelector(details, selector) { + let pselector = pselectors.get(selector); + if ( pselector === undefined ) { + pselector = new PSelector(JSON.parse(selector)); + pselectors.set(selector, pselector); + } + const nodes = pselector.exec(); + let modified = false; + for ( const node of nodes ) { + node.remove(); + modified = true; + } + if ( modified && logger.enabled ) { + logOne(details, 0, pselector.raw); + } + return modified; +} + +function applyCSSSelector(details, selector) { + const nodes = docRegister.querySelectorAll(selector); + let modified = false; + for ( const node of nodes ) { + node.remove(); + modified = true; + } + if ( modified && logger.enabled ) { + logOne(details, 0, selector); + } + return modified; +} + +function logError(writer, msg) { + logger.writeOne({ + realm: 'message', + type: 'error', + text: msg.replace('{who}', writer.properties.get('name') || '?') + }); +} + +htmlFilteringEngine.reset = function() { + filterDB.clear(); + pselectors.clear(); + duplicates.clear(); + acceptedCount = 0; + discardedCount = 0; +}; + +htmlFilteringEngine.freeze = function() { + duplicates.clear(); + filterDB.collectGarbage(); +}; + +htmlFilteringEngine.compile = function(parser, writer) { + const isException = parser.isException(); + const { raw, compiled } = parser.result; + if ( compiled === undefined ) { + return logError(writer, `Invalid HTML filter in {who}: ##${raw}`); + } + + writer.select('HTML_FILTERS'); + + // Only exception filters are allowed to be global. + if ( parser.hasOptions() === false ) { + if ( isException ) { + writer.push([ 64, '', 1, compiled ]); + } + return; + } + + const compiledFilters = []; + let hasOnlyNegated = true; + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + let kind = isException ? 0b01 : 0b00; + if ( not ) { + kind ^= 0b01; + } else { + hasOnlyNegated = false; + } + if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { + kind |= 0b10; + } + compiledFilters.push([ 64, hn, kind, compiled ]); + } + + // Not allowed since it's equivalent to forbidden generic HTML filters + if ( isException === false && hasOnlyNegated ) { + return logError(writer, `Invalid HTML filter in {who}: ##${raw}`); + } + + writer.pushMany(compiledFilters); +}; + +htmlFilteringEngine.fromCompiledContent = function(reader) { + // Don't bother loading filters if stream filtering is not supported. + if ( µb.canFilterResponseData === false ) { return; } + + reader.select('HTML_FILTERS'); + + while ( reader.next() ) { + acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( duplicates.has(fingerprint) ) { + discardedCount += 1; + continue; + } + duplicates.add(fingerprint); + const args = reader.args(); + filterDB.store(args[1], args[2], args[3]); + } +}; + +htmlFilteringEngine.retrieve = function(fctxt) { + const plains = new Set(); + const procedurals = new Set(); + const exceptions = new Set(); + const retrieveSets = [ plains, exceptions, procedurals, exceptions ]; + + const hostname = fctxt.getHostname(); + filterDB.retrieve(hostname, retrieveSets); + + const domain = fctxt.getDomain(); + const entity = entityFromDomain(domain); + const hostnameEntity = entity !== '' + ? `${hostname.slice(0, -domain.length)}${entity}` + : '*'; + filterDB.retrieve(hostnameEntity, retrieveSets, 1); + + if ( plains.size === 0 && procedurals.size === 0 ) { return; } + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not filter if the site is under an `allow` rule. + if ( + µb.userSettings.advancedUserEnabled && + sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 + ) { + return; + } + + const out = { plains, procedurals }; + + if ( exceptions.size === 0 ) { + return out; + } + + for ( const selector of exceptions ) { + if ( plains.has(selector) ) { + plains.delete(selector); + logOne(fctxt, 1, selector); + continue; + } + if ( procedurals.has(selector) ) { + procedurals.delete(selector); + logOne(fctxt, 1, JSON.parse(selector).raw); + continue; + } + } + + if ( plains.size !== 0 || procedurals.size !== 0 ) { + return out; + } +}; + +htmlFilteringEngine.apply = function(doc, details, selectors) { + docRegister = doc; + let modified = false; + for ( const selector of selectors.plains ) { + if ( applyCSSSelector(details, selector) ) { + modified = true; + } + } + for ( const selector of selectors.procedurals ) { + if ( applyProceduralSelector(details, selector) ) { + modified = true; + } + } + docRegister = undefined; + return modified; +}; + +htmlFilteringEngine.toSelfie = function() { + return filterDB.toSelfie(); +}; + +htmlFilteringEngine.fromSelfie = function(selfie) { + filterDB.fromSelfie(selfie); + pselectors.clear(); +}; + +/******************************************************************************/ + +export default htmlFilteringEngine; + +/******************************************************************************/ diff --git a/src/js/httpheader-filtering.js b/src/js/httpheader-filtering.js new file mode 100644 index 0000000..522ea21 --- /dev/null +++ b/src/js/httpheader-filtering.js @@ -0,0 +1,213 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2021-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import logger from './logger.js'; +import µb from './background.js'; +import { entityFromDomain } from './uri-utils.js'; +import { sessionFirewall } from './filtering-engines.js'; +import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js'; +import * as sfp from './static-filtering-parser.js'; + +/******************************************************************************/ + +const duplicates = new Set(); +const filterDB = new StaticExtFilteringHostnameDB(1); + +const $headers = new Set(); +const $exceptions = new Set(); + +let acceptedCount = 0; +let discardedCount = 0; + +const headerIndexFromName = function(name, headers, start = 0) { + for ( let i = start; i < headers.length; i++ ) { + if ( headers[i].name.toLowerCase() !== name ) { continue; } + return i; + } + return -1; +}; + +const logOne = function(isException, token, fctxt) { + fctxt.duplicate() + .setRealm('extended') + .setType('header') + .setFilter({ + modifier: true, + result: isException ? 2 : 1, + source: 'extended', + raw: `${(isException ? '#@#' : '##')}^responseheader(${token})` + }) + .toLogger(); +}; + +const httpheaderFilteringEngine = { + get acceptedCount() { + return acceptedCount; + }, + get discardedCount() { + return discardedCount; + } +}; + +httpheaderFilteringEngine.reset = function() { + filterDB.clear(); + duplicates.clear(); + acceptedCount = 0; + discardedCount = 0; +}; + +httpheaderFilteringEngine.freeze = function() { + duplicates.clear(); + filterDB.collectGarbage(); +}; + +httpheaderFilteringEngine.compile = function(parser, writer) { + writer.select('HTTPHEADER_FILTERS'); + + const isException = parser.isException(); + const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER); + const headerName = parser.getNodeString(root); + + // Tokenless is meaningful only for exception filters. + if ( headerName === '' && isException === false ) { return; } + + // Only exception filters are allowed to be global. + if ( parser.hasOptions() === false ) { + if ( isException ) { + writer.push([ 64, '', 1, headerName ]); + } + return; + } + + // https://github.com/gorhill/uBlock/issues/3375 + // Ignore instances of exception filter with negated hostnames, + // because there is no way to create an exception to an exception. + + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + let kind = 0; + if ( isException ) { + if ( not ) { continue; } + kind |= 1; + } else if ( not ) { + kind |= 1; + } + writer.push([ 64, hn, kind, headerName ]); + } +}; + +// 01234567890123456789 +// responseheader(name) +// ^ ^ +// 15 -1 + +httpheaderFilteringEngine.fromCompiledContent = function(reader) { + reader.select('HTTPHEADER_FILTERS'); + + while ( reader.next() ) { + acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( duplicates.has(fingerprint) ) { + discardedCount += 1; + continue; + } + duplicates.add(fingerprint); + const args = reader.args(); + if ( args.length < 4 ) { continue; } + filterDB.store(args[1], args[2], args[3]); + } +}; + +httpheaderFilteringEngine.apply = function(fctxt, headers) { + if ( filterDB.size === 0 ) { return; } + + const hostname = fctxt.getHostname(); + if ( hostname === '' ) { return; } + + const domain = fctxt.getDomain(); + let entity = entityFromDomain(domain); + if ( entity !== '' ) { + entity = `${hostname.slice(0, -domain.length)}${entity}`; + } else { + entity = '*'; + } + + $headers.clear(); + $exceptions.clear(); + + filterDB.retrieve(hostname, [ $headers, $exceptions ]); + filterDB.retrieve(entity, [ $headers, $exceptions ], 1); + if ( $headers.size === 0 ) { return; } + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not filter response headers if the site is under an `allow` rule. + if ( + µb.userSettings.advancedUserEnabled && + sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 + ) { + return; + } + + const hasGlobalException = $exceptions.has(''); + + let modified = false; + let i = 0; + + for ( const name of $headers ) { + const isExcepted = hasGlobalException || $exceptions.has(name); + if ( isExcepted ) { + if ( logger.enabled ) { + logOne(true, hasGlobalException ? '' : name, fctxt); + } + continue; + } + i = 0; + for (;;) { + i = headerIndexFromName(name, headers, i); + if ( i === -1 ) { break; } + headers.splice(i, 1); + if ( logger.enabled ) { + logOne(false, name, fctxt); + } + modified = true; + } + } + + return modified; +}; + +httpheaderFilteringEngine.toSelfie = function() { + return filterDB.toSelfie(); +}; + +httpheaderFilteringEngine.fromSelfie = function(selfie) { + filterDB.fromSelfie(selfie); +}; + +/******************************************************************************/ + +export default httpheaderFilteringEngine; + +/******************************************************************************/ diff --git a/src/js/i18n.js b/src/js/i18n.js new file mode 100644 index 0000000..6302b35 --- /dev/null +++ b/src/js/i18n.js @@ -0,0 +1,346 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +const i18n = + self.browser instanceof Object && + self.browser instanceof Element === false + ? self.browser.i18n + : self.chrome.i18n; + +/******************************************************************************/ + +function i18n$(...args) { + return i18n.getMessage(...args); +} + +/******************************************************************************/ + +const isBackgroundProcess = document.title === 'uBlock Origin Background Page'; + +if ( isBackgroundProcess !== true ) { + + // http://www.w3.org/International/questions/qa-scripts#directions + document.body.setAttribute( + 'dir', + ['ar', 'he', 'fa', 'ps', 'ur'].indexOf(i18n$('@@ui_locale')) !== -1 + ? 'rtl' + : 'ltr' + ); + + // https://github.com/gorhill/uBlock/issues/2084 + // Anything else than <a>, <b>, <code>, <em>, <i>, and <span> will + // be rendered as plain text. + // For <a>, only href attribute must be present, and it MUST starts with + // `https://`, and includes no single- or double-quotes. + // No HTML entities are allowed, there is code to handle existing HTML + // entities already present in translation files until they are all gone. + + const allowedTags = new Set([ + 'a', + 'b', + 'code', + 'em', + 'i', + 'span', + 'u', + ]); + + const expandHtmlEntities = (( ) => { + const entities = new Map([ + // TODO: Remove quote entities once no longer present in translation + // files. Other entities must stay. + [ '­', '\u00AD' ], + [ '“', '“' ], + [ '”', '”' ], + [ '‘', '‘' ], + [ '’', '’' ], + [ '<', '<' ], + [ '>', '>' ], + ]); + const decodeEntities = match => { + return entities.get(match) || match; + }; + return function(text) { + if ( text.indexOf('&') !== -1 ) { + text = text.replace(/&[a-z]+;/g, decodeEntities); + } + return text; + }; + })(); + + const safeTextToTextNode = function(text) { + return document.createTextNode(expandHtmlEntities(text)); + }; + + const sanitizeElement = function(node) { + if ( allowedTags.has(node.localName) === false ) { return null; } + node.removeAttribute('style'); + let child = node.firstElementChild; + while ( child !== null ) { + const next = child.nextElementSibling; + if ( sanitizeElement(child) === null ) { + child.remove(); + } + child = next; + } + return node; + }; + + const safeTextToDOM = function(text, parent) { + if ( text === '' ) { return; } + + // Fast path (most common). + if ( text.indexOf('<') === -1 ) { + const toInsert = safeTextToTextNode(text); + let toReplace = parent.childCount !== 0 + ? parent.firstChild + : null; + while ( toReplace !== null ) { + if ( toReplace.nodeType === 3 && toReplace.nodeValue === '_' ) { + break; + } + toReplace = toReplace.nextSibling; + } + if ( toReplace !== null ) { + parent.replaceChild(toInsert, toReplace); + } else { + parent.appendChild(toInsert); + } + return; + } + + // Slow path. + // `<p>` no longer allowed. Code below can be removed once all <p>'s are + // gone from translation files. + text = text.replace(/^<p>|<\/p>/g, '') + .replace(/<p>/g, '\n\n'); + // Parse allowed HTML tags. + const domParser = new DOMParser(); + const parsedDoc = domParser.parseFromString(text, 'text/html'); + let node = parsedDoc.body.firstChild; + while ( node !== null ) { + const next = node.nextSibling; + switch ( node.nodeType ) { + case 1: // element + if ( sanitizeElement(node) === null ) { break; } + parent.appendChild(node); + break; + case 3: // text + parent.appendChild(node); + break; + default: + break; + } + node = next; + } + }; + + i18n.safeTemplateToDOM = function(id, dict, parent) { + if ( parent === undefined ) { + parent = document.createDocumentFragment(); + } + let textin = i18n$(id); + if ( textin === '' ) { + return parent; + } + if ( textin.indexOf('{{') === -1 ) { + safeTextToDOM(textin, parent); + return parent; + } + const re = /\{\{\w+\}\}/g; + let textout = ''; + for (;;) { + let match = re.exec(textin); + if ( match === null ) { + textout += textin; + break; + } + textout += textin.slice(0, match.index); + let prop = match[0].slice(2, -2); + if ( dict.hasOwnProperty(prop) ) { + textout += dict[prop].replace(/</g, '<') + .replace(/>/g, '>'); + } else { + textout += prop; + } + textin = textin.slice(re.lastIndex); + } + safeTextToDOM(textout, parent); + return parent; + }; + + // Helper to deal with the i18n'ing of HTML files. + i18n.render = function(context) { + const docu = document; + const root = context || docu; + + for ( const elem of root.querySelectorAll('[data-i18n]') ) { + let text = i18n$(elem.getAttribute('data-i18n')); + if ( !text ) { continue; } + if ( text.indexOf('{{') === -1 ) { + safeTextToDOM(text, elem); + continue; + } + // Handle selector-based placeholders: these placeholders tell where + // existing child DOM element are to be positioned relative to the + // localized text nodes. + const parts = text.split(/(\{\{[^}]+\}\})/); + const fragment = document.createDocumentFragment(); + let textBefore = ''; + for ( let part of parts ) { + if ( part === '' ) { continue; } + if ( part.startsWith('{{') && part.endsWith('}}') ) { + // TODO: remove detection of ':' once it no longer appears + // in translation files. + const pos = part.indexOf(':'); + if ( pos !== -1 ) { + part = part.slice(0, pos) + part.slice(-2); + } + const selector = part.slice(2, -2); + let node; + // Ideally, the i18n strings explicitly refer to the + // class of the element to insert. However for now we + // will create a class from what is currently found in + // the placeholder and first try to lookup the resulting + // selector. This way we don't have to revisit all + // translations just for the sake of declaring the proper + // selector in the placeholder field. + if ( selector.charCodeAt(0) !== 0x2E /* '.' */ ) { + node = elem.querySelector(`.${selector}`); + } + if ( node instanceof Element === false ) { + node = elem.querySelector(selector); + } + if ( node instanceof Element ) { + safeTextToDOM(textBefore, fragment); + fragment.appendChild(node); + textBefore = ''; + continue; + } + } + textBefore += part; + } + if ( textBefore !== '' ) { + safeTextToDOM(textBefore, fragment); + } + elem.appendChild(fragment); + } + + for ( const elem of root.querySelectorAll('[data-i18n-title]') ) { + const text = i18n$(elem.getAttribute('data-i18n-title')); + if ( !text ) { continue; } + elem.setAttribute('title', expandHtmlEntities(text)); + } + + for ( const elem of root.querySelectorAll('[placeholder]') ) { + const text = i18n$(elem.getAttribute('placeholder')); + if ( text === '' ) { continue; } + elem.setAttribute('placeholder', text); + } + + for ( const elem of root.querySelectorAll('[data-i18n-tip]') ) { + const text = i18n$(elem.getAttribute('data-i18n-tip')) + .replace(/<br>/g, '\n') + .replace(/\n{3,}/g, '\n\n'); + elem.setAttribute('data-tip', text); + if ( elem.getAttribute('aria-label') === 'data-tip' ) { + elem.setAttribute('aria-label', text); + } + } + }; + + i18n.renderElapsedTimeToString = function(tstamp) { + let value = (Date.now() - tstamp) / 60000; + if ( value < 2 ) { + return i18n$('elapsedOneMinuteAgo'); + } + if ( value < 60 ) { + return i18n$('elapsedManyMinutesAgo').replace('{{value}}', Math.floor(value).toLocaleString()); + } + value /= 60; + if ( value < 2 ) { + return i18n$('elapsedOneHourAgo'); + } + if ( value < 24 ) { + return i18n$('elapsedManyHoursAgo').replace('{{value}}', Math.floor(value).toLocaleString()); + } + value /= 24; + if ( value < 2 ) { + return i18n$('elapsedOneDayAgo'); + } + return i18n$('elapsedManyDaysAgo').replace('{{value}}', Math.floor(value).toLocaleString()); + }; + + const unicodeFlagToImageSrc = new Map([ + [ '🇦🇱', 'al' ], [ '🇦🇷', 'ar' ], [ '🇦🇹', 'at' ], [ '🇧🇦', 'ba' ], + [ '🇧🇬', 'bg' ], [ '🇧🇷', 'br' ], [ '🇨🇦', 'ca' ], [ '🇨🇭', 'ch' ], + [ '🇨🇳', 'cn' ], [ '🇨🇴', 'co' ], [ '🇨🇾', 'cy' ], [ '🇨🇿', 'cz' ], + [ '🇩🇪', 'de' ], [ '🇩🇰', 'dk' ], [ '🇩🇿', 'dz' ], [ '🇪🇪', 'ee' ], + [ '🇪🇬', 'eg' ], [ '🇪🇸', 'es' ], [ '🇫🇮', 'fi' ], [ '🇫🇴', 'fo' ], + [ '🇫🇷', 'fr' ], [ '🇬🇷', 'gr' ], [ '🇭🇷', 'hr' ], [ '🇭🇺', 'hu' ], + [ '🇮🇩', 'id' ], [ '🇮🇱', 'il' ], [ '🇮🇳', 'in' ], [ '🇮🇷', 'ir' ], + [ '🇮🇸', 'is' ], [ '🇮🇹', 'it' ], [ '🇯🇵', 'jp' ], [ '🇰🇷', 'kr' ], + [ '🇰🇿', 'kz' ], [ '🇱🇰', 'lk' ], [ '🇱🇹', 'lt' ], [ '🇱🇻', 'lv' ], + [ '🇲🇦', 'ma' ], [ '🇲🇩', 'md' ], [ '🇲🇰', 'mk' ], [ '🇲🇽', 'mx' ], + [ '🇲🇾', 'my' ], [ '🇳🇱', 'nl' ], [ '🇳🇴', 'no' ], [ '🇳🇵', 'np' ], + [ '🇵🇱', 'pl' ], [ '🇵🇹', 'pt' ], [ '🇷🇴', 'ro' ], [ '🇷🇸', 'rs' ], + [ '🇷🇺', 'ru' ], [ '🇸🇦', 'sa' ], [ '🇸🇮', 'si' ], [ '🇸🇰', 'sk' ], + [ '🇸🇪', 'se' ], [ '🇸🇷', 'sr' ], [ '🇹🇭', 'th' ], [ '🇹🇯', 'tj' ], + [ '🇹🇼', 'tw' ], [ '🇹🇷', 'tr' ], [ '🇺🇦', 'ua' ], [ '🇺🇿', 'uz' ], + [ '🇻🇳', 'vn' ], [ '🇽🇰', 'xk' ], + ]); + const reUnicodeFlags = new RegExp( + Array.from(unicodeFlagToImageSrc).map(a => a[0]).join('|'), + 'gu' + ); + i18n.patchUnicodeFlags = function(text) { + const fragment = document.createDocumentFragment(); + let i = 0; + for (;;) { + const match = reUnicodeFlags.exec(text); + if ( match === null ) { break; } + if ( match.index > i ) { + fragment.append(text.slice(i, match.index)); + } + const img = document.createElement('img'); + const countryCode = unicodeFlagToImageSrc.get(match[0]); + img.src = `/img/flags-of-the-world/${countryCode}.png`; + img.title = countryCode; + img.classList.add('countryFlag'); + fragment.append(img, '\u200A'); + i = reUnicodeFlags.lastIndex; + } + if ( i < text.length ) { + fragment.append(text.slice(i)); + } + return fragment; + }; + + i18n.render(); +} + +/******************************************************************************/ + +export { i18n, i18n$ }; diff --git a/src/js/logger-ui-inspector.js b/src/js/logger-ui-inspector.js new file mode 100644 index 0000000..092baf8 --- /dev/null +++ b/src/js/logger-ui-inspector.js @@ -0,0 +1,710 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +import { dom, qs$, qsa$ } from './dom.js'; + +/******************************************************************************/ + +(( ) => { + +/******************************************************************************/ + +const logger = self.logger; +const showdomButton = qs$('#showdom'); +const inspector = qs$('#domInspector'); +const domTree = qs$('#domTree'); +const filterToIdMap = new Map(); + +let inspectedTabId = 0; +let inspectedURL = ''; +let inspectedHostname = ''; +let uidGenerator = 1; + +/******************************************************************************* + * + * How it works: + * + * 1. The logger/inspector is enabled from the logger window + * + * 2. The inspector content script is injected in the root frame of the tab + * currently selected in the logger + * + * 3. The inspector content script asks the logger/inspector to establish + * a two-way communication channel + * + * 3. The inspector content script embed an inspector frame in the document + * being inspected and waits for the inspector frame to be fully loaded + * + * 4. The inspector content script sends a messaging port object to the + * embedded inspector frame for a two-way communication channel between + * the inspector frame and the inspector content script + * + * 5. The inspector content script sends dom information to the + * logger/inspector + * + * */ + +const contentInspectorChannel = (( ) => { + let bcChannel; + let toContentPort; + + const start = ( ) => { + bcChannel = new globalThis.BroadcastChannel('contentInspectorChannel'); + bcChannel.onmessage = ev => { + const msg = ev.data || {}; + connect(msg.tabId, msg.frameId); + }; + browser.webNavigation.onDOMContentLoaded.addListener(onContentLoaded); + }; + + const shutdown = ( ) => { + browser.webNavigation.onDOMContentLoaded.removeListener(onContentLoaded); + disconnect(); + bcChannel.close(); + bcChannel.onmessage = null; + bcChannel = undefined; + }; + + const connect = (tabId, frameId) => { + disconnect(); + try { + toContentPort = browser.tabs.connect(tabId, { frameId }); + toContentPort.onMessage.addListener(onContentMessage); + toContentPort.onDisconnect.addListener(onContentDisconnect); + } catch(_) { + } + }; + + const disconnect = ( ) => { + if ( toContentPort === undefined ) { return; } + toContentPort.onMessage.removeListener(onContentMessage); + toContentPort.onDisconnect.removeListener(onContentDisconnect); + toContentPort.disconnect(); + toContentPort = undefined; + }; + + const send = msg => { + if ( toContentPort === undefined ) { return; } + toContentPort.postMessage(msg); + }; + + const onContentMessage = msg => { + if ( msg.what === 'domLayoutFull' ) { + inspectedURL = msg.url; + inspectedHostname = msg.hostname; + renderDOMFull(msg); + } else if ( msg.what === 'domLayoutIncremental' ) { + renderDOMIncremental(msg); + } + }; + + const onContentDisconnect = ( ) => { + disconnect(); + }; + + const onContentLoaded = details => { + if ( details.tabId !== inspectedTabId ) { return; } + if ( details.frameId !== 0 ) { return; } + disconnect(); + injectInspector(); + }; + + return { start, disconnect, send, shutdown }; +})(); + +/******************************************************************************/ + +const nodeFromDomEntry = entry => { + const li = document.createElement('li'); + dom.attr(li, 'id', entry.nid); + // expander/collapser + li.appendChild(document.createElement('span')); + // selector + let node = document.createElement('code'); + node.textContent = entry.sel; + li.appendChild(node); + // descendant count + let value = entry.cnt || 0; + node = document.createElement('span'); + node.textContent = value !== 0 ? value.toLocaleString() : ''; + dom.attr(node, 'data-cnt', value); + li.appendChild(node); + // cosmetic filter + if ( entry.filter === undefined ) { + return li; + } + node = document.createElement('code'); + dom.cl.add(node, 'filter'); + value = filterToIdMap.get(entry.filter); + if ( value === undefined ) { + value = `${uidGenerator}`; + filterToIdMap.set(entry.filter, value); + uidGenerator += 1; + } + dom.attr(node, 'data-filter-id', value); + node.textContent = entry.filter; + li.appendChild(node); + dom.cl.add(li, 'isCosmeticHide'); + return li; +}; + +/******************************************************************************/ + +const appendListItem = (ul, li) => { + ul.appendChild(li); + // Ancestor nodes of a node which is affected by a cosmetic filter will + // be marked as "containing cosmetic filters", for user convenience. + if ( dom.cl.has(li, 'isCosmeticHide') === false ) { return; } + for (;;) { + li = li.parentElement.parentElement; + if ( li === null ) { break; } + dom.cl.add(li, 'hasCosmeticHide'); + } +}; + +/******************************************************************************/ + +const renderDOMFull = response => { + const domTreeParent = domTree.parentElement; + let ul = domTreeParent.removeChild(domTree); + logger.removeAllChildren(domTree); + + filterToIdMap.clear(); + + let lvl = 0; + let li; + for ( const entry of response.layout ) { + if ( entry.lvl === lvl ) { + li = nodeFromDomEntry(entry); + appendListItem(ul, li); + continue; + } + if ( entry.lvl > lvl ) { + ul = document.createElement('ul'); + li.appendChild(ul); + dom.cl.add(li, 'branch'); + li = nodeFromDomEntry(entry); + appendListItem(ul, li); + lvl = entry.lvl; + continue; + } + // entry.lvl < lvl + while ( entry.lvl < lvl ) { + ul = li.parentNode; + li = ul.parentNode; + ul = li.parentNode; + lvl -= 1; + } + li = nodeFromDomEntry(entry); + appendListItem(ul, li); + } + while ( ul.parentNode !== null ) { + ul = ul.parentNode; + } + dom.cl.add(ul.firstElementChild, 'show'); + + domTreeParent.appendChild(domTree); +}; + +/******************************************************************************/ + +const patchIncremental = (from, delta) => { + let li = from.parentElement.parentElement; + const patchCosmeticHide = delta >= 0 && + dom.cl.has(from, 'isCosmeticHide') && + dom.cl.has(li, 'hasCosmeticHide') === false; + // Include descendants count when removing a node + if ( delta < 0 ) { + delta -= countFromNode(from); + } + for ( ; li.localName === 'li'; li = li.parentElement.parentElement ) { + const span = li.children[2]; + if ( delta !== 0 ) { + const cnt = countFromNode(li) + delta; + span.textContent = cnt !== 0 ? cnt.toLocaleString() : ''; + dom.attr(span, 'data-cnt', cnt); + } + if ( patchCosmeticHide ) { + dom.cl.add(li, 'hasCosmeticHide'); + } + } +}; + +/******************************************************************************/ + +const renderDOMIncremental = response => { + // Process each journal entry: + // 1 = node added + // -1 = node removed + const nodes = new Map(response.nodes); + let li = null; + let ul = null; + for ( const entry of response.journal ) { + // Remove node + if ( entry.what === -1 ) { + li = qs$(`#${entry.nid}`); + if ( li === null ) { continue; } + patchIncremental(li, -1); + li.parentNode.removeChild(li); + continue; + } + // Modify node + if ( entry.what === 0 ) { + // TODO: update selector/filter + continue; + } + // Add node as sibling + if ( entry.what === 1 && entry.l ) { + const previous = qs$(`#${entry.l}`); + // This should not happen + if ( previous === null ) { + // throw new Error('No left sibling!?'); + continue; + } + ul = previous.parentElement; + li = nodeFromDomEntry(nodes.get(entry.nid)); + ul.insertBefore(li, previous.nextElementSibling); + patchIncremental(li, 1); + continue; + } + // Add node as child + if ( entry.what === 1 && entry.u ) { + li = qs$(`#${entry.u}`); + // This should not happen + if ( li === null ) { + // throw new Error('No parent!?'); + continue; + } + ul = qs$(li, 'ul'); + if ( ul === null ) { + ul = document.createElement('ul'); + li.appendChild(ul); + dom.cl.add(li, 'branch'); + } + li = nodeFromDomEntry(nodes.get(entry.nid)); + ul.appendChild(li); + patchIncremental(li, 1); + continue; + } + } +}; + +/******************************************************************************/ + +const countFromNode = li => { + const span = li.children[2]; + const cnt = parseInt(dom.attr(span, 'data-cnt'), 10); + return isNaN(cnt) ? 0 : cnt; +}; + +/******************************************************************************/ + +const selectorFromNode = node => { + let selector = ''; + while ( node !== null ) { + if ( node.localName === 'li' ) { + const code = qs$(node, 'code'); + if ( code !== null ) { + selector = `${code.textContent} > ${selector}`; + if ( selector.includes('#') ) { break; } + } + } + node = node.parentElement; + } + return selector.slice(0, -3); +}; + +/******************************************************************************/ + +const selectorFromFilter = node => { + while ( node !== null ) { + if ( node.localName === 'li' ) { + const code = qs$(node, 'code:nth-of-type(2)'); + if ( code !== null ) { + return code.textContent; + } + } + node = node.parentElement; + } + return ''; +}; + +/******************************************************************************/ + +const nidFromNode = node => { + let li = node; + while ( li !== null ) { + if ( li.localName === 'li' ) { + return li.id || ''; + } + li = li.parentElement; + } + return ''; +}; + +/******************************************************************************/ + +const startDialog = (( ) => { + let dialog; + let textarea; + let hideSelectors = []; + let unhideSelectors = []; + + const parse = function() { + hideSelectors = []; + unhideSelectors = []; + + const re = /^([^#]*)(#@?#)(.+)$/; + for ( let line of textarea.value.split(/\s*\n\s*/) ) { + line = line.trim(); + if ( line === '' || line.charAt(0) === '!' ) { continue; } + const matches = re.exec(line); + if ( matches === null || matches.length !== 4 ) { continue; } + if ( inspectedHostname.lastIndexOf(matches[1]) === -1 ) { + continue; + } + if ( matches[2] === '##' ) { + hideSelectors.push(matches[3]); + } else { + unhideSelectors.push(matches[3]); + } + } + + showCommitted(); + }; + + const inputTimer = vAPI.defer.create(parse); + + const onInputChanged = ( ) => { + inputTimer.on(743); + }; + + const onClicked = function(ev) { + const target = ev.target; + + ev.stopPropagation(); + + if ( target.id === 'createCosmeticFilters' ) { + vAPI.messaging.send('loggerUI', { + what: 'createUserFilter', + filters: textarea.value, + }); + // Force a reload for the new cosmetic filter(s) to take effect + vAPI.messaging.send('loggerUI', { + what: 'reloadTab', + tabId: inspectedTabId, + }); + return stop(); + } + }; + + const showCommitted = function() { + contentInspectorChannel.send({ + what: 'showCommitted', + hide: hideSelectors.join(',\n'), + unhide: unhideSelectors.join(',\n') + }); + }; + + const showInteractive = function() { + contentInspectorChannel.send({ + what: 'showInteractive', + hide: hideSelectors.join(',\n'), + unhide: unhideSelectors.join(',\n') + }); + }; + + const start = function() { + dialog = logger.modalDialog.create('#cosmeticFilteringDialog', stop); + textarea = qs$(dialog, 'textarea'); + hideSelectors = []; + for ( const node of qsa$(domTree, 'code.off') ) { + if ( dom.cl.has(node, 'filter') ) { continue; } + hideSelectors.push(selectorFromNode(node)); + } + const taValue = []; + for ( const selector of hideSelectors ) { + taValue.push(inspectedHostname + '##' + selector); + } + const ids = new Set(); + for ( const node of qsa$(domTree, 'code.filter.off') ) { + const id = dom.attr(node, 'data-filter-id'); + if ( ids.has(id) ) { continue; } + ids.add(id); + unhideSelectors.push(node.textContent); + taValue.push(inspectedHostname + '#@#' + node.textContent); + } + textarea.value = taValue.join('\n'); + textarea.addEventListener('input', onInputChanged); + dialog.addEventListener('click', onClicked, true); + showCommitted(); + logger.modalDialog.show(); + }; + + const stop = function() { + inputTimer.off(); + showInteractive(); + textarea.removeEventListener('input', onInputChanged); + dialog.removeEventListener('click', onClicked, true); + dialog = undefined; + textarea = undefined; + hideSelectors = []; + unhideSelectors = []; + }; + + return start; +})(); + +/******************************************************************************/ + +const onClicked = ev => { + ev.stopPropagation(); + + if ( inspectedTabId === 0 ) { return; } + + const target = ev.target; + const parent = target.parentElement; + + // Expand/collapse branch + if ( + target.localName === 'span' && + parent instanceof HTMLLIElement && + dom.cl.has(parent, 'branch') && + target === parent.firstElementChild + ) { + const state = dom.cl.toggle(parent, 'show'); + if ( !state ) { + for ( const node of qsa$(parent, '.branch') ) { + dom.cl.remove(node, 'show'); + } + } + return; + } + + // Not a node or filter + if ( target.localName !== 'code' ) { return; } + + // Toggle cosmetic filter + if ( dom.cl.has(target, 'filter') ) { + contentInspectorChannel.send({ + what: 'toggleFilter', + original: false, + target: dom.cl.toggle(target, 'off'), + selector: selectorFromNode(target), + filter: selectorFromFilter(target), + nid: nidFromNode(target) + }); + dom.cl.toggle( + qsa$(inspector, `[data-filter-id="${dom.attr(target, 'data-filter-id')}"]`), + 'off', + dom.cl.has(target, 'off') + ); + } + // Toggle node + else { + contentInspectorChannel.send({ + what: 'toggleNodes', + original: true, + target: dom.cl.toggle(target, 'off') === false, + selector: selectorFromNode(target), + nid: nidFromNode(target) + }); + } + + const cantCreate = qs$(domTree, '.off') === null; + dom.cl.toggle(qs$(inspector, '.permatoolbar .revert'), 'disabled', cantCreate); + dom.cl.toggle(qs$(inspector, '.permatoolbar .commit'), 'disabled', cantCreate); +}; + +/******************************************************************************/ + +const onMouseOver = (( ) => { + let mouseoverTarget = null; + + const mouseoverTimer = vAPI.defer.create(( ) => { + contentInspectorChannel.send({ + what: 'highlightOne', + selector: selectorFromNode(mouseoverTarget), + nid: nidFromNode(mouseoverTarget), + scrollTo: true + }); + }); + + return ev => { + if ( inspectedTabId === 0 ) { return; } + // Convenience: skip real-time highlighting if shift key is pressed. + if ( ev.shiftKey ) { return; } + // Find closest `li` + const target = ev.target.closest('li'); + if ( target === mouseoverTarget ) { return; } + mouseoverTarget = target; + mouseoverTimer.on(50); + }; +})(); + +/******************************************************************************/ + +const currentTabId = ( ) => { + if ( dom.cl.has(showdomButton, 'active') === false ) { return 0; } + return logger.tabIdFromPageSelector(); +}; + +/******************************************************************************/ + +const injectInspector = (( ) => { + const timer = vAPI.defer.create(( ) => { + const tabId = currentTabId(); + if ( tabId <= 0 ) { return; } + inspectedTabId = tabId; + vAPI.messaging.send('loggerUI', { + what: 'scriptlet', + tabId, + scriptlet: 'dom-inspector', + }); + }); + return ( ) => { + shutdownInspector(); + timer.offon(353); + }; +})(); + +/******************************************************************************/ + +const shutdownInspector = ( ) => { + contentInspectorChannel.disconnect(); + logger.removeAllChildren(domTree); + dom.cl.remove(inspector, 'vExpanded'); + inspectedTabId = 0; +}; + +/******************************************************************************/ + +const onTabIdChanged = ( ) => { + const tabId = currentTabId(); + if ( tabId <= 0 ) { + return toggleOff(); + } + if ( inspectedTabId !== tabId ) { + injectInspector(); + } +}; + +/******************************************************************************/ + +const toggleVExpandView = ( ) => { + const branches = qsa$('#domTree li.branch.show > ul > li.branch:not(.show)'); + for ( const branch of branches ) { + dom.cl.add(branch, 'show'); + } +}; + +const toggleVCompactView = ( ) => { + const branches = qsa$('#domTree li.branch.show > ul > li:not(.show)'); + const tohideSet = new Set(); + for ( const branch of branches ) { + const node = branch.closest('li.branch.show'); + if ( node.id === 'n1' ) { continue; } + tohideSet.add(node); + } + const tohideList = Array.from(tohideSet); + let i = tohideList.length - 1; + while ( i > 0 ) { + if ( tohideList[i-1].contains(tohideList[i]) ) { + tohideList.splice(i-1, 1); + } else if ( tohideList[i].contains(tohideList[i-1]) ) { + tohideList.splice(i, 1); + } + i -= 1; + } + for ( const node of tohideList ) { + dom.cl.remove(node, 'show'); + } +}; + +const toggleHCompactView = ( ) => { + dom.cl.toggle(inspector, 'hCompact'); +}; + +/******************************************************************************/ + +const revert = ( ) => { + dom.cl.remove('#domTree .off', 'off'); + contentInspectorChannel.send({ what: 'resetToggledNodes' }); + dom.cl.add(qs$(inspector, '.permatoolbar .revert'), 'disabled'); + dom.cl.add(qs$(inspector, '.permatoolbar .commit'), 'disabled'); +}; + +/******************************************************************************/ + +const toggleOn = ( ) => { + dom.cl.add('#inspectors', 'dom'); + window.addEventListener('beforeunload', toggleOff); + document.addEventListener('tabIdChanged', onTabIdChanged); + domTree.addEventListener('click', onClicked, true); + domTree.addEventListener('mouseover', onMouseOver, true); + dom.on('#domInspector .vExpandToggler', 'click', toggleVExpandView); + dom.on('#domInspector .vCompactToggler', 'click', toggleVCompactView); + dom.on('#domInspector .hCompactToggler', 'click', toggleHCompactView); + dom.on('#domInspector .permatoolbar .revert', 'click', revert); + dom.on('#domInspector .permatoolbar .commit', 'click', startDialog); + contentInspectorChannel.start(); + injectInspector(); +}; + +/******************************************************************************/ + +const toggleOff = ( ) => { + dom.cl.remove(showdomButton, 'active'); + dom.cl.remove('#inspectors', 'dom'); + shutdownInspector(); + window.removeEventListener('beforeunload', toggleOff); + document.removeEventListener('tabIdChanged', onTabIdChanged); + domTree.removeEventListener('click', onClicked, true); + domTree.removeEventListener('mouseover', onMouseOver, true); + dom.off('#domInspector .vExpandToggler', 'click', toggleVExpandView); + dom.off('#domInspector .vCompactToggler', 'click', toggleVCompactView); + dom.off('#domInspector .hCompactToggler', 'click', toggleHCompactView); + dom.off('#domInspector .permatoolbar .revert', 'click', revert); + dom.off('#domInspector .permatoolbar .commit', 'click', startDialog); + contentInspectorChannel.shutdown(); + inspectedTabId = 0; +}; + +/******************************************************************************/ + +const toggle = ( ) => { + if ( dom.cl.toggle(showdomButton, 'active') ) { + toggleOn(); + } else { + toggleOff(); + } + logger.resize(); +}; + +dom.on(showdomButton, 'click', toggle); + +/******************************************************************************/ + +})(); diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js new file mode 100644 index 0000000..177632e --- /dev/null +++ b/src/js/logger-ui.js @@ -0,0 +1,3044 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { hostnameFromURI } from './uri-utils.js'; +import { i18n, i18n$ } from './i18n.js'; +import { dom, qs$, qsa$ } from './dom.js'; + +/******************************************************************************/ + +// TODO: fix the inconsistencies re. realm vs. filter source which have +// accumulated over time. + +const messaging = vAPI.messaging; +const logger = self.logger = { ownerId: Date.now() }; +const logDate = new Date(); +const logDateTimezoneOffset = logDate.getTimezoneOffset() * 60000; +const loggerEntries = []; + +const COLUMN_TIMESTAMP = 0; +const COLUMN_FILTER = 1; +const COLUMN_MESSAGE = 1; +const COLUMN_RESULT = 2; +const COLUMN_INITIATOR = 3; +const COLUMN_PARTYNESS = 4; +const COLUMN_METHOD = 5; +const COLUMN_TYPE = 6; +const COLUMN_URL = 7; + +let filteredLoggerEntries = []; +let filteredLoggerEntryVoidedCount = 0; + +let popupLoggerBox; +let popupLoggerTooltips; +let activeTabId = 0; +let selectedTabId = 0; +let netInspectorPaused = false; +let cnameOfEnabled = false; + +/******************************************************************************/ + +// Various helpers. + +const tabIdFromPageSelector = logger.tabIdFromPageSelector = function() { + const value = qs$('#pageSelector').value; + return value !== '_' ? (parseInt(value, 10) || 0) : activeTabId; +}; + +const tabIdFromAttribute = function(elem) { + const value = dom.attr(elem, 'data-tabid') || ''; + const tabId = parseInt(value, 10); + return isNaN(tabId) ? 0 : tabId; +}; + + +/******************************************************************************/ +/******************************************************************************/ + +const onStartMovingWidget = (( ) => { + let widget = null; + let ondone = null; + let mx0 = 0, my0 = 0; + let mx1 = 0, my1 = 0; + let l0 = 0, t0 = 0; + let pw = 0, ph = 0; + let cw = 0, ch = 0; + let timer; + + const xyFromEvent = ev => { + if ( ev.type.startsWith('mouse') ) { + return { x: ev.pageX, y: ev.pageY }; + } + const touch = ev.touches[0]; + return { x: touch.pageX, y: touch.pageY }; + }; + + const eatEvent = function(ev) { + ev.stopPropagation(); + if ( ev.touches !== undefined ) { return; } + ev.preventDefault(); + }; + + const move = ( ) => { + timer = undefined; + const l1 = Math.min(Math.max(l0 + mx1 - mx0, 0), Math.max(pw - cw, 0)); + if ( (l1+cw/2) < (pw/2) ) { + widget.style.left = `${l1/pw*100}%`; + widget.style.right = ''; + } else { + widget.style.right = `${(pw-l1-cw)/pw*100}%`; + widget.style.left = ''; + } + const t1 = Math.min(Math.max(t0 + my1 - my0, 0), Math.max(ph - ch, 0)); + widget.style.top = `${t1/ph*100}%`; + widget.style.bottom = ''; + }; + + const moveAsync = ev => { + if ( timer !== undefined ) { return; } + const coord = xyFromEvent(ev); + mx1 = coord.x; my1 = coord.y; + timer = self.requestAnimationFrame(move); + eatEvent(ev); + }; + + const stop = ev => { + if ( timer !== undefined ) { + self.cancelAnimationFrame(timer); + timer = undefined; + } + if ( widget === null ) { return; } + if ( widget.classList.contains('moving') === false ) { return; } + widget.classList.remove('moving'); + self.removeEventListener('mousemove', moveAsync, { capture: true }); + self.removeEventListener('touchmove', moveAsync, { capture: true }); + eatEvent(ev); + widget = null; + if ( ondone !== null ) { + ondone(); + ondone = null; + } + }; + + return function(ev, target, callback) { + if ( dom.cl.has(target, 'moving') ) { return; } + widget = target; + ondone = callback || null; + const coord = xyFromEvent(ev); + mx0 = coord.x; my0 = coord.y; + const widgetParent = widget.parentElement; + const crect = widget.getBoundingClientRect(); + const prect = widgetParent.getBoundingClientRect(); + pw = prect.width; ph = prect.height; + cw = crect.width; ch = crect.height; + l0 = crect.x - prect.x; t0 = crect.y - prect.y; + widget.classList.add('moving'); + self.addEventListener('mousemove', moveAsync, { capture: true }); + self.addEventListener('mouseup', stop, { capture: true, once: true }); + self.addEventListener('touchmove', moveAsync, { capture: true }); + self.addEventListener('touchend', stop, { capture: true, once: true }); + eatEvent(ev); + }; +})(); + +/******************************************************************************/ +/******************************************************************************/ + +// Current design allows for only one modal DOM-based dialog at any given time. +// +const modalDialog = (( ) => { + const overlay = qs$('#modalOverlay'); + const container = qs$('#modalOverlayContainer'); + const closeButton = qs$(overlay, ':scope .closeButton'); + let onDestroyed; + + const removeChildren = logger.removeAllChildren = function(node) { + while ( node.firstChild ) { + node.removeChild(node.firstChild); + } + }; + + const create = function(selector, destroyListener) { + const template = qs$(selector); + const dialog = dom.clone(template); + removeChildren(container); + container.appendChild(dialog); + onDestroyed = destroyListener; + return dialog; + }; + + const show = function() { + dom.cl.add(overlay, 'on'); + }; + + const destroy = function() { + dom.cl.remove(overlay, 'on'); + const dialog = container.firstElementChild; + removeChildren(container); + if ( typeof onDestroyed === 'function' ) { + onDestroyed(dialog); + } + onDestroyed = undefined; + }; + + const onClose = function(ev) { + if ( ev.target === overlay || ev.target === closeButton ) { + destroy(); + } + }; + dom.on(overlay, 'click', onClose); + dom.on(closeButton, 'click', onClose); + + return { create, show, destroy }; +})(); + +self.logger.modalDialog = modalDialog; + + +/******************************************************************************/ +/******************************************************************************/ + +const prettyRequestTypes = { + 'main_frame': 'doc', + 'stylesheet': 'css', + 'sub_frame': 'frame', + 'xmlhttprequest': 'xhr' +}; + +const uglyRequestTypes = { + 'doc': 'main_frame', + 'css': 'stylesheet', + 'frame': 'sub_frame', + 'xhr': 'xmlhttprequest' +}; + +let allTabIds = new Map(); +let allTabIdsToken; + +/******************************************************************************/ +/******************************************************************************/ + +const regexFromURLFilteringResult = function(result) { + const beg = result.indexOf(' '); + const end = result.indexOf(' ', beg + 1); + const url = result.slice(beg + 1, end); + if ( url === '*' ) { + return new RegExp('^.*$', 'gi'); + } + return new RegExp('^' + url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi'); +}; + +/******************************************************************************/ + +// Emphasize hostname in URL, as this is what matters in uMatrix's rules. + +const nodeFromURL = function(parent, url, re, type) { + const fragment = document.createDocumentFragment(); + if ( re === undefined ) { + fragment.textContent = url; + } else { + if ( typeof re === 'string' ) { + re = new RegExp(re.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'); + } + const matches = re.exec(url); + if ( matches === null || matches[0].length === 0 ) { + fragment.textContent = url; + } else { + if ( matches.index !== 0 ) { + fragment.appendChild( + document.createTextNode(url.slice(0, matches.index)) + ); + } + const b = document.createElement('b'); + b.textContent = url.slice(matches.index, re.lastIndex); + fragment.appendChild(b); + if ( re.lastIndex !== url.length ) { + fragment.appendChild( + document.createTextNode(url.slice(re.lastIndex)) + ); + } + } + } + if ( /^https?:\/\//.test(url) ) { + const a = document.createElement('a'); + let href = url; + switch ( type ) { + case 'css': + case 'doc': + case 'frame': + case 'object': + case 'other': + case 'script': + case 'xhr': + href = `code-viewer.html?url=${encodeURIComponent(href)}`; + break; + default: + break; + } + dom.attr(a, 'href', href); + dom.attr(a, 'target', '_blank'); + fragment.appendChild(a); + } + parent.appendChild(fragment); +}; + +/******************************************************************************/ + +const padTo2 = function(v) { + return v < 10 ? '0' + v : v; +}; + +const normalizeToStr = function(s) { + return typeof s === 'string' && s !== '' ? s : ''; +}; + +/******************************************************************************/ + +const LogEntry = function(details) { + if ( details instanceof Object === false ) { return; } + const receiver = LogEntry.prototype; + for ( const prop in receiver ) { + if ( + details.hasOwnProperty(prop) && + details[prop] !== receiver[prop] + ) { + this[prop] = details[prop]; + } + } + if ( details.aliasURL !== undefined ) { + this.aliased = true; + } + if ( this.tabDomain === '' ) { + this.tabDomain = this.tabHostname || ''; + } + if ( this.docDomain === '' ) { + this.docDomain = this.docHostname || ''; + } + if ( this.domain === '' ) { + this.domain = details.hostname || ''; + } +}; +LogEntry.prototype = { + aliased: false, + dead: false, + docDomain: '', + docHostname: '', + domain: '', + filter: undefined, + id: '', + method: '', + realm: '', + tabDomain: '', + tabHostname: '', + tabId: undefined, + textContent: '', + tstamp: 0, + type: '', + voided: false, +}; + +/******************************************************************************/ + +const createLogSeparator = function(details, text) { + const separator = new LogEntry(); + separator.tstamp = details.tstamp; + separator.realm = 'message'; + separator.tabId = details.tabId; + separator.type = 'tabLoad'; + separator.textContent = ''; + + const textContent = []; + logDate.setTime(separator.tstamp - logDateTimezoneOffset); + textContent.push( + // cell 0 + padTo2(logDate.getUTCHours()) + ':' + + padTo2(logDate.getUTCMinutes()) + ':' + + padTo2(logDate.getSeconds()), + // cell 1 + text + ); + separator.textContent = textContent.join('\t'); + + if ( details.voided ) { + separator.voided = true; + } + + return separator; +}; + +/******************************************************************************/ + +// TODO: once refactoring is mature, consider using push() instead of +// unshift(). This will require inverting the access logic +// throughout the code. +// +const processLoggerEntries = function(response) { + const entries = response.entries; + if ( entries.length === 0 ) { return; } + + const autoDeleteVoidedRows = qs$('#pageSelector').value === '_'; + const previousCount = filteredLoggerEntries.length; + + for ( const entry of entries ) { + const unboxed = JSON.parse(entry); + if ( unboxed.filter instanceof Object ){ + loggerStats.processFilter(unboxed.filter); + } + if ( netInspectorPaused ) { continue; } + const parsed = parseLogEntry(unboxed); + if ( + parsed.tabId !== undefined && + allTabIds.has(parsed.tabId) === false + ) { + if ( autoDeleteVoidedRows ) { continue; } + parsed.voided = true; + } + if ( + parsed.type === 'main_frame' && + parsed.aliased === false && ( + parsed.filter === undefined || + parsed.filter.modifier !== true + ) + ) { + const separator = createLogSeparator(parsed, unboxed.url); + loggerEntries.unshift(separator); + if ( rowFilterer.filterOne(separator) ) { + filteredLoggerEntries.unshift(separator); + if ( separator.voided ) { + filteredLoggerEntryVoidedCount += 1; + } + } + } + if ( cnameOfEnabled === false && parsed.aliased ) { + qs$('#filterExprCnameOf').style.display = ''; + cnameOfEnabled = true; + } + loggerEntries.unshift(parsed); + if ( rowFilterer.filterOne(parsed) ) { + filteredLoggerEntries.unshift(parsed); + if ( parsed.voided ) { + filteredLoggerEntryVoidedCount += 1; + } + } + } + + const addedCount = filteredLoggerEntries.length - previousCount; + if ( addedCount !== 0 ) { + viewPort.updateContent(addedCount); + rowJanitor.inserted(addedCount); + } +}; + +/******************************************************************************/ + +const parseLogEntry = function(details) { + // Patch realm until changed all over codebase to make this unnecessary + if ( details.realm === 'cosmetic' ) { + details.realm = 'extended'; + } + + const entry = new LogEntry(details); + + // Assemble the text content, i.e. the pre-built string which will be + // used to match logger output filtering expressions. + const textContent = []; + + // Cell 0 + logDate.setTime(details.tstamp - logDateTimezoneOffset); + textContent.push( + padTo2(logDate.getUTCHours()) + ':' + + padTo2(logDate.getUTCMinutes()) + ':' + + padTo2(logDate.getSeconds()) + ); + + // Cell 1 + if ( details.realm === 'message' ) { + textContent.push(details.text); + entry.textContent = textContent.join('\t'); + return entry; + } + + // Cell 1, 2 + if ( entry.filter !== undefined ) { + textContent.push(entry.filter.raw); + if ( entry.filter.result === 1 ) { + textContent.push('--'); + } else if ( entry.filter.result === 2 ) { + textContent.push('++'); + } else if ( entry.filter.result === 3 ) { + textContent.push('**'); + } else if ( entry.filter.source === 'redirect' ) { + textContent.push('<<'); + } else { + textContent.push(''); + } + } else { + textContent.push('', ''); + } + + // Cell 3 + textContent.push(normalizeToStr(entry.docHostname)); + + // Cell 4: partyness + if ( + entry.realm === 'network' && + typeof entry.domain === 'string' && + entry.domain !== '' + ) { + let partyness = ''; + if ( entry.tabDomain !== undefined ) { + if ( entry.tabId < 0 ) { + partyness += '0,'; + } + partyness += entry.domain === entry.tabDomain ? '1' : '3'; + } else { + partyness += '?'; + } + if ( entry.docDomain !== entry.tabDomain ) { + partyness += ','; + if ( entry.docDomain !== undefined ) { + partyness += entry.domain === entry.docDomain ? '1' : '3'; + } else { + partyness += '?'; + } + } + textContent.push(partyness); + } else { + textContent.push(''); + } + + // Cell 5: method + textContent.push(entry.method || ''); + + // Cell 6 + textContent.push( + normalizeToStr(prettyRequestTypes[entry.type] || entry.type) + ); + + // Cell 7 + textContent.push(normalizeToStr(details.url)); + + // Hidden cells -- useful for row-filtering purpose + + // Cell 8 + if ( entry.aliased ) { + textContent.push(`aliasURL=${details.aliasURL}`); + } + + entry.textContent = textContent.join('\t'); + return entry; +}; + +/******************************************************************************/ + +const viewPort = (( ) => { + const vwRenderer = qs$('#vwRenderer'); + const vwScroller = qs$('#vwScroller'); + const vwVirtualContent = qs$('#vwVirtualContent'); + const vwContent = qs$('#vwContent'); + const vwLineSizer = qs$('#vwLineSizer'); + const vwLogEntryTemplate = qs$('#logEntryTemplate > div'); + const vwEntries = []; + + const detailableRealms = new Set([ 'network', 'extended' ]); + + let vwHeight = 0; + let lineHeight = 0; + let wholeHeight = 0; + let lastTopPix = 0; + let lastTopRow = 0; + + const ViewEntry = function() { + this.div = document.createElement('div'); + this.div.className = 'logEntry'; + vwContent.appendChild(this.div); + this.logEntry = undefined; + }; + ViewEntry.prototype = { + dispose: function() { + vwContent.removeChild(this.div); + }, + }; + + const rowFromScrollTopPix = function(px) { + return lineHeight !== 0 ? Math.floor(px / lineHeight) : 0; + }; + + // This is called when the browser fired scroll events + const onScrollChanged = function() { + const newScrollTopPix = vwScroller.scrollTop; + const delta = newScrollTopPix - lastTopPix; + if ( delta === 0 ) { return; } + lastTopPix = newScrollTopPix; + if ( filteredLoggerEntries.length <= 2 ) { return; } + // No entries were rolled = all entries keep their current details + if ( rollLines(rowFromScrollTopPix(newScrollTopPix)) ) { + fillLines(); + } + positionLines(); + vwContent.style.top = `${lastTopPix}px`; + }; + + // Coalesce scroll events + const scrollTimer = vAPI.defer.create(onScrollChanged); + const onScroll = ( ) => { + scrollTimer.onvsync(1000/32); + }; + dom.on(vwScroller, 'scroll', onScroll, { passive: true }); + + const onLayoutChanged = function() { + vwHeight = vwRenderer.clientHeight; + vwContent.style.height = `${vwScroller.clientHeight}px`; + + const vExpanded = + dom.cl.has('#netInspector .vCompactToggler', 'vExpanded'); + + let newLineHeight = qs$(vwLineSizer, '.oneLine').clientHeight; + + if ( vExpanded ) { + newLineHeight *= loggerSettings.linesPerEntry; + } + + const lineCount = newLineHeight !== 0 + ? Math.ceil(vwHeight / newLineHeight) + 1 + : 0; + if ( lineCount > vwEntries.length ) { + do { + vwEntries.push(new ViewEntry()); + } while ( lineCount > vwEntries.length ); + } else if ( lineCount < vwEntries.length ) { + do { + vwEntries.pop().dispose(); + } while ( lineCount < vwEntries.length ); + } + + const cellWidths = Array.from( + qsa$(vwLineSizer, '.oneLine span') + ).map((el, i) => { + return loggerSettings.columns[i] !== false + ? el.clientWidth + 1 + : 0; + }); + const reservedWidth = + cellWidths[COLUMN_TIMESTAMP] + + cellWidths[COLUMN_RESULT] + + cellWidths[COLUMN_PARTYNESS] + + cellWidths[COLUMN_METHOD] + + cellWidths[COLUMN_TYPE]; + cellWidths[COLUMN_URL] = 0.5; + if ( cellWidths[COLUMN_FILTER] === 0 && cellWidths[COLUMN_INITIATOR] === 0 ) { + cellWidths[COLUMN_URL] = 1; + } else if ( cellWidths[COLUMN_FILTER] === 0 ) { + cellWidths[COLUMN_INITIATOR] = 0.35; + cellWidths[COLUMN_URL] = 0.65; + } else if ( cellWidths[COLUMN_INITIATOR] === 0 ) { + cellWidths[COLUMN_FILTER] = 0.35; + cellWidths[COLUMN_URL] = 0.65; + } else { + cellWidths[COLUMN_FILTER] = 0.25; + cellWidths[COLUMN_INITIATOR] = 0.25; + cellWidths[COLUMN_URL] = 0.5; + } + const style = qs$('#vwRendererRuntimeStyles'); + const cssRules = [ + '#vwContent .logEntry {', + ` height: ${newLineHeight}px;`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_TIMESTAMP+1}) {`, + ` width: ${cellWidths[COLUMN_TIMESTAMP]}px;`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_FILTER+1}) {`, + ` width: calc(calc(100% - ${reservedWidth}px) * ${cellWidths[COLUMN_FILTER]});`, + '}', + `#vwContent .logEntry > div.messageRealm > span:nth-of-type(${COLUMN_MESSAGE+1}) {`, + ` width: calc(100% - ${cellWidths[COLUMN_TIMESTAMP]}px);`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_RESULT+1}) {`, + ` width: ${cellWidths[COLUMN_RESULT]}px;`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_INITIATOR+1}) {`, + ` width: calc(calc(100% - ${reservedWidth}px) * ${cellWidths[COLUMN_INITIATOR]});`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_PARTYNESS+1}) {`, + ` width: ${cellWidths[COLUMN_PARTYNESS]}px;`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_METHOD+1}) {`, + ` width: ${cellWidths[COLUMN_METHOD]}px;`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_TYPE+1}) {`, + ` width: ${cellWidths[COLUMN_TYPE]}px;`, + '}', + `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_URL+1}) {`, + ` width: calc(calc(100% - ${reservedWidth}px) * ${cellWidths[COLUMN_URL]});`, + '}', + '', + ]; + for ( let i = 0; i < cellWidths.length; i++ ) { + if ( cellWidths[i] !== 0 ) { continue; } + cssRules.push( + `#vwContent .logEntry > div > span:nth-of-type(${i + 1}) {`, + ' display: none;', + '}' + ); + } + style.textContent = cssRules.join('\n'); + + lineHeight = newLineHeight; + positionLines(); + dom.cl.toggle('#netInspector', 'vExpanded', vExpanded); + + updateContent(0); + }; + + const resizeTimer = vAPI.defer.create(onLayoutChanged); + const updateLayout = function() { + resizeTimer.onvsync(1000/8); + }; + dom.on(window, 'resize', updateLayout, { passive: true }); + + updateLayout(); + + const renderFilterToSpan = function(span, filter) { + if ( filter.charCodeAt(0) !== 0x23 /* '#' */ ) { return false; } + const match = /^#@?#/.exec(filter); + if ( match === null ) { return false; } + let child = document.createElement('span'); + child.textContent = match[0]; + span.appendChild(child); + child = document.createElement('span'); + child.textContent = filter.slice(match[0].length); + span.appendChild(child); + return true; + }; + + const renderToDiv = function(vwEntry, i) { + if ( i >= filteredLoggerEntries.length ) { + vwEntry.logEntry = undefined; + return null; + } + + const details = filteredLoggerEntries[i]; + if ( vwEntry.logEntry === details ) { + return vwEntry.div.firstElementChild; + } + + vwEntry.logEntry = details; + + const cells = details.textContent.split('\t'); + const div = dom.clone(vwLogEntryTemplate); + const divcl = div.classList; + let span; + + // Realm + if ( details.realm !== undefined ) { + divcl.add(details.realm + 'Realm'); + } + + // Timestamp + span = div.children[COLUMN_TIMESTAMP]; + span.textContent = cells[COLUMN_TIMESTAMP]; + + // Tab id + if ( details.tabId !== undefined ) { + dom.attr(div, 'data-tabid', details.tabId); + if ( details.voided ) { + divcl.add('voided'); + } + } + + if ( details.realm === 'message' ) { + if ( details.type !== undefined ) { + dom.attr(div, 'data-type', details.type); + } + span = div.children[COLUMN_MESSAGE]; + span.textContent = cells[COLUMN_MESSAGE]; + return div; + } + + if ( detailableRealms.has(details.realm) ) { + divcl.add('canDetails'); + } + + // Filter + const filter = details.filter || undefined; + let filteringType; + if ( filter !== undefined ) { + if ( typeof filter.source === 'string' ) { + filteringType = filter.source; + } + if ( filteringType === 'static' ) { + divcl.add('canLookup'); + } else if ( details.realm === 'extended' ) { + divcl.toggle('canLookup', /^#@?#/.test(filter.raw)); + divcl.toggle('isException', filter.raw.startsWith('#@#')); + } + if ( filter.modifier === true ) { + dom.attr(div, 'data-modifier', ''); + } + } + span = div.children[COLUMN_FILTER]; + if ( renderFilterToSpan(span, cells[COLUMN_FILTER]) ) { + if ( /^\+js\(.*\)$/.test(span.children[1].textContent) ) { + divcl.add('scriptlet'); + } + } else { + span.textContent = cells[COLUMN_FILTER]; + } + + // Event + if ( cells[COLUMN_RESULT] === '--' ) { + dom.attr(div, 'data-status', '1'); + } else if ( cells[COLUMN_RESULT] === '++' ) { + dom.attr(div, 'data-status', '2'); + } else if ( cells[COLUMN_RESULT] === '**' ) { + dom.attr(div, 'data-status', '3'); + } else if ( cells[COLUMN_RESULT] === '<<' ) { + divcl.add('redirect'); + } + span = div.children[COLUMN_RESULT]; + span.textContent = cells[COLUMN_RESULT]; + + // Origins + if ( details.tabHostname ) { + dom.attr(div, 'data-tabhn', details.tabHostname); + } + if ( details.docHostname ) { + dom.attr(div, 'data-dochn', details.docHostname); + } + span = div.children[COLUMN_INITIATOR]; + span.textContent = cells[COLUMN_INITIATOR]; + + // Partyness + if ( + cells[COLUMN_PARTYNESS] !== '' && + details.realm === 'network' && + details.domain !== undefined + ) { + let text = `${details.tabDomain}`; + if ( details.docDomain !== details.tabDomain ) { + text += ` \u22ef ${details.docDomain}`; + } + text += ` \u21d2 ${details.domain}`; + dom.attr(div, 'data-parties', text); + } + span = div.children[COLUMN_PARTYNESS]; + span.textContent = cells[COLUMN_PARTYNESS]; + + // Method + span = div.children[COLUMN_METHOD]; + span.textContent = cells[COLUMN_METHOD]; + + // Type + span = div.children[COLUMN_TYPE]; + span.textContent = cells[COLUMN_TYPE]; + + // URL + let re; + if ( filteringType === 'static' ) { + re = new RegExp(filter.regex, 'gi'); + } else if ( filteringType === 'dynamicUrl' ) { + re = regexFromURLFilteringResult(filter.rule.join(' ')); + } + nodeFromURL(div.children[COLUMN_URL], cells[COLUMN_URL], re, cells[COLUMN_TYPE]); + + // Alias URL (CNAME, etc.) + if ( cells.length > 8 ) { + const pos = details.textContent.lastIndexOf('\taliasURL='); + if ( pos !== -1 ) { + dom.attr(div, 'data-aliasid', details.id); + } + } + + return div; + }; + + // The idea is that positioning DOM elements is faster than + // removing/inserting DOM elements. + const positionLines = function() { + if ( lineHeight === 0 ) { return; } + let y = -(lastTopPix % lineHeight); + for ( const vwEntry of vwEntries ) { + vwEntry.div.style.top = `${y}px`; + y += lineHeight; + } + }; + + const rollLines = function(topRow) { + let delta = topRow - lastTopRow; + let deltaLength = Math.abs(delta); + // No point rolling if no rows can be reused + if ( deltaLength > 0 && deltaLength < vwEntries.length ) { + if ( delta < 0 ) { // Move bottom rows to the top + vwEntries.unshift(...vwEntries.splice(delta)); + } else { // Move top rows to the bottom + vwEntries.push(...vwEntries.splice(0, delta)); + } + } + lastTopRow = topRow; + return delta; + }; + + const fillLines = function() { + let rowBeg = lastTopRow; + for ( const vwEntry of vwEntries ) { + const newDiv = renderToDiv(vwEntry, rowBeg); + const container = vwEntry.div; + const oldDiv = container.firstElementChild; + if ( newDiv !== null ) { + if ( oldDiv === null ) { + container.appendChild(newDiv); + } else if ( newDiv !== oldDiv ) { + container.removeChild(oldDiv); + container.appendChild(newDiv); + } + } else if ( oldDiv !== null ) { + container.removeChild(oldDiv); + } + rowBeg += 1; + } + }; + + const contentChanged = function(addedCount) { + lastTopRow += addedCount; + const newWholeHeight = Math.max( + filteredLoggerEntries.length * lineHeight, + vwRenderer.clientHeight + ); + if ( newWholeHeight !== wholeHeight ) { + vwVirtualContent.style.height = `${newWholeHeight}px`; + wholeHeight = newWholeHeight; + } + }; + + const updateContent = function(addedCount) { + contentChanged(addedCount); + // Content changed + if ( addedCount === 0 ) { + if ( + lastTopRow !== 0 && + lastTopRow + vwEntries.length > filteredLoggerEntries.length + ) { + lastTopRow = filteredLoggerEntries.length - vwEntries.length; + if ( lastTopRow < 0 ) { lastTopRow = 0; } + lastTopPix = lastTopRow * lineHeight; + vwContent.style.top = `${lastTopPix}px`; + vwScroller.scrollTop = lastTopPix; + positionLines(); + } + fillLines(); + return; + } + + // Content added + // Preserve scroll position + if ( lastTopPix === 0 ) { + rollLines(0); + positionLines(); + fillLines(); + return; + } + + // Preserve row position + lastTopPix += lineHeight * addedCount; + vwContent.style.top = `${lastTopPix}px`; + vwScroller.scrollTop = lastTopPix; + }; + + return { updateContent, updateLayout, }; +})(); + +/******************************************************************************/ + +const updateCurrentTabTitle = (( ) => { + const i18nCurrentTab = i18n$('loggerCurrentTab'); + + return ( ) => { + const select = qs$('#pageSelector'); + if ( select.value !== '_' || activeTabId === 0 ) { return; } + const opt0 = qs$(select, '[value="_"]'); + const opt1 = qs$(select, `[value="${activeTabId}"]`); + let text = i18nCurrentTab; + if ( opt1 !== null ) { + text += ' / ' + opt1.textContent; + } + opt0.textContent = text; + }; +})(); + +/******************************************************************************/ + +const synchronizeTabIds = function(newTabIds) { + const select = qs$('#pageSelector'); + const selectedTabValue = select.value; + const oldTabIds = allTabIds; + + // Collate removed tab ids. + const toVoid = new Set(); + for ( const tabId of oldTabIds.keys() ) { + if ( newTabIds.has(tabId) ) { continue; } + toVoid.add(tabId); + } + allTabIds = newTabIds; + + // Mark as "void" all logger entries which are linked to now invalid + // tab ids. + // When an entry is voided without being removed, we re-create a new entry + // in order to ensure the entry has a new identity. A new identify ensures + // that identity-based associations elsewhere are automatically + // invalidated. + if ( toVoid.size !== 0 ) { + const autoDeleteVoidedRows = selectedTabValue === '_'; + let rowVoided = false; + for ( let i = 0, n = loggerEntries.length; i < n; i++ ) { + const entry = loggerEntries[i]; + if ( toVoid.has(entry.tabId) === false ) { continue; } + if ( entry.voided ) { continue; } + rowVoided = entry.voided = true; + if ( autoDeleteVoidedRows ) { + entry.dead = true; + } + loggerEntries[i] = new LogEntry(entry); + } + if ( rowVoided ) { + rowFilterer.filterAll(); + } + } + + // Remove popup if it is currently bound to a removed tab. + if ( toVoid.has(popupManager.tabId) ) { + popupManager.toggleOff(); + } + + const tabIds = Array.from(newTabIds.keys()).sort(function(a, b) { + return newTabIds.get(a).localeCompare(newTabIds.get(b)); + }); + let j = 3; + for ( const tabId of tabIds ) { + if ( tabId <= 0 ) { continue; } + if ( j === select.options.length ) { + select.appendChild(document.createElement('option')); + } + const option = select.options[j]; + // Truncate too long labels. + option.textContent = newTabIds.get(tabId).slice(0, 80); + dom.attr(option, 'value', tabId); + if ( option.value === selectedTabValue ) { + select.selectedIndex = j; + dom.attr(option, 'selected', ''); + } else { + dom.attr(option, 'selected', null); + } + j += 1; + } + while ( j < select.options.length ) { + select.removeChild(select.options[j]); + } + if ( select.value !== selectedTabValue ) { + select.selectedIndex = 0; + select.value = ''; + dom.attr(select.options[0], 'selected', ''); + pageSelectorChanged(); + } + + updateCurrentTabTitle(); +}; + +/******************************************************************************/ + +const onLogBufferRead = function(response) { + if ( !response || response.unavailable ) { return; } + + // Disable tooltips? + if ( + popupLoggerTooltips === undefined && + response.tooltips !== undefined + ) { + popupLoggerTooltips = response.tooltips; + if ( popupLoggerTooltips === false ) { + dom.attr('[data-i18n-title]', 'title', ''); + } + } + + // Tab id of currently active tab + let activeTabIdChanged = false; + if ( response.activeTabId ) { + activeTabIdChanged = response.activeTabId !== activeTabId; + activeTabId = response.activeTabId; + } + + if ( Array.isArray(response.tabIds) ) { + response.tabIds = new Map(response.tabIds); + } + + // List of tab ids has changed + if ( response.tabIds !== undefined ) { + synchronizeTabIds(response.tabIds); + allTabIdsToken = response.tabIdsToken; + } + + if ( activeTabIdChanged ) { + pageSelectorFromURLHash(); + } + + processLoggerEntries(response); + + // Synchronize DOM with sent logger data + dom.cl.toggle(dom.html, 'colorBlind', response.colorBlind === true); + dom.cl.toggle('#clean', 'disabled', filteredLoggerEntryVoidedCount === 0); + dom.cl.toggle('#clear', 'disabled', filteredLoggerEntries.length === 0); +}; + +/******************************************************************************/ + +const readLogBuffer = (( ) => { + let reading = false; + + const readLogBufferNow = async function() { + if ( logger.ownerId === undefined ) { return; } + if ( reading ) { return; } + + reading = true; + + const msg = { + what: 'readAll', + ownerId: logger.ownerId, + tabIdsToken: allTabIdsToken, + }; + + // This is to detect changes in the position or size of the logger + // popup window (if in use). + if ( + popupLoggerBox instanceof Object && + ( + self.screenX !== popupLoggerBox.x || + self.screenY !== popupLoggerBox.y || + self.outerWidth !== popupLoggerBox.w || + self.outerHeight !== popupLoggerBox.h + ) + ) { + popupLoggerBox.x = self.screenX; + popupLoggerBox.y = self.screenY; + popupLoggerBox.w = self.outerWidth; + popupLoggerBox.h = self.outerHeight; + msg.popupLoggerBoxChanged = true; + } + + const response = await vAPI.messaging.send('loggerUI', msg); + + onLogBufferRead(response); + + reading = false; + + timer.on(1200); + }; + + const timer = vAPI.defer.create(readLogBufferNow); + + readLogBufferNow(); + + return ( ) => { + timer.on(1200); + }; +})(); + +/******************************************************************************/ + +const pageSelectorChanged = function() { + const select = qs$('#pageSelector'); + window.location.replace('#' + select.value); + pageSelectorFromURLHash(); +}; + +const pageSelectorFromURLHash = (( ) => { + let lastHash; + let lastSelectedTabId; + + return function() { + let hash = window.location.hash.slice(1); + let match = /^([^+]+)\+(.+)$/.exec(hash); + if ( match !== null ) { + hash = match[1]; + activeTabId = parseInt(match[2], 10) || 0; + window.location.hash = '#' + hash; + } + + if ( hash !== lastHash ) { + const select = qs$('#pageSelector'); + let option = qs$(select, `option[value="${hash}"]`); + if ( option === null ) { + hash = '0'; + option = select.options[0]; + } + select.selectedIndex = option.index; + select.value = option.value; + lastHash = hash; + } + + selectedTabId = hash === '_' + ? activeTabId + : parseInt(hash, 10) || 0; + + if ( lastSelectedTabId === selectedTabId ) { return; } + + rowFilterer.filterAll(); + document.dispatchEvent(new Event('tabIdChanged')); + updateCurrentTabTitle(); + dom.cl.toggle('.needdom', 'disabled', selectedTabId <= 0); + dom.cl.toggle('.needscope', 'disabled', selectedTabId <= 0); + lastSelectedTabId = selectedTabId; + }; +})(); + +/******************************************************************************/ + +const reloadTab = function(bypassCache = false) { + const tabId = tabIdFromPageSelector(); + if ( tabId <= 0 ) { return; } + messaging.send('loggerUI', { + what: 'reloadTab', + tabId, + bypassCache, + }); +}; + +dom.on('#refresh', 'click', ev => { + reloadTab(ev.ctrlKey || ev.metaKey || ev.shiftKey); +}); + +dom.on(document, 'keydown', ev => { + if ( ev.isComposing ) { return; } + let bypassCache = false; + switch ( ev.key ) { + case 'F5': + bypassCache = ev.ctrlKey || ev.metaKey || ev.shiftKey; + break; + case 'r': + if ( (ev.ctrlKey || ev.metaKey) !== true ) { return; } + break; + case 'R': + if ( (ev.ctrlKey || ev.metaKey) !== true ) { return; } + bypassCache = true; + break; + default: + return; + } + reloadTab(bypassCache); + ev.preventDefault(); + ev.stopPropagation(); +}, { capture: true }); + +/******************************************************************************/ +/******************************************************************************/ + +(( ) => { + const reRFC3986 = /^([^:\/?#]+:)?(\/\/[^\/?#]*)?([^?#]*)(\?[^#]*)?(#.*)?/; + const reSchemeOnly = /^[\w-]+:$/; + const staticFilterTypes = { + 'beacon': 'ping', + 'doc': 'document', + 'css': 'stylesheet', + 'frame': 'subdocument', + 'object_subrequest': 'object', + 'csp_report': 'other', + }; + const createdStaticFilters = {}; + const reIsExceptionFilter = /^@@|^[\w.-]*?#@#/; + + let dialog = null; + let targetRow = null; + let targetType; + let targetURLs = []; + let targetFrameHostname; + let targetPageHostname; + let targetTabId; + let targetDomain; + let targetPageDomain; + let targetFrameDomain; + + const uglyTypeFromSelector = pane => { + const prettyType = selectValue('select.type.' + pane); + if ( pane === 'static' ) { + return staticFilterTypes[prettyType] || prettyType; + } + return uglyRequestTypes[prettyType] || prettyType; + }; + + const selectNode = selector => { + return qs$(dialog, selector); + }; + + const selectValue = selector => { + return selectNode(selector).value || ''; + }; + + const staticFilterNode = ( ) => { + return qs$(dialog, 'div.panes > div.static textarea'); + }; + + const toExceptionFilter = (filter, extended) => { + if ( reIsExceptionFilter.test(filter) ) { return filter; } + return extended ? filter.replace('##', '#@#') : `@@${filter}`; + }; + + const onColorsReady = function(response) { + dom.cl.toggle(dom.body, 'dirty', response.dirty); + for ( const url in response.colors ) { + if ( response.colors.hasOwnProperty(url) === false ) { continue; } + const colorEntry = response.colors[url]; + const node = qs$(dialog, `.dynamic .entry .action[data-url="${url}"]`); + if ( node === null ) { continue; } + dom.cl.toggle(node, 'allow', colorEntry.r === 2); + dom.cl.toggle(node, 'noop', colorEntry.r === 3); + dom.cl.toggle(node, 'block', colorEntry.r === 1); + dom.cl.toggle(node, 'own', colorEntry.own); + } + }; + + const colorize = async function() { + const response = await messaging.send('loggerUI', { + what: 'getURLFilteringData', + context: selectValue('select.dynamic.origin'), + urls: targetURLs, + type: uglyTypeFromSelector('dynamic'), + }); + onColorsReady(response); + }; + + const parseStaticInputs = function() { + const options = []; + const block = selectValue('select.static.action') === ''; + let filter = ''; + if ( !block ) { + filter = '@@'; + } + let value = selectValue('select.static.url'); + if ( value !== '' ) { + if ( reSchemeOnly.test(value) ) { + value = `|${value}`; + } else { + if ( value.endsWith('/') ) { + value += '*'; + } else if ( /[/?]/.test(value) === false ) { + value += '^'; + } + value = `||${value}`; + } + } + filter += value; + value = selectValue('select.static.type'); + if ( value !== '' ) { + options.push(uglyTypeFromSelector('static')); + } + value = selectValue('select.static.origin'); + if ( value !== '' ) { + if ( value === targetDomain ) { + options.push('1p'); + } else { + options.push('domain=' + value); + } + } + if ( block && selectValue('select.static.importance') !== '' ) { + options.push('important'); + } + if ( options.length ) { + filter += '$' + options.join(','); + } + staticFilterNode().value = filter; + updateWidgets(); + }; + + const updateWidgets = function() { + const value = staticFilterNode().value; + dom.cl.toggle( + qs$(dialog, '#createStaticFilter'), + 'disabled', + createdStaticFilters.hasOwnProperty(value) || value === '' + ); + }; + + const onClick = async function(ev) { + const target = ev.target; + const tcl = target.classList; + + // Close entry tools + if ( tcl.contains('closeButton') ) { + ev.stopPropagation(); + toggleOff(); + return; + } + + // Select a pane + if ( tcl.contains('header') ) { + ev.stopPropagation(); + dom.attr(dialog, 'data-pane', dom.attr(target, 'data-pane')); + return; + } + + // Toggle temporary exception filter + if ( tcl.contains('exceptor') ) { + ev.stopPropagation(); + const filter = filterFromTargetRow(); + const status = await messaging.send('loggerUI', { + what: 'toggleInMemoryFilter', + filter: toExceptionFilter(filter, dom.cl.has(targetRow, 'extendedRealm')), + }); + const row = target.closest('div'); + dom.cl.toggle(row, 'exceptored', status); + return; + } + + // Create static filter + if ( target.id === 'createStaticFilter' ) { + ev.stopPropagation(); + const value = staticFilterNode().value; + // Avoid duplicates + if ( createdStaticFilters.hasOwnProperty(value) ) { return; } + createdStaticFilters[value] = true; + // https://github.com/uBlockOrigin/uBlock-issues/issues/1281#issuecomment-704217175 + // TODO: + // Figure a way to use the actual document URL. Currently using + // a synthetic URL derived from the document hostname. + if ( value !== '' ) { + messaging.send('loggerUI', { + what: 'createUserFilter', + autoComment: true, + filters: value, + docURL: `https://${targetFrameHostname}/`, + }); + } + updateWidgets(); + return; + } + + // Save url filtering rule(s) + if ( target.id === 'saveRules' ) { + ev.stopPropagation(); + await messaging.send('loggerUI', { + what: 'saveURLFilteringRules', + context: selectValue('select.dynamic.origin'), + urls: targetURLs, + type: uglyTypeFromSelector('dynamic'), + }); + colorize(); + return; + } + + const persist = !!ev.ctrlKey || !!ev.metaKey; + + // Remove url filtering rule + if ( tcl.contains('action') ) { + ev.stopPropagation(); + await messaging.send('loggerUI', { + what: 'setURLFilteringRule', + context: selectValue('select.dynamic.origin'), + url: dom.attr(target, 'data-url'), + type: uglyTypeFromSelector('dynamic'), + action: 0, + persist: persist, + }); + colorize(); + return; + } + + // add "allow" url filtering rule + if ( tcl.contains('allow') ) { + ev.stopPropagation(); + await messaging.send('loggerUI', { + what: 'setURLFilteringRule', + context: selectValue('select.dynamic.origin'), + url: dom.attr(target.parentNode, 'data-url'), + type: uglyTypeFromSelector('dynamic'), + action: 2, + persist: persist, + }); + colorize(); + return; + } + + // add "block" url filtering rule + if ( tcl.contains('noop') ) { + ev.stopPropagation(); + await messaging.send('loggerUI', { + what: 'setURLFilteringRule', + context: selectValue('select.dynamic.origin'), + url: dom.attr(target.parentNode, 'data-url'), + type: uglyTypeFromSelector('dynamic'), + action: 3, + persist: persist, + }); + colorize(); + return; + } + + // add "block" url filtering rule + if ( tcl.contains('block') ) { + ev.stopPropagation(); + await messaging.send('loggerUI', { + what: 'setURLFilteringRule', + context: selectValue('select.dynamic.origin'), + url: dom.attr(target.parentNode, 'data-url'), + type: uglyTypeFromSelector('dynamic'), + action: 1, + persist: persist, + }); + colorize(); + return; + } + + // Highlight corresponding element in target web page + if ( tcl.contains('picker') ) { + ev.stopPropagation(); + messaging.send('loggerUI', { + what: 'launchElementPicker', + tabId: targetTabId, + targetURL: 'img\t' + targetURLs[0], + select: true, + }); + return; + } + + // Reload tab associated with event + if ( tcl.contains('reload') ) { + ev.stopPropagation(); + messaging.send('loggerUI', { + what: 'reloadTab', + tabId: targetTabId, + bypassCache: ev.ctrlKey || ev.metaKey || ev.shiftKey, + }); + return; + } + }; + + const onSelectChange = function(ev) { + const tcl = ev.target.classList; + + if ( tcl.contains('dynamic') ) { + colorize(); + return; + } + + if ( tcl.contains('static') ) { + parseStaticInputs(); + return; + } + }; + + const onInputChange = function() { + updateWidgets(); + }; + + const createPreview = function(type, url) { + const cantPreview = + type !== 'image' || + dom.cl.has(targetRow, 'networkRealm') === false || + dom.attr(targetRow, 'data-status') === '1'; + + // Whether picker can be used + dom.cl.toggle( + qs$(dialog, '.picker'), + 'hide', + targetTabId < 0 || cantPreview + ); + + // Whether the resource can be previewed + if ( cantPreview ) { return; } + + const container = qs$(dialog, '.preview'); + dom.on(qs$(container, 'span'), 'click', ( ) => { + const preview = dom.create('img'); + dom.attr(preview, 'src', url); + container.replaceChild(preview, container.firstElementChild); + }, { once: true }); + + dom.cl.remove(container, 'hide'); + }; + + // https://github.com/gorhill/uBlock/issues/1511 + const shortenLongString = function(url, max) { + const urlLen = url.length; + if ( urlLen <= max ) { + return url; + } + const n = urlLen - max - 1; + const i = (urlLen - n) / 2 | 0; + return url.slice(0, i) + '…' + url.slice(i + n); + }; + + // Build list of candidate URLs + const createTargetURLs = function(url) { + const matches = reRFC3986.exec(url); + if ( matches === null ) { return []; } + if ( typeof matches[2] !== 'string' || matches[2].length === 0 ) { + return [ matches[1] ]; + } + // Shortest URL for a valid URL filtering rule + const urls = []; + const rootURL = matches[1] + matches[2]; + urls.unshift(rootURL); + const path = matches[3] || ''; + let pos = path.charAt(0) === '/' ? 1 : 0; + while ( pos < path.length ) { + pos = path.indexOf('/', pos); + if ( pos === -1 ) { + pos = path.length; + } else { + pos += 1; + } + urls.unshift(rootURL + path.slice(0, pos)); + } + const query = matches[4] || ''; + if ( query !== '' ) { + urls.unshift(rootURL + path + query); + } + return urls; + }; + + const filterFromTargetRow = function() { + return dom.text(targetRow.children[COLUMN_FILTER]); + }; + + const aliasURLFromID = function(id) { + if ( id === '' ) { return ''; } + for ( const entry of loggerEntries ) { + if ( entry.id !== id || entry.aliased ) { continue; } + const fields = entry.textContent.split('\t'); + return fields[COLUMN_URL] || ''; + } + return ''; + }; + + const toSummaryPaneFilterNode = async function(receiver, filter) { + receiver.children[COLUMN_FILTER].textContent = filter; + if ( dom.cl.has(targetRow, 'canLookup') === false ) { return; } + const isException = reIsExceptionFilter.test(filter); + let isExcepted = false; + if ( isException ) { + isExcepted = await messaging.send('loggerUI', { + what: 'hasInMemoryFilter', + filter: toExceptionFilter(filter, dom.cl.has(targetRow, 'extendedRealm')), + }); + } + if ( isException && isExcepted === false ) { return; } + dom.cl.toggle(receiver, 'exceptored', isExcepted); + receiver.children[2].style.visibility = ''; + }; + + const fillSummaryPaneFilterList = async function(rows) { + const rawFilter = targetRow.children[COLUMN_FILTER].textContent; + + const nodeFromFilter = function(filter, lists) { + const fragment = document.createDocumentFragment(); + const template = qs$('#filterFinderListEntry > span'); + for ( const list of lists ) { + const span = dom.clone(template); + let a = qs$(span, 'a:nth-of-type(1)'); + a.href += encodeURIComponent(list.assetKey); + a.append(i18n.patchUnicodeFlags(list.title)); + a = qs$(span, 'a:nth-of-type(2)'); + if ( list.supportURL ) { + dom.attr(a, 'href', list.supportURL); + } else { + a.style.display = 'none'; + } + if ( fragment.childElementCount !== 0 ) { + fragment.appendChild(document.createTextNode('\n')); + } + fragment.appendChild(span); + } + return fragment; + }; + + const handleResponse = function(response) { + if ( response instanceof Object === false ) { + response = {}; + } + let bestMatchFilter = ''; + for ( const filter in response ) { + if ( filter.length > bestMatchFilter.length ) { + bestMatchFilter = filter; + } + } + if ( + bestMatchFilter !== '' && + Array.isArray(response[bestMatchFilter]) + ) { + toSummaryPaneFilterNode(rows[0], bestMatchFilter); + rows[1].children[1].appendChild(nodeFromFilter( + bestMatchFilter, + response[bestMatchFilter] + )); + } + // https://github.com/gorhill/uBlock/issues/2179 + if ( rows[1].children[1].childElementCount === 0 ) { + i18n.safeTemplateToDOM( + 'loggerStaticFilteringFinderSentence2', + { filter: rawFilter }, + rows[1].children[1] + ); + } + }; + + if ( dom.cl.has(targetRow, 'networkRealm') ) { + const response = await messaging.send('loggerUI', { + what: 'listsFromNetFilter', + rawFilter: rawFilter, + }); + handleResponse(response); + } else if ( dom.cl.has(targetRow, 'extendedRealm') ) { + const response = await messaging.send('loggerUI', { + what: 'listsFromCosmeticFilter', + url: targetRow.children[COLUMN_URL].textContent, + rawFilter: rawFilter, + }); + handleResponse(response); + } + }; + + const fillSummaryPane = function() { + const rows = qsa$(dialog, '.pane.details > div'); + const tr = targetRow; + const trcl = tr.classList; + const trch = tr.children; + let text; + // Filter and context + text = filterFromTargetRow(); + if ( + (text !== '') && + (trcl.contains('extendedRealm') || trcl.contains('networkRealm')) + ) { + toSummaryPaneFilterNode(rows[0], text); + } else { + rows[0].style.display = 'none'; + } + // Rule + if ( + (text !== '') && + ( + trcl.contains('dynamicHost') || + trcl.contains('dynamicUrl') || + trcl.contains('switchRealm') + ) + ) { + rows[2].children[1].textContent = text; + } else { + rows[2].style.display = 'none'; + } + // Filter list + if ( trcl.contains('canLookup') ) { + fillSummaryPaneFilterList(rows); + } else { + rows[1].style.display = 'none'; + } + // Root and immediate contexts + const tabhn = dom.attr(tr, 'data-tabhn') || ''; + const dochn = dom.attr(tr, 'data-dochn') || ''; + if ( tabhn !== '' && tabhn !== dochn ) { + rows[3].children[1].textContent = tabhn; + } else { + rows[3].style.display = 'none'; + } + if ( dochn !== '' ) { + rows[4].children[1].textContent = dochn; + } else { + rows[4].style.display = 'none'; + } + // Partyness + text = dom.attr(tr, 'data-parties') || ''; + if ( text !== '' ) { + rows[5].children[1].textContent = `(${trch[COLUMN_PARTYNESS].textContent})\u2002${text}`; + } else { + rows[5].style.display = 'none'; + } + // Type + text = trch[COLUMN_TYPE].textContent; + if ( text !== '' ) { + rows[6].children[1].textContent = text; + } else { + rows[6].style.display = 'none'; + } + // URL + const canonicalURL = trch[COLUMN_URL].textContent; + if ( canonicalURL !== '' ) { + const attr = dom.attr(tr, 'data-status') || ''; + if ( attr !== '' ) { + dom.attr(rows[7], 'data-status', attr); + if ( tr.hasAttribute('data-modifier') ) { + dom.attr(rows[7], 'data-modifier', ''); + } + } + rows[7].children[1].appendChild(dom.clone(trch[COLUMN_URL])); + } else { + rows[7].style.display = 'none'; + } + // Alias URL + text = dom.attr(tr, 'data-aliasid'); + const aliasURL = text ? aliasURLFromID(text) : ''; + if ( aliasURL !== '' ) { + rows[8].children[1].textContent = + hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' + + hostnameFromURI(canonicalURL); + rows[9].children[1].textContent = aliasURL; + } else { + rows[8].style.display = 'none'; + rows[9].style.display = 'none'; + } + }; + + // Fill dynamic URL filtering pane + const fillDynamicPane = function() { + if ( dom.cl.has(targetRow, 'extendedRealm') ) { return; } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/662#issuecomment-509220702 + if ( targetType === 'doc' ) { return; } + + // https://github.com/gorhill/uBlock/issues/2469 + if ( targetURLs.length === 0 || reSchemeOnly.test(targetURLs[0]) ) { + return; + } + + // Fill context selector + let select = selectNode('select.dynamic.origin'); + fillOriginSelect(select, targetPageHostname, targetPageDomain); + const option = document.createElement('option'); + option.textContent = '*'; + dom.attr(option, 'value', '*'); + select.appendChild(option); + + // Fill type selector + select = selectNode('select.dynamic.type'); + select.options[0].textContent = targetType; + dom.attr(select.options[0], 'value', targetType); + select.selectedIndex = 0; + + // Fill entries + const menuEntryTemplate = qs$(dialog, '.dynamic .toolbar .entry'); + const tbody = qs$(dialog, '.dynamic .entries'); + for ( const targetURL of targetURLs ) { + const menuEntry = dom.clone(menuEntryTemplate); + dom.attr(menuEntry.children[0], 'data-url', targetURL); + menuEntry.children[1].textContent = shortenLongString(targetURL, 128); + tbody.appendChild(menuEntry); + } + + colorize(); + }; + + const fillOriginSelect = function(select, hostname, domain) { + const template = i18n$('loggerStaticFilteringSentencePartOrigin'); + let value = hostname; + for (;;) { + const option = document.createElement('option'); + dom.attr(option, 'value', value); + option.textContent = template.replace('{{origin}}', value); + select.appendChild(option); + if ( value === domain ) { break; } + const pos = value.indexOf('.'); + if ( pos === -1 ) { break; } + value = value.slice(pos + 1); + } + }; + + // Fill static filtering pane + const fillStaticPane = function() { + if ( dom.cl.has(targetRow, 'extendedRealm') ) { return; } + + const template = i18n$('loggerStaticFilteringSentence'); + const rePlaceholder = /\{\{[^}]+?\}\}/g; + const nodes = []; + let pos = 0; + for (;;) { + const match = rePlaceholder.exec(template); + if ( match === null ) { break; } + if ( pos !== match.index ) { + nodes.push(document.createTextNode(template.slice(pos, match.index))); + } + pos = rePlaceholder.lastIndex; + let select, option; + switch ( match[0] ) { + case '{{br}}': + nodes.push(document.createElement('br')); + break; + + case '{{action}}': + select = document.createElement('select'); + select.className = 'static action'; + option = document.createElement('option'); + dom.attr(option, 'value', ''); + option.textContent = i18n$('loggerStaticFilteringSentencePartBlock'); + select.appendChild(option); + option = document.createElement('option'); + dom.attr(option, 'value', '@@'); + option.textContent = i18n$('loggerStaticFilteringSentencePartAllow'); + select.appendChild(option); + nodes.push(select); + break; + + case '{{type}}': { + const filterType = staticFilterTypes[targetType] || targetType; + select = document.createElement('select'); + select.className = 'static type'; + option = document.createElement('option'); + dom.attr(option, 'value', filterType); + option.textContent = i18n$('loggerStaticFilteringSentencePartType').replace('{{type}}', filterType); + select.appendChild(option); + option = document.createElement('option'); + dom.attr(option, 'value', ''); + option.textContent = i18n$('loggerStaticFilteringSentencePartAnyType'); + select.appendChild(option); + nodes.push(select); + break; + } + case '{{url}}': + select = document.createElement('select'); + select.className = 'static url'; + for ( const targetURL of targetURLs ) { + const value = targetURL.replace(/^[a-z-]+:\/\//, ''); + option = document.createElement('option'); + dom.attr(option, 'value', value); + option.textContent = shortenLongString(value, 128); + select.appendChild(option); + } + nodes.push(select); + break; + + case '{{origin}}': + select = document.createElement('select'); + select.className = 'static origin'; + fillOriginSelect(select, targetFrameHostname, targetFrameDomain); + option = document.createElement('option'); + dom.attr(option, 'value', ''); + option.textContent = i18n$('loggerStaticFilteringSentencePartAnyOrigin'); + select.appendChild(option); + nodes.push(select); + break; + + case '{{importance}}': + select = document.createElement('select'); + select.className = 'static importance'; + option = document.createElement('option'); + dom.attr(option, 'value', ''); + option.textContent = i18n$('loggerStaticFilteringSentencePartNotImportant'); + select.appendChild(option); + option = document.createElement('option'); + dom.attr(option, 'value', 'important'); + option.textContent = i18n$('loggerStaticFilteringSentencePartImportant'); + select.appendChild(option); + nodes.push(select); + break; + + default: + break; + } + } + if ( pos < template.length ) { + nodes.push(document.createTextNode(template.slice(pos))); + } + const parent = qs$(dialog, 'div.panes > .static > div:first-of-type'); + for ( let i = 0; i < nodes.length; i++ ) { + parent.appendChild(nodes[i]); + } + parseStaticInputs(); + }; + + const moveDialog = ev => { + if ( ev.button !== 0 && ev.touches === undefined ) { return; } + const widget = qs$('#netInspector .entryTools'); + onStartMovingWidget(ev, widget, ( ) => { + vAPI.localStorage.setItem( + 'loggerUI.entryTools', + JSON.stringify({ + bottom: widget.style.bottom, + left: widget.style.left, + right: widget.style.right, + top: widget.style.top, + }) + ); + }); + }; + + const fillDialog = function(domains) { + dialog = dom.clone('#templates .netFilteringDialog'); + dom.cl.toggle( + dialog, + 'extendedRealm', + dom.cl.has(targetRow, 'extendedRealm') + ); + targetDomain = domains[0]; + targetPageDomain = domains[1]; + targetFrameDomain = domains[2]; + createPreview(targetType, targetURLs[0]); + fillSummaryPane(); + fillDynamicPane(); + fillStaticPane(); + dom.on(dialog, 'click', ev => { onClick(ev); }, true); + dom.on(dialog, 'change', onSelectChange, true); + dom.on(dialog, 'input', onInputChange, true); + const container = qs$('#netInspector .entryTools'); + if ( container.firstChild ) { + container.replaceChild(dialog, container.firstChild); + } else { + container.append(dialog); + } + const moveBand = qs$(dialog, '.moveBand'); + dom.on(moveBand, 'mousedown', moveDialog); + dom.on(moveBand, 'touchstart', moveDialog); + }; + + const toggleOn = async function(ev) { + targetRow = ev.target.closest('.canDetails'); + if ( targetRow === null ) { return; } + ev.stopPropagation(); + targetTabId = tabIdFromAttribute(targetRow); + targetType = targetRow.children[COLUMN_TYPE].textContent.trim() || ''; + targetURLs = createTargetURLs(targetRow.children[COLUMN_URL].textContent); + targetPageHostname = dom.attr(targetRow, 'data-tabhn') || ''; + targetFrameHostname = dom.attr(targetRow, 'data-dochn') || ''; + + // We need the root domain names for best user experience. + const domains = await messaging.send('loggerUI', { + what: 'getDomainNames', + targets: [ + targetURLs[0], + targetPageHostname, + targetFrameHostname + ], + }); + fillDialog(domains); + }; + + const toggleOff = function() { + const container = qs$('#netInspector .entryTools'); + if ( container.firstChild ) { + container.firstChild.remove(); + } + targetURLs = []; + targetRow = null; + dialog = null; + }; + + // Restore position of entry tools dialog + vAPI.localStorage.getItemAsync( + 'loggerUI.entryTools', + ).then(response => { + if ( typeof response !== 'string' ) { return; } + const settings = JSON.parse(response); + const widget = qs$('#netInspector .entryTools'); + widget.style.bottom = ''; + widget.style.left = settings.left || ''; + widget.style.right = settings.right || ''; + widget.style.top = settings.top || ''; + if ( /^-/.test(widget.style.top) ) { + widget.style.top = '0'; + } + }); + + dom.on( + '#netInspector', + 'click', + '.canDetails > span:not(:nth-of-type(4)):not(:nth-of-type(8))', + ev => { toggleOn(ev); } + ); + + dom.on( + '#netInspector', + 'click', + '.logEntry > div > span:nth-of-type(8) a', + ev => { + vAPI.messaging.send('codeViewer', { + what: 'gotoURL', + details: { + url: ev.target.getAttribute('href'), + select: true, + }, + }); + ev.preventDefault(); + ev.stopPropagation(); + } + ); +})(); + +/******************************************************************************/ +/******************************************************************************/ + +const rowFilterer = (( ) => { + const userFilters = []; + const builtinFilters = []; + + let masterFilterSwitch = true; + let filters = []; + + const parseInput = function() { + userFilters.length = 0; + + const rawParts = qs$('#filterInput > input').value.trim().split(/\s+/); + const n = rawParts.length; + const reStrs = []; + let not = false; + for ( let i = 0; i < n; i++ ) { + let rawPart = rawParts[i]; + if ( rawPart.charAt(0) === '!' ) { + if ( reStrs.length === 0 ) { + not = true; + } + rawPart = rawPart.slice(1); + } + let reStr = ''; + if ( rawPart.startsWith('/') && rawPart.endsWith('/') ) { + reStr = rawPart.slice(1, -1); + try { + new RegExp(reStr); + } catch(ex) { + reStr = ''; + } + } + if ( reStr === '' ) { + const hardBeg = rawPart.startsWith('|'); + if ( hardBeg ) { + rawPart = rawPart.slice(1); + } + const hardEnd = rawPart.endsWith('|'); + if ( hardEnd ) { + rawPart = rawPart.slice(0, -1); + } + // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions + reStr = rawPart.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // https://github.com/orgs/uBlockOrigin/teams/ublock-issues-volunteers/discussions/51 + // Be more flexible when interpreting leading/trailing pipes, + // as leading/trailing pipes are often used in static filters. + if ( hardBeg ) { + reStr = reStr !== '' ? '(?:^|\\s|\\|)' + reStr : '\\|'; + } + if ( hardEnd ) { + reStr += '(?:\\||\\s|$)'; + } + } + if ( reStr === '' ) { continue; } + reStrs.push(reStr); + if ( i < (n - 1) && rawParts[i + 1] === '||' ) { + i += 1; + continue; + } + reStr = reStrs.length === 1 ? reStrs[0] : reStrs.join('|'); + userFilters.push({ + re: new RegExp(reStr, 'i'), + r: !not + }); + reStrs.length = 0; + not = false; + } + filters = builtinFilters.concat(userFilters); + }; + + const filterOne = function(logEntry) { + if ( + logEntry.dead || + selectedTabId !== 0 && + ( + logEntry.tabId === undefined || + logEntry.tabId > 0 && logEntry.tabId !== selectedTabId + ) + ) { + return false; + } + + if ( masterFilterSwitch === false || filters.length === 0 ) { + return true; + } + + // Do not filter out tab load event, they help separate key sections + // of logger. + if ( logEntry.type === 'tabLoad' ) { return true; } + + for ( const f of filters ) { + if ( f.re.test(logEntry.textContent) !== f.r ) { return false; } + } + return true; + }; + + const filterAll = function() { + filteredLoggerEntries = []; + filteredLoggerEntryVoidedCount = 0; + for ( const entry of loggerEntries ) { + if ( filterOne(entry) === false ) { continue; } + filteredLoggerEntries.push(entry); + if ( entry.voided ) { + filteredLoggerEntryVoidedCount += 1; + } + } + viewPort.updateContent(0); + dom.cl.toggle('#filterButton', 'active', filters.length !== 0); + dom.cl.toggle('#clean', 'disabled', filteredLoggerEntryVoidedCount === 0); + dom.cl.toggle('#clear', 'disabled', filteredLoggerEntries.length === 0); + }; + + const onFilterChangedAsync = (( ) => { + const commit = ( ) => { + parseInput(); + filterAll(); + }; + const timer = vAPI.defer.create(commit); + return ( ) => { + timer.offon(750); + }; + })(); + + const onFilterButton = function() { + masterFilterSwitch = !masterFilterSwitch; + dom.cl.toggle('#netInspector', 'f', masterFilterSwitch); + filterAll(); + }; + + const onToggleExtras = function(ev) { + dom.cl.toggle(ev.target, 'expanded'); + }; + + const builtinFilterExpression = function() { + builtinFilters.length = 0; + const filtexElems = qsa$('#filterExprPicker [data-filtex]'); + const orExprs = []; + let not = false; + for ( const filtexElem of filtexElems ) { + const filtex = filtexElem.dataset.filtex; + const active = dom.cl.has(filtexElem, 'on'); + if ( filtex === '!' ) { + if ( orExprs.length !== 0 ) { + builtinFilters.push({ + re: new RegExp(orExprs.join('|')), + r: !not + }); + orExprs.length = 0; + } + not = active; + } else if ( active ) { + orExprs.push(filtex); + } + } + if ( orExprs.length !== 0 ) { + builtinFilters.push({ + re: new RegExp(orExprs.join('|')), + r: !not + }); + } + filters = builtinFilters.concat(userFilters); + dom.cl.toggle('#filterExprButton', 'active', builtinFilters.length !== 0); + filterAll(); + }; + + dom.on('#filterButton', 'click', onFilterButton); + dom.on('#filterInput > input', 'input', onFilterChangedAsync); + dom.on('#filterExprButton', 'click', onToggleExtras); + dom.on('#filterExprPicker', 'click', '[data-filtex]', ev => { + dom.cl.toggle(ev.target, 'on'); + builtinFilterExpression(); + }); + dom.on('#filterInput > input', 'drop', ev => { + const dropItem = item => { + if ( item.kind !== 'string' ) { return false; } + if ( item.type !== 'text/plain' ) { return false; } + item.getAsString(s => { + qs$('#filterInput > input').value = s; + parseInput(); + filterAll(); + }); + return true; + }; + for ( const item of ev.dataTransfer.items ) { + if ( dropItem(item) === false ) { continue; } + ev.preventDefault(); + break; + } + }); + + // https://github.com/gorhill/uBlock/issues/404 + // Ensure page state is in sync with the state of its various widgets. + parseInput(); + builtinFilterExpression(); + filterAll(); + + return { filterOne, filterAll }; +})(); + +/******************************************************************************/ + +// Discard logger entries to prevent undue memory usage growth. The criteria +// to discard are multiple and user configurable: +// +// - Max number of page load per distinct tab +// - Max number of entry per distinct tab +// - Max entry age + +const rowJanitor = (( ) => { + const tabIdToDiscard = new Set(); + const tabIdToLoadCountMap = new Map(); + const tabIdToEntryCountMap = new Map(); + + let rowIndex = 0; + + const discard = function(deadline) { + const opts = loggerSettings.discard; + const maxLoadCount = typeof opts.maxLoadCount === 'number' + ? opts.maxLoadCount + : 0; + const maxEntryCount = typeof opts.maxEntryCount === 'number' + ? opts.maxEntryCount + : 0; + const obsolete = typeof opts.maxAge === 'number' + ? Date.now() - opts.maxAge * 60000 + : 0; + + let i = rowIndex; + // TODO: below should not happen -- remove when confirmed. + if ( i >= loggerEntries.length ) { + i = 0; + } + + if ( i === 0 ) { + tabIdToDiscard.clear(); + tabIdToLoadCountMap.clear(); + tabIdToEntryCountMap.clear(); + } + + let idel = -1; + let bufferedTabId = 0; + let bufferedEntryCount = 0; + let modified = false; + + while ( i < loggerEntries.length ) { + + if ( i % 64 === 0 && deadline.timeRemaining() === 0 ) { break; } + + const entry = loggerEntries[i]; + const tabId = entry.tabId || 0; + + if ( entry.dead || tabIdToDiscard.has(tabId) ) { + if ( idel === -1 ) { idel = i; } + i += 1; + continue; + } + + if ( maxLoadCount !== 0 && entry.type === 'tabLoad' ) { + let count = (tabIdToLoadCountMap.get(tabId) || 0) + 1; + tabIdToLoadCountMap.set(tabId, count); + if ( count >= maxLoadCount ) { + tabIdToDiscard.add(tabId); + } + } + + if ( maxEntryCount !== 0 ) { + if ( bufferedTabId !== tabId ) { + if ( bufferedEntryCount !== 0 ) { + tabIdToEntryCountMap.set(bufferedTabId, bufferedEntryCount); + } + bufferedTabId = tabId; + bufferedEntryCount = tabIdToEntryCountMap.get(tabId) || 0; + } + bufferedEntryCount += 1; + if ( bufferedEntryCount >= maxEntryCount ) { + tabIdToDiscard.add(bufferedTabId); + } + } + + // Since entries in the logger are chronologically ordered, + // everything below obsolete is to be discarded. + if ( obsolete !== 0 && entry.tstamp <= obsolete ) { + if ( idel === -1 ) { idel = i; } + break; + } + + if ( idel !== -1 ) { + loggerEntries.copyWithin(idel, i); + loggerEntries.length -= i - idel; + idel = -1; + modified = true; + } + + i += 1; + } + + if ( idel !== -1 ) { + loggerEntries.length = idel; + modified = true; + } + + if ( i >= loggerEntries.length ) { i = 0; } + rowIndex = i; + + if ( rowIndex === 0 ) { + tabIdToDiscard.clear(); + tabIdToLoadCountMap.clear(); + tabIdToEntryCountMap.clear(); + } + + if ( modified === false ) { return; } + + rowFilterer.filterAll(); + }; + + const discardAsync = function(deadline) { + if ( deadline ) { + discard(deadline); + } + janitorTimer.onidle(1889); + }; + + const janitorTimer = vAPI.defer.create(discardAsync); + + // Clear voided entries from the logger's visible content. + // + // Voided entries should be visible only from the "All" option of the + // tab selector. + // + const clean = function() { + if ( filteredLoggerEntries.length === 0 ) { return; } + + let j = 0; + let targetEntry = filteredLoggerEntries[0]; + for ( const entry of loggerEntries ) { + if ( entry !== targetEntry ) { continue; } + if ( entry.voided ) { + entry.dead = true; + } + j += 1; + if ( j === filteredLoggerEntries.length ) { break; } + targetEntry = filteredLoggerEntries[j]; + } + rowFilterer.filterAll(); + }; + + // Clear the logger's visible content. + // + // "Unrelated" entries -- shown for convenience -- will be also cleared + // if and only if the filtered logger content is made entirely of unrelated + // entries. In effect, this means clicking a second time on the eraser will + // cause unrelated entries to also be cleared. + // + const clear = function() { + if ( filteredLoggerEntries.length === 0 ) { return; } + + let clearUnrelated = true; + if ( selectedTabId !== 0 ) { + for ( const entry of filteredLoggerEntries ) { + if ( entry.tabId === selectedTabId ) { + clearUnrelated = false; + break; + } + } + } + + let j = 0; + let targetEntry = filteredLoggerEntries[0]; + for ( const entry of loggerEntries ) { + if ( entry !== targetEntry ) { continue; } + if ( entry.tabId === selectedTabId || clearUnrelated ) { + entry.dead = true; + } + j += 1; + if ( j === filteredLoggerEntries.length ) { break; } + targetEntry = filteredLoggerEntries[j]; + } + rowFilterer.filterAll(); + }; + + discardAsync(); + + dom.on('#clean', 'click', clean); + dom.on('#clear', 'click', clear); + + return { + inserted: function(count) { + if ( rowIndex !== 0 ) { + rowIndex += count; + } + }, + }; +})(); + +/******************************************************************************/ + +const pauseNetInspector = function() { + netInspectorPaused = dom.cl.toggle('#netInspector', 'paused'); +}; + +/******************************************************************************/ + +const toggleVCompactView = function() { + dom.cl.toggle('#netInspector .vCompactToggler', 'vExpanded'); + viewPort.updateLayout(); +}; + +/******************************************************************************/ + +const popupManager = (( ) => { + let realTabId = 0; + let popup = null; + let popupObserver = null; + + const resizePopup = function() { + if ( popup === null ) { return; } + const popupBody = popup.contentWindow.document.body; + if ( popupBody.clientWidth !== 0 && popup.clientWidth !== popupBody.clientWidth ) { + popup.style.setProperty('width', popupBody.clientWidth + 'px'); + } + if ( popupBody.clientHeight !== 0 && popup.clientHeight !== popupBody.clientHeight ) { + popup.style.setProperty('height', popupBody.clientHeight + 'px'); + } + }; + + const onLoad = function() { + resizePopup(); + popupObserver.observe(popup.contentDocument.body, { + subtree: true, + attributes: true + }); + }; + + const setTabId = function(tabId) { + if ( popup === null ) { return; } + dom.attr(popup, 'src', `popup-fenix.html?portrait=1&tabId=${tabId}`); + }; + + const onTabIdChanged = function() { + const tabId = tabIdFromPageSelector(); + if ( tabId === 0 ) { return toggleOff(); } + realTabId = tabId; + setTabId(realTabId); + }; + + const toggleOn = function() { + const tabId = tabIdFromPageSelector(); + if ( tabId === 0 ) { return; } + realTabId = tabId; + + popup = qs$('#popupContainer'); + + dom.on(popup, 'load', onLoad); + popupObserver = new MutationObserver(resizePopup); + + const parent = qs$('#inspectors'); + const rect = parent.getBoundingClientRect(); + popup.style.setProperty('right', `${rect.right - parent.clientWidth}px`); + dom.cl.add(parent, 'popupOn'); + + dom.on(document, 'tabIdChanged', onTabIdChanged); + + setTabId(realTabId); + dom.cl.add('#showpopup', 'active'); + }; + + const toggleOff = function() { + dom.cl.remove('#showpopup', 'active'); + dom.off(document, 'tabIdChanged', onTabIdChanged); + dom.cl.remove('#inspectors', 'popupOn'); + dom.off(popup, 'load', onLoad); + popupObserver.disconnect(); + popupObserver = null; + dom.attr(popup, 'src', ''); + + realTabId = 0; + }; + + const api = { + get tabId() { return realTabId || 0; }, + toggleOff: function() { + if ( realTabId !== 0 ) { + toggleOff(); + } + } + }; + + dom.on('#showpopup', 'click', ( ) => { + void (realTabId === 0 ? toggleOn() : toggleOff()); + }); + + return api; +})(); + +/******************************************************************************/ + +// Filter hit stats' MVP ("minimum viable product") +// +const loggerStats = (( ) => { + const enabled = false; + const filterHits = new Map(); + let dialog; + let timer; + const makeRow = function() { + const div = document.createElement('div'); + div.appendChild(document.createElement('span')); + div.appendChild(document.createElement('span')); + return div; + }; + + const fillRow = function(div, entry) { + div.children[0].textContent = entry[1].toLocaleString(); + div.children[1].textContent = entry[0]; + }; + + const updateList = function() { + const sortedHits = Array.from(filterHits).sort((a, b) => { + return b[1] - a[1]; + }); + + const doc = document; + const parent = qs$(dialog, '.sortedEntries'); + let i = 0; + + // Reuse existing rows + for ( let iRow = 0; iRow < parent.childElementCount; iRow++ ) { + if ( i === sortedHits.length ) { break; } + fillRow(parent.children[iRow], sortedHits[i]); + i += 1; + } + + // Append new rows + if ( i < sortedHits.length ) { + const list = doc.createDocumentFragment(); + for ( ; i < sortedHits.length; i++ ) { + const div = makeRow(); + fillRow(div, sortedHits[i]); + list.appendChild(div); + } + parent.appendChild(list); + } + + // Remove extraneous rows + // [Should never happen at this point in this current + // bare-bone implementation] + }; + + const toggleOn = function() { + dialog = modalDialog.create( + '#loggerStatsDialog', + ( ) => { + dialog = undefined; + if ( timer !== undefined ) { + self.cancelIdleCallback(timer); + timer = undefined; + } + } + ); + updateList(); + modalDialog.show(); + }; + + dom.on('#loggerStats', 'click', toggleOn); + + return { + processFilter: function(filter) { + if ( enabled !== true ) { return; } + if ( filter.source !== 'static' && filter.source !== 'cosmetic' ) { + return; + } + filterHits.set(filter.raw, (filterHits.get(filter.raw) || 0) + 1); + if ( dialog === undefined || timer !== undefined ) { return; } + timer = self.requestIdleCallback( + ( ) => { + timer = undefined; + updateList(); + }, + { timeout: 2001 } + ); + } + }; +})(); + +/******************************************************************************/ + +(( ) => { + const lines = []; + const options = { + format: 'list', + encoding: 'markdown', + time: 'anonymous', + }; + let dialog; + + const collectLines = function() { + lines.length = 0; + let t0 = filteredLoggerEntries.length !== 0 + ? filteredLoggerEntries[filteredLoggerEntries.length - 1].tstamp + : 0; + for ( const entry of filteredLoggerEntries ) { + const text = entry.textContent; + const fields = []; + let i = 0; + let beg = text.indexOf('\t'); + if ( beg === 0 ) { continue; } + let timeField = text.slice(0, beg); + if ( options.time === 'anonymous' ) { + timeField = '+' + Math.round((entry.tstamp - t0) / 1000).toString(); + } + fields.push(timeField); + beg += 1; + while ( beg < text.length ) { + let end = text.indexOf('\t', beg); + if ( end === -1 ) { end = text.length; } + fields.push(text.slice(beg, end)); + beg = end + 1; + i += 1; + } + lines.push(fields); + } + }; + + const formatAsPlainTextTable = function() { + const outputAll = []; + for ( const fields of lines ) { + outputAll.push(fields.join('\t')); + } + outputAll.push(''); + return outputAll.join('\n'); + }; + + const formatAsMarkdownTable = function() { + const outputAll = []; + let fieldCount = 0; + for ( const fields of lines ) { + if ( fields.length <= 2 ) { continue; } + if ( fields.length > fieldCount ) { + fieldCount = fields.length; + } + const outputOne = []; + for ( let i = 0; i < fields.length; i++ ) { + const field = fields[i]; + let code = /\b(?:www\.|https?:\/\/)/.test(field) ? '`' : ''; + outputOne.push(` ${code}${field.replace(/\|/g, '\\|')}${code} `); + } + outputAll.push(outputOne.join('|')); + } + if ( fieldCount !== 0 ) { + outputAll.unshift( + `${' |'.repeat(fieldCount-1)} `, + `${':--- |'.repeat(fieldCount-1)}:--- ` + ); + } + return `<details><summary>Logger output</summary>\n\n|${outputAll.join('|\n|')}|\n</details>\n`; + }; + + const formatAsTable = function() { + if ( options.encoding === 'plain' ) { + return formatAsPlainTextTable(); + } + return formatAsMarkdownTable(); + }; + + const formatAsList = function() { + const outputAll = []; + for ( const fields of lines ) { + const outputOne = []; + for ( let i = 0; i < fields.length; i++ ) { + let str = fields[i]; + if ( str.length === 0 ) { continue; } + outputOne.push(str); + } + outputAll.push(outputOne.join('\n')); + } + let before, between, after; + if ( options.encoding === 'markdown' ) { + const code = '```'; + before = `<details><summary>Logger output</summary>\n\n${code}\n`; + between = `\n${code}\n${code}\n`; + after = `\n${code}\n</details>\n`; + } else { + before = ''; + between = '\n\n'; + after = '\n'; + } + return `${before}${outputAll.join(between)}${after}`; + }; + + const format = function() { + const output = qs$(dialog, '.output'); + if ( options.format === 'list' ) { + output.textContent = formatAsList(); + } else { + output.textContent = formatAsTable(); + } + }; + + const setRadioButton = function(group, value) { + if ( options.hasOwnProperty(group) === false ) { return; } + const groupEl = qs$(dialog, `[data-radio="${group}"]`); + const buttonEls = qsa$(groupEl, '[data-radio-item]'); + for ( const buttonEl of buttonEls ) { + dom.cl.toggle( + buttonEl, + 'on', + dom.attr(buttonEl, 'data-radio-item') === value + ); + } + options[group] = value; + }; + + const onOption = function(ev) { + const target = ev.target.closest('span[data-i18n]'); + if ( target === null ) { return; } + + // Copy to clipboard + if ( target.matches('.pushbutton') ) { + const textarea = qs$(dialog, 'textarea'); + textarea.focus(); + if ( textarea.selectionEnd === textarea.selectionStart ) { + textarea.select(); + } + document.execCommand('copy'); + ev.stopPropagation(); + return; + } + + // Radio buttons + const group = target.closest('[data-radio]'); + if ( group === null ) { return; } + if ( target.matches('span.on') ) { return; } + const item = target.closest('[data-radio-item]'); + if ( item === null ) { return; } + setRadioButton( + dom.attr(group, 'data-radio'), + dom.attr(item, 'data-radio-item') + ); + format(); + ev.stopPropagation(); + }; + + const toggleOn = function() { + dialog = modalDialog.create( + '#loggerExportDialog', + ( ) => { + dialog = undefined; + lines.length = 0; + } + ); + + setRadioButton('format', options.format); + setRadioButton('encoding', options.encoding); + + collectLines(); + format(); + + dom.on(qs$(dialog, '.options'), 'click', onOption, { capture: true }); + + modalDialog.show(); + }; + + dom.on('#loggerExport', 'click', toggleOn); +})(); + +/******************************************************************************/ + +// TODO: +// - Give some thoughts to: +// - an option to discard immediately filtered out new entries +// - max entry count _per load_ +// +const loggerSettings = (( ) => { + const settings = { + discard: { + maxAge: 240, // global + maxEntryCount: 2000, // per-tab + maxLoadCount: 20, // per-tab + }, + columns: [ true, true, true, true, true, true, true, true, true ], + linesPerEntry: 4, + }; + + vAPI.localStorage.getItemAsync('loggerSettings').then(value => { + try { + const stored = JSON.parse(value); + if ( typeof stored.discard.maxAge === 'number' ) { + settings.discard.maxAge = stored.discard.maxAge; + } + if ( typeof stored.discard.maxEntryCount === 'number' ) { + settings.discard.maxEntryCount = stored.discard.maxEntryCount; + } + if ( typeof stored.discard.maxLoadCount === 'number' ) { + settings.discard.maxLoadCount = stored.discard.maxLoadCount; + } + if ( typeof stored.linesPerEntry === 'number' ) { + settings.linesPerEntry = stored.linesPerEntry; + } + if ( Array.isArray(stored.columns) ) { + settings.columns = stored.columns; + } + } catch(ex) { + } + }); + + const valueFromInput = function(input, def) { + let value = parseInt(input.value, 10); + if ( isNaN(value) ) { value = def; } + const min = parseInt(dom.attr(input, 'min'), 10); + if ( isNaN(min) === false ) { + value = Math.max(value, min); + } + const max = parseInt(dom.attr(input, 'max'), 10); + if ( isNaN(max) === false ) { + value = Math.min(value, max); + } + return value; + }; + + const toggleOn = function() { + const dialog = modalDialog.create( + '#loggerSettingsDialog', + dialog => { + toggleOff(dialog); + } + ); + + // Number inputs + let inputs = qsa$(dialog, 'input[type="number"]'); + inputs[0].value = settings.discard.maxAge; + inputs[1].value = settings.discard.maxLoadCount; + inputs[2].value = settings.discard.maxEntryCount; + inputs[3].value = settings.linesPerEntry; + dom.on(inputs[3], 'input', ev => { + settings.linesPerEntry = valueFromInput(ev.target, 4); + viewPort.updateLayout(); + }); + + // Column checkboxs + const onColumnChanged = ev => { + const input = ev.target; + const i = parseInt(dom.attr(input, 'data-column'), 10); + settings.columns[i] = input.checked !== true; + viewPort.updateLayout(); + }; + inputs = qsa$(dialog, 'input[type="checkbox"][data-column]'); + for ( const input of inputs ) { + const i = parseInt(dom.attr(input, 'data-column'), 10); + input.checked = settings.columns[i] === false; + dom.on(input, 'change', onColumnChanged); + } + + modalDialog.show(); + }; + + const toggleOff = function(dialog) { + // Number inputs + let inputs = qsa$(dialog, 'input[type="number"]'); + settings.discard.maxAge = valueFromInput(inputs[0], 240); + settings.discard.maxLoadCount = valueFromInput(inputs[1], 25); + settings.discard.maxEntryCount = valueFromInput(inputs[2], 2000); + settings.linesPerEntry = valueFromInput(inputs[3], 4); + + // Column checkboxs + inputs = qsa$(dialog, 'input[type="checkbox"][data-column]'); + for ( const input of inputs ) { + const i = parseInt(dom.attr(input, 'data-column'), 10); + settings.columns[i] = input.checked !== true; + } + + vAPI.localStorage.setItem( + 'loggerSettings', + JSON.stringify(settings) + ); + + viewPort.updateLayout(); + }; + + dom.on('#loggerSettings', 'click', toggleOn); + + return settings; +})(); + +/******************************************************************************/ + +logger.resize = (function() { + let timer; + + const resize = function() { + const vrect = dom.body.getBoundingClientRect(); + for ( const elem of qsa$('.vscrollable') ) { + const crect = elem.getBoundingClientRect(); + const dh = crect.bottom - vrect.bottom; + if ( dh === 0 ) { continue; } + elem.style.height = Math.ceil(crect.height - dh) + 'px'; + } + }; + + const resizeAsync = function() { + if ( timer !== undefined ) { return; } + timer = self.requestAnimationFrame(( ) => { + timer = undefined; + resize(); + }); + }; + + resizeAsync(); + + dom.on(window, 'resize', resizeAsync, { passive: true }); + + return resizeAsync; +})(); + +/******************************************************************************/ + +const grabView = function() { + if ( logger.ownerId === undefined ) { + logger.ownerId = Date.now(); + } + readLogBuffer(); +}; + +const releaseView = function() { + if ( logger.ownerId === undefined ) { return; } + vAPI.messaging.send('loggerUI', { + what: 'releaseView', + ownerId: logger.ownerId, + }); + logger.ownerId = undefined; +}; + +dom.on(window, 'pagehide', releaseView); +dom.on(window, 'pageshow', grabView); +// https://bugzilla.mozilla.org/show_bug.cgi?id=1398625 +dom.on(window, 'beforeunload', releaseView); + +/******************************************************************************/ + +dom.on('#pageSelector', 'change', pageSelectorChanged); +dom.on('#netInspector .vCompactToggler', 'click', toggleVCompactView); +dom.on('#pause', 'click', pauseNetInspector); + +// https://github.com/gorhill/uBlock/issues/507 +// Ensure tab selector is in sync with URL hash +pageSelectorFromURLHash(); +dom.on(window, 'hashchange', pageSelectorFromURLHash); + +// Start to watch the current window geometry 2 seconds after the document +// is loaded, to be sure no spurious geometry changes will be triggered due +// to the window geometry pontentially not settling fast enough. +if ( self.location.search.includes('popup=1') ) { + dom.on(window, 'load', ( ) => { + vAPI.defer.once(2000).then(( ) => { + popupLoggerBox = { + x: self.screenX, + y: self.screenY, + w: self.outerWidth, + h: self.outerHeight, + }; + }); + }, { once: true }); +} + +/******************************************************************************/ diff --git a/src/js/logger.js b/src/js/logger.js new file mode 100644 index 0000000..5d1114f --- /dev/null +++ b/src/js/logger.js @@ -0,0 +1,88 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import { broadcastToAll } from './broadcast.js'; + +/******************************************************************************/ + +let buffer = null; +let lastReadTime = 0; +let writePtr = 0; + +// After 30 seconds without being read, the logger buffer will be considered +// unused, and thus disabled. +const logBufferObsoleteAfter = 30 * 1000; + +const janitorTimer = vAPI.defer.create(( ) => { + if ( buffer === null ) { return; } + if ( lastReadTime >= (Date.now() - logBufferObsoleteAfter) ) { + return janitorTimer.on(logBufferObsoleteAfter); + } + logger.enabled = false; + buffer = null; + writePtr = 0; + logger.ownerId = undefined; + broadcastToAll({ what: 'loggerDisabled' }); +}); + +const boxEntry = function(details) { + if ( details.tstamp === undefined ) { + details.tstamp = Date.now(); + } + return JSON.stringify(details); +}; + +const logger = { + enabled: false, + ownerId: undefined, + writeOne: function(details) { + if ( buffer === null ) { return; } + const box = boxEntry(details); + if ( writePtr === buffer.length ) { + buffer.push(box); + } else { + buffer[writePtr] = box; + } + writePtr += 1; + }, + readAll: function(ownerId) { + this.ownerId = ownerId; + if ( buffer === null ) { + this.enabled = true; + buffer = []; + janitorTimer.on(logBufferObsoleteAfter); + } + const out = buffer.slice(0, writePtr); + writePtr = 0; + lastReadTime = Date.now(); + return out; + }, +}; + +/******************************************************************************/ + +export default logger; + +/******************************************************************************/ diff --git a/src/js/lz4.js b/src/js/lz4.js new file mode 100644 index 0000000..608cdd8 --- /dev/null +++ b/src/js/lz4.js @@ -0,0 +1,190 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2018-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global lz4BlockCodec */ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; + +/******************************************************************************* + + Experimental support for storage compression. + + For background information on the topic, see: + https://github.com/uBlockOrigin/uBlock-issues/issues/141#issuecomment-407737186 + +**/ + +/******************************************************************************/ + +let promisedInstance; +let textEncoder, textDecoder; +let ttlCount = 0; +let ttlDelay = 60000; + +const init = function() { + ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000; + if ( promisedInstance === undefined ) { + let flavor; + if ( µb.hiddenSettings.disableWebAssembly === true ) { + flavor = 'js'; + } + promisedInstance = lz4BlockCodec.createInstance(flavor); + } + return promisedInstance; +}; + +// We can't shrink memory usage of lz4 codec instances, and in the +// current case memory usage can grow to a significant amount given +// that a single contiguous memory buffer is required to accommodate +// both input and output data. Thus a time-to-live implementation +// which will cause the wasm instance to be forgotten after enough +// time elapse without the instance being used. + +const destroy = function() { + //if ( lz4CodecInstance !== undefined ) { + // console.info( + // 'uBO: freeing lz4-block-codec instance (%s KB)', + // lz4CodecInstance.bytesInUse() >>> 10 + // ); + //} + promisedInstance = undefined; + textEncoder = textDecoder = undefined; + ttlCount = 0; +}; + +const ttlTimer = vAPI.defer.create(destroy); + +const ttlManage = function(count) { + ttlTimer.off(); + ttlCount += count; + if ( ttlCount > 0 ) { return; } + ttlTimer.on(ttlDelay); +}; + +const encodeValue = function(lz4CodecInstance, dataIn) { + if ( !lz4CodecInstance ) { return; } + //let t0 = window.performance.now(); + if ( textEncoder === undefined ) { + textEncoder = new TextEncoder(); + } + const inputArray = textEncoder.encode(dataIn); + const inputSize = inputArray.byteLength; + const outputArray = lz4CodecInstance.encodeBlock(inputArray, 8); + if ( outputArray instanceof Uint8Array === false ) { return; } + outputArray[0] = 0x18; + outputArray[1] = 0x4D; + outputArray[2] = 0x22; + outputArray[3] = 0x04; + outputArray[4] = (inputSize >>> 0) & 0xFF; + outputArray[5] = (inputSize >>> 8) & 0xFF; + outputArray[6] = (inputSize >>> 16) & 0xFF; + outputArray[7] = (inputSize >>> 24) & 0xFF; + //console.info( + // 'uBO: [%s] compressed %d KB => %d KB (%s%%) in %s ms', + // inputArray.byteLength >> 10, + // outputArray.byteLength >> 10, + // (outputArray.byteLength / inputArray.byteLength * 100).toFixed(0), + // (window.performance.now() - t0).toFixed(1) + //); + return outputArray; +}; + +const decodeValue = function(lz4CodecInstance, inputArray) { + if ( !lz4CodecInstance ) { return; } + //let t0 = window.performance.now(); + if ( + inputArray[0] !== 0x18 || inputArray[1] !== 0x4D || + inputArray[2] !== 0x22 || inputArray[3] !== 0x04 + ) { + console.error('decodeValue: invalid input array'); + return; + } + const outputSize = + (inputArray[4] << 0) | (inputArray[5] << 8) | + (inputArray[6] << 16) | (inputArray[7] << 24); + const outputArray = lz4CodecInstance.decodeBlock(inputArray, 8, outputSize); + if ( outputArray instanceof Uint8Array === false ) { return; } + if ( textDecoder === undefined ) { + textDecoder = new TextDecoder(); + } + const s = textDecoder.decode(outputArray); + //console.info( + // 'uBO: [%s] decompressed %d KB => %d KB (%s%%) in %s ms', + // inputArray.byteLength >>> 10, + // outputSize >>> 10, + // (inputArray.byteLength / outputSize * 100).toFixed(0), + // (window.performance.now() - t0).toFixed(1) + //); + return s; +}; + +const lz4Codec = { + // Arguments: + // dataIn: must be a string + // Returns: + // A Uint8Array, or the input string as is if compression is not + // possible. + encode: async function(dataIn, serialize = undefined) { + if ( typeof dataIn !== 'string' || dataIn.length < 4096 ) { + return dataIn; + } + ttlManage(1); + const lz4CodecInstance = await init(); + let dataOut = encodeValue(lz4CodecInstance, dataIn); + ttlManage(-1); + if ( serialize instanceof Function ) { + dataOut = await serialize(dataOut); + } + return dataOut || dataIn; + }, + // Arguments: + // dataIn: must be a Uint8Array + // Returns: + // A string, or the input argument as is if decompression is not + // possible. + decode: async function(dataIn, deserialize = undefined) { + if ( deserialize instanceof Function ) { + dataIn = await deserialize(dataIn); + } + if ( dataIn instanceof Uint8Array === false ) { + return dataIn; + } + ttlManage(1); + const lz4CodecInstance = await init(); + const dataOut = decodeValue(lz4CodecInstance, dataIn); + ttlManage(-1); + return dataOut || dataIn; + }, + relinquish: function() { + ttlDelay = 1; + ttlManage(0); + }, +}; + +/******************************************************************************/ + +export default lz4Codec; + +/******************************************************************************/ diff --git a/src/js/messaging.js b/src/js/messaging.js new file mode 100644 index 0000000..52242b3 --- /dev/null +++ b/src/js/messaging.js @@ -0,0 +1,2195 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; +import punycode from '../lib/punycode.js'; + +import { filteringBehaviorChanged } from './broadcast.js'; +import cacheStorage from './cachestorage.js'; +import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import htmlFilteringEngine from './html-filtering.js'; +import logger from './logger.js'; +import lz4Codec from './lz4.js'; +import io from './assets.js'; +import scriptletFilteringEngine from './scriptlet-filtering.js'; +import staticFilteringReverseLookup from './reverselookup.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import µb from './background.js'; +import webRequest from './traffic.js'; +import { denseBase64 } from './base64-custom.js'; +import { dnrRulesetFromRawLists } from './static-dnr-filtering.js'; +import { i18n$ } from './i18n.js'; +import { redirectEngine } from './redirect-engine.js'; +import * as sfp from './static-filtering-parser.js'; + +import { + permanentFirewall, + sessionFirewall, + permanentSwitches, + sessionSwitches, + permanentURLFiltering, + sessionURLFiltering, +} from './filtering-engines.js'; + +import { + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; + +import './benchmarks.js'; + +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/710 +// Listeners have a name and a "privileged" status. +// The nameless default handler is always deemed "privileged". +// Messages from privileged ports must never relayed to listeners +// which are not privileged. + +/******************************************************************************/ +/******************************************************************************/ + +// Default handler +// privileged + +{ +// >>>>> start of local scope + +const clickToLoad = function(request, sender) { + const { tabId, frameId } = sender; + if ( tabId === undefined || frameId === undefined ) { return false; } + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore === null ) { return false; } + pageStore.clickToLoad(frameId, request.frameURL); + return true; +}; + +const getDomainNames = function(targets) { + return targets.map(target => { + if ( typeof target !== 'string' ) { return ''; } + return target.indexOf('/') !== -1 + ? domainFromURI(target) || '' + : domainFromHostname(target) || target; + }); +}; + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'getAssetContent': + // https://github.com/chrisaljoudi/uBlock/issues/417 + io.get(request.url, { + dontCache: true, + needSourceURL: true, + }).then(result => { + result.trustedSource = µb.isTrustedList(result.assetKey); + callback(result); + }); + return; + + case 'listsFromNetFilter': + staticFilteringReverseLookup.fromNetFilter( + request.rawFilter + ).then(response => { + callback(response); + }); + return; + + case 'listsFromCosmeticFilter': + staticFilteringReverseLookup.fromExtendedFilter( + request + ).then(response => { + callback(response); + }); + return; + + case 'reloadAllFilters': + µb.loadFilterLists().then(( ) => { callback(); }); + return; + + case 'scriptlet': + vAPI.tabs.executeScript(request.tabId, { + file: `/js/scriptlets/${request.scriptlet}.js` + }).then(result => { + callback(result); + }); + return; + + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'applyFilterListSelection': + response = µb.applyFilterListSelection(request); + break; + + case 'clickToLoad': + response = clickToLoad(request, sender); + break; + + case 'createUserFilter': + µb.createUserFilters(request); + break; + + case 'getAppData': + response = { + name: browser.runtime.getManifest().name, + version: vAPI.app.version, + canBenchmark: µb.hiddenSettings.benchmarkDatasetURL !== 'unset', + }; + break; + + case 'getDomainNames': + response = getDomainNames(request.targets); + break; + + case 'getTrustedScriptletTokens': + response = redirectEngine.getTrustedScriptletTokens(); + break; + + case 'getWhitelist': + response = { + whitelist: µb.arrayFromWhitelist(µb.netWhitelist), + whitelistDefault: µb.netWhitelistDefault, + reBadHostname: µb.reWhitelistBadHostname.source, + reHostnameExtractor: µb.reWhitelistHostnameExtractor.source + }; + break; + + case 'launchElementPicker': + // Launched from some auxiliary pages, clear context menu coords. + µb.epickerArgs.mouse = false; + µb.elementPickerExec(request.tabId, 0, request.targetURL, request.zap); + break; + + case 'loggerDisabled': + µb.clearInMemoryFilters(); + break; + + case 'gotoURL': + µb.openNewTab(request.details); + break; + + case 'readyToFilter': + response = µb.readyToFilter; + break; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1954 + // In case of document-blocked page, navigate to blocked URL instead + // of forcing a reload. + case 'reloadTab': { + if ( vAPI.isBehindTheSceneTabId(request.tabId) ) { break; } + const { tabId, bypassCache, url, select } = request; + vAPI.tabs.get(tabId).then(tab => { + if ( url && tab && url !== tab.url ) { + vAPI.tabs.replace(tabId, url); + } else { + vAPI.tabs.reload(tabId, bypassCache === true); + } + }); + if ( select && vAPI.tabs.select ) { + vAPI.tabs.select(tabId); + } + break; + } + case 'setWhitelist': + µb.netWhitelist = µb.whitelistFromString(request.whitelist); + µb.saveWhitelist(); + filteringBehaviorChanged(); + break; + + case 'toggleHostnameSwitch': + µb.toggleHostnameSwitch(request); + break; + + case 'uiAccentStylesheet': + µb.uiAccentStylesheet = request.stylesheet; + break; + + case 'uiStyles': + response = { + uiAccentCustom: µb.userSettings.uiAccentCustom, + uiAccentCustom0: µb.userSettings.uiAccentCustom0, + uiAccentStylesheet: µb.uiAccentStylesheet, + uiStyles: µb.hiddenSettings.uiStyles, + uiTheme: µb.userSettings.uiTheme, + }; + break; + + case 'userSettings': + response = µb.changeUserSettings(request.name, request.value); + if ( response instanceof Object ) { + if ( vAPI.net.canUncloakCnames !== true ) { + response.cnameUncloakEnabled = undefined; + } + response.canLeakLocalIPAddresses = + vAPI.browserSettings.canLeakLocalIPAddresses === true; + } + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.setup(onMessage); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// popupPanel +// privileged + +{ +// >>>>> start of local scope + +const createCounts = ( ) => { + return { + blocked: { any: 0, frame: 0, script: 0 }, + allowed: { any: 0, frame: 0, script: 0 }, + }; +}; + +const getHostnameDict = function(hostnameDetailsMap, out) { + const hnDict = Object.create(null); + const cnMap = []; + + const createDictEntry = (domain, hostname, details) => { + const cname = vAPI.net.canonicalNameFromHostname(hostname); + if ( cname !== undefined ) { + cnMap.push([ cname, hostname ]); + } + hnDict[hostname] = { domain, counts: details.counts }; + }; + + for ( const hnDetails of hostnameDetailsMap.values() ) { + const hostname = hnDetails.hostname; + if ( hnDict[hostname] !== undefined ) { continue; } + const domain = domainFromHostname(hostname) || hostname; + const dnDetails = + hostnameDetailsMap.get(domain) || { counts: createCounts() }; + if ( hnDict[domain] === undefined ) { + createDictEntry(domain, domain, dnDetails); + } + if ( hostname === domain ) { continue; } + createDictEntry(domain, hostname, hnDetails); + } + + out.hostnameDict = hnDict; + out.cnameMap = cnMap; +}; + +const firewallRuleTypes = [ + '*', + 'image', + '3p', + 'inline-script', + '1p-script', + '3p-script', + '3p-frame', +]; + +const getFirewallRules = function(src, out) { + const ruleset = out.firewallRules = {}; + const df = sessionFirewall; + + for ( const type of firewallRuleTypes ) { + const r = df.lookupRuleData('*', '*', type); + if ( r === undefined ) { continue; } + ruleset[`/ * ${type}`] = r; + } + if ( typeof src !== 'string' ) { return; } + + for ( const type of firewallRuleTypes ) { + const r = df.lookupRuleData(src, '*', type); + if ( r === undefined ) { continue; } + ruleset[`. * ${type}`] = r; + } + + const { hostnameDict } = out; + for ( const des in hostnameDict ) { + let r = df.lookupRuleData('*', des, '*'); + if ( r !== undefined ) { ruleset[`/ ${des} *`] = r; } + r = df.lookupRuleData(src, des, '*'); + if ( r !== undefined ) { ruleset[`. ${des} *`] = r; } + } +}; + +const popupDataFromTabId = function(tabId, tabTitle) { + const tabContext = µb.tabContextManager.mustLookup(tabId); + const rootHostname = tabContext.rootHostname; + const µbus = µb.userSettings; + const µbhs = µb.hiddenSettings; + const r = { + advancedUserEnabled: µbus.advancedUserEnabled, + appName: vAPI.app.name, + appVersion: vAPI.app.version, + colorBlindFriendly: µbus.colorBlindFriendly, + cosmeticFilteringSwitch: false, + firewallPaneMinimized: µbus.firewallPaneMinimized, + globalAllowedRequestCount: µb.localSettings.allowedRequestCount, + globalBlockedRequestCount: µb.localSettings.blockedRequestCount, + fontSize: µbhs.popupFontSize, + godMode: µbhs.filterAuthorMode, + netFilteringSwitch: false, + rawURL: tabContext.rawURL, + pageURL: tabContext.normalURL, + pageHostname: rootHostname, + pageDomain: tabContext.rootDomain, + popupBlockedCount: 0, + popupPanelSections: µbus.popupPanelSections, + popupPanelDisabledSections: µbhs.popupPanelDisabledSections, + popupPanelLockedSections: µbhs.popupPanelLockedSections, + popupPanelHeightMode: µbhs.popupPanelHeightMode, + tabId, + tabTitle, + tooltipsDisabled: µbus.tooltipsDisabled, + hasUnprocessedRequest: vAPI.net && vAPI.net.hasUnprocessedRequest(tabId), + }; + + if ( µbhs.uiPopupConfig !== 'unset' ) { + r.uiPopupConfig = µbhs.uiPopupConfig; + } + + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore ) { + r.pageCounts = pageStore.counts; + r.netFilteringSwitch = pageStore.getNetFilteringSwitch(); + getHostnameDict(pageStore.getAllHostnameDetails(), r); + r.contentLastModified = pageStore.contentLastModified; + getFirewallRules(rootHostname, r); + r.canElementPicker = isNetworkURI(r.rawURL); + r.noPopups = sessionSwitches.evaluateZ( + 'no-popups', + rootHostname + ); + r.popupBlockedCount = pageStore.popupBlockedCount; + r.noCosmeticFiltering = sessionSwitches.evaluateZ( + 'no-cosmetic-filtering', + rootHostname + ); + r.noLargeMedia = sessionSwitches.evaluateZ( + 'no-large-media', + rootHostname + ); + r.largeMediaCount = pageStore.largeMediaCount; + r.noRemoteFonts = sessionSwitches.evaluateZ( + 'no-remote-fonts', + rootHostname + ); + r.remoteFontCount = pageStore.remoteFontCount; + r.noScripting = sessionSwitches.evaluateZ( + 'no-scripting', + rootHostname + ); + } else { + r.hostnameDict = {}; + getFirewallRules(undefined, r); + } + + r.matrixIsDirty = sessionFirewall.hasSameRules( + permanentFirewall, + rootHostname, + r.hostnameDict + ) === false; + if ( r.matrixIsDirty === false ) { + r.matrixIsDirty = sessionSwitches.hasSameRules( + permanentSwitches, + rootHostname + ) === false; + } + return r; +}; + +const popupDataFromRequest = async function(request) { + if ( request.tabId ) { + return popupDataFromTabId(request.tabId, ''); + } + + // Still no target tab id? Use currently selected tab. + const tab = await vAPI.tabs.getCurrent(); + let tabId = ''; + let tabTitle = ''; + if ( tab instanceof Object ) { + tabId = tab.id; + tabTitle = tab.title || ''; + } + return popupDataFromTabId(tabId, tabTitle); +}; + +const getElementCount = async function(tabId, what) { + const results = await vAPI.tabs.executeScript(tabId, { + allFrames: true, + file: `/js/scriptlets/dom-survey-${what}.js`, + runAt: 'document_end', + }); + + let total = 0; + for ( const count of results ) { + if ( typeof count !== 'number' ) { continue; } + if ( count === -1 ) { return -1; } + total += count; + } + + return total; +}; + +const launchReporter = async function(request) { + const pageStore = µb.pageStoreFromTabId(request.tabId); + if ( pageStore === null ) { return; } + if ( pageStore.hasUnprocessedRequest ) { + request.popupPanel.hasUnprocessedRequest = true; + } + + const entries = await io.getUpdateAges({ + filters: µb.selectedFilterLists.slice() + }); + const shouldUpdateLists = []; + for ( const entry of entries ) { + if ( entry.age < (2 * 60 * 60 * 1000) ) { continue; } + shouldUpdateLists.push(entry.assetKey); + } + + // https://github.com/gorhill/uBlock/commit/6efd8eb#commitcomment-107523558 + // Important: for whatever reason, not using `document_start` causes the + // Promise returned by `tabs.executeScript()` to resolve only when the + // associated tab is closed. + const cosmeticSurveyResults = await vAPI.tabs.executeScript(request.tabId, { + allFrames: true, + file: '/js/scriptlets/cosmetic-report.js', + matchAboutBlank: true, + runAt: 'document_start', + }); + + const filters = cosmeticSurveyResults.reduce((a, v) => { + if ( Array.isArray(v) ) { a.push(...v); } + return a; + }, []); + // Remove duplicate, truncate too long filters. + if ( filters.length !== 0 ) { + request.popupPanel.extended = Array.from( + new Set(filters.map(s => s.length <= 64 ? s : `${s.slice(0, 64)}…`)) + ); + } + + const supportURL = new URL(vAPI.getURL('support.html')); + supportURL.searchParams.set('pageURL', request.pageURL); + supportURL.searchParams.set('popupPanel', JSON.stringify(request.popupPanel)); + if ( shouldUpdateLists.length ) { + supportURL.searchParams.set('shouldUpdateLists', JSON.stringify(shouldUpdateLists)); + } + return supportURL.href; +}; + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'getHiddenElementCount': + getElementCount(request.tabId, 'elements').then(count => { + callback(count); + }); + return; + + case 'getScriptCount': + getElementCount(request.tabId, 'scripts').then(count => { + callback(count); + }); + return; + + case 'getPopupData': + popupDataFromRequest(request).then(popupData => { + callback(popupData); + }); + return; + + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'dismissUnprocessedRequest': + vAPI.net.removeUnprocessedRequest(request.tabId); + µb.updateToolbarIcon(request.tabId, 0b110); + break; + + case 'hasPopupContentChanged': { + const pageStore = µb.pageStoreFromTabId(request.tabId); + const lastModified = pageStore ? pageStore.contentLastModified : 0; + response = lastModified !== request.contentLastModified; + break; + } + + case 'launchReporter': { + launchReporter(request).then(url => { + if ( typeof url !== 'string' ) { return; } + µb.openNewTab({ url, select: true, index: -1 }); + }); + break; + } + + case 'revertFirewallRules': + // TODO: use Set() to message around sets of hostnames + sessionFirewall.copyRules( + permanentFirewall, + request.srcHostname, + Object.assign(Object.create(null), request.desHostnames) + ); + sessionSwitches.copyRules( + permanentSwitches, + request.srcHostname + ); + // https://github.com/gorhill/uBlock/issues/188 + cosmeticFilteringEngine.removeFromSelectorCache( + request.srcHostname, + 'net' + ); + µb.updateToolbarIcon(request.tabId, 0b100); + response = popupDataFromTabId(request.tabId); + break; + + case 'saveFirewallRules': + // TODO: use Set() to message around sets of hostnames + if ( + permanentFirewall.copyRules( + sessionFirewall, + request.srcHostname, + Object.assign(Object.create(null), request.desHostnames) + ) + ) { + µb.savePermanentFirewallRules(); + } + if ( + permanentSwitches.copyRules( + sessionSwitches, + request.srcHostname + ) + ) { + µb.saveHostnameSwitches(); + } + break; + + case 'toggleHostnameSwitch': + µb.toggleHostnameSwitch(request); + response = popupDataFromTabId(request.tabId); + break; + + case 'toggleFirewallRule': + µb.toggleFirewallRule(request); + response = popupDataFromTabId(request.tabId); + break; + + case 'toggleNetFiltering': { + const pageStore = µb.pageStoreFromTabId(request.tabId); + if ( pageStore ) { + pageStore.toggleNetFilteringSwitch( + request.url, + request.scope, + request.state + ); + µb.updateToolbarIcon(request.tabId, 0b111); + } + break; + } + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'popupPanel', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// contentscript +// unprivileged + +{ +// >>>>> start of local scope + +const retrieveContentScriptParameters = async function(sender, request) { + if ( µb.readyToFilter !== true ) { return; } + const { tabId, frameId } = sender; + if ( tabId === undefined || frameId === undefined ) { return; } + + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore === null || pageStore.getNetFilteringSwitch() === false ) { + return; + } + + // A content script may not always be able to successfully look up the + // effective context, hence in such case we try again to look up here + // using cached information about embedded frames. + if ( frameId !== 0 && request.url.startsWith('about:') ) { + request.url = pageStore.getEffectiveFrameURL(sender); + } + + const noSpecificCosmeticFiltering = + pageStore.shouldApplySpecificCosmeticFilters(frameId) === false; + const noGenericCosmeticFiltering = + pageStore.shouldApplyGenericCosmeticFilters(frameId) === false; + + const response = { + collapseBlocked: µb.userSettings.collapseBlocked, + noGenericCosmeticFiltering, + noSpecificCosmeticFiltering, + }; + + request.tabId = tabId; + request.frameId = frameId; + request.hostname = hostnameFromURI(request.url); + request.domain = domainFromHostname(request.hostname); + request.entity = entityFromDomain(request.domain); + + const scf = response.specificCosmeticFilters = + cosmeticFilteringEngine.retrieveSpecificSelectors(request, response); + + // The procedural filterer's code is loaded only when needed and must be + // present before returning response to caller. + if ( + scf.proceduralFilters.length !== 0 || ( + logger.enabled && ( + scf.convertedProceduralFilters.length !== 0 || + scf.exceptedFilters.length !== 0 + ) + ) + ) { + await vAPI.tabs.executeScript(tabId, { + allFrames: false, + file: '/js/contentscript-extra.js', + frameId, + matchAboutBlank: true, + runAt: 'document_start', + }); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/688#issuecomment-748179731 + // For non-network URIs, scriptlet injection is deferred to here. The + // effective URL is available here in `request.url`. + if ( logger.enabled || request.needScriptlets ) { + const scriptletDetails = scriptletFilteringEngine.injectNow(request); + if ( scriptletDetails !== undefined ) { + scriptletFilteringEngine.toLogger(request, scriptletDetails); + if ( request.needScriptlets ) { + response.scriptletDetails = scriptletDetails; + } + } + } + + // https://github.com/NanoMeow/QuickReports/issues/6#issuecomment-414516623 + // Inject as early as possible to make the cosmetic logger code less + // sensitive to the removal of DOM nodes which may match injected + // cosmetic filters. + if ( logger.enabled ) { + if ( + noSpecificCosmeticFiltering === false || + noGenericCosmeticFiltering === false + ) { + vAPI.tabs.executeScript(tabId, { + allFrames: false, + file: '/js/scriptlets/cosmetic-logger.js', + frameId, + matchAboutBlank: true, + runAt: 'document_start', + }); + } + } + + return response; +}; + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'retrieveContentScriptParameters': + return retrieveContentScriptParameters( + sender, + request + ).then(response => { + callback(response); + }); + default: + break; + } + + const pageStore = µb.pageStoreFromTabId(sender.tabId); + + // Sync + let response; + + switch ( request.what ) { + case 'cosmeticFiltersInjected': + cosmeticFilteringEngine.addToSelectorCache(request); + break; + + case 'disableGenericCosmeticFilteringSurveyor': + cosmeticFilteringEngine.disableSurveyor(request); + break; + + case 'getCollapsibleBlockedRequests': + response = { + id: request.id, + hash: request.hash, + netSelectorCacheCountMax: + cosmeticFilteringEngine.netSelectorCacheCountMax, + }; + if ( + µb.userSettings.collapseBlocked && + pageStore && pageStore.getNetFilteringSwitch() + ) { + pageStore.getBlockedResources(request, response); + } + break; + + case 'maybeGoodPopup': + µb.maybeGoodPopup.tabId = sender.tabId; + µb.maybeGoodPopup.url = request.url; + break; + + case 'shouldRenderNoscriptTags': + if ( pageStore === null ) { break; } + const fctxt = µb.filteringContext.fromTabId(sender.tabId); + if ( pageStore.filterScripting(fctxt, undefined) ) { + vAPI.tabs.executeScript(sender.tabId, { + file: '/js/scriptlets/noscript-spoof.js', + frameId: sender.frameId, + runAt: 'document_end', + }); + } + break; + + case 'retrieveGenericCosmeticSelectors': + request.tabId = sender.tabId; + request.frameId = sender.frameId; + response = { + result: cosmeticFilteringEngine.retrieveGenericSelectors(request), + }; + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'contentscript', + listener: onMessage, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// elementPicker +// unprivileged + +{ +// >>>>> start of local scope + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + // The procedural filterer must be present in case the user wants to + // type-in custom filters. + case 'elementPickerArguments': + return vAPI.tabs.executeScript(sender.tabId, { + allFrames: false, + file: '/js/contentscript-extra.js', + frameId: sender.frameId, + matchAboutBlank: true, + runAt: 'document_start', + }).then(( ) => { + callback({ + target: µb.epickerArgs.target, + mouse: µb.epickerArgs.mouse, + zap: µb.epickerArgs.zap, + eprom: µb.epickerArgs.eprom, + pickerURL: vAPI.getURL( + `/web_accessible_resources/epicker-ui.html?secret=${vAPI.warSecret.short()}` + ), + }); + µb.epickerArgs.target = ''; + }); + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'elementPickerEprom': + µb.epickerArgs.eprom = request; + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'elementPicker', + listener: onMessage, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// cloudWidget +// privileged + +{ +// >>>>> start of local scope + +const fromBase64 = function(encoded) { + if ( typeof encoded !== 'string' ) { + return Promise.resolve(encoded); + } + let u8array; + try { + u8array = denseBase64.decode(encoded); + } catch(ex) { + } + return Promise.resolve(u8array !== undefined ? u8array : encoded); +}; + +const toBase64 = function(data) { + const value = data instanceof Uint8Array + ? denseBase64.encode(data) + : data; + return Promise.resolve(value); +}; + +const compress = function(json) { + return lz4Codec.encode(json, toBase64); +}; + +const decompress = function(encoded) { + return lz4Codec.decode(encoded, fromBase64); +}; + +const onMessage = function(request, sender, callback) { + // Cloud storage support is optional. + if ( µb.cloudStorageSupported !== true ) { + callback(); + return; + } + + // Async + switch ( request.what ) { + case 'cloudGetOptions': + vAPI.cloud.getOptions(function(options) { + options.enabled = µb.userSettings.cloudStorageEnabled === true; + callback(options); + }); + return; + + case 'cloudSetOptions': + vAPI.cloud.setOptions(request.options, callback); + return; + + case 'cloudPull': + request.decode = decompress; + return vAPI.cloud.pull(request).then(result => { + callback(result); + }); + + case 'cloudPush': + if ( µb.hiddenSettings.cloudStorageCompression ) { + request.encode = compress; + } + return vAPI.cloud.push(request).then(result => { + callback(result); + }); + + case 'cloudUsed': + return vAPI.cloud.used(request.datakey).then(result => { + callback(result); + }); + + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + // For when cloud storage is disabled. + case 'cloudPull': + // fallthrough + case 'cloudPush': + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'cloudWidget', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// dashboard +// privileged + +{ +// >>>>> start of local scope + +// Settings +const getLocalData = async function() { + const data = Object.assign({}, µb.restoreBackupSettings); + data.storageUsed = await µb.getBytesInUse(); + data.cloudStorageSupported = µb.cloudStorageSupported; + data.privacySettingsSupported = µb.privacySettingsSupported; + return data; +}; + +const backupUserData = async function() { + const userFilters = await µb.loadUserFilters(); + + const userData = { + timeStamp: Date.now(), + version: vAPI.app.version, + userSettings: + µb.getModifiedSettings(µb.userSettings, µb.userSettingsDefault), + selectedFilterLists: µb.selectedFilterLists, + hiddenSettings: + µb.getModifiedSettings(µb.hiddenSettings, µb.hiddenSettingsDefault), + whitelist: µb.arrayFromWhitelist(µb.netWhitelist), + dynamicFilteringString: permanentFirewall.toString(), + urlFilteringString: permanentURLFiltering.toString(), + hostnameSwitchesString: permanentSwitches.toString(), + userFilters: userFilters.content, + }; + + const filename = i18n$('aboutBackupFilename') + .replace('{{datetime}}', µb.dateNowToSensibleString()) + .replace(/ +/g, '_'); + µb.restoreBackupSettings.lastBackupFile = filename; + µb.restoreBackupSettings.lastBackupTime = Date.now(); + vAPI.storage.set(µb.restoreBackupSettings); + + const localData = await getLocalData(); + + return { localData, userData }; +}; + +const restoreUserData = async function(request) { + const userData = request.userData; + + // https://github.com/LiCybora/NanoDefenderFirefox/issues/196 + // Backup data could be from Chromium platform or from an older + // Firefox version. + if ( + vAPI.webextFlavor.soup.has('firefox') && + vAPI.app.intFromVersion(userData.version) <= 1031003011 + ) { + userData.hostnameSwitchesString += '\nno-csp-reports: * true'; + } + + // List of external lists is meant to be a string. + if ( Array.isArray(userData.externalLists) ) { + userData.externalLists = userData.externalLists.join('\n'); + } + + // https://github.com/chrisaljoudi/uBlock/issues/1102 + // Ensure all currently cached assets are flushed from storage AND memory. + io.rmrf(); + + // If we are going to restore all, might as well wipe out clean local + // storages + await Promise.all([ + cacheStorage.clear(), + vAPI.storage.clear(), + ]); + + // Restore block stats + µb.saveLocalSettings(); + + // Restore user data + vAPI.storage.set(userData.userSettings); + + // Restore advanced settings. + let hiddenSettings = userData.hiddenSettings; + if ( hiddenSettings instanceof Object === false ) { + hiddenSettings = µb.hiddenSettingsFromString( + userData.hiddenSettingsString || '' + ); + } + // Discard unknown setting or setting with default value. + for ( const key in hiddenSettings ) { + if ( + µb.hiddenSettingsDefault.hasOwnProperty(key) === false || + hiddenSettings[key] === µb.hiddenSettingsDefault[key] + ) { + delete hiddenSettings[key]; + } + } + + // Whitelist directives can be represented as an array or as a + // (eventually to be deprecated) string. + let whitelist = userData.whitelist; + if ( + Array.isArray(whitelist) === false && + typeof userData.netWhitelist === 'string' && + userData.netWhitelist !== '' + ) { + whitelist = userData.netWhitelist.split('\n'); + } + vAPI.storage.set({ + hiddenSettings, + netWhitelist: whitelist || [], + dynamicFilteringString: userData.dynamicFilteringString || '', + urlFilteringString: userData.urlFilteringString || '', + hostnameSwitchesString: userData.hostnameSwitchesString || '', + lastRestoreFile: request.file || '', + lastRestoreTime: Date.now(), + lastBackupFile: '', + lastBackupTime: 0 + }); + µb.saveUserFilters(userData.userFilters); + if ( Array.isArray(userData.selectedFilterLists) ) { + await µb.saveSelectedFilterLists(userData.selectedFilterLists); + } + + vAPI.app.restart(); +}; + +// Remove all stored data but keep global counts, people can become +// quite attached to numbers +const resetUserData = async function() { + await Promise.all([ + cacheStorage.clear(), + vAPI.storage.clear(), + ]); + + await µb.saveLocalSettings(); + + vAPI.app.restart(); +}; + +// Filter lists +const prepListEntries = function(entries) { + for ( const k in entries ) { + if ( entries.hasOwnProperty(k) === false ) { continue; } + const entry = entries[k]; + if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { + entry.supportName = hostnameFromURI(entry.supportURL); + } else if ( typeof entry.homeURL === 'string' && entry.homeURL !== '' ) { + const hn = hostnameFromURI(entry.homeURL); + entry.supportURL = `http://${hn}/`; + entry.supportName = domainFromHostname(hn); + } + } +}; + +const getLists = async function(callback) { + const r = { + autoUpdate: µb.userSettings.autoUpdate, + available: null, + cache: null, + cosmeticFilterCount: cosmeticFilteringEngine.getFilterCount(), + current: µb.availableFilterLists, + ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, + isUpdating: io.isUpdating(), + netFilterCount: staticNetFilteringEngine.getFilterCount(), + parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters, + suspendUntilListsAreLoaded: µb.userSettings.suspendUntilListsAreLoaded, + userFiltersPath: µb.userFiltersPath + }; + const [ lists, metadata ] = await Promise.all([ + µb.getAvailableLists(), + io.metadata(), + ]); + r.available = lists; + prepListEntries(r.available); + r.cache = metadata; + prepListEntries(r.cache); + callback(r); +}; + +// My filters + +// TODO: also return origin of embedded frames? +const getOriginHints = function() { + const out = new Set(); + for ( const tabId of µb.pageStores.keys() ) { + if ( tabId === -1 ) { continue; } + const tabContext = µb.tabContextManager.lookup(tabId); + if ( tabContext === null ) { continue; } + let { rootDomain, rootHostname } = tabContext; + if ( rootDomain.endsWith('-scheme') ) { continue; } + const isPunycode = rootHostname.includes('xn--'); + out.add(isPunycode ? punycode.toUnicode(rootDomain) : rootDomain); + if ( rootHostname === rootDomain ) { continue; } + out.add(isPunycode ? punycode.toUnicode(rootHostname) : rootHostname); + } + return Array.from(out); +}; + +// My rules +const getRules = function() { + return { + permanentRules: + permanentFirewall.toArray().concat( + permanentSwitches.toArray(), + permanentURLFiltering.toArray() + ), + sessionRules: + sessionFirewall.toArray().concat( + sessionSwitches.toArray(), + sessionURLFiltering.toArray() + ), + pslSelfie: publicSuffixList.toSelfie(), + }; +}; + +const modifyRuleset = function(details) { + let swRuleset, hnRuleset, urlRuleset; + if ( details.permanent ) { + swRuleset = permanentSwitches; + hnRuleset = permanentFirewall; + urlRuleset = permanentURLFiltering; + } else { + swRuleset = sessionSwitches; + hnRuleset = sessionFirewall; + urlRuleset = sessionURLFiltering; + } + let toRemove = new Set(details.toRemove.trim().split(/\s*[\n\r]+\s*/)); + for ( let rule of toRemove ) { + if ( rule === '' ) { continue; } + let parts = rule.split(/\s+/); + if ( hnRuleset.removeFromRuleParts(parts) === false ) { + if ( swRuleset.removeFromRuleParts(parts) === false ) { + urlRuleset.removeFromRuleParts(parts); + } + } + } + let toAdd = new Set(details.toAdd.trim().split(/\s*[\n\r]+\s*/)); + for ( let rule of toAdd ) { + if ( rule === '' ) { continue; } + let parts = rule.split(/\s+/); + if ( hnRuleset.addFromRuleParts(parts) === false ) { + if ( swRuleset.addFromRuleParts(parts) === false ) { + urlRuleset.addFromRuleParts(parts); + } + } + } + if ( details.permanent ) { + if ( swRuleset.changed ) { + µb.saveHostnameSwitches(); + swRuleset.changed = false; + } + if ( hnRuleset.changed ) { + µb.savePermanentFirewallRules(); + hnRuleset.changed = false; + } + if ( urlRuleset.changed ) { + µb.savePermanentURLFilteringRules(); + urlRuleset.changed = false; + } + } +}; + +// Support +const getSupportData = async function() { + const diffArrays = function(modified, original) { + const modifiedSet = new Set(modified); + const originalSet = new Set(original); + let added = []; + let removed = []; + for ( const item of modifiedSet ) { + if ( originalSet.has(item) ) { continue; } + added.push(item); + } + for ( const item of originalSet ) { + if ( modifiedSet.has(item) ) { continue; } + removed.push(item); + } + if ( added.length === 0 ) { + added = undefined; + } + if ( removed.length === 0 ) { + removed = undefined; + } + if ( added !== undefined || removed !== undefined ) { + return { added, removed }; + } + }; + + const modifiedUserSettings = µb.getModifiedSettings( + µb.userSettings, + µb.userSettingsDefault + ); + + const modifiedHiddenSettings = µb.getModifiedSettings( + µb.hiddenSettings, + µb.hiddenSettingsDefault + ); + + let filterset = []; + const userFilters = await µb.loadUserFilters(); + for ( const line of userFilters.content.split(/\s*\n+\s*/) ) { + if ( /^($|![^#])/.test(line) ) { continue; } + filterset.push(line); + } + + const now = Date.now(); + + const formatDelayFromNow = list => { + const time = list.writeTime; + if ( typeof time !== 'number' || time === 0 ) { return 'never'; } + if ( (time || 0) === 0 ) { return '?'; } + const delayInSec = (now - time) / 1000; + const days = (delayInSec / 86400) | 0; + const hours = (delayInSec % 86400) / 3600 | 0; + const minutes = (delayInSec % 3600) / 60 | 0; + const parts = []; + if ( days > 0 ) { parts.push(`${days}d`); } + if ( hours > 0 ) { parts.push(`${hours}h`); } + if ( minutes > 0 ) { parts.push(`${minutes}m`); } + if ( parts.length === 0 ) { parts.push('now'); } + const out = parts.join('.'); + if ( list.diffUpdated ) { return `${out} Δ`; } + return out; + }; + + const lists = µb.availableFilterLists; + let defaultListset = {}; + let addedListset = {}; + let removedListset = {}; + for ( const listKey in lists ) { + if ( lists.hasOwnProperty(listKey) === false ) { continue; } + const list = lists[listKey]; + if ( list.content !== 'filters' ) { continue; } + const used = µb.selectedFilterLists.includes(listKey); + const listDetails = []; + if ( used ) { + if ( typeof list.entryCount === 'number' ) { + listDetails.push(`${list.entryCount}-${list.entryCount-list.entryUsedCount}`); + } + listDetails.push(formatDelayFromNow(list)); + } + if ( list.isDefault || listKey === µb.userFiltersPath ) { + if ( used ) { + defaultListset[listKey] = listDetails.join(', '); + } else { + removedListset[listKey] = null; + } + } else if ( used ) { + addedListset[listKey] = listDetails.join(', '); + } + } + if ( Object.keys(defaultListset).length === 0 ) { + defaultListset = undefined; + } + if ( Object.keys(addedListset).length === 0 ) { + addedListset = undefined; + } else { + const added = Object.keys(addedListset); + const truncated = added.slice(12); + for ( const key of truncated ) { + delete addedListset[key]; + } + if ( truncated.length !== 0 ) { + addedListset[`[${truncated.length} lists not shown]`] = '[too many]'; + } + } + if ( Object.keys(removedListset).length === 0 ) { + removedListset = undefined; + } + + let browserFamily = (( ) => { + if ( vAPI.webextFlavor.soup.has('firefox') ) { return 'Firefox'; } + if ( vAPI.webextFlavor.soup.has('chromium') ) { return 'Chromium'; } + return 'Unknown'; + })(); + if ( vAPI.webextFlavor.soup.has('mobile') ) { + browserFamily += ' Mobile'; + } + + return { + [`${vAPI.app.name}`]: `${vAPI.app.version}`, + [`${browserFamily}`]: `${vAPI.webextFlavor.major}`, + 'filterset (summary)': { + network: staticNetFilteringEngine.getFilterCount(), + cosmetic: cosmeticFilteringEngine.getFilterCount(), + scriptlet: scriptletFilteringEngine.getFilterCount(), + html: htmlFilteringEngine.getFilterCount(), + }, + 'listset (total-discarded, last-updated)': { + removed: removedListset, + added: addedListset, + default: defaultListset, + }, + 'filterset (user)': filterset, + trustedset: diffArrays( + µb.arrayFromWhitelist(µb.netWhitelist), + µb.netWhitelistDefault + ), + switchRuleset: diffArrays( + sessionSwitches.toArray(), + µb.hostnameSwitchesDefault + ), + hostRuleset: diffArrays( + sessionFirewall.toArray(), + µb.dynamicFilteringDefault + ), + urlRuleset: diffArrays( + sessionURLFiltering.toArray(), + [] + ), + 'userSettings': modifiedUserSettings, + 'hiddenSettings': modifiedHiddenSettings, + supportStats: µb.supportStats, + }; +}; + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'backupUserData': + return backupUserData().then(data => { + callback(data); + }); + + case 'getLists': + return µb.isReadyPromise.then(( ) => { + getLists(callback); + }); + + case 'getLocalData': + return getLocalData().then(localData => { + callback(localData); + }); + + case 'getSupportData': { + getSupportData().then(response => { + callback(response); + }); + return; + } + + case 'readUserFilters': + return µb.loadUserFilters().then(result => { + result.trustedSource = µb.isTrustedList(µb.userFiltersPath); + callback(result); + }); + + case 'writeUserFilters': + return µb.saveUserFilters(request.content).then(result => { + callback(result); + }); + + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'dashboardConfig': + response = { + noDashboard: µb.noDashboard, + }; + break; + + case 'getAutoCompleteDetails': + response = {}; + if ( (request.hintUpdateToken || 0) === 0 ) { + response.redirectResources = redirectEngine.getResourceDetails(); + response.preparseDirectiveEnv = vAPI.webextFlavor.env.slice(); + response.preparseDirectiveHints = sfp.utils.preparser.getHints(); + } + if ( request.hintUpdateToken !== µb.pageStoresToken ) { + response.originHints = getOriginHints(); + response.hintUpdateToken = µb.pageStoresToken; + } + break; + + case 'getRules': + response = getRules(); + break; + + case 'modifyRuleset': + // https://github.com/chrisaljoudi/uBlock/issues/772 + cosmeticFilteringEngine.removeFromSelectorCache('*'); + modifyRuleset(request); + response = getRules(); + break; + + case 'supportUpdateNow': { + const { assetKeys } = request; + if ( assetKeys.length === 0 ) { return; } + for ( const assetKey of assetKeys ) { + io.purge(assetKey); + } + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100 }); + break; + } + + case 'listsUpdateNow': { + const { assetKeys, preferOrigin = false } = request; + if ( assetKeys.length === 0 ) { return; } + for ( const assetKey of assetKeys ) { + io.purge(assetKey); + } + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: preferOrigin !== true }); + break; + } + + case 'readHiddenSettings': + response = { + 'default': µb.hiddenSettingsDefault, + 'admin': µb.hiddenSettingsAdmin, + 'current': µb.hiddenSettings, + }; + break; + + case 'restoreUserData': + restoreUserData(request); + break; + + case 'resetUserData': + resetUserData(); + break; + + case 'updateNow': + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: true }); + break; + + case 'writeHiddenSettings': + µb.changeHiddenSettings(µb.hiddenSettingsFromString(request.content)); + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'dashboard', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// loggerUI +// privileged + +{ +// >>>>> start of local scope + +const extensionOriginURL = vAPI.getURL(''); +const documentBlockedURL = vAPI.getURL('document-blocked.html'); + +const getLoggerData = async function(details, activeTabId, callback) { + const response = { + activeTabId, + colorBlind: µb.userSettings.colorBlindFriendly, + entries: logger.readAll(details.ownerId), + tabIdsToken: µb.pageStoresToken, + tooltips: µb.userSettings.tooltipsDisabled === false + }; + if ( µb.pageStoresToken !== details.tabIdsToken ) { + response.tabIds = []; + for ( const [ tabId, pageStore ] of µb.pageStores ) { + const { rawURL, title } = pageStore; + if ( rawURL.startsWith(extensionOriginURL) ) { + if ( rawURL.startsWith(documentBlockedURL) === false ) { continue; } + } + response.tabIds.push([ tabId, title ]); + } + } + if ( activeTabId ) { + const pageStore = µb.pageStoreFromTabId(activeTabId); + const rawURL = pageStore && pageStore.rawURL; + if ( + rawURL === null || + rawURL.startsWith(extensionOriginURL) && + rawURL.startsWith(documentBlockedURL) === false + ) { + response.activeTabId = undefined; + } + } + if ( details.popupLoggerBoxChanged && vAPI.windows instanceof Object ) { + const tabs = await vAPI.tabs.query({ + url: vAPI.getURL('/logger-ui.html?popup=1') + }); + if ( tabs.length !== 0 ) { + const win = await vAPI.windows.get(tabs[0].windowId); + if ( win === null ) { return; } + vAPI.localStorage.setItem('popupLoggerBox', JSON.stringify({ + left: win.left, + top: win.top, + width: win.width, + height: win.height, + })); + } + } + callback(response); +}; + +const getURLFilteringData = function(details) { + const colors = {}; + const response = { + dirty: false, + colors: colors + }; + const suf = sessionURLFiltering; + const puf = permanentURLFiltering; + const urls = details.urls; + const context = details.context; + const type = details.type; + for ( const url of urls ) { + const colorEntry = colors[url] = { r: 0, own: false }; + if ( suf.evaluateZ(context, url, type).r !== 0 ) { + colorEntry.r = suf.r; + colorEntry.own = suf.r !== 0 && + suf.context === context && + suf.url === url && + suf.type === type; + } + if ( response.dirty ) { continue; } + puf.evaluateZ(context, url, type); + const pown = ( + puf.r !== 0 && + puf.context === context && + puf.url === url && + puf.type === type + ); + response.dirty = colorEntry.own !== pown || colorEntry.r !== puf.r; + } + return response; +}; + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'readAll': + if ( logger.ownerId !== undefined && logger.ownerId !== request.ownerId ) { + return callback({ unavailable: true }); + } + vAPI.tabs.getCurrent().then(tab => { + getLoggerData(request, tab && tab.id, callback); + }); + return; + + case 'toggleInMemoryFilter': { + const promise = µb.hasInMemoryFilter(request.filter) + ? µb.removeInMemoryFilter(request.filter) + : µb.addInMemoryFilter(request.filter); + promise.then(status => { callback(status); }); + return; + } + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'hasInMemoryFilter': + response = µb.hasInMemoryFilter(request.filter); + break; + + case 'releaseView': + if ( request.ownerId !== logger.ownerId ) { break; } + logger.ownerId = undefined; + µb.clearInMemoryFilters(); + break; + + case 'saveURLFilteringRules': + response = permanentURLFiltering.copyRules( + sessionURLFiltering, + request.context, + request.urls, + request.type + ); + if ( response ) { + µb.savePermanentURLFilteringRules(); + } + break; + + case 'setURLFilteringRule': + µb.toggleURLFilteringRule(request); + break; + + case 'getURLFilteringData': + response = getURLFilteringData(request); + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'loggerUI', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// domInspectorContent +// unprivileged + +{ +// >>>>> start of local scope + +const onMessage = (request, sender, callback) => { + // Async + switch ( request.what ) { + default: + break; + } + // Sync + let response; + switch ( request.what ) { + case 'getInspectorArgs': + const bc = new globalThis.BroadcastChannel('contentInspectorChannel'); + bc.postMessage({ + what: 'contentInspectorChannel', + tabId: sender.tabId || 0, + frameId: sender.frameId || 0, + }); + response = { + inspectorURL: vAPI.getURL( + `/web_accessible_resources/dom-inspector.html?secret=${vAPI.warSecret.short()}` + ), + }; + break; + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'domInspectorContent', + listener: onMessage, + privileged: false, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// documentBlocked +// privileged + +{ +// >>>>> start of local scope + +const onMessage = function(request, sender, callback) { + const tabId = sender.tabId || 0; + + // Async + switch ( request.what ) { + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'closeThisTab': + vAPI.tabs.remove(tabId); + break; + + case 'temporarilyWhitelistDocument': + webRequest.strictBlockBypass(request.hostname); + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'documentBlocked', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// devTools +// privileged + +{ +// >>>>> start of local scope + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'purgeAllCaches': + µb.getBytesInUse().then(bytesInUseBefore => + io.remove(/./).then(( ) => + µb.getBytesInUse().then(bytesInUseAfter => { + callback([ + `Storage used before: ${µb.formatCount(bytesInUseBefore)}B`, + `Storage used after: ${µb.formatCount(bytesInUseAfter)}B`, + ].join('\n')); + }) + ) + ); + return; + + case 'snfeBenchmark': + µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => { + callback(result); + }); + return; + + case 'snfeToDNR': { + const listPromises = []; + const listNames = []; + for ( const assetKey of µb.selectedFilterLists ) { + listPromises.push( + io.get(assetKey, { dontCache: true }).then(details => { + listNames.push(assetKey); + return { name: assetKey, text: details.content }; + }) + ); + } + const options = { + extensionPaths: redirectEngine.getResourceDetails().filter(e => + typeof e[1].extensionPath === 'string' && e[1].extensionPath !== '' + ).map(e => + [ e[0], e[1].extensionPath ] + ), + env: vAPI.webextFlavor.env, + }; + const t0 = Date.now(); + dnrRulesetFromRawLists(listPromises, options).then(result => { + const { network } = result; + const replacer = (k, v) => { + if ( k.startsWith('__') ) { return; } + if ( Array.isArray(v) ) { + return v.sort(); + } + if ( v instanceof Object ) { + const sorted = {}; + for ( const kk of Object.keys(v).sort() ) { + sorted[kk] = v[kk]; + } + return sorted; + } + return v; + }; + const isUnsupported = rule => + rule._error !== undefined; + const isRegex = rule => + rule.condition !== undefined && + rule.condition.regexFilter !== undefined; + const isRedirect = rule => + rule.action !== undefined && + rule.action.type === 'redirect' && + rule.action.redirect.extensionPath !== undefined; + const isCsp = rule => + rule.action !== undefined && + rule.action.type === 'modifyHeaders'; + const isRemoveparam = rule => + rule.action !== undefined && + rule.action.type === 'redirect' && + rule.action.redirect.transform !== undefined; + const runtime = Date.now() - t0; + const { ruleset } = network; + const good = ruleset.filter(rule => + isUnsupported(rule) === false && + isRegex(rule) === false && + isRedirect(rule) === false && + isCsp(rule) === false && + isRemoveparam(rule) === false + ); + const unsupported = ruleset.filter(rule => + isUnsupported(rule) + ); + const regexes = ruleset.filter(rule => + isUnsupported(rule) === false && + isRegex(rule) && + isRedirect(rule) === false && + isCsp(rule) === false && + isRemoveparam(rule) === false + ); + const redirects = ruleset.filter(rule => + isUnsupported(rule) === false && + isRedirect(rule) + ); + const headers = ruleset.filter(rule => + isUnsupported(rule) === false && + isCsp(rule) + ); + const removeparams = ruleset.filter(rule => + isUnsupported(rule) === false && + isRemoveparam(rule) + ); + const out = [ + `dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`, + `Run time: ${runtime} ms`, + `Filters count: ${network.filterCount}`, + `Accepted filter count: ${network.acceptedFilterCount}`, + `Rejected filter count: ${network.rejectedFilterCount}`, + `Un-DNR-able filter count: ${unsupported.length}`, + `Resulting DNR rule count: ${ruleset.length}`, + ]; + out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`); + out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`); + out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`); + out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`); + out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`); + out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`); + out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`); + if ( result.specificCosmetic ) { + out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`); + for ( const details of result.specificCosmetic ) { + out.push(` ${JSON.stringify(details)}`); + } + } else { + out.push(' Cosmetic filters: 0'); + } + callback(out.join('\n')); + }); + return; + } + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'snfeDump': + response = staticNetFilteringEngine.dump(); + break; + + case 'cfeDump': + response = cosmeticFilteringEngine.dump(); + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'devTools', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + +// Channel: +// scriptlets +// unprivileged + +{ +// >>>>> start of local scope + +const logCosmeticFilters = function(tabId, details) { + if ( logger.enabled === false ) { return; } + + const filter = { source: 'cosmetic', raw: '' }; + const fctxt = µb.filteringContext.duplicate(); + fctxt.fromTabId(tabId) + .setRealm('cosmetic') + .setType('dom') + .setURL(details.frameURL) + .setDocOriginFromURL(details.frameURL) + .setFilter(filter); + for ( const selector of details.matchedSelectors.sort() ) { + filter.raw = selector; + fctxt.toLogger(); + } +}; + +const logCSPViolations = function(pageStore, request) { + if ( logger.enabled === false || pageStore === null ) { + return false; + } + if ( request.violations.length === 0 ) { + return true; + } + + const fctxt = µb.filteringContext.duplicate(); + fctxt.fromTabId(pageStore.tabId) + .setRealm('network') + .setDocOriginFromURL(request.docURL) + .setURL(request.docURL); + + let cspData = pageStore.extraData.get('cspData'); + if ( cspData === undefined ) { + cspData = new Map(); + + const staticDirectives = + staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'csp'); + if ( staticDirectives !== undefined ) { + for ( const directive of staticDirectives ) { + if ( directive.result !== 1 ) { continue; } + cspData.set(directive.value, directive.logData()); + } + } + + fctxt.type = 'inline-script'; + fctxt.filter = undefined; + if ( pageStore.filterRequest(fctxt) === 1 ) { + cspData.set(µb.cspNoInlineScript, fctxt.filter); + } + + fctxt.type = 'script'; + fctxt.filter = undefined; + if ( pageStore.filterScripting(fctxt, true) === 1 ) { + cspData.set(µb.cspNoScripting, fctxt.filter); + } + + fctxt.type = 'inline-font'; + fctxt.filter = undefined; + if ( pageStore.filterRequest(fctxt) === 1 ) { + cspData.set(µb.cspNoInlineFont, fctxt.filter); + } + + if ( cspData.size === 0 ) { return false; } + + pageStore.extraData.set('cspData', cspData); + } + + const typeMap = logCSPViolations.policyDirectiveToTypeMap; + for ( const json of request.violations ) { + const violation = JSON.parse(json); + let type = typeMap.get(violation.directive); + if ( type === undefined ) { continue; } + const logData = cspData.get(violation.policy); + if ( logData === undefined ) { continue; } + if ( /^[\w.+-]+:\/\//.test(violation.url) === false ) { + violation.url = request.docURL; + if ( type === 'script' ) { type = 'inline-script'; } + else if ( type === 'font' ) { type = 'inline-font'; } + } + // The resource was blocked as a result of applying a CSP directive + // elsewhere rather than to the resource itself. + logData.modifier = undefined; + fctxt.setURL(violation.url) + .setType(type) + .setFilter(logData) + .toLogger(); + } + + return true; +}; + +logCSPViolations.policyDirectiveToTypeMap = new Map([ + [ 'img-src', 'image' ], + [ 'connect-src', 'xmlhttprequest' ], + [ 'font-src', 'font' ], + [ 'frame-src', 'sub_frame' ], + [ 'media-src', 'media' ], + [ 'object-src', 'object' ], + [ 'script-src', 'script' ], + [ 'script-src-attr', 'script' ], + [ 'script-src-elem', 'script' ], + [ 'style-src', 'stylesheet' ], + [ 'style-src-attr', 'stylesheet' ], + [ 'style-src-elem', 'stylesheet' ], +]); + +const onMessage = function(request, sender, callback) { + const tabId = sender.tabId || 0; + const pageStore = µb.pageStoreFromTabId(tabId); + + // Async + switch ( request.what ) { + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'inlinescriptFound': + if ( logger.enabled && pageStore !== null ) { + const fctxt = µb.filteringContext.duplicate(); + fctxt.fromTabId(tabId) + .setType('inline-script') + .setURL(request.docURL) + .setDocOriginFromURL(request.docURL); + if ( pageStore.filterRequest(fctxt) === 0 ) { + fctxt.setRealm('network').toLogger(); + } + } + break; + + case 'logCosmeticFilteringData': + logCosmeticFilters(tabId, request); + break; + + case 'securityPolicyViolation': + response = logCSPViolations(pageStore, request); + break; + + case 'temporarilyAllowLargeMediaElement': + if ( pageStore !== null ) { + pageStore.allowLargeMediaElementsUntil = Date.now() + 5000; + } + break; + + case 'subscribeTo': + // https://github.com/uBlockOrigin/uBlock-issues/issues/1797 + if ( /^(file|https?):\/\//.test(request.location) === false ) { break; } + const url = encodeURIComponent(request.location); + const title = encodeURIComponent(request.title); + const hash = µb.selectedFilterLists.indexOf(request.location) !== -1 + ? '#subscribed' + : ''; + vAPI.tabs.open({ + url: `/asset-viewer.html?url=${url}&title=${title}&subscribe=1${hash}`, + select: true, + }); + break; + + case 'updateLists': + const listkeys = request.listkeys.split(',').filter(s => s !== ''); + if ( listkeys.length === 0 ) { return; } + if ( listkeys.includes('all') ) { + io.purge(/./, 'public_suffix_list.dat'); + } else { + for ( const listkey of listkeys ) { + io.purge(listkey); + } + } + µb.openNewTab({ + url: 'dashboard.html#3p-filters.html', + select: true, + }); + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: request.auto }); + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'scriptlets', + listener: onMessage, +}); + +// <<<<< end of local scope +} + + +/******************************************************************************/ +/******************************************************************************/ diff --git a/src/js/mrucache.js b/src/js/mrucache.js new file mode 100644 index 0000000..9a16047 --- /dev/null +++ b/src/js/mrucache.js @@ -0,0 +1,58 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +export class MRUCache { + constructor(maxSize) { + this.maxSize = maxSize; + this.array = []; + this.map = new Map(); + this.resetTime = Date.now(); + } + add(key, value) { + const found = this.map.has(key); + this.map.set(key, value); + if ( found ) { return; } + if ( this.array.length === this.maxSize ) { + this.map.delete(this.array.pop()); + } + this.array.unshift(key); + } + remove(key) { + if ( this.map.delete(key) === false ) { return; } + this.array.splice(this.array.indexOf(key), 1); + } + lookup(key) { + const value = this.map.get(key); + if ( value === undefined ) { return; } + if ( this.array[0] === key ) { return value; } + const i = this.array.indexOf(key); + this.array.copyWithin(1, 0, i); + this.array[0] = key; + return value; + } + reset() { + this.array = []; + this.map.clear(); + this.resetTime = Date.now(); + } +} diff --git a/src/js/pagestore.js b/src/js/pagestore.js new file mode 100644 index 0000000..907e747 --- /dev/null +++ b/src/js/pagestore.js @@ -0,0 +1,1140 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import contextMenu from './contextmenu.js'; +import logger from './logger.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import µb from './background.js'; +import webext from './webext.js'; +import { orphanizeString } from './text-utils.js'; +import { redirectEngine } from './redirect-engine.js'; + +import { + sessionFirewall, + sessionSwitches, + sessionURLFiltering, +} from './filtering-engines.js'; + +import { + domainFromHostname, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; + +/******************************************************************************* + +A PageRequestStore object is used to store net requests in two ways: + +To record distinct net requests +To create a log of net requests + +**/ + +/******************************************************************************/ + +const NetFilteringResultCache = class { + constructor() { + this.pruneTimer = vAPI.defer.create(( ) => { + this.prune(); + }); + this.init(); + } + + init() { + this.blocked = new Map(); + this.results = new Map(); + this.hash = 0; + return this; + } + + // https://github.com/gorhill/uBlock/issues/3619 + // Don't collapse redirected resources + rememberResult(fctxt, result) { + if ( fctxt.tabId <= 0 ) { return; } + if ( this.results.size === 0 ) { + this.pruneAsync(); + } + const key = `${fctxt.getDocHostname()} ${fctxt.type} ${fctxt.url}`; + this.results.set(key, { + result, + redirectURL: fctxt.redirectURL, + logData: fctxt.filter, + tstamp: Date.now() + }); + if ( result !== 1 || fctxt.redirectURL !== undefined ) { return; } + const now = Date.now(); + this.blocked.set(key, now); + this.hash = now; + } + + rememberBlock(fctxt) { + if ( fctxt.tabId <= 0 ) { return; } + if ( this.blocked.size === 0 ) { + this.pruneAsync(); + } + if ( fctxt.redirectURL !== undefined ) { return; } + const now = Date.now(); + this.blocked.set( + `${fctxt.getDocHostname()} ${fctxt.type} ${fctxt.url}`, + now + ); + this.hash = now; + } + + forgetResult(docHostname, type, url) { + const key = `${docHostname} ${type} ${url}`; + this.results.delete(key); + this.blocked.delete(key); + } + + empty() { + this.blocked.clear(); + this.results.clear(); + this.hash = 0; + this.pruneTimer.off(); + } + + prune() { + const obsolete = Date.now() - this.shelfLife; + for ( const entry of this.blocked ) { + if ( entry[1] <= obsolete ) { + this.results.delete(entry[0]); + this.blocked.delete(entry[0]); + } + } + for ( const entry of this.results ) { + if ( entry[1].tstamp <= obsolete ) { + this.results.delete(entry[0]); + } + } + if ( this.blocked.size !== 0 || this.results.size !== 0 ) { + this.pruneAsync(); + } + } + + pruneAsync() { + this.pruneTimer.on(this.shelfLife); + } + + lookupResult(fctxt) { + const entry = this.results.get( + fctxt.getDocHostname() + ' ' + + fctxt.type + ' ' + + fctxt.url + ); + if ( entry === undefined ) { return; } + // We need to use a new WAR secret if one is present since WAR secrets + // can only be used once. + if ( + entry.redirectURL !== undefined && + entry.redirectURL.startsWith(this.extensionOriginURL) + ) { + const redirectURL = new URL(entry.redirectURL); + redirectURL.searchParams.set('secret', vAPI.warSecret.short()); + entry.redirectURL = redirectURL.href; + } + return entry; + } + + lookupAllBlocked(hostname) { + const result = []; + for ( const entry of this.blocked ) { + const pos = entry[0].indexOf(' '); + if ( entry[0].slice(0, pos) === hostname ) { + result[result.length] = entry[0].slice(pos + 1); + } + } + return result; + } + + static factory() { + return new NetFilteringResultCache(); + } +}; + +NetFilteringResultCache.prototype.shelfLife = 15000; +NetFilteringResultCache.prototype.extensionOriginURL = vAPI.getURL('/'); + +/******************************************************************************/ + +// Frame stores are used solely to associate a URL with a frame id. + +const FrameStore = class { + constructor(frameURL, parentId) { + this.init(frameURL, parentId); + } + + init(frameURL, parentId) { + this.t0 = Date.now(); + this.parentId = parentId; + this.exceptCname = undefined; + this.clickToLoad = false; + this.rawURL = frameURL; + if ( frameURL !== undefined ) { + this.hostname = hostnameFromURI(frameURL); + this.domain = domainFromHostname(this.hostname) || this.hostname; + } + // Evaluated on-demand + // - 0b01: specific cosmetic filtering + // - 0b10: generic cosmetic filtering + this._cosmeticFilteringBits = undefined; + return this; + } + + dispose() { + this.rawURL = this.hostname = this.domain = ''; + if ( FrameStore.junkyard.length < FrameStore.junkyardMax ) { + FrameStore.junkyard.push(this); + } + return null; + } + + updateURL(url) { + if ( typeof url !== 'string' ) { return; } + this.rawURL = url; + this.hostname = hostnameFromURI(url); + this.domain = domainFromHostname(this.hostname) || this.hostname; + this._cosmeticFilteringBits = undefined; + } + + getCosmeticFilteringBits(tabId) { + if ( this._cosmeticFilteringBits !== undefined ) { + return this._cosmeticFilteringBits; + } + this._cosmeticFilteringBits = 0b11; + { + const result = staticNetFilteringEngine.matchRequestReverse( + 'specifichide', + this.rawURL + ); + if ( result !== 0 && logger.enabled ) { + µb.filteringContext + .duplicate() + .fromTabId(tabId) + .setURL(this.rawURL) + .setDocOriginFromURL(this.rawURL) + .setRealm('network') + .setType('specifichide') + .setFilter(staticNetFilteringEngine.toLogData()) + .toLogger(); + } + if ( result === 2 ) { + this._cosmeticFilteringBits &= ~0b01; + } + } + { + const result = staticNetFilteringEngine.matchRequestReverse( + 'generichide', + this.rawURL + ); + if ( result !== 0 && logger.enabled ) { + µb.filteringContext + .duplicate() + .fromTabId(tabId) + .setURL(this.rawURL) + .setDocOriginFromURL(this.rawURL) + .setRealm('network') + .setType('generichide') + .setFilter(staticNetFilteringEngine.toLogData()) + .toLogger(); + } + if ( result === 2 ) { + this._cosmeticFilteringBits &= ~0b10; + } + } + return this._cosmeticFilteringBits; + } + + shouldApplySpecificCosmeticFilters(tabId) { + return (this.getCosmeticFilteringBits(tabId) & 0b01) !== 0; + } + + shouldApplyGenericCosmeticFilters(tabId) { + return (this.getCosmeticFilteringBits(tabId) & 0b10) !== 0; + } + + static factory(frameURL, parentId = -1) { + const entry = FrameStore.junkyard.pop(); + if ( entry === undefined ) { + return new FrameStore(frameURL, parentId); + } + return entry.init(frameURL, parentId); + } +}; + +// To mitigate memory churning +FrameStore.junkyard = []; +FrameStore.junkyardMax = 50; + +/******************************************************************************/ + +const CountDetails = class { + constructor() { + this.allowed = { any: 0, frame: 0, script: 0 }; + this.blocked = { any: 0, frame: 0, script: 0 }; + } + reset() { + const { allowed, blocked } = this; + blocked.any = blocked.frame = blocked.script = + allowed.any = allowed.frame = allowed.script = 0; + } + inc(blocked, type = undefined) { + const stat = blocked ? this.blocked : this.allowed; + if ( type !== undefined ) { stat[type] += 1; } + stat.any += 1; + } +}; + +const HostnameDetails = class { + constructor(hostname) { + this.counts = new CountDetails(); + this.init(hostname); + } + init(hostname) { + this.hostname = hostname; + this.counts.reset(); + } + dispose() { + this.hostname = ''; + if ( HostnameDetails.junkyard.length < HostnameDetails.junkyardMax ) { + HostnameDetails.junkyard.push(this); + } + } +}; + +HostnameDetails.junkyard = []; +HostnameDetails.junkyardMax = 100; + +const HostnameDetailsMap = class extends Map { + reset() { + this.clear(); + } + dispose() { + for ( const item of this.values() ) { + item.dispose(); + } + this.reset(); + } +}; + +/******************************************************************************/ + +const PageStore = class { + constructor(tabId, details) { + this.extraData = new Map(); + this.journal = []; + this.journalLastCommitted = this.journalLastUncommitted = -1; + this.journalLastUncommittedOrigin = undefined; + this.netFilteringCache = NetFilteringResultCache.factory(); + this.hostnameDetailsMap = new HostnameDetailsMap(); + this.counts = new CountDetails(); + this.journalTimer = vAPI.defer.create(( ) => { + this.journalProcess(); + }); + this.largeMediaTimer = vAPI.defer.create(( ) => { + this.injectLargeMediaElementScriptlet(); + }); + this.init(tabId, details); + } + + static factory(tabId, details) { + let entry = PageStore.junkyard.pop(); + if ( entry === undefined ) { + entry = new PageStore(tabId, details); + } else { + entry.init(tabId, details); + } + return entry; + } + + // https://github.com/gorhill/uBlock/issues/3201 + // The context is used to determine whether we report behavior change + // to the logger. + + init(tabId, details) { + const tabContext = µb.tabContextManager.mustLookup(tabId); + this.tabId = tabId; + + // If we are navigating from-to same site, remember whether large + // media elements were temporarily allowed. + if ( + typeof this.allowLargeMediaElementsUntil !== 'number' || + tabContext.rootHostname !== this.tabHostname + ) { + this.allowLargeMediaElementsUntil = Date.now(); + } + + this.tabHostname = tabContext.rootHostname; + this.rawURL = tabContext.rawURL; + this.hostnameDetailsMap.reset(); + this.contentLastModified = 0; + this.logData = undefined; + this.counts.reset(); + this.remoteFontCount = 0; + this.popupBlockedCount = 0; + this.largeMediaCount = 0; + this.allowLargeMediaElementsRegex = undefined; + this.extraData.clear(); + + this.frameAddCount = 0; + this.frames = new Map(); + this.setFrameURL({ url: tabContext.rawURL }); + + if ( this.titleFromDetails(details) === false ) { + this.title = tabContext.rawURL; + } + + // Evaluated on-demand + this._noCosmeticFiltering = undefined; + + // Remember if the webpage was potentially improperly filtered, for + // reporting purpose. + this.hasUnprocessedRequest = vAPI.net.hasUnprocessedRequest(tabId); + + return this; + } + + reuse(context, details) { + // When force refreshing a page, the page store data needs to be reset. + + // If the hostname changes, we can't merely just update the context. + const tabContext = µb.tabContextManager.mustLookup(this.tabId); + if ( tabContext.rootHostname !== this.tabHostname ) { + context = ''; + } + + // If URL changes without a page reload (more and more common), then + // we need to keep all that we collected for reuse. In particular, + // not doing so was causing a problem in `videos.foxnews.com`: + // clicking a video thumbnail would not work, because the frame + // hierarchy structure was flushed from memory, while not really being + // flushed on the page. + if ( context === 'tabUpdated' ) { + // As part of https://github.com/chrisaljoudi/uBlock/issues/405 + // URL changed, force a re-evaluation of filtering switch + this.rawURL = tabContext.rawURL; + this.setFrameURL({ url: this.rawURL }); + this.titleFromDetails(details); + return this; + } + + // A new page is completely reloaded from scratch, reset all. + this.largeMediaTimer.off(); + this.disposeFrameStores(); + this.init(this.tabId, details); + return this; + } + + dispose() { + this.tabHostname = ''; + this.title = ''; + this.rawURL = ''; + this.hostnameDetailsMap.dispose(); + this.netFilteringCache.empty(); + this.allowLargeMediaElementsUntil = Date.now(); + this.allowLargeMediaElementsRegex = undefined; + this.largeMediaTimer.off(); + this.disposeFrameStores(); + this.journalTimer.off(); + this.journal = []; + this.journalLastUncommittedOrigin = undefined; + this.journalLastCommitted = this.journalLastUncommitted = -1; + if ( PageStore.junkyard.length < PageStore.junkyardMax ) { + PageStore.junkyard.push(this); + } + return null; + } + + titleFromDetails(details) { + if ( + details instanceof Object === false || + details.title === undefined + ) { + return false; + } + this.title = orphanizeString(details.title.slice(0, 128)); + return true; + } + + disposeFrameStores() { + for ( const frameStore of this.frames.values() ) { + frameStore.dispose(); + } + this.frames.clear(); + } + + getFrameStore(frameId) { + return this.frames.get(frameId) || null; + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1858 + // Mind that setFrameURL() can be called from navigation event handlers. + setFrameURL(details) { + let { frameId, url, parentFrameId } = details; + if ( frameId === undefined ) { frameId = 0; } + if ( parentFrameId === undefined ) { parentFrameId = -1; } + let frameStore = this.frames.get(frameId); + if ( frameStore !== undefined ) { + if ( url === frameStore.rawURL ) { + frameStore.parentId = parentFrameId; + } else { + frameStore.init(url, parentFrameId); + } + return frameStore; + } + frameStore = FrameStore.factory(url, parentFrameId); + this.frames.set(frameId, frameStore); + this.frameAddCount += 1; + if ( url.startsWith('about:') ) { + frameStore.updateURL(this.getEffectiveFrameURL({ frameId })); + } + if ( (this.frameAddCount & 0b111111) === 0 ) { + this.pruneFrames(); + } + return frameStore; + } + + getEffectiveFrameURL(sender) { + let { frameId } = sender; + for (;;) { + const frameStore = this.getFrameStore(frameId); + if ( frameStore === null ) { break; } + if ( frameStore.rawURL.startsWith('about:') === false ) { + return frameStore.rawURL; + } + frameId = frameStore.parentId; + if ( frameId === -1 ) { break; } + } + return sender.frameURL; + } + + // There is no event to tell us a specific subframe has been removed from + // the main document. The code below will remove subframes which are no + // longer present in the root document. Removing obsolete subframes is + // not a critical task, so this is executed just once on a while, to avoid + // bloated dictionary of subframes. + // A TTL is used to avoid race conditions when new iframes are added + // through the webRequest API but still not yet visible through the + // webNavigation API. + async pruneFrames() { + let entries; + try { + entries = await webext.webNavigation.getAllFrames({ + tabId: this.tabId + }); + } catch(ex) { + } + if ( Array.isArray(entries) === false ) { return; } + const toKeep = new Set(); + for ( const { frameId } of entries ) { + toKeep.add(frameId); + } + const obsolete = Date.now() - 60000; + for ( const [ frameId, { t0 } ] of this.frames ) { + if ( toKeep.has(frameId) || t0 >= obsolete ) { continue; } + this.frames.delete(frameId); + } + } + + getNetFilteringSwitch() { + return µb.tabContextManager + .mustLookup(this.tabId) + .getNetFilteringSwitch(); + } + + toggleNetFilteringSwitch(url, scope, state) { + µb.toggleNetFilteringSwitch(url, scope, state); + this.netFilteringCache.empty(); + } + + shouldApplyCosmeticFilters(frameId = 0) { + if ( this._noCosmeticFiltering === undefined ) { + this._noCosmeticFiltering = this.getNetFilteringSwitch() === false; + if ( this._noCosmeticFiltering === false ) { + this._noCosmeticFiltering = sessionSwitches.evaluateZ( + 'no-cosmetic-filtering', + this.tabHostname + ) === true; + if ( this._noCosmeticFiltering && logger.enabled ) { + µb.filteringContext + .duplicate() + .fromTabId(this.tabId) + .setURL(this.rawURL) + .setRealm('cosmetic') + .setType('dom') + .setFilter(sessionSwitches.toLogData()) + .toLogger(); + } + } + } + if ( this._noCosmeticFiltering ) { return false; } + if ( frameId === -1 ) { return true; } + // Cosmetic filtering can be effectively disabled when both specific + // and generic cosmetic filters are disabled. + return this.shouldApplySpecificCosmeticFilters(frameId) || + this.shouldApplyGenericCosmeticFilters(frameId); + } + + shouldApplySpecificCosmeticFilters(frameId) { + if ( this.shouldApplyCosmeticFilters(-1) === false ) { return false; } + const frameStore = this.getFrameStore(frameId); + if ( frameStore === null ) { return false; } + return frameStore.shouldApplySpecificCosmeticFilters(this.tabId); + } + + shouldApplyGenericCosmeticFilters(frameId) { + if ( this.shouldApplyCosmeticFilters(-1) === false ) { return false; } + const frameStore = this.getFrameStore(frameId); + if ( frameStore === null ) { return false; } + return frameStore.shouldApplyGenericCosmeticFilters(this.tabId); + } + + // https://github.com/gorhill/uBlock/issues/2105 + // Be sure to always include the current page's hostname -- it might not + // be present when the page itself is pulled from the browser's + // short-term memory cache. + getAllHostnameDetails() { + if ( + this.hostnameDetailsMap.has(this.tabHostname) === false && + isNetworkURI(this.rawURL) + ) { + this.hostnameDetailsMap.set( + this.tabHostname, + new HostnameDetails(this.tabHostname) + ); + } + return this.hostnameDetailsMap; + } + + injectLargeMediaElementScriptlet() { + vAPI.tabs.executeScript(this.tabId, { + file: '/js/scriptlets/load-large-media-interactive.js', + allFrames: true, + runAt: 'document_idle', + }); + contextMenu.update(this.tabId); + } + + temporarilyAllowLargeMediaElements(state) { + this.largeMediaCount = 0; + contextMenu.update(this.tabId); + if ( state ) { + this.allowLargeMediaElementsUntil = 0; + this.allowLargeMediaElementsRegex = undefined; + } else { + this.allowLargeMediaElementsUntil = Date.now(); + } + vAPI.tabs.executeScript(this.tabId, { + file: '/js/scriptlets/load-large-media-all.js', + allFrames: true, + }); + } + + // https://github.com/gorhill/uBlock/issues/2053 + // There is no way around using journaling to ensure we deal properly with + // potentially out of order navigation events vs. network request events. + journalAddRequest(fctxt, result) { + const hostname = fctxt.getHostname(); + if ( hostname === '' ) { return; } + this.journal.push(hostname, result, fctxt.itype); + this.journalTimer.on(µb.hiddenSettings.requestJournalProcessPeriod); + } + + journalAddRootFrame(type, url) { + if ( type === 'committed' ) { + this.journalLastCommitted = this.journal.length; + if ( + this.journalLastUncommitted !== -1 && + this.journalLastUncommitted < this.journalLastCommitted && + this.journalLastUncommittedOrigin === hostnameFromURI(url) + ) { + this.journalLastCommitted = this.journalLastUncommitted; + } + } else if ( type === 'uncommitted' ) { + const newOrigin = hostnameFromURI(url); + if ( + this.journalLastUncommitted === -1 || + this.journalLastUncommittedOrigin !== newOrigin + ) { + this.journalLastUncommitted = this.journal.length; + this.journalLastUncommittedOrigin = newOrigin; + } + } + this.journalTimer.offon(µb.hiddenSettings.requestJournalProcessPeriod); + } + + journalProcess() { + this.journalTimer.off(); + + const journal = this.journal; + const pivot = Math.max(0, this.journalLastCommitted); + const now = Date.now(); + const { SCRIPT, SUB_FRAME, OBJECT } = µb.FilteringContext; + let aggregateAllowed = 0; + let aggregateBlocked = 0; + + // Everything after pivot originates from current page. + for ( let i = pivot; i < journal.length; i += 3 ) { + const hostname = journal[i+0]; + let hnDetails = this.hostnameDetailsMap.get(hostname); + if ( hnDetails === undefined ) { + hnDetails = new HostnameDetails(hostname); + this.hostnameDetailsMap.set(hostname, hnDetails); + this.contentLastModified = now; + } + const blocked = journal[i+1] === 1; + const itype = journal[i+2]; + if ( itype === SCRIPT ) { + hnDetails.counts.inc(blocked, 'script'); + this.counts.inc(blocked, 'script'); + } else if ( itype === SUB_FRAME || itype === OBJECT ) { + hnDetails.counts.inc(blocked, 'frame'); + this.counts.inc(blocked, 'frame'); + } else { + hnDetails.counts.inc(blocked); + this.counts.inc(blocked); + } + if ( blocked ) { + aggregateBlocked += 1; + } else { + aggregateAllowed += 1; + } + } + this.journalLastUncommitted = this.journalLastCommitted = -1; + + // https://github.com/chrisaljoudi/uBlock/issues/905#issuecomment-76543649 + // No point updating the badge if it's not being displayed. + if ( aggregateBlocked !== 0 && µb.userSettings.showIconBadge ) { + µb.updateToolbarIcon(this.tabId, 0x02); + } + + // Everything before pivot does not originate from current page -- we + // still need to bump global blocked/allowed counts. + for ( let i = 0; i < pivot; i += 3 ) { + if ( journal[i+1] === 1 ) { + aggregateBlocked += 1; + } else { + aggregateAllowed += 1; + } + } + if ( aggregateAllowed !== 0 || aggregateBlocked !== 0 ) { + µb.localSettings.blockedRequestCount += aggregateBlocked; + µb.localSettings.allowedRequestCount += aggregateAllowed; + µb.localSettingsLastModified = now; + } + journal.length = 0; + } + + filterRequest(fctxt) { + fctxt.filter = undefined; + fctxt.redirectURL = undefined; + + if ( this.getNetFilteringSwitch(fctxt) === false ) { + return 0; + } + + if ( + fctxt.itype === fctxt.CSP_REPORT && + this.filterCSPReport(fctxt) === 1 + ) { + return 1; + } + + if ( + (fctxt.itype & fctxt.FONT_ANY) !== 0 && + this.filterFont(fctxt) === 1 ) + { + return 1; + } + + if ( + fctxt.itype === fctxt.SCRIPT && + this.filterScripting(fctxt, true) === 1 + ) { + return 1; + } + + const cacheableResult = + this.cacheableResults.has(fctxt.itype) && + fctxt.aliasURL === undefined; + + if ( cacheableResult ) { + const entry = this.netFilteringCache.lookupResult(fctxt); + if ( entry !== undefined ) { + fctxt.redirectURL = entry.redirectURL; + fctxt.filter = entry.logData; + return entry.result; + } + } + + const requestType = fctxt.type; + const loggerEnabled = logger.enabled; + + // Dynamic URL filtering. + let result = sessionURLFiltering.evaluateZ( + fctxt.getTabHostname(), + fctxt.url, + requestType + ); + if ( result !== 0 && loggerEnabled ) { + fctxt.filter = sessionURLFiltering.toLogData(); + } + + // Dynamic hostname/type filtering. + if ( result === 0 && µb.userSettings.advancedUserEnabled ) { + result = sessionFirewall.evaluateCellZY( + fctxt.getTabHostname(), + fctxt.getHostname(), + requestType + ); + if ( result !== 0 && result !== 3 && loggerEnabled ) { + fctxt.filter = sessionFirewall.toLogData(); + } + } + + // Static filtering has lowest precedence. + const snfe = staticNetFilteringEngine; + if ( result === 0 || result === 3 ) { + result = snfe.matchRequest(fctxt); + if ( result !== 0 ) { + if ( loggerEnabled ) { + fctxt.setFilter(snfe.toLogData()); + } + // https://github.com/uBlockOrigin/uBlock-issues/issues/943 + // Blanket-except blocked aliased canonical hostnames? + if ( + result === 1 && + fctxt.aliasURL !== undefined && + snfe.isBlockImportant() === false && + this.shouldExceptCname(fctxt) + ) { + return 2; + } + } + } + + // Click-to-load? + // When frameId is not -1, the resource is always sub_frame. + if ( result === 1 && fctxt.frameId !== -1 ) { + const frameStore = this.getFrameStore(fctxt.frameId); + if ( frameStore !== null && frameStore.clickToLoad ) { + result = 2; + if ( loggerEnabled ) { + fctxt.pushFilter({ + result, + source: 'network', + raw: 'click-to-load', + }); + } + } + } + + // Modifier(s)? + // A modifier is an action which transform the original network request. + // https://github.com/gorhill/uBlock/issues/949 + // Redirect blocked request? + // https://github.com/uBlockOrigin/uBlock-issues/issues/760 + // Redirect non-blocked request? + if ( (fctxt.itype & fctxt.INLINE_ANY) === 0 ) { + if ( result === 1 ) { + this.redirectBlockedRequest(fctxt); + } else { + this.redirectNonBlockedRequest(fctxt); + } + } + + if ( cacheableResult ) { + this.netFilteringCache.rememberResult(fctxt, result); + } else if ( result === 1 && this.collapsibleResources.has(fctxt.itype) ) { + this.netFilteringCache.rememberBlock(fctxt); + } + + return result; + } + + filterOnHeaders(fctxt, headers) { + fctxt.filter = undefined; + + if ( this.getNetFilteringSwitch(fctxt) === false ) { return 0; } + + let result = staticNetFilteringEngine.matchHeaders(fctxt, headers); + if ( result === 0 ) { return 0; } + + const loggerEnabled = logger.enabled; + if ( loggerEnabled ) { + fctxt.filter = staticNetFilteringEngine.toLogData(); + } + + // Dynamic filtering allow rules + // URL filtering + if ( + result === 1 && + sessionURLFiltering.evaluateZ( + fctxt.getTabHostname(), + fctxt.url, + fctxt.type + ) === 2 + ) { + result = 2; + if ( loggerEnabled ) { + fctxt.filter = sessionURLFiltering.toLogData(); + } + } + // Hostname filtering + if ( + result === 1 && + µb.userSettings.advancedUserEnabled && + sessionFirewall.evaluateCellZY( + fctxt.getTabHostname(), + fctxt.getHostname(), + fctxt.type + ) === 2 + ) { + result = 2; + if ( loggerEnabled ) { + fctxt.filter = sessionFirewall.toLogData(); + } + } + + return result; + } + + redirectBlockedRequest(fctxt) { + const directives = staticNetFilteringEngine.redirectRequest(redirectEngine, fctxt); + if ( directives === undefined ) { return; } + if ( logger.enabled !== true ) { return; } + fctxt.pushFilters(directives.map(a => a.logData())); + if ( fctxt.redirectURL === undefined ) { return; } + fctxt.pushFilter({ + source: 'redirect', + raw: directives[directives.length-1].value + }); + } + + redirectNonBlockedRequest(fctxt) { + const transformDirectives = staticNetFilteringEngine.transformRequest(fctxt); + const pruneDirectives = fctxt.redirectURL === undefined && + staticNetFilteringEngine.hasQuery(fctxt) && + staticNetFilteringEngine.filterQuery(fctxt) || + undefined; + if ( transformDirectives === undefined && pruneDirectives === undefined ) { return; } + if ( logger.enabled !== true ) { return; } + if ( transformDirectives !== undefined ) { + fctxt.pushFilters(transformDirectives.map(a => a.logData())); + } + if ( pruneDirectives !== undefined ) { + fctxt.pushFilters(pruneDirectives.map(a => a.logData())); + } + if ( fctxt.redirectURL === undefined ) { return; } + fctxt.pushFilter({ + source: 'redirect', + raw: fctxt.redirectURL + }); + } + + filterCSPReport(fctxt) { + if ( + sessionSwitches.evaluateZ( + 'no-csp-reports', + fctxt.getHostname() + ) + ) { + if ( logger.enabled ) { + fctxt.filter = sessionSwitches.toLogData(); + } + return 1; + } + return 0; + } + + filterFont(fctxt) { + if ( fctxt.itype === fctxt.FONT ) { + this.remoteFontCount += 1; + } + if ( + sessionSwitches.evaluateZ( + 'no-remote-fonts', + fctxt.getTabHostname() + ) !== false + ) { + if ( logger.enabled ) { + fctxt.filter = sessionSwitches.toLogData(); + } + return 1; + } + return 0; + } + + filterScripting(fctxt, netFiltering) { + fctxt.filter = undefined; + if ( netFiltering === undefined ) { + netFiltering = this.getNetFilteringSwitch(fctxt); + } + if ( + netFiltering === false || + sessionSwitches.evaluateZ( + 'no-scripting', + fctxt.getTabHostname() + ) === false + ) { + return 0; + } + if ( logger.enabled ) { + fctxt.filter = sessionSwitches.toLogData(); + } + return 1; + } + + // The caller is responsible to check whether filtering is enabled or not. + filterLargeMediaElement(fctxt, size) { + fctxt.filter = undefined; + + if ( this.allowLargeMediaElementsUntil === 0 ) { + return 0; + } + // Disregard large media elements previously allowed: for example, to + // seek inside a previously allowed audio/video. + if ( + this.allowLargeMediaElementsRegex instanceof RegExp && + this.allowLargeMediaElementsRegex.test(fctxt.url) + ) { + return 0; + } + if ( Date.now() < this.allowLargeMediaElementsUntil ) { + const sources = this.allowLargeMediaElementsRegex instanceof RegExp + ? [ this.allowLargeMediaElementsRegex.source ] + : []; + sources.push('^' + µb.escapeRegex(fctxt.url)); + this.allowLargeMediaElementsRegex = new RegExp(sources.join('|')); + return 0; + } + if ( + sessionSwitches.evaluateZ( + 'no-large-media', + fctxt.getTabHostname() + ) !== true + ) { + this.allowLargeMediaElementsUntil = 0; + return 0; + } + if ( (size >>> 10) < µb.userSettings.largeMediaSize ) { + return 0; + } + + this.largeMediaCount += 1; + this.largeMediaTimer.on(500); + + if ( logger.enabled ) { + fctxt.filter = sessionSwitches.toLogData(); + } + + return 1; + } + + clickToLoad(frameId, frameURL) { + let frameStore = this.getFrameStore(frameId); + if ( frameStore === null ) { + frameStore = this.setFrameURL({ frameId, url: frameURL }); + } + this.netFilteringCache.forgetResult( + this.tabHostname, + 'sub_frame', + frameURL + ); + frameStore.clickToLoad = true; + } + + shouldExceptCname(fctxt) { + let exceptCname; + let frameStore; + if ( fctxt.docId !== undefined ) { + frameStore = this.getFrameStore(fctxt.docId); + if ( frameStore instanceof Object ) { + exceptCname = frameStore.exceptCname; + } + } + if ( exceptCname === undefined ) { + const result = staticNetFilteringEngine.matchRequestReverse( + 'cname', + frameStore instanceof Object + ? frameStore.rawURL + : fctxt.getDocOrigin() + ); + exceptCname = result === 2 + ? staticNetFilteringEngine.toLogData() + : false; + if ( frameStore instanceof Object ) { + frameStore.exceptCname = exceptCname; + } + } + if ( exceptCname === false ) { return false; } + if ( exceptCname instanceof Object ) { + fctxt.setFilter(exceptCname); + } + return true; + } + + getBlockedResources(request, response) { + const normalURL = µb.normalizeTabURL(this.tabId, request.frameURL); + const resources = request.resources; + const fctxt = µb.filteringContext; + fctxt.fromTabId(this.tabId) + .setDocOriginFromURL(normalURL); + // Force some resources to go through the filtering engine in order to + // populate the blocked-resources cache. This is required because for + // some resources it's not possible to detect whether they were blocked + // content script-side (i.e. `iframes` -- unlike `img`). + if ( Array.isArray(resources) && resources.length !== 0 ) { + for ( const resource of resources ) { + this.filterRequest( + fctxt.setType(resource.type).setURL(resource.url) + ); + } + } + if ( this.netFilteringCache.hash === response.hash ) { return; } + response.hash = this.netFilteringCache.hash; + response.blockedResources = + this.netFilteringCache.lookupAllBlocked(fctxt.getDocHostname()); + } +}; + +PageStore.prototype.cacheableResults = new Set([ + µb.FilteringContext.SUB_FRAME, +]); + +PageStore.prototype.collapsibleResources = new Set([ + µb.FilteringContext.IMAGE, + µb.FilteringContext.MEDIA, + µb.FilteringContext.OBJECT, + µb.FilteringContext.SUB_FRAME, +]); + +// To mitigate memory churning +PageStore.junkyard = []; +PageStore.junkyardMax = 10; + +/******************************************************************************/ + +export { PageStore }; diff --git a/src/js/popup-fenix.js b/src/js/popup-fenix.js new file mode 100644 index 0000000..b44b923 --- /dev/null +++ b/src/js/popup-fenix.js @@ -0,0 +1,1530 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import punycode from '../lib/punycode.js'; +import { i18n$ } from './i18n.js'; +import { dom, qs$, qsa$ } from './dom.js'; + +/******************************************************************************/ + +let popupFontSize = 'unset'; +vAPI.localStorage.getItemAsync('popupFontSize').then(value => { + if ( typeof value !== 'string' || value === 'unset' ) { return; } + document.body.style.setProperty('--font-size', value); + popupFontSize = value; +}); + +// https://github.com/chrisaljoudi/uBlock/issues/996 +// Experimental: mitigate glitchy popup UI: immediately set the firewall +// pane visibility to its last known state. By default the pane is hidden. +vAPI.localStorage.getItemAsync('popupPanelSections').then(bits => { + if ( typeof bits !== 'number' ) { return; } + setSections(bits); +}); + +/******************************************************************************/ + +const messaging = vAPI.messaging; +const scopeToSrcHostnameMap = { + '/': '*', + '.': '' +}; +const hostnameToSortableTokenMap = new Map(); +const statsStr = i18n$('popupBlockedStats'); +const domainsHitStr = i18n$('popupHitDomainCount'); + +let popupData = {}; +let dfPaneBuilt = false; +let dfHotspots = null; +const allHostnameRows = []; +let cachedPopupHash = ''; + +// https://github.com/gorhill/uBlock/issues/2550 +// Solution inspired from +// - https://bugs.chromium.org/p/chromium/issues/detail?id=683314 +// - https://bugzilla.mozilla.org/show_bug.cgi?id=1332714#c17 +// Confusable character set from: +// - http://unicode.org/cldr/utility/list-unicodeset.jsp?a=%5B%D0%B0%D1%81%D4%81%D0%B5%D2%BB%D1%96%D1%98%D3%8F%D0%BE%D1%80%D4%9B%D1%95%D4%9D%D1%85%D1%83%D1%8A%D0%AC%D2%BD%D0%BF%D0%B3%D1%B5%D1%A1%5D&g=gc&i= +// Linked from: +// - https://www.chromium.org/developers/design-documents/idn-in-google-chrome +const reCyrillicNonAmbiguous = /[\u0400-\u042b\u042d-\u042f\u0431\u0432\u0434\u0436-\u043d\u0442\u0444\u0446-\u0449\u044b-\u0454\u0457\u0459-\u0460\u0462-\u0474\u0476-\u04ba\u04bc\u04be-\u04ce\u04d0-\u0500\u0502-\u051a\u051c\u051e-\u052f]/; +const reCyrillicAmbiguous = /[\u042c\u0430\u0433\u0435\u043e\u043f\u0440\u0441\u0443\u0445\u044a\u0455\u0456\u0458\u0461\u0475\u04bb\u04bd\u04cf\u0501\u051b\u051d]/; + +/******************************************************************************/ + +const cachePopupData = function(data) { + popupData = {}; + scopeToSrcHostnameMap['.'] = ''; + hostnameToSortableTokenMap.clear(); + + if ( typeof data !== 'object' ) { + return popupData; + } + popupData = data; + popupData.cnameMap = new Map(popupData.cnameMap); + scopeToSrcHostnameMap['.'] = popupData.pageHostname || ''; + const hostnameDict = popupData.hostnameDict; + if ( typeof hostnameDict !== 'object' ) { + return popupData; + } + for ( const hostname in hostnameDict ) { + if ( hostnameDict.hasOwnProperty(hostname) === false ) { continue; } + let domain = hostnameDict[hostname].domain; + let prefix = hostname.slice(0, 0 - domain.length - 1); + // Prefix with space char for 1st-party hostnames: this ensure these + // will come first in list. + if ( domain === popupData.pageDomain ) { + domain = '\u0020'; + } + hostnameToSortableTokenMap.set( + hostname, + domain + ' ' + prefix.split('.').reverse().join('.') + ); + } + return popupData; +}; + +/******************************************************************************/ + +const hashFromPopupData = function(reset = false) { + // It makes no sense to offer to refresh the behind-the-scene scope + if ( popupData.pageHostname === 'behind-the-scene' ) { + dom.cl.remove(dom.body, 'needReload'); + return; + } + + const hasher = []; + const rules = popupData.firewallRules; + for ( const key in rules ) { + const rule = rules[key]; + if ( rule === undefined ) { continue; } + hasher.push(rule); + } + hasher.sort(); + hasher.push( + dom.cl.has('body', 'off'), + dom.cl.has('#no-large-media', 'on'), + dom.cl.has('#no-cosmetic-filtering', 'on'), + dom.cl.has('#no-remote-fonts', 'on'), + dom.cl.has('#no-scripting', 'on') + ); + + const hash = hasher.join(''); + if ( reset ) { + cachedPopupHash = hash; + } + dom.cl.toggle(dom.body, 'needReload', + hash !== cachedPopupHash || popupData.hasUnprocessedRequest === true + ); +}; + +/******************************************************************************/ + +// greater-than-zero test + +const gtz = n => typeof n === 'number' && n > 0; + +/******************************************************************************/ + +const formatNumber = function(count) { + if ( typeof count !== 'number' ) { return ''; } + if ( count < 1e6 ) { return count.toLocaleString(); } + + if ( + intlNumberFormat === undefined && + Intl.NumberFormat instanceof Function + ) { + const intl = new Intl.NumberFormat(undefined, { + notation: 'compact', + maximumSignificantDigits: 4 + }); + if ( + intl.resolvedOptions instanceof Function && + intl.resolvedOptions().hasOwnProperty('notation') + ) { + intlNumberFormat = intl; + } + } + + if ( intlNumberFormat ) { + return intlNumberFormat.format(count); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1027#issuecomment-629696676 + // For platforms which do not support proper number formatting, use + // a poor's man compact form, which unfortunately is not i18n-friendly. + count /= 1000000; + if ( count >= 100 ) { + count = Math.floor(count * 10) / 10; + } else if ( count > 10 ) { + count = Math.floor(count * 100) / 100; + } else { + count = Math.floor(count * 1000) / 1000; + } + return (count).toLocaleString(undefined) + '\u2009M'; +}; + +let intlNumberFormat; + +/******************************************************************************/ + +const safePunycodeToUnicode = function(hn) { + const pretty = punycode.toUnicode(hn); + return pretty === hn || + reCyrillicAmbiguous.test(pretty) === false || + reCyrillicNonAmbiguous.test(pretty) + ? pretty + : hn; +}; + +/******************************************************************************/ + +const updateFirewallCellCount = function(cells, allowed, blocked) { + for ( const cell of cells ) { + if ( gtz(allowed) ) { + dom.attr(cell, 'data-acount', + Math.min(Math.ceil(Math.log(allowed + 1) / Math.LN10), 3) + ); + } else { + dom.attr(cell, 'data-acount', '0'); + } + if ( gtz(blocked) ) { + dom.attr(cell, 'data-bcount', + Math.min(Math.ceil(Math.log(blocked + 1) / Math.LN10), 3) + ); + } else { + dom.attr(cell, 'data-bcount', '0'); + } + } +}; + +/******************************************************************************/ + +const updateFirewallCellRule = function(cells, scope, des, type, rule) { + const ruleParts = rule !== undefined ? rule.split(' ') : undefined; + + for ( const cell of cells ) { + if ( ruleParts === undefined ) { + dom.attr(cell, 'class', null); + continue; + } + + const action = updateFirewallCellRule.actionNames[ruleParts[3]]; + dom.attr(cell, 'class', `${action}Rule`); + + // Use dark shade visual cue if the rule is specific to the cell. + if ( + (ruleParts[1] !== '*' || ruleParts[2] === type) && + (ruleParts[1] === des) && + (ruleParts[0] === scopeToSrcHostnameMap[scope]) + + ) { + dom.cl.add(cell, 'ownRule'); + } + } +}; + +updateFirewallCellRule.actionNames = { '1': 'block', '2': 'allow', '3': 'noop' }; + +/******************************************************************************/ + +const updateAllFirewallCells = function(doRules = true, doCounts = true) { + const { pageDomain } = popupData; + const rowContainer = qs$('#firewall'); + const rows = qsa$(rowContainer, '#firewall > [data-des][data-type]'); + + let a1pScript = 0, b1pScript = 0; + let a3pScript = 0, b3pScript = 0; + let a3pFrame = 0, b3pFrame = 0; + + for ( const row of rows ) { + const des = dom.attr(row, 'data-des'); + const type = dom.attr(row, 'data-type'); + if ( doRules ) { + updateFirewallCellRule( + qsa$(row, ':scope > span[data-src="/"]'), + '/', + des, + type, + popupData.firewallRules[`/ ${des} ${type}`] + ); + } + const cells = qsa$(row, ':scope > span[data-src="."]'); + if ( doRules ) { + updateFirewallCellRule( + cells, + '.', + des, + type, + popupData.firewallRules[`. ${des} ${type}`] + ); + } + if ( des === '*' || type !== '*' ) { continue; } + if ( doCounts === false ) { continue; } + const hnDetails = popupData.hostnameDict[des]; + if ( hnDetails === undefined ) { + updateFirewallCellCount(cells); + continue; + } + const { allowed, blocked } = hnDetails.counts; + updateFirewallCellCount([ cells[0] ], allowed.any, blocked.any); + const { totals } = hnDetails; + if ( totals !== undefined ) { + updateFirewallCellCount([ cells[1] ], totals.allowed.any, totals.blocked.any); + } + if ( hnDetails.domain === pageDomain ) { + a1pScript += allowed.script; b1pScript += blocked.script; + } else { + a3pScript += allowed.script; b3pScript += blocked.script; + a3pFrame += allowed.frame; b3pFrame += blocked.frame; + } + } + + if ( doCounts ) { + const fromType = type => + qsa$(`#firewall > [data-des="*"][data-type="${type}"] > [data-src="."]`); + updateFirewallCellCount(fromType('1p-script'), a1pScript, b1pScript); + updateFirewallCellCount(fromType('3p-script'), a3pScript, b3pScript); + dom.cl.toggle(rowContainer, 'has3pScript', a3pScript !== 0 || b3pScript !== 0); + updateFirewallCellCount(fromType('3p-frame'), a3pFrame, b3pFrame); + dom.cl.toggle(rowContainer, 'has3pFrame', a3pFrame !== 0 || b3pFrame !== 0); + } + + dom.cl.toggle(dom.body, 'needSave', popupData.matrixIsDirty === true); +}; + +/******************************************************************************/ + +// Compute statistics useful only to firewall entries -- we need to call +// this only when overview pane needs to be rendered. + +const expandHostnameStats = ( ) => { + let dnDetails; + for ( const des of allHostnameRows ) { + const hnDetails = popupData.hostnameDict[des]; + const { domain, counts } = hnDetails; + const isDomain = des === domain; + const { allowed: hnAllowed, blocked: hnBlocked } = counts; + if ( isDomain ) { + dnDetails = hnDetails; + dnDetails.totals = JSON.parse(JSON.stringify(dnDetails.counts)); + } else { + const { allowed: dnAllowed, blocked: dnBlocked } = dnDetails.totals; + dnAllowed.any += hnAllowed.any; + dnBlocked.any += hnBlocked.any; + } + hnDetails.hasScript = hnAllowed.script !== 0 || hnBlocked.script !== 0; + dnDetails.hasScript = dnDetails.hasScript || hnDetails.hasScript; + hnDetails.hasFrame = hnAllowed.frame !== 0 || hnBlocked.frame !== 0; + dnDetails.hasFrame = dnDetails.hasFrame || hnDetails.hasFrame; + } +}; + +/******************************************************************************/ + +const buildAllFirewallRows = function() { + // Do this before removing the rows + if ( dfHotspots === null ) { + dfHotspots = qs$('#actionSelector'); + dom.on(dfHotspots, 'click', setFirewallRuleHandler); + } + dfHotspots.remove(); + + // This must be called before we create the rows. + expandHostnameStats(); + + // Update incrementally: reuse existing rows if possible. + const rowContainer = qs$('#firewall'); + const toAppend = document.createDocumentFragment(); + const rowTemplate = qs$('#templates > div[data-des=""][data-type="*"]'); + const { cnameMap, hostnameDict, pageDomain, pageHostname } = popupData; + + let row = qs$(rowContainer, 'div[data-des="*"][data-type="3p-frame"] + div'); + + for ( const des of allHostnameRows ) { + if ( row === null ) { + row = dom.clone(rowTemplate); + toAppend.appendChild(row); + } + dom.attr(row, 'data-des', des); + + const hnDetails = hostnameDict[des] || {}; + const isDomain = des === hnDetails.domain; + const prettyDomainName = des.includes('xn--') + ? punycode.toUnicode(des) + : des; + const isPunycoded = prettyDomainName !== des; + + if ( isDomain && row.childElementCount < 4 ) { + row.append(dom.clone(row.children[2])); + } else if ( isDomain === false && row.childElementCount === 4 ) { + row.children[3].remove(); + } + + const span = qs$(row, 'span:first-of-type'); + dom.text(qs$(span, ':scope > span > span'), prettyDomainName); + + const classList = row.classList; + + let desExtra = ''; + if ( classList.toggle('isCname', cnameMap.has(des)) ) { + desExtra = punycode.toUnicode(cnameMap.get(des)); + } else if ( + isDomain && isPunycoded && + reCyrillicAmbiguous.test(prettyDomainName) && + reCyrillicNonAmbiguous.test(prettyDomainName) === false + ) { + desExtra = des; + } + dom.text(qs$(span, 'sub'), desExtra); + + classList.toggle('isRootContext', des === pageHostname); + classList.toggle('is3p', hnDetails.domain !== pageDomain); + classList.toggle('isDomain', isDomain); + classList.toggle('hasSubdomains', isDomain && hnDetails.hasSubdomains); + classList.toggle('isSubdomain', !isDomain); + const { counts } = hnDetails; + classList.toggle('allowed', gtz(counts.allowed.any)); + classList.toggle('blocked', gtz(counts.blocked.any)); + const { totals } = hnDetails; + classList.toggle('totalAllowed', gtz(totals && totals.allowed.any)); + classList.toggle('totalBlocked', gtz(totals && totals.blocked.any)); + classList.toggle('hasScript', hnDetails.hasScript === true); + classList.toggle('hasFrame', hnDetails.hasFrame === true); + classList.toggle('expandException', expandExceptions.has(hnDetails.domain)); + + row = row.nextElementSibling; + } + + // Remove unused trailing rows + if ( row !== null ) { + while ( row.nextElementSibling !== null ) { + row.nextElementSibling.remove(); + } + row.remove(); + } + + // Add new rows all at once + if ( toAppend.childElementCount !== 0 ) { + rowContainer.append(toAppend); + } + + if ( dfPaneBuilt !== true && popupData.advancedUserEnabled ) { + dom.on('#firewall', 'click', 'span[data-src]', unsetFirewallRuleHandler); + dom.on('#firewall', 'mouseenter', 'span[data-src]', mouseenterCellHandler); + dom.on('#firewall', 'mouseleave', 'span[data-src]', mouseleaveCellHandler); + dfPaneBuilt = true; + } + + updateAllFirewallCells(); +}; + +/******************************************************************************/ + +const hostnameCompare = function(a, b) { + let ha = a; + if ( !reIP.test(ha) ) { + ha = hostnameToSortableTokenMap.get(ha) || ' '; + } + let hb = b; + if ( !reIP.test(hb) ) { + hb = hostnameToSortableTokenMap.get(hb) || ' '; + } + const ca = ha.charCodeAt(0); + const cb = hb.charCodeAt(0); + return ca !== cb ? ca - cb : ha.localeCompare(hb); +}; + +const reIP = /(\d|\])$/; + +/******************************************************************************/ + +function filterFirewallRows() { + const firewallElem = qs$('#firewall'); + const elems = qsa$('#firewall .filterExpressions span[data-expr]'); + let not = false; + for ( const elem of elems ) { + const on = dom.cl.has(elem, 'on'); + switch ( elem.dataset.expr ) { + case 'not': + not = on; + break; + case 'blocked': + dom.cl.toggle(firewallElem, 'showBlocked', !not && on); + dom.cl.toggle(firewallElem, 'hideBlocked', not && on); + break; + case 'allowed': + dom.cl.toggle(firewallElem, 'showAllowed', !not && on); + dom.cl.toggle(firewallElem, 'hideAllowed', not && on); + break; + case 'script': + dom.cl.toggle(firewallElem, 'show3pScript', !not && on); + dom.cl.toggle(firewallElem, 'hide3pScript', not && on); + break; + case 'frame': + dom.cl.toggle(firewallElem, 'show3pFrame', !not && on); + dom.cl.toggle(firewallElem, 'hide3pFrame', not && on); + break; + default: + break; + } + } +} + +dom.on('#firewall .filterExpressions', 'click', 'span[data-expr]', ev => { + const target = ev.target; + dom.cl.toggle(target, 'on'); + switch ( target.dataset.expr ) { + case 'blocked': + if ( dom.cl.has(target, 'on') === false ) { break; } + dom.cl.remove('#firewall .filterExpressions span[data-expr="allowed"]', 'on'); + break; + case 'allowed': + if ( dom.cl.has(target, 'on') === false ) { break; } + dom.cl.remove('#firewall .filterExpressions span[data-expr="blocked"]', 'on'); + break; + } + filterFirewallRows(); + const elems = qsa$('#firewall .filterExpressions span[data-expr]'); + const filters = Array.from(elems) .map(el => dom.cl.has(el, 'on') ? '1' : '0'); + filters.unshift('00'); + vAPI.localStorage.setItem('firewallFilters', filters.join(' ')); +}); + +{ + vAPI.localStorage.getItemAsync('firewallFilters').then(v => { + if ( v === null ) { return; } + const filters = v.split(' '); + if ( filters.shift() !== '00' ) { return; } + if ( filters.every(v => v === '0') ) { return; } + const elems = qsa$('#firewall .filterExpressions span[data-expr]'); + for ( let i = 0; i < elems.length; i++ ) { + if ( filters[i] === '0' ) { continue; } + dom.cl.add(elems[i], 'on'); + } + filterFirewallRows(); + }); +} + +/******************************************************************************/ + +const renderPrivacyExposure = function() { + const allDomains = {}; + let allDomainCount = 0; + let touchedDomainCount = 0; + + allHostnameRows.length = 0; + + // Sort hostnames. First-party hostnames must always appear at the top + // of the list. + const { hostnameDict } = popupData; + const desHostnameDone = new Set(); + const keys = Object.keys(hostnameDict).sort(hostnameCompare); + for ( const des of keys ) { + // Specific-type rules -- these are built-in + if ( des === '*' || desHostnameDone.has(des) ) { continue; } + const hnDetails = hostnameDict[des]; + const { domain, counts } = hnDetails; + if ( allDomains.hasOwnProperty(domain) === false ) { + allDomains[domain] = false; + allDomainCount += 1; + } + if ( gtz(counts.allowed.any) ) { + if ( allDomains[domain] === false ) { + allDomains[domain] = true; + touchedDomainCount += 1; + } + } + const dnDetails = hostnameDict[domain]; + if ( dnDetails !== undefined ) { + if ( des !== domain ) { + dnDetails.hasSubdomains = true; + } else if ( dnDetails.hasSubdomains === undefined ) { + dnDetails.hasSubdomains = false; + } + } + allHostnameRows.push(des); + desHostnameDone.add(des); + } + + const summary = domainsHitStr + .replace('{{count}}', touchedDomainCount.toLocaleString()) + .replace('{{total}}', allDomainCount.toLocaleString()); + dom.text('[data-i18n^="popupDomainsConnected"] + span', summary); +}; + +/******************************************************************************/ + +const updateHnSwitches = function() { + dom.cl.toggle('#no-popups', 'on', popupData.noPopups === true); + dom.cl.toggle('#no-large-media', 'on', popupData.noLargeMedia === true); + dom.cl.toggle('#no-cosmetic-filtering', 'on',popupData.noCosmeticFiltering === true); + dom.cl.toggle('#no-remote-fonts', 'on', popupData.noRemoteFonts === true); + dom.cl.toggle('#no-scripting', 'on', popupData.noScripting === true); +}; + +/******************************************************************************/ + +// Assume everything has to be done incrementally. + +const renderPopup = function() { + if ( popupData.tabTitle ) { + document.title = popupData.appName + ' - ' + popupData.tabTitle; + } + + const isFiltering = popupData.netFilteringSwitch; + + dom.cl.toggle(dom.body, 'advancedUser', popupData.advancedUserEnabled === true); + dom.cl.toggle(dom.body, 'off', popupData.pageURL === '' || isFiltering !== true); + dom.cl.toggle(dom.body, 'needSave', popupData.matrixIsDirty === true); + + // The hostname information below the power switch + { + const [ elemHn, elemDn ] = qs$('#hostname').children; + const { pageDomain, pageHostname } = popupData; + if ( pageDomain !== '' ) { + dom.text(elemDn, safePunycodeToUnicode(pageDomain)); + dom.text(elemHn, pageHostname !== pageDomain + ? safePunycodeToUnicode(pageHostname.slice(0, -pageDomain.length - 1)) + '.' + : '' + ); + } else { + dom.text(elemDn, ''); + dom.text(elemHn, ''); + } + } + + dom.cl.toggle( + '#basicTools', + 'canPick', + popupData.canElementPicker === true && isFiltering + ); + + let blocked, total; + if ( popupData.pageCounts !== undefined ) { + const counts = popupData.pageCounts; + blocked = counts.blocked.any; + total = blocked + counts.allowed.any; + } else { + blocked = 0; + total = 0; + } + let text; + if ( total === 0 ) { + text = formatNumber(0); + } else { + text = statsStr.replace('{{count}}', formatNumber(blocked)) + .replace('{{percent}}', formatNumber(Math.floor(blocked * 100 / total))); + } + dom.text('[data-i18n^="popupBlockedOnThisPage"] + span', text); + + blocked = popupData.globalBlockedRequestCount; + total = popupData.globalAllowedRequestCount + blocked; + if ( total === 0 ) { + text = formatNumber(0); + } else { + text = statsStr.replace('{{count}}', formatNumber(blocked)) + .replace('{{percent}}', formatNumber(Math.floor(blocked * 100 / total))); + } + dom.text('[data-i18n^="popupBlockedSinceInstall"] + span', text); + + // This will collate all domains, touched or not + renderPrivacyExposure(); + + // Extra tools + updateHnSwitches(); + + // Report popup count on badge + total = popupData.popupBlockedCount; + dom.text( + '#no-popups .fa-icon-badge', + total ? Math.min(total, 99).toLocaleString() : '' + ); + + // Report large media count on badge + total = popupData.largeMediaCount; + dom.text( + '#no-large-media .fa-icon-badge', + total ? Math.min(total, 99).toLocaleString() : '' + ); + + // Report remote font count on badge + total = popupData.remoteFontCount; + dom.text( + '#no-remote-fonts .fa-icon-badge', + total ? Math.min(total, 99).toLocaleString() : '' + ); + + // Unprocesseed request(s) warning + dom.cl.toggle(dom.root, 'warn', popupData.hasUnprocessedRequest === true); + + dom.cl.toggle(dom.html, 'colorBlind', popupData.colorBlindFriendly === true); + + setGlobalExpand(popupData.firewallPaneMinimized === false, true); + + // Build dynamic filtering pane only if in use + if ( (computedSections() & sectionFirewallBit) !== 0 ) { + buildAllFirewallRows(); + } + + renderTooltips(); +}; + +/******************************************************************************/ + +dom.on('.dismiss', 'click', ( ) => { + messaging.send('popupPanel', { + what: 'dismissUnprocessedRequest', + tabId: popupData.tabId, + }).then(( ) => { + popupData.hasUnprocessedRequest = false; + dom.cl.remove(dom.root, 'warn'); + }); +}); + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/2889 +// Use tooltip for ARIA purpose. + +const renderTooltips = function(selector) { + for ( const [ key, details ] of tooltipTargetSelectors ) { + if ( selector !== undefined && key !== selector ) { continue; } + const elem = qs$(key); + if ( elem.hasAttribute('title') === false ) { continue; } + const text = i18n$( + details.i18n + + (qs$(details.state) === null ? '1' : '2') + ); + dom.attr(elem, 'aria-label', text); + dom.attr(elem, 'title', text); + } +}; + +const tooltipTargetSelectors = new Map([ + [ + '#switch', + { + state: 'body.off', + i18n: 'popupPowerSwitchInfo', + } + ], + [ + '#no-popups', + { + state: '#no-popups.on', + i18n: 'popupTipNoPopups' + } + ], + [ + '#no-large-media', + { + state: '#no-large-media.on', + i18n: 'popupTipNoLargeMedia' + } + ], + [ + '#no-cosmetic-filtering', + { + state: '#no-cosmetic-filtering.on', + i18n: 'popupTipNoCosmeticFiltering' + } + ], + [ + '#no-remote-fonts', + { + state: '#no-remote-fonts.on', + i18n: 'popupTipNoRemoteFonts' + } + ], + [ + '#no-scripting', + { + state: '#no-scripting.on', + i18n: 'popupTipNoScripting' + } + ], +]); + +/******************************************************************************/ + +// All rendering code which need to be executed only once. + +let renderOnce = function() { + renderOnce = function(){}; + + if ( popupData.fontSize !== popupFontSize ) { + popupFontSize = popupData.fontSize; + if ( popupFontSize !== 'unset' ) { + dom.body.style.setProperty('--font-size', popupFontSize); + vAPI.localStorage.setItem('popupFontSize', popupFontSize); + } else { + dom.body.style.removeProperty('--font-size'); + vAPI.localStorage.removeItem('popupFontSize'); + } + } + + dom.text('#version', popupData.appVersion); + + setSections(computedSections()); + + if ( popupData.uiPopupConfig !== undefined ) { + dom.attr(dom.body, 'data-ui', popupData.uiPopupConfig); + } + + dom.cl.toggle(dom.body, 'no-tooltips', popupData.tooltipsDisabled === true); + if ( popupData.tooltipsDisabled === true ) { + dom.attr('[title]', 'title', null); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/22 + if ( popupData.advancedUserEnabled !== true ) { + dom.attr('#firewall [title][data-src]', 'title', null); + } + + // This must be done the firewall is populated + if ( popupData.popupPanelHeightMode === 1 ) { + dom.cl.add(dom.body, 'vMin'); + } + + // Prevent non-advanced user opting into advanced user mode from harming + // themselves by disabling by default features generally suitable to + // filter list maintainers and actual advanced users. + if ( popupData.godMode ) { + dom.cl.add(dom.body, 'godMode'); + } +}; + +/******************************************************************************/ + +const renderPopupLazy = (( ) => { + let mustRenderCosmeticFilteringBadge = true; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/756 + // Launch potentially expensive hidden elements-counting scriptlet on + // demand only. + { + const sw = qs$('#no-cosmetic-filtering'); + const badge = qs$(sw, ':scope .fa-icon-badge'); + dom.text(badge, '\u22EF'); + + const render = ( ) => { + if ( mustRenderCosmeticFilteringBadge === false ) { return; } + mustRenderCosmeticFilteringBadge = false; + if ( dom.cl.has(sw, 'hnSwitchBusy') ) { return; } + dom.cl.add(sw, 'hnSwitchBusy'); + messaging.send('popupPanel', { + what: 'getHiddenElementCount', + tabId: popupData.tabId, + }).then(count => { + let text; + if ( (count || 0) === 0 ) { + text = ''; + } else if ( count === -1 ) { + text = '?'; + } else { + text = Math.min(count, 99).toLocaleString(); + } + dom.text(badge, text); + dom.cl.remove(sw, 'hnSwitchBusy'); + }); + }; + + dom.on(sw, 'mouseenter', render, { passive: true }); + } + + return async function() { + const count = await messaging.send('popupPanel', { + what: 'getScriptCount', + tabId: popupData.tabId, + }); + dom.text( + '#no-scripting .fa-icon-badge', + (count || 0) !== 0 ? Math.min(count, 99).toLocaleString() : '' + ); + mustRenderCosmeticFilteringBadge = true; + }; +})(); + +/******************************************************************************/ + +const toggleNetFilteringSwitch = function(ev) { + if ( !popupData || !popupData.pageURL ) { return; } + messaging.send('popupPanel', { + what: 'toggleNetFiltering', + url: popupData.pageURL, + scope: ev.ctrlKey || ev.metaKey ? 'page' : '', + state: dom.cl.toggle(dom.body, 'off') === false, + tabId: popupData.tabId, + }); + renderTooltips('#switch'); + hashFromPopupData(); +}; + +/******************************************************************************/ + +const gotoZap = function() { + messaging.send('popupPanel', { + what: 'launchElementPicker', + tabId: popupData.tabId, + zap: true, + }); + + vAPI.closePopup(); +}; + +/******************************************************************************/ + +const gotoPick = function() { + messaging.send('popupPanel', { + what: 'launchElementPicker', + tabId: popupData.tabId, + }); + + vAPI.closePopup(); +}; + +/******************************************************************************/ + +const gotoReport = function() { + const popupPanel = { + blocked: popupData.pageCounts.blocked.any, + }; + const reportedStates = [ + { name: 'enabled', prop: 'netFilteringSwitch', expected: true }, + { name: 'no-cosmetic-filtering', prop: 'noCosmeticFiltering', expected: false }, + { name: 'no-large-media', prop: 'noLargeMedia', expected: false }, + { name: 'no-popups', prop: 'noPopups', expected: false }, + { name: 'no-remote-fonts', prop: 'noRemoteFonts', expected: false }, + { name: 'no-scripting', prop: 'noScripting', expected: false }, + { name: 'can-element-picker', prop: 'canElementPicker', expected: true }, + ]; + for ( const { name, prop, expected } of reportedStates ) { + if ( popupData[prop] === expected ) { continue; } + popupPanel[name] = !expected; + } + if ( hostnameToSortableTokenMap.size !== 0 ) { + const network = {}; + const hostnames = + Array.from(hostnameToSortableTokenMap.keys()).sort(hostnameCompare); + for ( const hostname of hostnames ) { + const entry = popupData.hostnameDict[hostname]; + const count = entry.counts.blocked.any; + if ( count === 0 ) { continue; } + const domain = entry.domain; + if ( network[domain] === undefined ) { + network[domain] = 0; + } + network[domain] += count; + } + if ( Object.keys(network).length !== 0 ) { + popupPanel.network = network; + } + } + messaging.send('popupPanel', { + what: 'launchReporter', + tabId: popupData.tabId, + pageURL: popupData.rawURL, + popupPanel, + }); + + vAPI.closePopup(); +}; + +/******************************************************************************/ + +const gotoURL = function(ev) { + if ( this.hasAttribute('href') === false ) { return; } + + ev.preventDefault(); + + let url = dom.attr(ev.target, 'href'); + if ( + url === 'logger-ui.html#_' && + typeof popupData.tabId === 'number' + ) { + url += '+' + popupData.tabId; + } + + messaging.send('popupPanel', { + what: 'gotoURL', + details: { + url: url, + select: true, + index: -1, + shiftKey: ev.shiftKey + }, + }); + + vAPI.closePopup(); +}; + +/******************************************************************************/ + +// The popup panel is made of sections. Visibility of sections can +// be toggled on/off. + +const maxNumberOfSections = 6; +const sectionFirewallBit = 0b10000; + +const computedSections = ( ) => + popupData.popupPanelSections & + ~popupData.popupPanelDisabledSections | + popupData.popupPanelLockedSections; + +const sectionBitsFromAttribute = function() { + const attr = document.body.dataset.more; + if ( attr === '' ) { return 0; } + let bits = 0; + for ( const c of attr ) { + bits |= 1 << (c.charCodeAt(0) - 97); + } + return bits; +}; + +const sectionBitsToAttribute = function(bits) { + const attr = []; + for ( let i = 0; i < maxNumberOfSections; i++ ) { + const bit = 1 << i; + if ( (bits & bit) === 0 ) { continue; } + attr.push(String.fromCharCode(97 + i)); + } + return attr.join(''); +}; + +const setSections = function(bits) { + const value = sectionBitsToAttribute(bits); + const min = sectionBitsToAttribute(popupData.popupPanelLockedSections); + const max = sectionBitsToAttribute( + (1 << maxNumberOfSections) - 1 & ~popupData.popupPanelDisabledSections + ); + document.body.dataset.more = value; + dom.cl.toggle('#lessButton', 'disabled', value === min); + dom.cl.toggle('#moreButton', 'disabled', value === max); +}; + +const toggleSections = function(more) { + const offbits = ~popupData.popupPanelDisabledSections; + const onbits = popupData.popupPanelLockedSections; + let currentBits = sectionBitsFromAttribute(); + let newBits = currentBits; + for ( let i = 0; i < maxNumberOfSections; i++ ) { + const bit = 1 << (more ? i : maxNumberOfSections - i - 1); + if ( more ) { + newBits |= bit; + } else { + newBits &= ~bit; + } + newBits = newBits & offbits | onbits; + if ( newBits !== currentBits ) { break; } + } + if ( newBits === currentBits ) { return; } + + setSections(newBits); + + popupData.popupPanelSections = newBits; + messaging.send('popupPanel', { + what: 'userSettings', + name: 'popupPanelSections', + value: newBits, + }); + + // https://github.com/chrisaljoudi/uBlock/issues/996 + // Remember the last state of the firewall pane. This allows to + // configure the popup size early next time it is opened, which means a + // less glitchy popup at open time. + vAPI.localStorage.setItem('popupPanelSections', newBits); + + // Dynamic filtering pane may not have been built yet + if ( (newBits & sectionFirewallBit) !== 0 && dfPaneBuilt === false ) { + buildAllFirewallRows(); + } +}; + +dom.on('#moreButton', 'click', ( ) => { toggleSections(true); }); +dom.on('#lessButton', 'click', ( ) => { toggleSections(false); }); + +/******************************************************************************/ + +const mouseenterCellHandler = function(ev) { + const target = ev.target; + if ( dom.cl.has(target, 'ownRule') ) { return; } + target.appendChild(dfHotspots); +}; + +const mouseleaveCellHandler = function() { + dfHotspots.remove(); +}; + +/******************************************************************************/ + +const setFirewallRule = async function(src, des, type, action, persist) { + // This can happen on pages where uBlock does not work + if ( + typeof popupData.pageHostname !== 'string' || + popupData.pageHostname === '' + ) { + return; + } + + const response = await messaging.send('popupPanel', { + what: 'toggleFirewallRule', + tabId: popupData.tabId, + pageHostname: popupData.pageHostname, + srcHostname: src, + desHostname: des, + requestType: type, + action: action, + persist: persist, + }); + + // Remove action widget if an own rule has been set, this allows to click + // again immediately to remove the rule. + if ( action !== 0 ) { + dfHotspots.remove(); + } + + cachePopupData(response); + updateAllFirewallCells(true, false); + hashFromPopupData(); +}; + +/******************************************************************************/ + +const unsetFirewallRuleHandler = function(ev) { + const cell = ev.target; + const row = cell.closest('[data-des]'); + setFirewallRule( + dom.attr(cell, 'data-src') === '/' ? '*' : popupData.pageHostname, + dom.attr(row, 'data-des'), + dom.attr(row, 'data-type'), + 0, + ev.ctrlKey || ev.metaKey + ); + cell.appendChild(dfHotspots); +}; + +/******************************************************************************/ + +const setFirewallRuleHandler = function(ev) { + const hotspot = ev.target; + const cell = hotspot.closest('[data-src]'); + if ( cell === null ) { return; } + const row = cell.closest('[data-des]'); + let action = 0; + if ( hotspot.id === 'dynaAllow' ) { + action = 2; + } else if ( hotspot.id === 'dynaNoop' ) { + action = 3; + } else { + action = 1; + } + setFirewallRule( + dom.attr(cell, 'data-src') === '/' ? '*' : popupData.pageHostname, + dom.attr(row, 'data-des'), + dom.attr(row, 'data-type'), + action, + ev.ctrlKey || ev.metaKey + ); + dfHotspots.remove(); +}; + +/******************************************************************************/ + +const reloadTab = function(bypassCache = false) { + // Preemptively clear the unprocessed-requests status since we know for sure + // the page is being reloaded in this code path. + if ( popupData.hasUnprocessedRequest === true ) { + messaging.send('popupPanel', { + what: 'dismissUnprocessedRequest', + tabId: popupData.tabId, + }).then(( ) => { + popupData.hasUnprocessedRequest = false; + dom.cl.remove(dom.root, 'warn'); + }); + } + + messaging.send('popupPanel', { + what: 'reloadTab', + tabId: popupData.tabId, + url: popupData.rawURL, + select: vAPI.webextFlavor.soup.has('mobile'), + bypassCache, + }); + + // Polling will take care of refreshing the popup content + // https://github.com/chrisaljoudi/uBlock/issues/748 + // User forces a reload, assume the popup has to be updated regardless + // if there were changes or not. + popupData.contentLastModified = -1; + + // Reset popup state hash to current state. + hashFromPopupData(true); +}; + +dom.on('#refresh', 'click', ev => { + reloadTab(ev.ctrlKey || ev.metaKey || ev.shiftKey); +}); + +// https://github.com/uBlockOrigin/uBlock-issues/issues/672 +dom.on(document, 'keydown', ev => { + if ( ev.isComposing ) { return; } + let bypassCache = false; + switch ( ev.key ) { + case 'F5': + bypassCache = ev.ctrlKey || ev.metaKey || ev.shiftKey; + break; + case 'r': + if ( (ev.ctrlKey || ev.metaKey) !== true ) { return; } + break; + case 'R': + if ( (ev.ctrlKey || ev.metaKey) !== true ) { return; } + bypassCache = true; + break; + default: + return; + } + reloadTab(bypassCache); + ev.preventDefault(); + ev.stopPropagation(); +}, { capture: true }); + +/******************************************************************************/ + +const expandExceptions = new Set(); + +vAPI.localStorage.getItemAsync('popupExpandExceptions').then(exceptions => { + try { + if ( Array.isArray(exceptions) === false ) { return; } + for ( const exception of exceptions ) { + expandExceptions.add(exception); + } + } + catch(ex) { + } +}); + +const saveExpandExceptions = function() { + vAPI.localStorage.setItem( + 'popupExpandExceptions', + Array.from(expandExceptions) + ); +}; + +const setGlobalExpand = function(state, internal = false) { + dom.cl.remove('.expandException', 'expandException'); + if ( state ) { + dom.cl.add('#firewall', 'expanded'); + } else { + dom.cl.remove('#firewall', 'expanded'); + } + if ( internal ) { return; } + popupData.firewallPaneMinimized = !state; + expandExceptions.clear(); + saveExpandExceptions(); + messaging.send('popupPanel', { + what: 'userSettings', + name: 'firewallPaneMinimized', + value: popupData.firewallPaneMinimized, + }); +}; + +const setSpecificExpand = function(domain, state, internal = false) { + const elems = qsa$(`[data-des="${domain}"],[data-des$=".${domain}"]`); + if ( state ) { + dom.cl.add(elems, 'expandException'); + } else { + dom.cl.remove(elems, 'expandException'); + } + if ( internal ) { return; } + if ( state ) { + expandExceptions.add(domain); + } else { + expandExceptions.delete(domain); + } + saveExpandExceptions(); +}; + +dom.on('[data-i18n="popupAnyRulePrompt"]', 'click', ev => { + // Special display mode: in its own tab/window, with no vertical restraint. + // Useful to take snapshots of the whole list of domains -- example: + // https://github.com/gorhill/uBlock/issues/736#issuecomment-178879944 + if ( ev.shiftKey && ev.ctrlKey ) { + messaging.send('popupPanel', { + what: 'gotoURL', + details: { + url: `popup-fenix.html?tabId=${popupData.tabId}&intab=1`, + select: true, + index: -1, + }, + }); + vAPI.closePopup(); + return; + } + + setGlobalExpand(dom.cl.has('#firewall', 'expanded') === false); +}); + +dom.on('#firewall', 'click', '.isDomain[data-type="*"] > span:first-of-type', ev => { + const div = ev.target.closest('[data-des]'); + if ( div === null ) { return; } + setSpecificExpand( + dom.attr(div, 'data-des'), + dom.cl.has(div, 'expandException') === false + ); +}); + +/******************************************************************************/ + +const saveFirewallRules = function() { + messaging.send('popupPanel', { + what: 'saveFirewallRules', + srcHostname: popupData.pageHostname, + desHostnames: popupData.hostnameDict, + }); + dom.cl.remove(dom.body, 'needSave'); +}; + +/******************************************************************************/ + +const revertFirewallRules = async function() { + dom.cl.remove(dom.body, 'needSave'); + const response = await messaging.send('popupPanel', { + what: 'revertFirewallRules', + srcHostname: popupData.pageHostname, + desHostnames: popupData.hostnameDict, + tabId: popupData.tabId, + }); + cachePopupData(response); + updateAllFirewallCells(true, false); + updateHnSwitches(); + hashFromPopupData(); +}; + +/******************************************************************************/ + +const toggleHostnameSwitch = async function(ev) { + const target = ev.currentTarget; + const switchName = dom.attr(target, 'id'); + if ( !switchName ) { return; } + // For touch displays, process click only if the switch is not "busy". + if ( + vAPI.webextFlavor.soup.has('mobile') && + dom.cl.has(target, 'hnSwitchBusy') + ) { + return; + } + dom.cl.toggle(target, 'on'); + renderTooltips(`#${switchName}`); + + const response = await messaging.send('popupPanel', { + what: 'toggleHostnameSwitch', + name: switchName, + hostname: popupData.pageHostname, + state: dom.cl.has(target, 'on'), + tabId: popupData.tabId, + persist: ev.ctrlKey || ev.metaKey, + }); + + cachePopupData(response); + hashFromPopupData(); + + dom.cl.toggle(dom.body, 'needSave', popupData.matrixIsDirty === true); +}; + +/******************************************************************************* + + Double tap ctrl key: toggle god mode + +*/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/2145 +// Ignore events from auto-repeating keys + +{ + let eventCount = 0; + let eventTime = 0; + + dom.on(document, 'keydown', ev => { + if ( ev.key !== 'Control' ) { + eventCount = 0; + return; + } + if ( ev.repeat ) { return; } + const now = Date.now(); + if ( (now - eventTime) >= 500 ) { + eventCount = 0; + } + eventCount += 1; + eventTime = now; + if ( eventCount < 2 ) { return; } + eventCount = 0; + dom.cl.toggle(dom.body, 'godMode'); + }); +} + + +/******************************************************************************/ + +// Poll for changes. +// +// I couldn't find a better way to be notified of changes which can affect +// popup content, as the messaging API doesn't support firing events accurately +// from the main extension process to a specific auxiliary extension process: +// +// - broadcasting() is not an option given there could be a lot of tabs opened, +// and maybe even many frames within these tabs, i.e. unacceptable overhead +// regardless of whether the popup is opened or not. +// +// - Modifying the messaging API is not an option, as this would require +// revisiting all platform-specific code to support targeted broadcasting, +// which who knows could be not so trivial for some platforms. +// +// A well done polling is a better anyways IMO, I prefer that data is pulled +// on demand rather than forcing the main process to assume a client may need +// it and thus having to push it all the time unconditionally. + +const pollForContentChange = (( ) => { + const pollCallback = async function() { + const response = await messaging.send('popupPanel', { + what: 'hasPopupContentChanged', + tabId: popupData.tabId, + contentLastModified: popupData.contentLastModified, + }); + if ( response ) { + await getPopupData(popupData.tabId); + return; + } + poll(); + }; + + const pollTimer = vAPI.defer.create(pollCallback); + + const poll = function() { + pollTimer.on(1500); + }; + + return poll; +})(); + +/******************************************************************************/ + +const getPopupData = async function(tabId, first = false) { + const response = await messaging.send('popupPanel', { + what: 'getPopupData', + tabId, + }); + + cachePopupData(response); + renderOnce(); + renderPopup(); + renderPopupLazy(); // low priority rendering + hashFromPopupData(first); + pollForContentChange(); +}; + +/******************************************************************************/ + +// Popup DOM is assumed to be loaded at this point -- because this script +// is loaded after everything else. + +{ + // Extract the tab id of the page for this popup. If there's no tab id + // specified in the query string, it will default to current tab. + const selfURL = new URL(self.location.href); + const tabId = parseInt(selfURL.searchParams.get('tabId'), 10) || null; + + const nextFrames = async n => { + for ( let i = 0; i < n; i++ ) { + await new Promise(resolve => { + self.requestAnimationFrame(( ) => { resolve(); }); + }); + } + }; + + // The purpose of the following code is to reset to a vertical layout + // should the viewport not be enough wide to accommodate the horizontal + // layout. + // To avoid querying a spurious viewport width -- it happens sometimes, + // somehow -- we delay layout-changing operations to the next paint + // frames. + // Force a layout recalculation by querying the body width. To be + // honest, I have no clue if this makes a difference in the end. + // https://gist.github.com/paulirish/5d52fb081b3570c81e3a + // Use a tolerance proportional to the sum of the width of the panes + // when testing against viewport width. + const checkViewport = async function() { + if ( + dom.cl.has(dom.root, 'mobile') || + selfURL.searchParams.get('portrait') + ) { + dom.cl.add(dom.root, 'portrait'); + dom.cl.remove(dom.root, 'desktop'); + } else if ( dom.cl.has(dom.root, 'desktop') ) { + await nextFrames(8); + const main = qs$('#main'); + const firewall = qs$('#firewall'); + const minWidth = (main.offsetWidth + firewall.offsetWidth) / 1.1; + if ( + selfURL.searchParams.get('portrait') || + window.innerWidth < minWidth + ) { + dom.cl.add(dom.root, 'portrait'); + } + } + if ( dom.cl.has(dom.root, 'portrait') ) { + const panes = qs$('#panes'); + const sticky = qs$('#sticky'); + const stickyParent = sticky.parentElement; + if ( stickyParent !== panes ) { + panes.prepend(sticky); + } + } + if ( selfURL.searchParams.get('intab') !== null ) { + dom.cl.add(dom.root, 'intab'); + } + await nextFrames(1); + dom.cl.remove(dom.body, 'loading'); + }; + + getPopupData(tabId, true).then(( ) => { + if ( document.readyState !== 'complete' ) { + dom.on(self, 'load', ( ) => { checkViewport(); }, { once: true }); + } else { + checkViewport(); + } + }); +} + +/******************************************************************************/ + +dom.on('#switch', 'click', toggleNetFilteringSwitch); +dom.on('#gotoZap', 'click', gotoZap); +dom.on('#gotoPick', 'click', gotoPick); +dom.on('#gotoReport', 'click', gotoReport); +dom.on('.hnSwitch', 'click', ev => { toggleHostnameSwitch(ev); }); +dom.on('#saveRules', 'click', saveFirewallRules); +dom.on('#revertRules', 'click', ( ) => { revertFirewallRules(); }); +dom.on('a[href]', 'click', gotoURL); + +/******************************************************************************/ diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js new file mode 100644 index 0000000..2f58066 --- /dev/null +++ b/src/js/redirect-engine.js @@ -0,0 +1,494 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import redirectableResources from './redirect-resources.js'; + +import { + LineIterator, + orphanizeString, +} from './text-utils.js'; + +/******************************************************************************/ + +const extToMimeMap = new Map([ + [ 'css', 'text/css' ], + [ 'fn', 'fn/javascript' ], // invented mime type for internal use + [ 'gif', 'image/gif' ], + [ 'html', 'text/html' ], + [ 'js', 'text/javascript' ], + [ 'json', 'application/json' ], + [ 'mp3', 'audio/mp3' ], + [ 'mp4', 'video/mp4' ], + [ 'png', 'image/png' ], + [ 'txt', 'text/plain' ], + [ 'xml', 'text/xml' ], +]); + +const typeToMimeMap = new Map([ + [ 'main_frame', 'text/html' ], + [ 'other', 'text/plain' ], + [ 'script', 'text/javascript' ], + [ 'stylesheet', 'text/css' ], + [ 'sub_frame', 'text/html' ], + [ 'xmlhttprequest', 'text/plain' ], +]); + +const validMimes = new Set(extToMimeMap.values()); + +const mimeFromName = name => { + const match = /\.([^.]+)$/.exec(name); + if ( match === null ) { return ''; } + return extToMimeMap.get(match[1]); +}; + +const removeTopCommentBlock = text => { + return text.replace(/^\/\*[\S\s]+?\n\*\/\s*/, ''); +}; + +// vAPI.warSecret is optional, it could be absent in some environments, +// i.e. nodejs for example. Probably the best approach is to have the +// "web_accessible_resources secret" added outside by the client of this +// module, but for now I just want to remove an obstacle to modularization. +const warSecret = typeof vAPI === 'object' && vAPI !== null + ? vAPI.warSecret.short + : ( ) => ''; + +const RESOURCES_SELFIE_VERSION = 7; +const RESOURCES_SELFIE_NAME = 'compiled/redirectEngine/resources'; + +/******************************************************************************/ +/******************************************************************************/ + +class RedirectEntry { + constructor() { + this.mime = ''; + this.data = ''; + this.warURL = undefined; + this.params = undefined; + this.requiresTrust = false; + this.world = 'MAIN'; + this.dependencies = []; + } + + // Prevent redirection to web accessible resources when the request is + // of type 'xmlhttprequest', because XMLHttpRequest.responseURL would + // cause leakage of extension id. See: + // - https://stackoverflow.com/a/8056313 + // - https://bugzilla.mozilla.org/show_bug.cgi?id=998076 + // https://www.reddit.com/r/uBlockOrigin/comments/cpxm1v/ + // User-supplied resources may already be base64 encoded. + + toURL(fctxt, asDataURI = false) { + if ( + this.warURL !== undefined && + asDataURI !== true && + fctxt instanceof Object && + fctxt.type !== 'xmlhttprequest' + ) { + const params = []; + const secret = warSecret(); + if ( secret !== '' ) { params.push(`secret=${secret}`); } + if ( this.params !== undefined ) { + for ( const name of this.params ) { + const value = fctxt[name]; + if ( value === undefined ) { continue; } + params.push(`${name}=${encodeURIComponent(value)}`); + } + } + let url = `${this.warURL}`; + if ( params.length !== 0 ) { + url += `?${params.join('&')}`; + } + return url; + } + if ( this.data === undefined ) { return; } + // https://github.com/uBlockOrigin/uBlock-issues/issues/701 + if ( this.data === '' ) { + const mime = typeToMimeMap.get(fctxt.type); + if ( mime === '' ) { return; } + return `data:${mime},`; + } + if ( this.data.startsWith('data:') === false ) { + if ( this.mime.indexOf(';') === -1 ) { + this.data = `data:${this.mime};base64,${btoa(this.data)}`; + } else { + this.data = `data:${this.mime},${this.data}`; + } + } + return this.data; + } + + toContent() { + if ( this.data.startsWith('data:') ) { + const pos = this.data.indexOf(','); + const base64 = this.data.endsWith(';base64', pos); + this.data = this.data.slice(pos + 1); + if ( base64 ) { + this.data = atob(this.data); + } + } + return this.data; + } + + static fromDetails(details) { + const r = new RedirectEntry(); + Object.assign(r, details); + return r; + } +} + +/******************************************************************************/ +/******************************************************************************/ + +class RedirectEngine { + constructor() { + this.aliases = new Map(); + this.resources = new Map(); + this.reset(); + this.modifyTime = Date.now(); + } + + reset() { + } + + freeze() { + } + + tokenToURL( + fctxt, + token, + asDataURI = false + ) { + const entry = this.resources.get(this.aliases.get(token) || token); + if ( entry === undefined ) { return; } + return entry.toURL(fctxt, asDataURI); + } + + tokenToDNR(token) { + const entry = this.resources.get(this.aliases.get(token) || token); + if ( entry === undefined ) { return; } + if ( entry.warURL === undefined ) { return; } + return entry.warURL; + } + + hasToken(token) { + if ( token === 'none' ) { return true; } + const asDataURI = token.charCodeAt(0) === 0x25 /* '%' */; + if ( asDataURI ) { + token = token.slice(1); + } + return this.resources.get(this.aliases.get(token) || token) !== undefined; + } + + tokenRequiresTrust(token) { + const entry = this.resources.get(this.aliases.get(token) || token); + return entry && entry.requiresTrust === true || false; + } + + async toSelfie() { + } + + async fromSelfie() { + return true; + } + + contentFromName(name, mime = '') { + const entry = this.resources.get(this.aliases.get(name) || name); + if ( entry === undefined ) { return; } + if ( entry.mime.startsWith(mime) === false ) { return; } + return { + js: entry.toContent(), + world: entry.world, + dependencies: entry.dependencies.slice(), + }; + } + + // https://github.com/uBlockOrigin/uAssets/commit/deefe8755511 + // Consider 'none' a reserved keyword, to be used to disable redirection. + // https://github.com/uBlockOrigin/uBlock-issues/issues/1419 + // Append newlines to raw text to ensure processing of trailing resource. + + resourcesFromString(text) { + const lineIter = new LineIterator( + removeTopCommentBlock(text) + '\n\n' + ); + const reNonEmptyLine = /\S/; + let fields, encoded, details; + + while ( lineIter.eot() === false ) { + const line = lineIter.next(); + if ( line.startsWith('#') ) { continue; } + if ( line.startsWith('// ') ) { continue; } + + if ( fields === undefined ) { + if ( line === '' ) { continue; } + // Modern parser + if ( line.startsWith('/// ') ) { + const name = line.slice(4).trim(); + fields = [ name, mimeFromName(name) ]; + continue; + } + // Legacy parser + const head = line.trim().split(/\s+/); + if ( head.length !== 2 ) { continue; } + if ( head[0] === 'none' ) { continue; } + let pos = head[1].indexOf(';'); + if ( pos === -1 ) { pos = head[1].length; } + if ( validMimes.has(head[1].slice(0, pos)) === false ) { + continue; + } + encoded = head[1].indexOf(';') !== -1; + fields = head; + continue; + } + + if ( line.startsWith('/// ') ) { + if ( details === undefined ) { + details = []; + } + const [ prop, value ] = line.slice(4).trim().split(/\s+/); + if ( value !== undefined ) { + details.push({ prop, value }); + } + continue; + } + + if ( reNonEmptyLine.test(line) ) { + fields.push(encoded ? line.trim() : line); + continue; + } + + // No more data, add the resource. + const name = this.aliases.get(fields[0]) || fields[0]; + const mime = fields[1]; + const data = orphanizeString( + fields.slice(2).join(encoded ? '' : '\n') + ); + this.resources.set(name, RedirectEntry.fromDetails({ mime, data })); + if ( Array.isArray(details) ) { + const resource = this.resources.get(name); + for ( const { prop, value } of details ) { + switch ( prop ) { + case 'alias': + this.aliases.set(value, name); + break; + case 'world': + if ( /^isolated$/i.test(value) === false ) { break; } + resource.world = 'ISOLATED'; + break; + case 'dependency': + if ( this.resources.has(value) === false ) { break; } + resource.dependencies.push(value); + break; + default: + break; + } + } + } + + fields = undefined; + details = undefined; + } + + this.modifyTime = Date.now(); + } + + loadBuiltinResources(fetcher) { + this.resources = new Map(); + this.aliases = new Map(); + + const fetches = [ + import('/assets/resources/scriptlets.js').then(module => { + for ( const scriptlet of module.builtinScriptlets ) { + const details = {}; + details.mime = mimeFromName(scriptlet.name); + details.data = scriptlet.fn.toString(); + for ( const [ k, v ] of Object.entries(scriptlet) ) { + if ( k === 'fn' ) { continue; } + details[k] = v; + } + const entry = RedirectEntry.fromDetails(details); + this.resources.set(details.name, entry); + if ( Array.isArray(details.aliases) === false ) { continue; } + for ( const alias of details.aliases ) { + this.aliases.set(alias, details.name); + } + } + this.modifyTime = Date.now(); + }), + ]; + + const store = (name, data = undefined) => { + const details = redirectableResources.get(name); + const entry = RedirectEntry.fromDetails({ + mime: mimeFromName(name), + data, + warURL: `/web_accessible_resources/${name}`, + params: details.params, + }); + this.resources.set(name, entry); + if ( details.alias === undefined ) { return; } + if ( Array.isArray(details.alias) ) { + for ( const alias of details.alias ) { + this.aliases.set(alias, name); + } + } else { + this.aliases.set(details.alias, name); + } + }; + + const processBlob = (name, blob) => { + return new Promise(resolve => { + const reader = new FileReader(); + reader.onload = ( ) => { + store(name, reader.result); + resolve(); + }; + reader.onabort = reader.onerror = ( ) => { + resolve(); + }; + reader.readAsDataURL(blob); + }); + }; + + const processText = (name, text) => { + store(name, removeTopCommentBlock(text)); + }; + + const process = result => { + const match = /^\/web_accessible_resources\/([^?]+)/.exec(result.url); + if ( match === null ) { return; } + const name = match[1]; + return result.content instanceof Blob + ? processBlob(name, result.content) + : processText(name, result.content); + }; + + for ( const [ name, details ] of redirectableResources ) { + if ( typeof details.data !== 'string' ) { + store(name); + continue; + } + fetches.push( + fetcher(`/web_accessible_resources/${name}`, { + responseType: details.data + }).then( + result => process(result) + ) + ); + } + + return Promise.all(fetches); + } + + getResourceDetails() { + const out = new Map([ + [ 'none', { canInject: false, canRedirect: true, aliasOf: '' } ], + ]); + for ( const [ name, entry ] of this.resources ) { + out.set(name, { + canInject: typeof entry.data === 'string', + canRedirect: entry.warURL !== undefined, + aliasOf: '', + extensionPath: entry.warURL, + }); + } + for ( const [ alias, name ] of this.aliases ) { + const original = out.get(name); + if ( original === undefined ) { continue; } + const aliased = Object.assign({}, original); + aliased.aliasOf = name; + out.set(alias, aliased); + } + return Array.from(out).sort((a, b) => { + return a[0].localeCompare(b[0]); + }); + } + + getTrustedScriptletTokens() { + const out = []; + const isTrustedScriptlet = entry => { + if ( entry.requiresTrust !== true ) { return false; } + if ( entry.warURL !== undefined ) { return false; } + if ( typeof entry.data !== 'string' ) { return false; } + if ( entry.name.endsWith('.js') === false ) { return false; } + return true; + }; + for ( const [ name, entry ] of this.resources ) { + if ( isTrustedScriptlet(entry) === false ) { continue; } + out.push(name.slice(0, -3)); + } + for ( const [ alias, name ] of this.aliases ) { + if ( out.includes(name.slice(0, -3)) === false ) { continue; } + out.push(alias.slice(0, -3)); + } + return out; + } + + selfieFromResources(storage) { + storage.put( + RESOURCES_SELFIE_NAME, + JSON.stringify({ + version: RESOURCES_SELFIE_VERSION, + aliases: Array.from(this.aliases), + resources: Array.from(this.resources), + }) + ); + } + + async resourcesFromSelfie(storage) { + const result = await storage.get(RESOURCES_SELFIE_NAME); + let selfie; + try { + selfie = JSON.parse(result.content); + } catch(ex) { + } + if ( + selfie instanceof Object === false || + selfie.version !== RESOURCES_SELFIE_VERSION || + Array.isArray(selfie.resources) === false + ) { + return false; + } + this.aliases = new Map(selfie.aliases); + this.resources = new Map(); + for ( const [ token, entry ] of selfie.resources ) { + this.resources.set(token, RedirectEntry.fromDetails(entry)); + } + return true; + } + + invalidateResourcesSelfie(storage) { + storage.remove(RESOURCES_SELFIE_NAME); + } +} + +/******************************************************************************/ + +const redirectEngine = new RedirectEngine(); + +export { redirectEngine }; + +/******************************************************************************/ diff --git a/src/js/redirect-resources.js b/src/js/redirect-resources.js new file mode 100644 index 0000000..b8577e3 --- /dev/null +++ b/src/js/redirect-resources.js @@ -0,0 +1,182 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// The resources referenced below are found in ./web_accessible_resources/ +// +// The content of the resources which declare a `data` property will be loaded +// in memory, and converted to a suitable internal format depending on the +// type of the loaded data. The `data` property allows for manual injection +// through `+js(...)`, or for redirection to a data: URI when a redirection +// to a web accessible resource is not desirable. + +export default new Map([ + [ '1x1.gif', { + alias: '1x1-transparent.gif', + data: 'blob', + } ], + [ '2x2.png', { + alias: '2x2-transparent.png', + data: 'blob', + } ], + [ '3x2.png', { + alias: '3x2-transparent.png', + data: 'blob', + } ], + [ '32x32.png', { + alias: '32x32-transparent.png', + data: 'blob', + } ], + [ 'amazon_ads.js', { + alias: 'amazon-adsystem.com/aax2/amzn_ads.js', + data: 'text', + } ], + [ 'amazon_apstag.js', { + } ], + [ 'ampproject_v0.js', { + alias: 'ampproject.org/v0.js', + } ], + [ 'chartbeat.js', { + alias: 'static.chartbeat.com/chartbeat.js', + } ], + [ 'click2load.html', { + params: [ 'aliasURL', 'url' ], + } ], + [ 'doubleclick_instream_ad_status.js', { + alias: 'doubleclick.net/instream/ad_status.js', + data: 'text', + } ], + [ 'empty', { + data: 'text', // Important! + } ], + [ 'fingerprint2.js', { + data: 'text', + } ], + [ 'fingerprint3.js', { + data: 'text', + } ], + [ 'google-analytics_analytics.js', { + alias: [ + 'google-analytics.com/analytics.js', + 'googletagmanager_gtm.js', + 'googletagmanager.com/gtm.js' + ], + data: 'text', + } ], + [ 'google-analytics_cx_api.js', { + alias: 'google-analytics.com/cx/api.js', + } ], + [ 'google-analytics_ga.js', { + alias: 'google-analytics.com/ga.js', + data: 'text', + } ], + [ 'google-analytics_inpage_linkid.js', { + alias: 'google-analytics.com/inpage_linkid.js', + } ], + [ 'google-ima.js', { + alias: 'google-ima3', /* adguard compatibility */ + } ], + [ 'googlesyndication_adsbygoogle.js', { + alias: [ + 'googlesyndication.com/adsbygoogle.js', + 'googlesyndication-adsbygoogle', /* adguard compatibility */ + ], + data: 'text', + } ], + [ 'googletagservices_gpt.js', { + alias: [ + 'googletagservices.com/gpt.js', + 'googletagservices-gpt', /* adguard compatibility */ + ], + data: 'text', + } ], + [ 'hd-main.js', { + } ], + [ 'nobab.js', { + alias: [ 'bab-defuser.js', 'prevent-bab.js' ], + data: 'text', + } ], + [ 'nobab2.js', { + data: 'text', + } ], + [ 'noeval.js', { + data: 'text', + } ], + [ 'noeval-silent.js', { + alias: 'silent-noeval.js', + data: 'text', + } ], + [ 'nofab.js', { + alias: 'fuckadblock.js-3.2.0', + data: 'text', + } ], + [ 'noop-0.1s.mp3', { + alias: [ 'noopmp3-0.1s', 'abp-resource:blank-mp3' ], + data: 'blob', + } ], + [ 'noop-0.5s.mp3', { + } ], + [ 'noop-1s.mp4', { + alias: [ 'noopmp4-1s', 'abp-resource:blank-mp4' ], + data: 'blob', + } ], + [ 'noop.css', { + data: 'text', + } ], + [ 'noop.html', { + alias: 'noopframe', + } ], + [ 'noop.js', { + alias: [ 'noopjs', 'abp-resource:blank-js' ], + data: 'text', + } ], + [ 'noop.json', { + alias: [ 'noopjson' ], + data: 'text', + } ], + [ 'noop.txt', { + alias: 'nooptext', + data: 'text', + } ], + [ 'noop-vmap1.0.xml', { + alias: 'noopvmap-1.0', + data: 'text', + } ], + [ 'outbrain-widget.js', { + alias: 'widgets.outbrain.com/outbrain.js', + } ], + [ 'popads.js', { + alias: [ 'popads.net.js', 'prevent-popads-net.js' ], + data: 'text', + } ], + [ 'popads-dummy.js', { + data: 'text', + } ], + [ 'prebid-ads.js', { + data: 'text', + } ], + [ 'scorecardresearch_beacon.js', { + alias: 'scorecardresearch.com/beacon.js', + } ], +]); diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js new file mode 100644 index 0000000..37b8b65 --- /dev/null +++ b/src/js/reverselookup-worker.js @@ -0,0 +1,287 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +let listEntries = Object.create(null); + +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/2092 +// Order of ids matters + +const extractBlocks = function(content, ...ids) { + const out = []; + for ( const id of ids ) { + const pattern = `#block-start-${id}\n`; + let beg = content.indexOf(pattern); + if ( beg === -1 ) { continue; } + beg += pattern.length; + const end = content.indexOf(`#block-end-${id}`, beg); + out.push(content.slice(beg, end)); + } + return out.join('\n'); +}; + +/******************************************************************************/ + +// https://github.com/MajkiIT/polish-ads-filter/issues/14768#issuecomment-536006312 +// Avoid reporting badfilter-ed filters. + +const fromNetFilter = function(details) { + const lists = []; + const compiledFilter = details.compiledFilter; + + for ( const assetKey in listEntries ) { + const entry = listEntries[assetKey]; + if ( entry === undefined ) { continue; } + if ( entry.networkContent === undefined ) { + entry.networkContent = extractBlocks(entry.content, 'NETWORK_FILTERS:GOOD'); + } + const content = entry.networkContent; + let pos = 0; + for (;;) { + pos = content.indexOf(compiledFilter, pos); + if ( pos === -1 ) { break; } + // We need an exact match. + // https://github.com/gorhill/uBlock/issues/1392 + // https://github.com/gorhill/uBlock/issues/835 + const notFound = pos !== 0 && content.charCodeAt(pos - 1) !== 0x0A; + pos += compiledFilter.length; + if ( + notFound || + pos !== content.length && content.charCodeAt(pos) !== 0x0A + ) { + continue; + } + lists.push({ + assetKey: assetKey, + title: entry.title, + supportURL: entry.supportURL + }); + break; + } + } + + const response = {}; + response[details.rawFilter] = lists; + + self.postMessage({ id: details.id, response }); +}; + +/******************************************************************************/ + +// Looking up filter lists from a cosmetic filter is a bit more complicated +// than with network filters: +// +// The filter is its raw representation, not its compiled version. This is +// because the cosmetic filtering engine can't translate a live cosmetic +// filter into its compiled version. Reason is I do not want to burden +// cosmetic filtering with the resource overhead of being able to recompile +// live cosmetic filters. I want the cosmetic filtering code to be left +// completely unaffected by reverse lookup requirements. +// +// Mainly, given a CSS selector and a hostname as context, we will derive +// various versions of compiled filters and see if there are matches. This +// way the whole CPU cost is incurred by the reverse lookup code -- in a +// worker thread, and the cosmetic filtering engine incurs no cost at all. +// +// For this though, the reverse lookup code here needs some knowledge of +// the inners of the cosmetic filtering engine. +// FilterContainer.fromCompiledContent() is our reference code to create +// the various compiled versions. + +const fromExtendedFilter = function(details) { + const match = /^#@?#\^?/.exec(details.rawFilter); + const prefix = match[0]; + const exception = prefix.charAt(1) === '@'; + const selector = details.rawFilter.slice(prefix.length); + const isHtmlFilter = prefix.endsWith('^'); + const hostname = details.hostname; + + // The longer the needle, the lower the number of false positives. + // https://github.com/uBlockOrigin/uBlock-issues/issues/1139 + // Mind that there is no guarantee a selector has `\w` characters. + const needle = selector.match(/\w+|\*/g).reduce(function(a, b) { + return a.length > b.length ? a : b; + }); + + const regexFromLabels = (prefix, hn, suffix) => + new RegExp( + prefix + + hn.split('.').reduce((acc, item) => `(${acc}\\.)?${item}`) + + suffix + ); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/803 + // Support looking up selectors of the form `*##...` + const reHostname = regexFromLabels('^', hostname, '$'); + let reEntity; + { + const domain = details.domain; + const pos = domain.indexOf('.'); + if ( pos !== -1 ) { + reEntity = regexFromLabels( + '^(', + hostname.slice(0, pos + hostname.length - domain.length), + '\\.)?\\*$' + ); + } + } + + const hostnameMatches = hn => { + if ( hn === '' ) { return true; } + if ( hn.charCodeAt(0) === 0x2F /* / */ ) { + return (new RegExp(hn.slice(1,-1))).test(hostname); + } + if ( reHostname.test(hn) ) { return true; } + if ( reEntity === undefined ) { return false; } + if ( reEntity.test(hn) ) { return true; } + return false; + }; + + const response = Object.create(null); + + for ( const assetKey in listEntries ) { + const entry = listEntries[assetKey]; + if ( entry === undefined ) { continue; } + if ( entry.extendedContent === undefined ) { + entry.extendedContent = extractBlocks( + entry.content, + 'COSMETIC_FILTERS:SPECIFIC', + 'COSMETIC_FILTERS:GENERIC', + 'SCRIPTLET_FILTERS', + 'HTML_FILTERS', + 'HTTPHEADER_FILTERS' + ); + } + const content = entry.extendedContent; + let found; + let pos = 0; + while ( (pos = content.indexOf(needle, pos)) !== -1 ) { + let beg = content.lastIndexOf('\n', pos); + if ( beg === -1 ) { beg = 0; } + let end = content.indexOf('\n', pos); + if ( end === -1 ) { end = content.length; } + pos = end; + const fargs = JSON.parse(content.slice(beg, end)); + const filterType = fargs[0]; + + // https://github.com/gorhill/uBlock/issues/2763 + if ( filterType === 0 && details.ignoreGeneric ) { continue; } + + // Do not confuse cosmetic filters with HTML ones. + if ( (filterType === 64) !== isHtmlFilter ) { continue; } + + switch ( filterType ) { + // Lowly generic cosmetic filters + case 0: + if ( exception ) { break; } + if ( fargs[2] !== selector ) { break; } + found = prefix + selector; + break; + // Highly generic cosmetic filters + case 4: // simple highly generic + case 5: // complex highly generic + if ( exception ) { break; } + if ( fargs[1] !== selector ) { break; } + found = prefix + selector; + break; + // Specific cosmetic filtering + // Generic exception + case 8: + // HTML filtering + // Response header filtering + case 64: { + if ( exception !== ((fargs[2] & 0b001) !== 0) ) { break; } + const isProcedural = (fargs[2] & 0b010) !== 0; + if ( + isProcedural === false && fargs[3] !== selector || + isProcedural && JSON.parse(fargs[3]).raw !== selector + ) { + break; + } + if ( hostnameMatches(fargs[1]) === false ) { break; } + // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ + // Ignore match if specific cosmetic filters are disabled + if ( + filterType === 8 && + exception === false && + details.ignoreSpecific + ) { + break; + } + found = fargs[1] + prefix + selector; + break; + } + // Scriptlet injection + case 32: + if ( exception !== ((fargs[2] & 0b001) !== 0) ) { break; } + if ( fargs[3] !== details.compiled ) { break; } + if ( hostnameMatches(fargs[1]) ) { + found = fargs[1] + prefix + selector; + } + break; + } + if ( found !== undefined ) { + if ( response[found] === undefined ) { + response[found] = []; + } + response[found].push({ + assetKey: assetKey, + title: entry.title, + supportURL: entry.supportURL + }); + break; + } + } + } + + self.postMessage({ id: details.id, response }); +}; + +/******************************************************************************/ + +self.onmessage = function(e) { + const msg = e.data; + + switch ( msg.what ) { + case 'resetLists': + listEntries = Object.create(null); + break; + + case 'setList': + listEntries[msg.details.assetKey] = msg.details; + break; + + case 'fromNetFilter': + fromNetFilter(msg); + break; + + case 'fromExtendedFilter': + fromExtendedFilter(msg); + break; + } +}; + +/******************************************************************************/ diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js new file mode 100644 index 0000000..c21ca4b --- /dev/null +++ b/src/js/reverselookup.js @@ -0,0 +1,223 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import staticNetFilteringEngine from './static-net-filtering.js'; +import µb from './background.js'; +import { CompiledListWriter } from './static-filtering-io.js'; +import { i18n$ } from './i18n.js'; +import * as sfp from './static-filtering-parser.js'; + +import { + domainFromHostname, + hostnameFromURI, +} from './uri-utils.js'; + +/******************************************************************************/ + +const pendingResponses = new Map(); + +let worker = null; +let needLists = true; +let messageId = 1; + +const onWorkerMessage = function(e) { + const msg = e.data; + const resolver = pendingResponses.get(msg.id); + pendingResponses.delete(msg.id); + resolver(msg.response); +}; + +const stopWorker = function() { + workerTTLTimer.off(); + if ( worker === null ) { return; } + worker.terminate(); + worker = null; + needLists = true; + for ( const resolver of pendingResponses.values() ) { + resolver(); + } + pendingResponses.clear(); +}; + +const workerTTLTimer = vAPI.defer.create(stopWorker); +const workerTTL = { min: 5 }; + +const initWorker = function() { + if ( worker === null ) { + worker = new Worker('js/reverselookup-worker.js'); + worker.onmessage = onWorkerMessage; + } + + // The worker will be shutdown after n minutes without being used. + workerTTLTimer.offon(workerTTL); + + if ( needLists === false ) { + return Promise.resolve(); + } + needLists = false; + + const entries = new Map(); + + const onListLoaded = function(details) { + const entry = entries.get(details.assetKey); + + // https://github.com/gorhill/uBlock/issues/536 + // Use assetKey when there is no filter list title. + + worker.postMessage({ + what: 'setList', + details: { + assetKey: details.assetKey, + title: entry.title || details.assetKey, + supportURL: entry.supportURL, + content: details.content + } + }); + }; + + for ( const listKey in µb.availableFilterLists ) { + if ( µb.availableFilterLists.hasOwnProperty(listKey) === false ) { + continue; + } + const entry = µb.availableFilterLists[listKey]; + if ( entry.off === true ) { continue; } + entries.set(listKey, { + title: listKey !== µb.userFiltersPath ? + entry.title : + i18n$('1pPageName'), + supportURL: entry.supportURL || '' + }); + } + if ( entries.size === 0 ) { + return Promise.resolve(); + } + + const promises = []; + for ( const listKey of entries.keys() ) { + promises.push( + µb.getCompiledFilterList(listKey).then(details => { + onListLoaded(details); + }) + ); + } + return Promise.all(promises); +}; + +const fromNetFilter = async function(rawFilter) { + if ( typeof rawFilter !== 'string' || rawFilter === '' ) { return; } + + const writer = new CompiledListWriter(); + const parser = new sfp.AstFilterParser({ + trustedSource: true, + maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); + parser.parse(rawFilter); + + const compiler = staticNetFilteringEngine.createCompiler(); + if ( compiler.compile(parser, writer) === false ) { return; } + + await initWorker(); + + const id = messageId++; + worker.postMessage({ + what: 'fromNetFilter', + id, + compiledFilter: writer.last(), + rawFilter, + }); + + return new Promise(resolve => { + pendingResponses.set(id, resolve); + }); +}; + +const fromExtendedFilter = async function(details) { + if ( + typeof details.rawFilter !== 'string' || + details.rawFilter === '' + ) { + return; + } + + await initWorker(); + + const id = messageId++; + const hostname = hostnameFromURI(details.url); + + const parser = new sfp.AstFilterParser({ + trustedSource: true, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); + parser.parse(details.rawFilter); + let compiled; + if ( parser.isScriptletFilter() ) { + compiled = JSON.stringify(parser.getScriptletArgs()); + } + + worker.postMessage({ + what: 'fromExtendedFilter', + id, + domain: domainFromHostname(hostname), + hostname, + ignoreGeneric: + staticNetFilteringEngine.matchRequestReverse( + 'generichide', + details.url + ) === 2, + ignoreSpecific: + staticNetFilteringEngine.matchRequestReverse( + 'specifichide', + details.url + ) === 2, + rawFilter: details.rawFilter, + compiled, + }); + + return new Promise(resolve => { + pendingResponses.set(id, resolve); + }); +}; + +// This tells the worker that filter lists may have changed. + +const resetLists = function() { + needLists = true; + if ( worker === null ) { return; } + worker.postMessage({ what: 'resetLists' }); +}; + +/******************************************************************************/ + +const staticFilteringReverseLookup = { + fromNetFilter, + fromExtendedFilter, + resetLists, + shutdown: stopWorker +}; + +export default staticFilteringReverseLookup; + +/******************************************************************************/ diff --git a/src/js/scriptlet-filtering-core.js b/src/js/scriptlet-filtering-core.js new file mode 100644 index 0000000..125eb87 --- /dev/null +++ b/src/js/scriptlet-filtering-core.js @@ -0,0 +1,300 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import { redirectEngine as reng } from './redirect-engine.js'; +import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js'; + +/******************************************************************************/ + +// Increment when internal representation changes +const VERSION = 1; + +const $scriptlets = new Set(); +const $exceptions = new Set(); +const $mainWorldMap = new Map(); +const $isolatedWorldMap = new Map(); + +/******************************************************************************/ + +const normalizeRawFilter = (parser, sourceIsTrusted = false) => { + const args = parser.getScriptletArgs(); + if ( args.length !== 0 ) { + let token = `${args[0]}.js`; + if ( reng.aliases.has(token) ) { + token = reng.aliases.get(token); + } + if ( parser.isException() !== true ) { + if ( sourceIsTrusted !== true ) { + if ( reng.tokenRequiresTrust(token) ) { return; } + } + } + args[0] = token.slice(0, -3); + } + return JSON.stringify(args); +}; + +const lookupScriptlet = (rawToken, mainMap, isolatedMap, debug = false) => { + if ( mainMap.has(rawToken) || isolatedMap.has(rawToken) ) { return; } + const args = JSON.parse(rawToken); + const token = `${args[0]}.js`; + const details = reng.contentFromName(token, 'text/javascript'); + if ( details === undefined ) { return; } + const targetWorldMap = details.world !== 'ISOLATED' ? mainMap : isolatedMap; + const content = patchScriptlet(details.js, args.slice(1)); + const dependencies = details.dependencies || []; + while ( dependencies.length !== 0 ) { + const token = dependencies.shift(); + if ( targetWorldMap.has(token) ) { continue; } + const details = reng.contentFromName(token, 'fn/javascript') || + reng.contentFromName(token, 'text/javascript'); + if ( details === undefined ) { continue; } + targetWorldMap.set(token, details.js); + if ( Array.isArray(details.dependencies) === false ) { continue; } + dependencies.push(...details.dependencies); + } + targetWorldMap.set(rawToken, [ + 'try {', + '// >>>> scriptlet start', + content, + '// <<<< scriptlet end', + '} catch (e) {', + debug ? 'console.error(e);' : '', + '}', + ].join('\n')); +}; + +// Fill-in scriptlet argument placeholders. +const patchScriptlet = (content, arglist) => { + if ( content.startsWith('function') && content.endsWith('}') ) { + content = `(${content})({{args}});`; + } + for ( let i = 0; i < arglist.length; i++ ) { + content = content.replace(`{{${i+1}}}`, arglist[i]); + } + return content.replace('{{args}}', + JSON.stringify(arglist).slice(1,-1).replace(/\$/g, '$$$') + ); +}; + +const decompile = json => { + const args = JSON.parse(json).map(s => s.replace(/,/g, '\\,')); + if ( args.length === 0 ) { return '+js()'; } + return `+js(${args.join(', ')})`; +}; + +/******************************************************************************/ + +export class ScriptletFilteringEngine { + constructor() { + this.acceptedCount = 0; + this.discardedCount = 0; + this.scriptletDB = new StaticExtFilteringHostnameDB(1, VERSION); + this.duplicates = new Set(); + } + + getFilterCount() { + return this.scriptletDB.size; + } + + reset() { + this.scriptletDB.clear(); + this.duplicates.clear(); + this.acceptedCount = 0; + this.discardedCount = 0; + } + + freeze() { + this.duplicates.clear(); + this.scriptletDB.collectGarbage(); + } + + // parser: instance of AstFilterParser from static-filtering-parser.js + // writer: instance of CompiledListWriter from static-filtering-io.js + compile(parser, writer) { + writer.select('SCRIPTLET_FILTERS'); + + // Only exception filters are allowed to be global. + const isException = parser.isException(); + const normalized = normalizeRawFilter(parser, writer.properties.get('trustedSource')); + + // Can fail if there is a mismatch with trust requirement + if ( normalized === undefined ) { return; } + + // Tokenless is meaningful only for exception filters. + if ( normalized === '[]' && isException === false ) { return; } + + if ( parser.hasOptions() === false ) { + if ( isException ) { + writer.push([ 32, '', 1, normalized ]); + } + return; + } + + // https://github.com/gorhill/uBlock/issues/3375 + // Ignore instances of exception filter with negated hostnames, + // because there is no way to create an exception to an exception. + + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + let kind = 0; + if ( isException ) { + if ( not ) { continue; } + kind |= 1; + } else if ( not ) { + kind |= 1; + } + writer.push([ 32, hn, kind, normalized ]); + } + } + + // writer: instance of CompiledListReader from static-filtering-io.js + fromCompiledContent(reader) { + reader.select('SCRIPTLET_FILTERS'); + + while ( reader.next() ) { + this.acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( this.duplicates.has(fingerprint) ) { + this.discardedCount += 1; + continue; + } + this.duplicates.add(fingerprint); + const args = reader.args(); + if ( args.length < 4 ) { continue; } + this.scriptletDB.store(args[1], args[2], args[3]); + } + } + + toSelfie() { + return this.scriptletDB.toSelfie(); + } + + fromSelfie(selfie) { + if ( selfie instanceof Object === false ) { return false; } + if ( selfie.version !== VERSION ) { return false; } + this.scriptletDB.fromSelfie(selfie); + return true; + } + + retrieve(request, options = {}) { + if ( this.scriptletDB.size === 0 ) { return; } + + $scriptlets.clear(); + $exceptions.clear(); + + const { hostname } = request; + + this.scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]); + const entity = request.entity !== '' + ? `${hostname.slice(0, -request.domain.length)}${request.entity}` + : '*'; + this.scriptletDB.retrieve(entity, [ $scriptlets, $exceptions ], 1); + if ( $scriptlets.size === 0 ) { return; } + + // Wholly disable scriptlet injection? + if ( $exceptions.has('[]') ) { + return { filters: '#@#+js()' }; + } + + for ( const token of $exceptions ) { + if ( $scriptlets.has(token) ) { + $scriptlets.delete(token); + } else { + $exceptions.delete(token); + } + } + + for ( const token of $scriptlets ) { + lookupScriptlet(token, $mainWorldMap, $isolatedWorldMap, options.debug); + } + + const mainWorldCode = []; + for ( const js of $mainWorldMap.values() ) { + mainWorldCode.push(js); + } + + const isolatedWorldCode = []; + for ( const js of $isolatedWorldMap.values() ) { + isolatedWorldCode.push(js); + } + + const scriptletDetails = { + mainWorld: mainWorldCode.join('\n\n'), + isolatedWorld: isolatedWorldCode.join('\n\n'), + filters: [ + ...Array.from($scriptlets).map(s => `##${decompile(s)}`), + ...Array.from($exceptions).map(s => `#@#${decompile(s)}`), + ].join('\n'), + }; + $mainWorldMap.clear(); + $isolatedWorldMap.clear(); + + if ( scriptletDetails.mainWorld === '' ) { + if ( scriptletDetails.isolatedWorld === '' ) { + return { filters: scriptletDetails.filters }; + } + } + + const scriptletGlobals = options.scriptletGlobals || []; + + if ( options.debug ) { + scriptletGlobals.push([ 'canDebug', true ]); + } + + return { + mainWorld: scriptletDetails.mainWorld === '' ? '' : [ + '(function() {', + '// >>>> start of private namespace', + '', + options.debugScriptlets ? 'debugger;' : ';', + '', + // For use by scriptlets to share local data among themselves + `const scriptletGlobals = new Map(${JSON.stringify(scriptletGlobals, null, 2)});`, + '', + scriptletDetails.mainWorld, + '', + '// <<<< end of private namespace', + '})();', + ].join('\n'), + isolatedWorld: scriptletDetails.isolatedWorld === '' ? '' : [ + 'function() {', + '// >>>> start of private namespace', + '', + options.debugScriptlets ? 'debugger;' : ';', + '', + // For use by scriptlets to share local data among themselves + `const scriptletGlobals = new Map(${JSON.stringify(scriptletGlobals, null, 2)});`, + '', + scriptletDetails.isolatedWorld, + '', + '// <<<< end of private namespace', + '}', + ].join('\n'), + filters: scriptletDetails.filters, + }; + } +} + +/******************************************************************************/ diff --git a/src/js/scriptlet-filtering.js b/src/js/scriptlet-filtering.js new file mode 100644 index 0000000..10da19f --- /dev/null +++ b/src/js/scriptlet-filtering.js @@ -0,0 +1,328 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; +import logger from './logger.js'; +import { onBroadcast } from './broadcast.js'; +import { redirectEngine as reng } from './redirect-engine.js'; +import { sessionFirewall } from './filtering-engines.js'; +import { MRUCache } from './mrucache.js'; +import { ScriptletFilteringEngine } from './scriptlet-filtering-core.js'; + +import { + domainFromHostname, + entityFromDomain, + hostnameFromURI, +} from './uri-utils.js'; + +/******************************************************************************/ + +const contentScriptRegisterer = new (class { + constructor() { + this.hostnameToDetails = new Map(); + if ( browser.contentScripts === undefined ) { return; } + onBroadcast(msg => { + if ( msg.what !== 'filteringBehaviorChanged' ) { return; } + if ( msg.direction > 0 ) { return; } + if ( msg.hostname ) { return this.flush(msg.hostname); } + this.reset(); + }); + } + register(hostname, code) { + if ( browser.contentScripts === undefined ) { return false; } + if ( hostname === '' ) { return false; } + const details = this.hostnameToDetails.get(hostname); + if ( details !== undefined ) { + if ( code === details.code ) { + return details.handle instanceof Promise === false; + } + details.handle.unregister(); + this.hostnameToDetails.delete(hostname); + } + const promise = browser.contentScripts.register({ + js: [ { code } ], + allFrames: true, + matches: [ `*://*.${hostname}/*` ], + matchAboutBlank: true, + runAt: 'document_start', + }).then(handle => { + this.hostnameToDetails.set(hostname, { handle, code }); + }).catch(( ) => { + this.hostnameToDetails.delete(hostname); + }); + this.hostnameToDetails.set(hostname, { handle: promise, code }); + return false; + } + unregister(hostname) { + if ( this.hostnameToDetails.size === 0 ) { return; } + const details = this.hostnameToDetails.get(hostname); + if ( details === undefined ) { return; } + this.hostnameToDetails.delete(hostname); + this.unregisterHandle(details.handle); + } + flush(hostname) { + if ( hostname === '*' ) { return this.reset(); } + for ( const hn of this.hostnameToDetails.keys() ) { + if ( hn.endsWith(hostname) === false ) { continue; } + const pos = hn.length - hostname.length; + if ( pos !== 0 && hn.charCodeAt(pos-1) !== 0x2E /* . */ ) { continue; } + this.unregister(hn); + } + } + reset() { + if ( this.hostnameToDetails.size === 0 ) { return; } + for ( const details of this.hostnameToDetails.values() ) { + this.unregisterHandle(details.handle); + } + this.hostnameToDetails.clear(); + } + unregisterHandle(handle) { + if ( handle instanceof Promise ) { + handle.then(handle => { handle.unregister(); }); + } else { + handle.unregister(); + } + } +})(); + +/******************************************************************************/ + +const mainWorldInjector = (( ) => { + const parts = [ + '(', + function(injector, details) { + if ( typeof self.uBO_scriptletsInjected === 'string' ) { return; } + const doc = document; + if ( doc.location === null ) { return; } + const hostname = doc.location.hostname; + if ( hostname !== '' && details.hostname !== hostname ) { return; } + injector(doc, details); + return 0; + }.toString(), + ')(', + vAPI.scriptletsInjector, ', ', + 'json-slot', + ');', + ]; + return { + parts, + jsonSlot: parts.indexOf('json-slot'), + assemble: function(hostname, scriptlets, filters) { + this.parts[this.jsonSlot] = JSON.stringify({ + hostname, + scriptlets, + filters, + }); + return this.parts.join(''); + }, + }; +})(); + +const isolatedWorldInjector = (( ) => { + const parts = [ + '(', + function(details) { + if ( self.uBO_isolatedScriptlets === 'done' ) { return; } + const doc = document; + if ( doc.location === null ) { return; } + const hostname = doc.location.hostname; + if ( hostname !== '' && details.hostname !== hostname ) { return; } + const isolatedScriptlets = function(){}; + isolatedScriptlets(); + self.uBO_isolatedScriptlets = 'done'; + return 0; + }.toString(), + ')(', + 'json-slot', + ');', + ]; + return { + parts, + jsonSlot: parts.indexOf('json-slot'), + assemble: function(hostname, scriptlets) { + this.parts[this.jsonSlot] = JSON.stringify({ hostname }); + const code = this.parts.join(''); + // Manually substitute noop function with scriptlet wrapper + // function, so as to not suffer instances of special + // replacement characters `$`,`\` when using String.replace() + // with scriptlet code. + const match = /function\(\)\{\}/.exec(code); + return code.slice(0, match.index) + + scriptlets + + code.slice(match.index + match[0].length); + }, + }; +})(); + +/******************************************************************************/ + +export class ScriptletFilteringEngineEx extends ScriptletFilteringEngine { + constructor() { + super(); + this.warOrigin = vAPI.getURL('/web_accessible_resources'); + this.warSecret = undefined; + this.scriptletCache = new MRUCache(32); + this.isDevBuild = undefined; + onBroadcast(msg => { + if ( msg.what !== 'hiddenSettingsChanged' ) { return; } + this.scriptletCache.reset(); + this.isDevBuild = undefined; + }); + } + + reset() { + super.reset(); + this.warSecret = vAPI.warSecret.long(this.warSecret); + this.scriptletCache.reset(); + contentScriptRegisterer.reset(); + } + + freeze() { + super.freeze(); + this.warSecret = vAPI.warSecret.long(this.warSecret); + this.scriptletCache.reset(); + contentScriptRegisterer.reset(); + } + + retrieve(request) { + const { hostname } = request; + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not inject scriptlets if the site is under an `allow` rule. + if ( µb.userSettings.advancedUserEnabled ) { + if ( sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 ) { + return; + } + } + + if ( this.scriptletCache.resetTime < reng.modifyTime ) { + this.warSecret = vAPI.warSecret.long(this.warSecret); + this.scriptletCache.reset(); + } + + let scriptletDetails = this.scriptletCache.lookup(hostname); + if ( scriptletDetails !== undefined ) { + return scriptletDetails || undefined; + } + + if ( this.isDevBuild === undefined ) { + this.isDevBuild = vAPI.webextFlavor.soup.has('devbuild') || + µb.hiddenSettings.filterAuthorMode; + } + + if ( this.warSecret === undefined ) { + this.warSecret = vAPI.warSecret.long(); + } + + const options = { + scriptletGlobals: [ + [ 'warOrigin', this.warOrigin ], + [ 'warSecret', this.warSecret ], + ], + debug: this.isDevBuild, + debugScriptlets: µb.hiddenSettings.debugScriptlets, + }; + + scriptletDetails = super.retrieve(request, options); + + this.scriptletCache.add(hostname, scriptletDetails || null); + + return scriptletDetails; + } + + injectNow(details) { + if ( typeof details.frameId !== 'number' ) { return; } + + const request = { + tabId: details.tabId, + frameId: details.frameId, + url: details.url, + hostname: hostnameFromURI(details.url), + domain: undefined, + entity: undefined + }; + + request.domain = domainFromHostname(request.hostname); + request.entity = entityFromDomain(request.domain); + + const scriptletDetails = this.retrieve(request); + if ( scriptletDetails === undefined ) { + contentScriptRegisterer.unregister(request.hostname); + return; + } + + const contentScript = []; + if ( µb.hiddenSettings.debugScriptletInjector ) { + contentScript.push('debugger'); + } + const { mainWorld = '', isolatedWorld = '', filters } = scriptletDetails; + if ( mainWorld !== '' ) { + contentScript.push(mainWorldInjector.assemble(request.hostname, mainWorld, filters)); + } + if ( isolatedWorld !== '' ) { + contentScript.push(isolatedWorldInjector.assemble(request.hostname, isolatedWorld)); + } + + const code = contentScript.join('\n\n'); + + const isAlreadyInjected = contentScriptRegisterer.register(request.hostname, code); + if ( isAlreadyInjected !== true ) { + vAPI.tabs.executeScript(details.tabId, { + code, + frameId: details.frameId, + matchAboutBlank: true, + runAt: 'document_start', + }); + } + + return scriptletDetails; + } + + toLogger(request, details) { + if ( details === undefined ) { return; } + if ( logger.enabled !== true ) { return; } + if ( typeof details.filters !== 'string' ) { return; } + const fctxt = µb.filteringContext + .duplicate() + .fromTabId(request.tabId) + .setRealm('extended') + .setType('scriptlet') + .setURL(request.url) + .setDocOriginFromURL(request.url); + for ( const raw of details.filters.split('\n') ) { + fctxt.setFilter({ source: 'extended', raw }).toLogger(); + } + } +} + +/******************************************************************************/ + +const scriptletFilteringEngine = new ScriptletFilteringEngineEx(); + +export default scriptletFilteringEngine; + +/******************************************************************************/ diff --git a/src/js/scriptlets/cosmetic-logger.js b/src/js/scriptlets/cosmetic-logger.js new file mode 100644 index 0000000..5d1f1b9 --- /dev/null +++ b/src/js/scriptlets/cosmetic-logger.js @@ -0,0 +1,365 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +(( ) => { +// >>>>>>>> start of private namespace + +/******************************************************************************/ + +if ( typeof vAPI !== 'object' ) { return; } +if ( vAPI.domWatcher instanceof Object === false ) { return; } + +const reHasCSSCombinators = /[ >+~]/; +const simpleDeclarativeSet = new Set(); +let simpleDeclarativeStr; +const complexDeclarativeSet = new Set(); +let complexDeclarativeStr; +const proceduralDict = new Map(); +const exceptionDict = new Map(); +let exceptionStr; +const proceduralExceptionDict = new Map(); +const nodesToProcess = new Set(); +const loggedSelectors = new Set(); + +/******************************************************************************/ + +const rePseudoElements = /:(?::?after|:?before|:[a-z-]+)$/; + +function hasSelector(selector, context = document) { + try { + return context.querySelector(selector) !== null; + } + catch(ex) { + } + return false; +} + +function safeMatchSelector(selector, context) { + const safeSelector = rePseudoElements.test(selector) + ? selector.replace(rePseudoElements, '') + : selector; + try { + return context.matches(safeSelector); + } + catch(ex) { + } + return false; +} + +function safeQuerySelector(selector, context = document) { + const safeSelector = rePseudoElements.test(selector) + ? selector.replace(rePseudoElements, '') + : selector; + try { + return context.querySelector(safeSelector); + } + catch(ex) { + } + return null; +} + +function safeGroupSelectors(selectors) { + const arr = Array.isArray(selectors) + ? selectors + : Array.from(selectors); + return arr.map(s => { + return rePseudoElements.test(s) + ? s.replace(rePseudoElements, '') + : s; + }).join(',\n'); +} + +/******************************************************************************/ + +function processDeclarativeSimple(node, out) { + if ( simpleDeclarativeSet.size === 0 ) { return; } + if ( simpleDeclarativeStr === undefined ) { + simpleDeclarativeStr = safeGroupSelectors(simpleDeclarativeSet); + } + if ( + (node === document || node.matches(simpleDeclarativeStr) === false) && + (hasSelector(simpleDeclarativeStr, node) === false) + ) { + return; + } + for ( const selector of simpleDeclarativeSet ) { + if ( + (node === document || safeMatchSelector(selector, node) === false) && + (safeQuerySelector(selector, node) === null) + ) { + continue; + } + out.push(`##${selector}`); + simpleDeclarativeSet.delete(selector); + simpleDeclarativeStr = undefined; + loggedSelectors.add(selector); + } +} + +/******************************************************************************/ + +function processDeclarativeComplex(out) { + if ( complexDeclarativeSet.size === 0 ) { return; } + if ( complexDeclarativeStr === undefined ) { + complexDeclarativeStr = safeGroupSelectors(complexDeclarativeSet); + } + if ( hasSelector(complexDeclarativeStr) === false ) { return; } + for ( const selector of complexDeclarativeSet ) { + if ( safeQuerySelector(selector) === null ) { continue; } + out.push(`##${selector}`); + complexDeclarativeSet.delete(selector); + complexDeclarativeStr = undefined; + loggedSelectors.add(selector); + } +} + +/******************************************************************************/ + +function processProcedural(out) { + if ( proceduralDict.size === 0 ) { return; } + for ( const [ raw, pselector ] of proceduralDict ) { + if ( pselector.converted ) { + if ( safeQuerySelector(pselector.selector) === null ) { continue; } + } else if ( pselector.hit === false && pselector.exec().length === 0 ) { + continue; + } + out.push(`##${raw}`); + proceduralDict.delete(raw); + } +} + +/******************************************************************************/ + +function processExceptions(out) { + if ( exceptionDict.size === 0 ) { return; } + if ( exceptionStr === undefined ) { + exceptionStr = safeGroupSelectors(exceptionDict.keys()); + } + if ( hasSelector(exceptionStr) === false ) { return; } + for ( const [ selector, raw ] of exceptionDict ) { + if ( safeQuerySelector(selector) === null ) { continue; } + out.push(`#@#${raw}`); + exceptionDict.delete(selector); + exceptionStr = undefined; + loggedSelectors.add(raw); + } +} + +/******************************************************************************/ + +function processProceduralExceptions(out) { + if ( proceduralExceptionDict.size === 0 ) { return; } + for ( const exception of proceduralExceptionDict.values() ) { + if ( exception.test() === false ) { continue; } + out.push(`#@#${exception.raw}`); + proceduralExceptionDict.delete(exception.raw); + } +} + +/******************************************************************************/ + +const processTimer = new vAPI.SafeAnimationFrame(( ) => { + //console.time('dom logger/scanning for matches'); + processTimer.clear(); + if ( nodesToProcess.size === 0 ) { return; } + + if ( nodesToProcess.size !== 1 && nodesToProcess.has(document) ) { + nodesToProcess.clear(); + nodesToProcess.add(document); + } + + const toLog = []; + if ( simpleDeclarativeSet.size !== 0 ) { + for ( const node of nodesToProcess ) { + processDeclarativeSimple(node, toLog); + } + } + + processDeclarativeComplex(toLog); + processProcedural(toLog); + processExceptions(toLog); + processProceduralExceptions(toLog); + + nodesToProcess.clear(); + + if ( toLog.length === 0 ) { return; } + + const location = vAPI.effectiveSelf.location; + + vAPI.messaging.send('scriptlets', { + what: 'logCosmeticFilteringData', + frameURL: location.href, + frameHostname: location.hostname, + matchedSelectors: toLog, + }); + //console.timeEnd('dom logger/scanning for matches'); +}); + +/******************************************************************************/ + +const attributeObserver = new MutationObserver(mutations => { + if ( nodesToProcess.has(document) ) { return; } + for ( const mutation of mutations ) { + const node = mutation.target; + if ( node.nodeType !== 1 ) { continue; } + nodesToProcess.add(node); + } + if ( nodesToProcess.size !== 0 ) { + processTimer.start(100); + } +}); + +/******************************************************************************/ + +const handlers = { + onFiltersetChanged: function(changes) { + //console.time('dom logger/filterset changed'); + for ( const block of (changes.declarative || []) ) { + for ( const selector of block.split(',\n') ) { + if ( loggedSelectors.has(selector) ) { continue; } + if ( reHasCSSCombinators.test(selector) ) { + complexDeclarativeSet.add(selector); + complexDeclarativeStr = undefined; + } else { + simpleDeclarativeSet.add(selector); + simpleDeclarativeStr = undefined; + } + } + } + if ( + Array.isArray(changes.procedural) && + changes.procedural.length !== 0 + ) { + for ( const selector of changes.procedural ) { + proceduralDict.set(selector.raw, selector); + } + } + if ( Array.isArray(changes.exceptions) ) { + for ( const selector of changes.exceptions ) { + if ( loggedSelectors.has(selector) ) { continue; } + if ( selector.charCodeAt(0) !== 0x7B /* '{' */ ) { + exceptionDict.set(selector, selector); + continue; + } + const details = JSON.parse(selector); + if ( + details.action !== undefined && + details.tasks === undefined && + details.action[0] === 'style' + ) { + exceptionDict.set(details.selector, details.raw); + continue; + } + proceduralExceptionDict.set( + details.raw, + vAPI.domFilterer.createProceduralFilter(details) + ); + } + exceptionStr = undefined; + } + nodesToProcess.clear(); + nodesToProcess.add(document); + processTimer.start(1); + //console.timeEnd('dom logger/filterset changed'); + }, + + onDOMCreated: function() { + if ( vAPI.domFilterer instanceof Object === false ) { + return shutdown(); + } + handlers.onFiltersetChanged(vAPI.domFilterer.getAllSelectors()); + vAPI.domFilterer.addListener(handlers); + attributeObserver.observe(document.body, { + attributes: true, + subtree: true + }); + }, + + onDOMChanged: function(addedNodes) { + if ( nodesToProcess.has(document) ) { return; } + for ( const node of addedNodes ) { + if ( node.parentNode === null ) { continue; } + nodesToProcess.add(node); + } + if ( nodesToProcess.size !== 0 ) { + processTimer.start(100); + } + } +}; + +vAPI.domWatcher.addListener(handlers); + +/******************************************************************************/ + +const broadcastHandler = msg => { + if ( msg.what === 'loggerDisabled' ) { + shutdown(); + } +}; + +browser.runtime.onMessage.addListener(broadcastHandler); + +/******************************************************************************/ + +function shutdown() { + browser.runtime.onMessage.removeListener(broadcastHandler); + processTimer.clear(); + attributeObserver.disconnect(); + if ( typeof vAPI !== 'object' ) { return; } + if ( vAPI.domFilterer instanceof Object ) { + vAPI.domFilterer.removeListener(handlers); + } + if ( vAPI.domWatcher instanceof Object ) { + vAPI.domWatcher.removeListener(handlers); + } +} + +/******************************************************************************/ + +// <<<<<<<< end of private namespace +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; + diff --git a/src/js/scriptlets/cosmetic-off.js b/src/js/scriptlets/cosmetic-off.js new file mode 100644 index 0000000..f1301e2 --- /dev/null +++ b/src/js/scriptlets/cosmetic-off.js @@ -0,0 +1,48 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +if ( typeof vAPI === 'object' && vAPI.domFilterer ) { + vAPI.domFilterer.toggle(false); +} + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/cosmetic-on.js b/src/js/scriptlets/cosmetic-on.js new file mode 100644 index 0000000..7b30976 --- /dev/null +++ b/src/js/scriptlets/cosmetic-on.js @@ -0,0 +1,48 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +if ( typeof vAPI === 'object' && vAPI.domFilterer ) { + vAPI.domFilterer.toggle(true); +} + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/cosmetic-report.js b/src/js/scriptlets/cosmetic-report.js new file mode 100644 index 0000000..a968d4d --- /dev/null +++ b/src/js/scriptlets/cosmetic-report.js @@ -0,0 +1,142 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +(( ) => { +// >>>>>>>> start of private namespace + +/******************************************************************************/ + +if ( typeof vAPI !== 'object' ) { return; } +if ( typeof vAPI.domFilterer !== 'object' ) { return; } +if ( vAPI.domFilterer === null ) { return; } + +/******************************************************************************/ + +const rePseudoElements = /:(?::?after|:?before|:[a-z-]+)$/; + +const hasSelector = selector => { + try { + return document.querySelector(selector) !== null; + } + catch(ex) { + } + return false; +}; + +const safeQuerySelector = selector => { + const safeSelector = rePseudoElements.test(selector) + ? selector.replace(rePseudoElements, '') + : selector; + try { + return document.querySelector(safeSelector); + } + catch(ex) { + } + return null; +}; + +const safeGroupSelectors = selectors => { + const arr = Array.isArray(selectors) + ? selectors + : Array.from(selectors); + return arr.map(s => { + return rePseudoElements.test(s) + ? s.replace(rePseudoElements, '') + : s; + }).join(',\n'); +}; + +const allSelectors = vAPI.domFilterer.getAllSelectors(); +const matchedSelectors = []; + +if ( Array.isArray(allSelectors.declarative) ) { + const declarativeSet = new Set(); + for ( const block of allSelectors.declarative ) { + for ( const selector of block.split(',\n') ) { + declarativeSet.add(selector); + } + } + if ( hasSelector(safeGroupSelectors(declarativeSet)) ) { + for ( const selector of declarativeSet ) { + if ( safeQuerySelector(selector) === null ) { continue; } + matchedSelectors.push(`##${selector}`); + } + } +} + +if ( + Array.isArray(allSelectors.procedural) && + allSelectors.procedural.length !== 0 +) { + for ( const pselector of allSelectors.procedural ) { + if ( pselector.hit === false && pselector.exec().length === 0 ) { continue; } + matchedSelectors.push(`##${pselector.raw}`); + } +} + +if ( Array.isArray(allSelectors.exceptions) ) { + const exceptionDict = new Map(); + for ( const selector of allSelectors.exceptions ) { + if ( selector.charCodeAt(0) !== 0x7B /* '{' */ ) { + exceptionDict.set(selector, selector); + continue; + } + const details = JSON.parse(selector); + if ( + details.action !== undefined && + details.tasks === undefined && + details.action[0] === 'style' + ) { + exceptionDict.set(details.selector, details.raw); + continue; + } + const pselector = vAPI.domFilterer.createProceduralFilter(details); + if ( pselector.test() === false ) { continue; } + matchedSelectors.push(`#@#${pselector.raw}`); + } + if ( + exceptionDict.size !== 0 && + hasSelector(safeGroupSelectors(exceptionDict.keys())) + ) { + for ( const [ selector, raw ] of exceptionDict ) { + if ( safeQuerySelector(selector) === null ) { continue; } + matchedSelectors.push(`#@#${raw}`); + } + } +} + +if ( typeof self.uBO_scriptletsInjected === 'string' ) { + matchedSelectors.push(...self.uBO_scriptletsInjected.split('\n')); +} + +if ( matchedSelectors.length === 0 ) { return; } + +return matchedSelectors; + +/******************************************************************************/ + +// <<<<<<<< end of private namespace +})(); + diff --git a/src/js/scriptlets/dom-inspector.js b/src/js/scriptlets/dom-inspector.js new file mode 100644 index 0000000..b5317d5 --- /dev/null +++ b/src/js/scriptlets/dom-inspector.js @@ -0,0 +1,924 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ +/******************************************************************************/ + +(async ( ) => { + +/******************************************************************************/ + +if ( typeof vAPI !== 'object' ) { return; } +if ( typeof vAPI === null ) { return; } +if ( vAPI.domFilterer instanceof Object === false ) { return; } + +if ( vAPI.inspectorFrame ) { return; } +vAPI.inspectorFrame = true; + +const inspectorUniqueId = vAPI.randomToken(); + +const nodeToIdMap = new WeakMap(); // No need to iterate + +let blueNodes = []; +const roRedNodes = new Map(); // node => current cosmetic filter +const rwRedNodes = new Set(); // node => new cosmetic filter (toggle node) +const rwGreenNodes = new Set(); // node => new exception cosmetic filter (toggle filter) +//const roGreenNodes = new Map(); // node => current exception cosmetic filter (can't toggle) + +const reHasCSSCombinators = /[ >+~]/; + +/******************************************************************************/ + +const domLayout = (( ) => { + const skipTagNames = new Set([ + 'br', 'head', 'link', 'meta', 'script', 'style', 'title' + ]); + const resourceAttrNames = new Map([ + [ 'a', 'href' ], + [ 'iframe', 'src' ], + [ 'img', 'src' ], + [ 'object', 'data' ] + ]); + + let idGenerator = 1; + + // This will be used to uniquely identify nodes across process. + + const newNodeId = node => { + const nid = `n${(idGenerator++).toString(36)}`; + nodeToIdMap.set(node, nid); + return nid; + }; + + const selectorFromNode = node => { + const tag = node.localName; + let selector = CSS.escape(tag); + // Id + if ( typeof node.id === 'string' ) { + let str = node.id.trim(); + if ( str !== '' ) { + selector += `#${CSS.escape(str)}`; + } + } + // Class + const cl = node.classList; + if ( cl ) { + for ( let i = 0; i < cl.length; i++ ) { + selector += `.${CSS.escape(cl[i])}`; + } + } + // Tag-specific attributes + const attr = resourceAttrNames.get(tag); + if ( attr !== undefined ) { + let str = node.getAttribute(attr) || ''; + str = str.trim(); + const pos = str.startsWith('data:') ? 5 : str.search(/[#?]/); + let sw = ''; + if ( pos !== -1 ) { + str = str.slice(0, pos); + sw = '^'; + } + if ( str !== '' ) { + selector += `[${attr}${sw}="${CSS.escape(str, true)}"]`; + } + } + return selector; + }; + + function DomRoot() { + this.nid = newNodeId(document.body); + this.lvl = 0; + this.sel = 'body'; + this.cnt = 0; + this.filter = roRedNodes.get(document.body); + } + + function DomNode(node, level) { + this.nid = newNodeId(node); + this.lvl = level; + this.sel = selectorFromNode(node); + this.cnt = 0; + this.filter = roRedNodes.get(node); + } + + const domNodeFactory = (level, node) => { + const localName = node.localName; + if ( skipTagNames.has(localName) ) { return null; } + // skip uBlock's own nodes + if ( node === inspectorFrame ) { return null; } + if ( level === 0 && localName === 'body' ) { + return new DomRoot(); + } + return new DomNode(node, level); + }; + + // Collect layout data + + const getLayoutData = ( ) => { + const layout = []; + const stack = []; + let lvl = 0; + let node = document.documentElement; + if ( node === null ) { return layout; } + + for (;;) { + const domNode = domNodeFactory(lvl, node); + if ( domNode !== null ) { + layout.push(domNode); + } + // children + if ( domNode !== null && node.firstElementChild !== null ) { + stack.push(node); + lvl += 1; + node = node.firstElementChild; + continue; + } + // sibling + if ( node instanceof Element ) { + if ( node.nextElementSibling === null ) { + do { + node = stack.pop(); + if ( !node ) { break; } + lvl -= 1; + } while ( node.nextElementSibling === null ); + if ( !node ) { break; } + } + node = node.nextElementSibling; + } + } + + return layout; + }; + + // Descendant count for each node. + + const patchLayoutData = layout => { + const stack = []; + let ptr; + let lvl = 0; + let i = layout.length; + + while ( i-- ) { + const domNode = layout[i]; + if ( domNode.lvl === lvl ) { + stack[ptr] += 1; + continue; + } + if ( domNode.lvl > lvl ) { + while ( lvl < domNode.lvl ) { + stack.push(0); + lvl += 1; + } + ptr = lvl - 1; + stack[ptr] += 1; + continue; + } + // domNode.lvl < lvl + const cnt = stack.pop(); + domNode.cnt = cnt; + lvl -= 1; + ptr = lvl - 1; + stack[ptr] += cnt + 1; + } + return layout; + }; + + // Track and report mutations of the DOM + + let mutationObserver = null; + let mutationTimer; + let addedNodelists = []; + let removedNodelist = []; + + const previousElementSiblingId = node => { + let sibling = node; + for (;;) { + sibling = sibling.previousElementSibling; + if ( sibling === null ) { return null; } + if ( skipTagNames.has(sibling.localName) ) { continue; } + return nodeToIdMap.get(sibling); + } + }; + + const journalFromBranch = (root, newNodes, newNodeToIdMap) => { + let node = root.firstElementChild; + while ( node !== null ) { + const domNode = domNodeFactory(undefined, node); + if ( domNode !== null ) { + newNodeToIdMap.set(domNode.nid, domNode); + newNodes.push(node); + } + // down + if ( node.firstElementChild !== null ) { + node = node.firstElementChild; + continue; + } + // right + if ( node.nextElementSibling !== null ) { + node = node.nextElementSibling; + continue; + } + // up then right + for (;;) { + if ( node.parentElement === root ) { return; } + node = node.parentElement; + if ( node.nextElementSibling !== null ) { + node = node.nextElementSibling; + break; + } + } + } + }; + + const journalFromMutations = ( ) => { + mutationTimer = undefined; + + // This is used to temporarily hold all added nodes, before resolving + // their node id and relative position. + const newNodes = []; + const journalEntries = []; + const newNodeToIdMap = new Map(); + + for ( const nodelist of addedNodelists ) { + for ( const node of nodelist ) { + if ( node.nodeType !== 1 ) { continue; } + if ( node.parentElement === null ) { continue; } + cosmeticFilterMapper.incremental(node); + const domNode = domNodeFactory(undefined, node); + if ( domNode !== null ) { + newNodeToIdMap.set(domNode.nid, domNode); + newNodes.push(node); + } + journalFromBranch(node, newNodes, newNodeToIdMap); + } + } + addedNodelists = []; + for ( const nodelist of removedNodelist ) { + for ( const node of nodelist ) { + if ( node.nodeType !== 1 ) { continue; } + const nid = nodeToIdMap.get(node); + if ( nid === undefined ) { continue; } + journalEntries.push({ what: -1, nid }); + } + } + removedNodelist = []; + for ( const node of newNodes ) { + journalEntries.push({ + what: 1, + nid: nodeToIdMap.get(node), + u: nodeToIdMap.get(node.parentElement), + l: previousElementSiblingId(node) + }); + } + + if ( journalEntries.length === 0 ) { return; } + + contentInspectorChannel.toLogger({ + what: 'domLayoutIncremental', + url: window.location.href, + hostname: window.location.hostname, + journal: journalEntries, + nodes: Array.from(newNodeToIdMap) + }); + }; + + const onMutationObserved = mutationRecords => { + for ( const record of mutationRecords ) { + if ( record.addedNodes.length !== 0 ) { + addedNodelists.push(record.addedNodes); + } + if ( record.removedNodes.length !== 0 ) { + removedNodelist.push(record.removedNodes); + } + } + if ( mutationTimer === undefined ) { + mutationTimer = vAPI.setTimeout(journalFromMutations, 1000); + } + }; + + // API + + const getLayout = ( ) => { + cosmeticFilterMapper.reset(); + mutationObserver = new MutationObserver(onMutationObserved); + mutationObserver.observe(document.body, { + childList: true, + subtree: true + }); + + return { + what: 'domLayoutFull', + url: window.location.href, + hostname: window.location.hostname, + layout: patchLayoutData(getLayoutData()) + }; + }; + + const reset = ( ) => { + shutdown(); + }; + + const shutdown = ( ) => { + if ( mutationTimer !== undefined ) { + clearTimeout(mutationTimer); + mutationTimer = undefined; + } + if ( mutationObserver !== null ) { + mutationObserver.disconnect(); + mutationObserver = null; + } + addedNodelists = []; + removedNodelist = []; + }; + + return { + get: getLayout, + reset, + shutdown, + }; +})(); + +/******************************************************************************/ +/******************************************************************************/ + +const cosmeticFilterMapper = (( ) => { + const nodesFromStyleTag = rootNode => { + const filterMap = roRedNodes; + const details = vAPI.domFilterer.getAllSelectors(); + + // Declarative selectors. + for ( const block of (details.declarative || []) ) { + for ( const selector of block.split(',\n') ) { + let nodes; + if ( reHasCSSCombinators.test(selector) ) { + nodes = document.querySelectorAll(selector); + } else { + if ( + filterMap.has(rootNode) === false && + rootNode.matches(selector) + ) { + filterMap.set(rootNode, selector); + } + nodes = rootNode.querySelectorAll(selector); + } + for ( const node of nodes ) { + if ( filterMap.has(node) ) { continue; } + filterMap.set(node, selector); + } + } + } + + // Procedural selectors. + for ( const entry of (details.procedural || []) ) { + const nodes = entry.exec(); + for ( const node of nodes ) { + // Upgrade declarative selector to procedural one + filterMap.set(node, entry.raw); + } + } + }; + + const incremental = rootNode => { + nodesFromStyleTag(rootNode); + }; + + const reset = ( ) => { + roRedNodes.clear(); + if ( document.documentElement !== null ) { + incremental(document.documentElement); + } + }; + + const shutdown = ( ) => { + vAPI.domFilterer.toggle(true); + }; + + return { + incremental, + reset, + shutdown, + }; +})(); + +/******************************************************************************/ + +const elementsFromSelector = function(selector, context) { + if ( !context ) { + context = document; + } + if ( selector.indexOf(':') !== -1 ) { + const out = elementsFromSpecialSelector(selector); + if ( out !== undefined ) { return out; } + } + // plain CSS selector + try { + return context.querySelectorAll(selector); + } catch (ex) { + } + return []; +}; + +const elementsFromSpecialSelector = function(selector) { + const out = []; + let matches = /^(.+?):has\((.+?)\)$/.exec(selector); + if ( matches !== null ) { + let nodes; + try { + nodes = document.querySelectorAll(matches[1]); + } catch(ex) { + nodes = []; + } + for ( const node of nodes ) { + if ( node.querySelector(matches[2]) === null ) { continue; } + out.push(node); + } + return out; + } + + matches = /^:xpath\((.+?)\)$/.exec(selector); + if ( matches === null ) { return; } + const xpr = document.evaluate( + matches[1], + document, + null, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null + ); + let i = xpr.snapshotLength; + while ( i-- ) { + out.push(xpr.snapshotItem(i)); + } + return out; +}; + +/******************************************************************************/ + +const highlightElements = ( ) => { + const paths = []; + + const path = []; + for ( const elem of rwRedNodes.keys() ) { + if ( elem === inspectorFrame ) { continue; } + if ( rwGreenNodes.has(elem) ) { continue; } + if ( typeof elem.getBoundingClientRect !== 'function' ) { continue; } + const rect = elem.getBoundingClientRect(); + const xl = rect.left; + const w = rect.width; + const yt = rect.top; + const h = rect.height; + const ws = w.toFixed(1); + const poly = 'M' + xl.toFixed(1) + ' ' + yt.toFixed(1) + + 'h' + ws + + 'v' + h.toFixed(1) + + 'h-' + ws + + 'z'; + path.push(poly); + } + paths.push(path.join('') || 'M0 0'); + + path.length = 0; + for ( const elem of rwGreenNodes ) { + if ( typeof elem.getBoundingClientRect !== 'function' ) { continue; } + const rect = elem.getBoundingClientRect(); + const xl = rect.left; + const w = rect.width; + const yt = rect.top; + const h = rect.height; + const ws = w.toFixed(1); + const poly = 'M' + xl.toFixed(1) + ' ' + yt.toFixed(1) + + 'h' + ws + + 'v' + h.toFixed(1) + + 'h-' + ws + + 'z'; + path.push(poly); + } + paths.push(path.join('') || 'M0 0'); + + path.length = 0; + for ( const elem of roRedNodes.keys() ) { + if ( elem === inspectorFrame ) { continue; } + if ( rwGreenNodes.has(elem) ) { continue; } + if ( typeof elem.getBoundingClientRect !== 'function' ) { continue; } + const rect = elem.getBoundingClientRect(); + const xl = rect.left; + const w = rect.width; + const yt = rect.top; + const h = rect.height; + const ws = w.toFixed(1); + const poly = 'M' + xl.toFixed(1) + ' ' + yt.toFixed(1) + + 'h' + ws + + 'v' + h.toFixed(1) + + 'h-' + ws + + 'z'; + path.push(poly); + } + paths.push(path.join('') || 'M0 0'); + + path.length = 0; + for ( const elem of blueNodes ) { + if ( elem === inspectorFrame ) { continue; } + if ( typeof elem.getBoundingClientRect !== 'function' ) { continue; } + const rect = elem.getBoundingClientRect(); + const xl = rect.left; + const w = rect.width; + const yt = rect.top; + const h = rect.height; + const ws = w.toFixed(1); + const poly = 'M' + xl.toFixed(1) + ' ' + yt.toFixed(1) + + 'h' + ws + + 'v' + h.toFixed(1) + + 'h-' + ws + + 'z'; + path.push(poly); + } + paths.push(path.join('') || 'M0 0'); + + contentInspectorChannel.toFrame({ + what: 'svgPaths', + paths, + }); +}; + +/******************************************************************************/ + +const onScrolled = (( ) => { + let timer; + return ( ) => { + if ( timer ) { return; } + timer = window.requestAnimationFrame(( ) => { + timer = undefined; + highlightElements(); + }); + }; +})(); + +const onMouseOver = ( ) => { + if ( blueNodes.length === 0 ) { return; } + blueNodes = []; + highlightElements(); +}; + +/******************************************************************************/ + +const selectNodes = (selector, nid) => { + const nodes = elementsFromSelector(selector); + if ( nid === '' ) { return nodes; } + for ( const node of nodes ) { + if ( nodeToIdMap.get(node) === nid ) { + return [ node ]; + } + } + return []; +}; + +/******************************************************************************/ + +const nodesFromFilter = selector => { + const out = []; + for ( const entry of roRedNodes ) { + if ( entry[1] === selector ) { + out.push(entry[0]); + } + } + return out; +}; + +/******************************************************************************/ + +const toggleExceptions = (nodes, targetState) => { + for ( const node of nodes ) { + if ( targetState ) { + rwGreenNodes.add(node); + } else { + rwGreenNodes.delete(node); + } + } +}; + +const toggleFilter = (nodes, targetState) => { + for ( const node of nodes ) { + if ( targetState ) { + rwRedNodes.delete(node); + } else { + rwRedNodes.add(node); + } + } +}; + +const resetToggledNodes = ( ) => { + rwGreenNodes.clear(); + rwRedNodes.clear(); +}; + +/******************************************************************************/ + +const startInspector = ( ) => { + const onReady = ( ) => { + window.addEventListener('scroll', onScrolled, { + capture: true, + passive: true, + }); + window.addEventListener('mouseover', onMouseOver, { + capture: true, + passive: true, + }); + contentInspectorChannel.toLogger(domLayout.get()); + vAPI.domFilterer.toggle(false, highlightElements); + }; + if ( document.readyState === 'loading' ) { + document.addEventListener('DOMContentLoaded', onReady, { once: true }); + } else { + onReady(); + } +}; + +/******************************************************************************/ + +const shutdownInspector = ( ) => { + cosmeticFilterMapper.shutdown(); + domLayout.shutdown(); + window.removeEventListener('scroll', onScrolled, { + capture: true, + passive: true, + }); + window.removeEventListener('mouseover', onMouseOver, { + capture: true, + passive: true, + }); + contentInspectorChannel.shutdown(); + if ( inspectorFrame ) { + inspectorFrame.remove(); + inspectorFrame = null; + } + vAPI.userStylesheet.remove(inspectorCSS); + vAPI.userStylesheet.apply(); + vAPI.inspectorFrame = false; +}; + +/******************************************************************************/ +/******************************************************************************/ + +const onMessage = request => { + switch ( request.what ) { + case 'startInspector': + startInspector(); + break; + + case 'quitInspector': + shutdownInspector(); + break; + + case 'commitFilters': + highlightElements(); + break; + + case 'domLayout': + domLayout.get(); + highlightElements(); + break; + + case 'highlightMode': + break; + + case 'highlightOne': + blueNodes = selectNodes(request.selector, request.nid); + if ( blueNodes.length !== 0 ) { + blueNodes[0].scrollIntoView({ + behavior: 'smooth', + block: 'nearest', + inline: 'nearest', + }); + } + highlightElements(); + break; + + case 'resetToggledNodes': + resetToggledNodes(); + highlightElements(); + break; + + case 'showCommitted': + blueNodes = []; + // TODO: show only the new filters and exceptions. + highlightElements(); + break; + + case 'showInteractive': + blueNodes = []; + highlightElements(); + break; + + case 'toggleFilter': { + const nodes = selectNodes(request.selector, request.nid); + if ( nodes.length !== 0 ) { + nodes[0].scrollIntoView({ + behavior: 'smooth', + block: 'nearest', + inline: 'nearest', + }); + } + toggleExceptions(nodesFromFilter(request.filter), request.target); + highlightElements(); + break; + } + case 'toggleNodes': { + const nodes = selectNodes(request.selector, request.nid); + if ( nodes.length !== 0 ) { + nodes[0].scrollIntoView({ + behavior: 'smooth', + block: 'nearest', + inline: 'nearest', + }); + } + toggleFilter(nodes, request.target); + highlightElements(); + break; + } + default: + break; + } +}; + +/******************************************************************************* + * + * Establish two-way communication with logger/inspector window and + * inspector frame + * + * */ + +const contentInspectorChannel = (( ) => { + let toLoggerPort; + let toFramePort; + + const toLogger = msg => { + if ( toLoggerPort === undefined ) { return; } + try { + toLoggerPort.postMessage(msg); + } catch(_) { + shutdownInspector(); + } + }; + + const onLoggerMessage = msg => { + onMessage(msg); + }; + + const onLoggerDisconnect = ( ) => { + shutdownInspector(); + }; + + const onLoggerConnect = port => { + browser.runtime.onConnect.removeListener(onLoggerConnect); + toLoggerPort = port; + port.onMessage.addListener(onLoggerMessage); + port.onDisconnect.addListener(onLoggerDisconnect); + }; + + const toFrame = msg => { + if ( toFramePort === undefined ) { return; } + toFramePort.postMessage(msg); + }; + + const shutdown = ( ) => { + if ( toFramePort !== undefined ) { + toFrame({ what: 'quitInspector' }); + toFramePort.onmessage = null; + toFramePort.close(); + toFramePort = undefined; + } + if ( toLoggerPort !== undefined ) { + toLoggerPort.onMessage.removeListener(onLoggerMessage); + toLoggerPort.onDisconnect.removeListener(onLoggerDisconnect); + toLoggerPort.disconnect(); + toLoggerPort = undefined; + } + browser.runtime.onConnect.removeListener(onLoggerConnect); + }; + + const start = async ( ) => { + browser.runtime.onConnect.addListener(onLoggerConnect); + const inspectorArgs = await vAPI.messaging.send('domInspectorContent', { + what: 'getInspectorArgs', + }); + if ( typeof inspectorArgs !== 'object' ) { return; } + if ( inspectorArgs === null ) { return; } + return new Promise(resolve => { + const iframe = document.createElement('iframe'); + iframe.setAttribute(inspectorUniqueId, ''); + document.documentElement.append(iframe); + iframe.addEventListener('load', ( ) => { + iframe.setAttribute(`${inspectorUniqueId}-loaded`, ''); + const channel = new MessageChannel(); + toFramePort = channel.port1; + toFramePort.onmessage = ev => { + const msg = ev.data || {}; + if ( msg.what !== 'startInspector' ) { return; } + }; + iframe.contentWindow.postMessage( + { what: 'startInspector' }, + inspectorArgs.inspectorURL, + [ channel.port2 ] + ); + resolve(iframe); + }, { once: true }); + iframe.contentWindow.location = inspectorArgs.inspectorURL; + }); + }; + + return { start, toLogger, toFrame, shutdown }; +})(); + + +// Install DOM inspector widget +const inspectorCSSStyle = [ + 'background: transparent', + 'border: 0', + 'border-radius: 0', + 'box-shadow: none', + 'color-scheme: light dark', + 'display: block', + 'filter: none', + 'height: 100%', + 'left: 0', + 'margin: 0', + 'max-height: none', + 'max-width: none', + 'min-height: unset', + 'min-width: unset', + 'opacity: 1', + 'outline: 0', + 'padding: 0', + 'pointer-events: none', + 'position: fixed', + 'top: 0', + 'transform: none', + 'visibility: hidden', + 'width: 100%', + 'z-index: 2147483647', + '' +].join(' !important;\n'); + +const inspectorCSS = ` +:root > [${inspectorUniqueId}] { + ${inspectorCSSStyle} +} +:root > [${inspectorUniqueId}-loaded] { + visibility: visible !important; +} +`; + +vAPI.userStylesheet.add(inspectorCSS); +vAPI.userStylesheet.apply(); + +let inspectorFrame = await contentInspectorChannel.start(); +if ( inspectorFrame instanceof HTMLIFrameElement === false ) { + return shutdownInspector(); +} + +startInspector(); + +/******************************************************************************/ + +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/dom-survey-elements.js b/src/js/scriptlets/dom-survey-elements.js new file mode 100644 index 0000000..1582596 --- /dev/null +++ b/src/js/scriptlets/dom-survey-elements.js @@ -0,0 +1,72 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/756 +// Keep in mind CPU usage with large DOM and/or filterset. + +(( ) => { + if ( typeof vAPI !== 'object' ) { return; } + + const t0 = Date.now(); + + if ( vAPI.domSurveyElements instanceof Object === false ) { + vAPI.domSurveyElements = { + busy: false, + hiddenElementCount: Number.NaN, + surveyTime: t0, + }; + } + const surveyResults = vAPI.domSurveyElements; + + if ( surveyResults.busy ) { return; } + surveyResults.busy = true; + + if ( surveyResults.surveyTime < vAPI.domMutationTime ) { + surveyResults.hiddenElementCount = Number.NaN; + } + surveyResults.surveyTime = t0; + + if ( isNaN(surveyResults.hiddenElementCount) ) { + surveyResults.hiddenElementCount = (( ) => { + if ( vAPI.domFilterer instanceof Object === false ) { return 0; } + const details = vAPI.domFilterer.getAllSelectors(0b11); + if ( + Array.isArray(details.declarative) === false || + details.declarative.length === 0 + ) { + return 0; + } + return document.querySelectorAll( + details.declarative.join(',\n') + ).length; + })(); + } + + surveyResults.busy = false; + + // IMPORTANT: This is returned to the injector, so this MUST be + // the last statement. + return surveyResults.hiddenElementCount; +})(); diff --git a/src/js/scriptlets/dom-survey-scripts.js b/src/js/scriptlets/dom-survey-scripts.js new file mode 100644 index 0000000..e5300ff --- /dev/null +++ b/src/js/scriptlets/dom-survey-scripts.js @@ -0,0 +1,126 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// Scriptlets to count the number of script tags in a document. + +(( ) => { + if ( typeof vAPI !== 'object' ) { return; } + + const t0 = Date.now(); + + if ( vAPI.domSurveyScripts instanceof Object === false ) { + vAPI.domSurveyScripts = { + busy: false, + scriptCount: -1, + surveyTime: t0, + }; + } + const surveyResults = vAPI.domSurveyScripts; + + if ( surveyResults.busy ) { return; } + surveyResults.busy = true; + + if ( surveyResults.surveyTime < vAPI.domMutationTime ) { + surveyResults.scriptCount = -1; + } + surveyResults.surveyTime = t0; + + if ( surveyResults.scriptCount === -1 ) { + const reInlineScript = /^(data:|blob:|$)/; + let inlineScriptCount = 0; + let scriptCount = 0; + for ( const script of document.scripts ) { + if ( reInlineScript.test(script.src) ) { + inlineScriptCount = 1; + continue; + } + scriptCount += 1; + if ( scriptCount === 99 ) { break; } + } + scriptCount += inlineScriptCount; + if ( scriptCount !== 0 ) { + surveyResults.scriptCount = scriptCount; + } + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/756 + // Keep trying to find inline script-like instances but only if we + // have the time-budget to do so. + if ( surveyResults.scriptCount === -1 ) { + if ( document.querySelector('a[href^="javascript:"]') !== null ) { + surveyResults.scriptCount = 1; + } + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1756 + // Mind that there might be no body element. + if ( surveyResults.scriptCount === -1 && document.body !== null ) { + surveyResults.scriptCount = 0; + const onHandlers = new Set([ + 'onabort', 'onblur', 'oncancel', 'oncanplay', + 'oncanplaythrough', 'onchange', 'onclick', 'onclose', + 'oncontextmenu', 'oncuechange', 'ondblclick', 'ondrag', + 'ondragend', 'ondragenter', 'ondragexit', 'ondragleave', + 'ondragover', 'ondragstart', 'ondrop', 'ondurationchange', + 'onemptied', 'onended', 'onerror', 'onfocus', + 'oninput', 'oninvalid', 'onkeydown', 'onkeypress', + 'onkeyup', 'onload', 'onloadeddata', 'onloadedmetadata', + 'onloadstart', 'onmousedown', 'onmouseenter', 'onmouseleave', + 'onmousemove', 'onmouseout', 'onmouseover', 'onmouseup', + 'onwheel', 'onpause', 'onplay', 'onplaying', + 'onprogress', 'onratechange', 'onreset', 'onresize', + 'onscroll', 'onseeked', 'onseeking', 'onselect', + 'onshow', 'onstalled', 'onsubmit', 'onsuspend', + 'ontimeupdate', 'ontoggle', 'onvolumechange', 'onwaiting', + 'onafterprint', 'onbeforeprint', 'onbeforeunload', 'onhashchange', + 'onlanguagechange', 'onmessage', 'onoffline', 'ononline', + 'onpagehide', 'onpageshow', 'onrejectionhandled', 'onpopstate', + 'onstorage', 'onunhandledrejection', 'onunload', + 'oncopy', 'oncut', 'onpaste' + ]); + const nodeIter = document.createNodeIterator( + document.body, + NodeFilter.SHOW_ELEMENT + ); + for (;;) { + const node = nodeIter.nextNode(); + if ( node === null ) { break; } + if ( node.hasAttributes() === false ) { continue; } + for ( const attr of node.getAttributeNames() ) { + if ( onHandlers.has(attr) === false ) { continue; } + surveyResults.scriptCount = 1; + break; + } + } + } + + surveyResults.busy = false; + + // IMPORTANT: This is returned to the injector, so this MUST be + // the last statement. + if ( surveyResults.scriptCount !== -1 ) { + return surveyResults.scriptCount; + } +})(); diff --git a/src/js/scriptlets/epicker.js b/src/js/scriptlets/epicker.js new file mode 100644 index 0000000..80489e8 --- /dev/null +++ b/src/js/scriptlets/epicker.js @@ -0,0 +1,1356 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CSS */ + +'use strict'; + +/******************************************************************************/ +/******************************************************************************/ + +(async ( ) => { + +/******************************************************************************/ + +if ( typeof vAPI !== 'object' ) { return; } +if ( typeof vAPI === null ) { return; } + +if ( vAPI.pickerFrame ) { return; } +vAPI.pickerFrame = true; + +const pickerUniqueId = vAPI.randomToken(); + +const reCosmeticAnchor = /^#(\$|\?|\$\?)?#/; + +const netFilterCandidates = []; +const cosmeticFilterCandidates = []; + +let targetElements = []; +let candidateElements = []; +let bestCandidateFilter = null; + +const lastNetFilterSession = window.location.host + window.location.pathname; +let lastNetFilterHostname = ''; +let lastNetFilterUnion = ''; + +const hideBackgroundStyle = 'background-image:none!important;'; + +/******************************************************************************/ + +const safeQuerySelectorAll = function(node, selector) { + if ( node !== null ) { + try { + return node.querySelectorAll(selector); + } catch (e) { + } + } + return []; +}; + +/******************************************************************************/ + +const getElementBoundingClientRect = function(elem) { + let rect = typeof elem.getBoundingClientRect === 'function' + ? elem.getBoundingClientRect() + : { height: 0, left: 0, top: 0, width: 0 }; + + // https://github.com/gorhill/uBlock/issues/1024 + // Try not returning an empty bounding rect. + if ( rect.width !== 0 && rect.height !== 0 ) { + return rect; + } + if ( elem.shadowRoot instanceof DocumentFragment ) { + return getElementBoundingClientRect(elem.shadowRoot); + } + + let left = rect.left, + right = left + rect.width, + top = rect.top, + bottom = top + rect.height; + + for ( const child of elem.children ) { + rect = getElementBoundingClientRect(child); + if ( rect.width === 0 || rect.height === 0 ) { continue; } + if ( rect.left < left ) { left = rect.left; } + if ( rect.right > right ) { right = rect.right; } + if ( rect.top < top ) { top = rect.top; } + if ( rect.bottom > bottom ) { bottom = rect.bottom; } + } + + return { + bottom, + height: bottom - top, + left, + right, + top, + width: right - left + }; +}; + +/******************************************************************************/ + +const highlightElements = function(elems, force) { + // To make mouse move handler more efficient + if ( + (force !== true) && + (elems.length === targetElements.length) && + (elems.length === 0 || elems[0] === targetElements[0]) + ) { + return; + } + targetElements = []; + + const ow = self.innerWidth; + const oh = self.innerHeight; + const islands = []; + + for ( const elem of elems ) { + if ( elem === pickerFrame ) { continue; } + targetElements.push(elem); + const rect = getElementBoundingClientRect(elem); + // Ignore offscreen areas + if ( + rect.left > ow || rect.top > oh || + rect.left + rect.width < 0 || rect.top + rect.height < 0 + ) { + continue; + } + islands.push( + `M${rect.left} ${rect.top}h${rect.width}v${rect.height}h-${rect.width}z` + ); + } + + pickerFramePort.postMessage({ + what: 'svgPaths', + ocean: `M0 0h${ow}v${oh}h-${ow}z`, + islands: islands.join(''), + }); +}; + +/******************************************************************************/ + +const mergeStrings = function(urls) { + if ( urls.length === 0 ) { return ''; } + if ( + urls.length === 1 || + self.diff_match_patch instanceof Function === false + ) { + return urls[0]; + } + const differ = new self.diff_match_patch(); + let merged = urls[0]; + for ( let i = 1; i < urls.length; i++ ) { + // The differ works at line granularity: we insert a linefeed after + // each character to trick the differ to work at character granularity. + const diffs = differ.diff_main( + urls[i].split('').join('\n'), + merged.split('').join('\n') + ); + const result = []; + for ( const diff of diffs ) { + if ( diff[0] !== 0 ) { + result.push('*'); + } else { + result.push(diff[1].replace(/\n+/g, '')); + } + merged = result.join(''); + } + } + // Keep usage of wildcards to a sane level, too many of them can cause + // high overhead filters + merged = merged.replace(/^\*+$/, '') + .replace(/\*{2,}/g, '*') + .replace(/([^*]{1,3}\*)(?:[^*]{1,3}\*)+/g, '$1'); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1494 + let pos = merged.indexOf('/'); + if ( pos === -1 ) { pos = merged.length; } + return merged.slice(0, pos).includes('*') ? urls[0] : merged; +}; + +/******************************************************************************/ + +// Remove fragment part from a URL. + +const trimFragmentFromURL = function(url) { + const pos = url.indexOf('#'); + return pos !== -1 ? url.slice(0, pos) : url; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1897 +// Ignore `data:` URI, they can't be handled by an HTTP observer. + +const backgroundImageURLFromElement = function(elem) { + const style = window.getComputedStyle(elem); + const bgImg = style.backgroundImage || ''; + const matches = /^url\((["']?)([^"']+)\1\)$/.exec(bgImg); + const url = matches !== null && matches.length === 3 ? matches[2] : ''; + return url.lastIndexOf('data:', 0) === -1 + ? trimFragmentFromURL(url.slice(0, 1024)) + : ''; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1725#issuecomment-226479197 +// Limit returned string to 1024 characters. +// Also, return only URLs which will be seen by an HTTP observer. +// https://github.com/uBlockOrigin/uBlock-issues/issues/2260 +// Maybe get to the actual URL indirectly. +const resourceURLsFromElement = function(elem) { + const urls = []; + const tagName = elem.localName; + const prop = netFilter1stSources[tagName]; + if ( prop === undefined ) { + const url = backgroundImageURLFromElement(elem); + if ( url !== '' ) { urls.push(url); } + return urls; + } + let s = elem[prop]; + if ( s instanceof SVGAnimatedString ) { + s = s.baseVal; + } + if ( typeof s === 'string' && /^https?:\/\//.test(s) ) { + urls.push(trimFragmentFromURL(s.slice(0, 1024))); + } + resourceURLsFromSrcset(elem, urls); + resourceURLsFromPicture(elem, urls); + return urls; +}; + +// https://html.spec.whatwg.org/multipage/images.html#parsing-a-srcset-attribute +// https://github.com/uBlockOrigin/uBlock-issues/issues/1071 +const resourceURLsFromSrcset = function(elem, out) { + let srcset = elem.srcset; + if ( typeof srcset !== 'string' || srcset === '' ) { return; } + for(;;) { + // trim whitespace + srcset = srcset.trim(); + if ( srcset.length === 0 ) { break; } + // abort in case of leading comma + if ( /^,/.test(srcset) ) { break; } + // collect and consume all non-whitespace characters + let match = /^\S+/.exec(srcset); + if ( match === null ) { break; } + srcset = srcset.slice(match.index + match[0].length); + let url = match[0]; + // consume descriptor, if any + if ( /,$/.test(url) ) { + url = url.replace(/,$/, ''); + if ( /,$/.test(url) ) { break; } + } else { + match = /^[^,]*(?:\(.+?\))?[^,]*(?:,|$)/.exec(srcset); + if ( match === null ) { break; } + srcset = srcset.slice(match.index + match[0].length); + } + const parsedURL = new URL(url, document.baseURI); + if ( parsedURL.pathname.length === 0 ) { continue; } + out.push(trimFragmentFromURL(parsedURL.href)); + } +}; + +// https://github.com/uBlockOrigin/uBlock-issues/issues/2069#issuecomment-1080600661 +// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture +const resourceURLsFromPicture = function(elem, out) { + if ( elem.localName === 'source' ) { return; } + const picture = elem.parentElement; + if ( picture === null || picture.localName !== 'picture' ) { return; } + const sources = picture.querySelectorAll(':scope > source'); + for ( const source of sources ) { + const urls = resourceURLsFromElement(source); + if ( urls.length === 0 ) { continue; } + out.push(...urls); + } +}; + +/******************************************************************************/ + +const netFilterFromUnion = function(patternIn, out) { + // Reset reference filter when dealing with unrelated URLs + const currentHostname = self.location.hostname; + if ( + lastNetFilterUnion === '' || + currentHostname === '' || + currentHostname !== lastNetFilterHostname + ) { + lastNetFilterHostname = currentHostname; + lastNetFilterUnion = patternIn; + vAPI.messaging.send('elementPicker', { + what: 'elementPickerEprom', + lastNetFilterSession, + lastNetFilterHostname, + lastNetFilterUnion, + }); + return; + } + + // Related URLs + lastNetFilterHostname = currentHostname; + let patternOut = mergeStrings([ patternIn, lastNetFilterUnion ]); + if ( patternOut !== '/*' && patternOut !== patternIn ) { + const filter = `||${patternOut}`; + if ( out.indexOf(filter) === -1 ) { + out.push(filter); + } + lastNetFilterUnion = patternOut; + } + + // Remember across element picker sessions + vAPI.messaging.send('elementPicker', { + what: 'elementPickerEprom', + lastNetFilterSession, + lastNetFilterHostname, + lastNetFilterUnion, + }); +}; + +/******************************************************************************/ + +// Extract the best possible net filter, i.e. as specific as possible. + +const netFilterFromElement = function(elem) { + if ( elem === null ) { return 0; } + if ( elem.nodeType !== 1 ) { return 0; } + const urls = resourceURLsFromElement(elem); + if ( urls.length === 0 ) { return 0; } + + if ( candidateElements.indexOf(elem) === -1 ) { + candidateElements.push(elem); + } + + const candidates = netFilterCandidates; + const len = candidates.length; + + for ( let i = 0; i < urls.length; i++ ) { + urls[i] = urls[i].replace(/^https?:\/\//, ''); + } + const pattern = mergeStrings(urls); + + + if ( bestCandidateFilter === null && elem.matches('html,body') === false ) { + bestCandidateFilter = { + type: 'net', + filters: candidates, + slot: candidates.length + }; + } + + candidates.push(`||${pattern}`); + + // Suggest a less narrow filter if possible + const pos = pattern.indexOf('?'); + if ( pos !== -1 ) { + candidates.push(`||${pattern.slice(0, pos)}`); + } + + // Suggest a filter which is a result of combining more than one URL. + netFilterFromUnion(pattern, candidates); + + return candidates.length - len; +}; + +const netFilter1stSources = { + 'audio': 'src', + 'embed': 'src', + 'iframe': 'src', + 'img': 'src', + 'image': 'href', + 'object': 'data', + 'source': 'src', + 'video': 'src' +}; + +const filterTypes = { + 'audio': 'media', + 'embed': 'object', + 'iframe': 'subdocument', + 'img': 'image', + 'object': 'object', + 'video': 'media', +}; + +/******************************************************************************/ + +// Extract the best possible cosmetic filter, i.e. as specific as possible. + +// https://github.com/gorhill/uBlock/issues/1725 +// Also take into account the `src` attribute for `img` elements -- and limit +// the value to the 1024 first characters. + +const cosmeticFilterFromElement = function(elem) { + if ( elem === null ) { return 0; } + if ( elem.nodeType !== 1 ) { return 0; } + if ( noCosmeticFiltering ) { return 0; } + + if ( candidateElements.indexOf(elem) === -1 ) { + candidateElements.push(elem); + } + + let selector = ''; + + // Id + let v = typeof elem.id === 'string' && CSS.escape(elem.id); + if ( v ) { + selector = '#' + v; + } + + // Class(es) + v = elem.classList; + if ( v ) { + let i = v.length || 0; + while ( i-- ) { + selector += '.' + CSS.escape(v.item(i)); + } + } + + // Tag name + const tagName = CSS.escape(elem.localName); + + // Use attributes if still no selector found. + // https://github.com/gorhill/uBlock/issues/1901 + // Trim attribute value, this may help in case of malformed HTML. + // + // https://github.com/uBlockOrigin/uBlock-issues/issues/1923 + // Escape unescaped `"` in attribute values + if ( selector === '' ) { + let attributes = [], attr; + switch ( tagName ) { + case 'a': + v = elem.getAttribute('href'); + if ( v ) { + v = v.trim().replace(/\?.*$/, ''); + if ( v.length ) { + attributes.push({ k: 'href', v: v }); + } + } + break; + case 'iframe': + case 'img': + v = elem.getAttribute('src'); + if ( v && v.length !== 0 ) { + v = v.trim(); + if ( v.startsWith('data:') ) { + let pos = v.indexOf(','); + if ( pos !== -1 ) { + v = v.slice(0, pos + 1); + } + } else if ( v.startsWith('blob:') ) { + v = new URL(v.slice(5)); + v.pathname = ''; + v = 'blob:' + v.href; + } + attributes.push({ k: 'src', v: v.slice(0, 256) }); + break; + } + v = elem.getAttribute('alt'); + if ( v && v.length !== 0 ) { + attributes.push({ k: 'alt', v: v }); + break; + } + break; + default: + break; + } + while ( (attr = attributes.pop()) ) { + if ( attr.v.length === 0 ) { continue; } + const w = attr.v.replace(/([^\\])"/g, '$1\\"'); + v = elem.getAttribute(attr.k); + if ( attr.v === v ) { + selector += `[${attr.k}="${w}"]`; + } else if ( v.startsWith(attr.v) ) { + selector += `[${attr.k}^="${w}"]`; + } else { + selector += `[${attr.k}*="${w}"]`; + } + } + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/17 + // If selector is ambiguous at this point, add the element name to + // further narrow it down. + const parentNode = elem.parentNode; + if ( + selector === '' || + safeQuerySelectorAll(parentNode, `:scope > ${selector}`).length > 1 + ) { + selector = tagName + selector; + } + + // https://github.com/chrisaljoudi/uBlock/issues/637 + // If the selector is still ambiguous at this point, further narrow using + // `nth-of-type`. It is preferable to use `nth-of-type` as opposed to + // `nth-child`, as `nth-of-type` is less volatile. + if ( safeQuerySelectorAll(parentNode, `:scope > ${selector}`).length > 1 ) { + let i = 1; + while ( elem.previousSibling !== null ) { + elem = elem.previousSibling; + if ( + typeof elem.localName === 'string' && + elem.localName === tagName + ) { + i++; + } + } + selector += `:nth-of-type(${i})`; + } + + if ( bestCandidateFilter === null ) { + bestCandidateFilter = { + type: 'cosmetic', + filters: cosmeticFilterCandidates, + slot: cosmeticFilterCandidates.length + }; + } + + cosmeticFilterCandidates.push(`##${selector}`); + + return 1; +}; + +/******************************************************************************/ + +const filtersFrom = function(x, y) { + bestCandidateFilter = null; + netFilterCandidates.length = 0; + cosmeticFilterCandidates.length = 0; + candidateElements.length = 0; + + // We need at least one element. + let first = null; + if ( typeof x === 'number' ) { + first = elementFromPoint(x, y); + } else if ( x instanceof HTMLElement ) { + first = x; + x = undefined; + } + + // https://github.com/gorhill/uBlock/issues/1545 + // Network filter candidates from all other elements found at [x,y]. + // https://www.reddit.com/r/uBlockOrigin/comments/qmjk36/ + // Extract network candidates first. + if ( typeof x === 'number' ) { + const magicAttr = `${pickerUniqueId}-clickblind`; + pickerFrame.setAttribute(magicAttr, ''); + const elems = document.elementsFromPoint(x, y); + pickerFrame.removeAttribute(magicAttr); + for ( const elem of elems ) { + netFilterFromElement(elem); + } + } else if ( first !== null ) { + netFilterFromElement(first); + } + + // Cosmetic filter candidates from ancestors. + // https://github.com/gorhill/uBlock/issues/2519 + // https://github.com/uBlockOrigin/uBlock-issues/issues/17 + // Prepend `body` if full selector is ambiguous. + let elem = first; + while ( elem && elem !== document.body ) { + cosmeticFilterFromElement(elem); + elem = elem.parentNode; + } + // The body tag is needed as anchor only when the immediate child + // uses `nth-of-type`. + let i = cosmeticFilterCandidates.length; + if ( i !== 0 ) { + const selector = cosmeticFilterCandidates[i-1].slice(2); + if ( safeQuerySelectorAll(document.body, selector).length > 1 ) { + cosmeticFilterCandidates.push('##body'); + } + } + + // https://github.com/gorhill/uBlock/commit/ebaa8a8bb28aef043a68c99965fe6c128a3fe5e4#commitcomment-63818019 + // If still no best candidate, just use whatever is available in network + // filter candidates -- which may have been previously skipped in favor + // of cosmetic filters. + if ( bestCandidateFilter === null && netFilterCandidates.length !== 0 ) { + bestCandidateFilter = { + type: 'net', + filters: netFilterCandidates, + slot: 0 + }; + } + + return netFilterCandidates.length + cosmeticFilterCandidates.length; +}; + +/******************************************************************************* + + filterToDOMInterface.queryAll + @desc Look-up all the HTML elements matching the filter passed in + argument. + @param string, a cosmetic or network filter. + @param function, called once all items matching the filter have been + collected. + @return array, or undefined if the filter is invalid. + + filterToDOMInterface.preview + @desc Apply/unapply filter to the DOM. + @param string, a cosmetic of network filter, or literal false to remove + the effects of the filter on the DOM. + @return undefined. + + TODO: need to be revised once I implement chained cosmetic operators. + +*/ + +const filterToDOMInterface = (( ) => { + const reHnAnchorPrefix = '^[\\w-]+://(?:[^/?#]+\\.)?'; + const reCaret = '(?:[^%.0-9a-z_-]|$)'; + const rePseudoElements = /:(?::?after|:?before|:[a-z-]+)$/; + + // Net filters: we need to lookup manually -- translating into a foolproof + // CSS selector is just not possible. + // + // https://github.com/chrisaljoudi/uBlock/issues/945 + // Transform into a regular expression, this allows the user to + // edit and insert wildcard(s) into the proposed filter. + // https://www.reddit.com/r/uBlockOrigin/comments/c5do7w/ + // Better handling of pure hostname filters. Also, discard single + // alphanumeric character filters. + const fromNetworkFilter = function(filter) { + const out = []; + if ( /^[0-9a-z]$/i.test(filter) ) { return out; } + let reStr = ''; + if ( + filter.length > 2 && + filter.startsWith('/') && + filter.endsWith('/') + ) { + reStr = filter.slice(1, -1); + } else if ( /^\w[\w.-]*[a-z]$/i.test(filter) ) { + reStr = reHnAnchorPrefix + + filter.toLowerCase().replace(/\./g, '\\.') + + reCaret; + } else { + let rePrefix = '', reSuffix = ''; + if ( filter.startsWith('||') ) { + rePrefix = reHnAnchorPrefix; + filter = filter.slice(2); + } else if ( filter.startsWith('|') ) { + rePrefix = '^'; + filter = filter.slice(1); + } + if ( filter.endsWith('|') ) { + reSuffix = '$'; + filter = filter.slice(0, -1); + } + reStr = rePrefix + + filter.replace(/[.+?${}()|[\]\\]/g, '\\$&') + .replace(/\*+/g, '.*') + .replace(/\^/g, reCaret) + + reSuffix; + } + let reFilter = null; + try { + reFilter = new RegExp(reStr, 'i'); + } + catch (e) { + return out; + } + + // Lookup by tag names. + // https://github.com/uBlockOrigin/uBlock-issues/issues/2260 + // Maybe get to the actual URL indirectly. + const elems = document.querySelectorAll( + Object.keys(netFilter1stSources).join() + ); + for ( const elem of elems ) { + const srcProp = netFilter1stSources[elem.localName]; + let src = elem[srcProp]; + if ( src instanceof SVGAnimatedString ) { + src = src.baseVal; + } + if ( + typeof src === 'string' && + reFilter.test(src) || + typeof elem.currentSrc === 'string' && + reFilter.test(elem.currentSrc) + ) { + out.push({ + elem, + src: srcProp, + opt: filterTypes[elem.localName], + style: vAPI.hideStyle, + }); + } + } + + // Find matching background image in current set of candidate elements. + for ( const elem of candidateElements ) { + if ( reFilter.test(backgroundImageURLFromElement(elem)) ) { + out.push({ + elem, + bg: true, + opt: 'image', + style: hideBackgroundStyle, + }); + } + } + + return out; + }; + + // Cosmetic filters: these are straight CSS selectors. + // + // https://github.com/uBlockOrigin/uBlock-issues/issues/389 + // Test filter using comma-separated list to better detect invalid CSS + // selectors. + // + // https://github.com/gorhill/uBlock/issues/2515 + // Remove trailing pseudo-element when querying. + const fromPlainCosmeticFilter = function(raw) { + let elems; + try { + document.documentElement.matches(`${raw},\na`); + elems = document.querySelectorAll( + raw.replace(rePseudoElements, '') + ); + } + catch (e) { + return; + } + const out = []; + for ( const elem of elems ) { + if ( elem === pickerFrame ) { continue; } + out.push({ elem, raw, style: vAPI.hideStyle }); + } + return out; + }; + + // https://github.com/gorhill/uBlock/issues/1772 + // Handle procedural cosmetic filters. + // + // https://github.com/gorhill/uBlock/issues/2515 + // Remove trailing pseudo-element when querying. + const fromCompiledCosmeticFilter = function(raw) { + if ( noCosmeticFiltering ) { return; } + if ( typeof raw !== 'string' ) { return; } + let elems, style; + try { + const o = JSON.parse(raw); + elems = vAPI.domFilterer.createProceduralFilter(o).exec(); + switch ( o.action && o.action[0] || '' ) { + case '': + case 'remove': + style = vAPI.hideStyle; + break; + case 'style': + style = o.action[1]; + break; + default: + break; + } + } catch(ex) { + return; + } + if ( !elems ) { return; } + const out = []; + for ( const elem of elems ) { + out.push({ elem, raw, style }); + } + return out; + }; + + vAPI.epickerStyleProxies = vAPI.epickerStyleProxies || new Map(); + + let lastFilter; + let lastResultset; + let previewing = false; + + const queryAll = function(details) { + let { filter, compiled } = details; + filter = filter.trim(); + if ( filter === lastFilter ) { return lastResultset; } + unapply(); + if ( filter === '' || filter === '!' ) { + lastFilter = ''; + lastResultset = undefined; + return; + } + lastFilter = filter; + if ( reCosmeticAnchor.test(filter) === false ) { + lastResultset = fromNetworkFilter(filter); + if ( previewing ) { apply(); } + return lastResultset; + } + lastResultset = fromPlainCosmeticFilter(compiled); + if ( lastResultset ) { + if ( previewing ) { apply(); } + return lastResultset; + } + // Procedural cosmetic filter + lastResultset = fromCompiledCosmeticFilter(compiled); + if ( previewing ) { apply(); } + return lastResultset; + }; + + const apply = function() { + unapply(); + if ( Array.isArray(lastResultset) === false ) { return; } + const rootElem = document.documentElement; + for ( const { elem, style } of lastResultset ) { + if ( elem === pickerFrame ) { continue; } + if ( style === undefined ) { continue; } + if ( elem === rootElem && style === vAPI.hideStyle ) { continue; } + let styleToken = vAPI.epickerStyleProxies.get(style); + if ( styleToken === undefined ) { + styleToken = vAPI.randomToken(); + vAPI.epickerStyleProxies.set(style, styleToken); + vAPI.userStylesheet.add(`[${styleToken}]\n{${style}}`, true); + } + elem.setAttribute(styleToken, ''); + } + }; + + const unapply = function() { + for ( const styleToken of vAPI.epickerStyleProxies.values() ) { + for ( const elem of document.querySelectorAll(`[${styleToken}]`) ) { + elem.removeAttribute(styleToken); + } + } + }; + + // https://www.reddit.com/r/uBlockOrigin/comments/c62irc/ + // Support injecting the cosmetic filters into the DOM filterer + // immediately rather than wait for the next page load. + const preview = function(state, permanent = false) { + previewing = state !== false; + if ( previewing === false ) { + return unapply(); + } + if ( Array.isArray(lastResultset) === false ) { return; } + if ( permanent === false || reCosmeticAnchor.test(lastFilter) === false ) { + return apply(); + } + if ( noCosmeticFiltering ) { return; } + const cssSelectors = new Set(); + const proceduralSelectors = new Set(); + for ( const { raw } of lastResultset ) { + if ( raw.startsWith('{') ) { + proceduralSelectors.add(raw); + } else { + cssSelectors.add(raw); + } + } + if ( cssSelectors.size !== 0 ) { + vAPI.domFilterer.addCSS( + `${Array.from(cssSelectors).join('\n')}\n{${vAPI.hideStyle}}`, + { mustInject: true } + ); + } + if ( proceduralSelectors.size !== 0 ) { + vAPI.domFilterer.addProceduralSelectors( + Array.from(proceduralSelectors) + ); + } + }; + + return { preview, queryAll }; +})(); + +/******************************************************************************/ + +const onOptimizeCandidates = function(details) { + const { candidates } = details; + const results = []; + for ( const paths of candidates ) { + let count = Number.MAX_SAFE_INTEGER; + let selector = ''; + for ( let i = 0, n = paths.length; i < n; i++ ) { + const s = paths.slice(n - i - 1).join(''); + const elems = document.querySelectorAll(s); + if ( elems.length < count ) { + selector = s; + count = elems.length; + } + } + results.push({ selector: `##${selector}`, count }); + } + // Sort by most match count and shortest selector to least match count and + // longest selector. + results.sort((a, b) => { + const r = b.count - a.count; + if ( r !== 0 ) { return r; } + return a.selector.length - b.selector.length; + }); + pickerFramePort.postMessage({ + what: 'candidatesOptimized', + candidates: results.map(a => a.selector), + slot: details.slot, + }); +}; + +/******************************************************************************/ + +const showDialog = function(options) { + pickerFramePort.postMessage({ + what: 'showDialog', + url: self.location.href, + netFilters: netFilterCandidates, + cosmeticFilters: cosmeticFilterCandidates, + filter: bestCandidateFilter, + options, + }); +}; + +/******************************************************************************/ + +const elementFromPoint = (( ) => { + let lastX, lastY; + + return (x, y) => { + if ( x !== undefined ) { + lastX = x; lastY = y; + } else if ( lastX !== undefined ) { + x = lastX; y = lastY; + } else { + return null; + } + if ( !pickerFrame ) { return null; } + const magicAttr = `${pickerUniqueId}-clickblind`; + pickerFrame.setAttribute(magicAttr, ''); + let elem = document.elementFromPoint(x, y); + if ( + elem === null || /* to skip following tests */ + elem === document.body || + elem === document.documentElement || ( + pickerBootArgs.zap !== true && + noCosmeticFiltering && + resourceURLsFromElement(elem).length === 0 + ) + ) { + elem = null; + } + // https://github.com/uBlockOrigin/uBlock-issues/issues/380 + pickerFrame.removeAttribute(magicAttr); + return elem; + }; +})(); + +/******************************************************************************/ + +const highlightElementAtPoint = function(mx, my) { + const elem = elementFromPoint(mx, my); + highlightElements(elem ? [ elem ] : []); +}; + +/******************************************************************************/ + +const filterElementAtPoint = function(mx, my, broad) { + if ( filtersFrom(mx, my) === 0 ) { return; } + showDialog({ broad }); +}; + +/******************************************************************************/ + +// https://www.reddit.com/r/uBlockOrigin/comments/bktxtb/scrolling_doesnt_work/emn901o +// Override 'fixed' position property on body element if present. + +// With touch-driven devices, first highlight the element and remove only +// when tapping again the highlighted area. + +const zapElementAtPoint = function(mx, my, options) { + if ( options.highlight ) { + const elem = elementFromPoint(mx, my); + if ( elem ) { + highlightElements([ elem ]); + } + return; + } + + let elemToRemove = targetElements.length !== 0 && targetElements[0] || null; + if ( elemToRemove === null && mx !== undefined ) { + elemToRemove = elementFromPoint(mx, my); + } + + if ( elemToRemove instanceof Element === false ) { return; } + + const getStyleValue = (elem, prop) => { + const style = window.getComputedStyle(elem); + return style ? style[prop] : ''; + }; + + // Heuristic to detect scroll-locking: remove such lock when detected. + let maybeScrollLocked = elemToRemove.shadowRoot instanceof DocumentFragment; + if ( maybeScrollLocked === false ) { + let elem = elemToRemove; + do { + maybeScrollLocked = + parseInt(getStyleValue(elem, 'zIndex'), 10) >= 1000 || + getStyleValue(elem, 'position') === 'fixed'; + elem = elem.parentElement; + } while ( elem !== null && maybeScrollLocked === false ); + } + if ( maybeScrollLocked ) { + const doc = document; + if ( getStyleValue(doc.body, 'overflowY') === 'hidden' ) { + doc.body.style.setProperty('overflow', 'auto', 'important'); + } + if ( getStyleValue(doc.body, 'position') === 'fixed' ) { + doc.body.style.setProperty('position', 'initial', 'important'); + } + if ( getStyleValue(doc.documentElement, 'position') === 'fixed' ) { + doc.documentElement.style.setProperty('position', 'initial', 'important'); + } + if ( getStyleValue(doc.documentElement, 'overflowY') === 'hidden' ) { + doc.documentElement.style.setProperty('overflow', 'auto', 'important'); + } + } + elemToRemove.remove(); + highlightElementAtPoint(mx, my); +}; + +/******************************************************************************/ + +const onKeyPressed = function(ev) { + // Delete + if ( + (ev.key === 'Delete' || ev.key === 'Backspace') && + pickerBootArgs.zap + ) { + ev.stopPropagation(); + ev.preventDefault(); + zapElementAtPoint(); + return; + } + // Esc + if ( ev.key === 'Escape' || ev.which === 27 ) { + ev.stopPropagation(); + ev.preventDefault(); + filterToDOMInterface.preview(false); + quitPicker(); + return; + } +}; + +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/190 +// May need to dynamically adjust the height of the overlay + new position +// of highlighted elements. + +const onViewportChanged = function() { + highlightElements(targetElements, true); +}; + +/******************************************************************************/ + +// Auto-select a specific target, if any, and if possible + +const startPicker = function() { + pickerFrame.focus(); + + self.addEventListener('scroll', onViewportChanged, { passive: true }); + self.addEventListener('resize', onViewportChanged, { passive: true }); + self.addEventListener('keydown', onKeyPressed, true); + + // Try using mouse position + if ( + pickerBootArgs.mouse && + vAPI.mouseClick instanceof Object && + typeof vAPI.mouseClick.x === 'number' && + vAPI.mouseClick.x > 0 + ) { + if ( filtersFrom(vAPI.mouseClick.x, vAPI.mouseClick.y) !== 0 ) { + return showDialog(); + } + } + + // No mouse position available, use suggested target + const target = pickerBootArgs.target || ''; + const pos = target.indexOf('\t'); + if ( pos === -1 ) { return; } + + const srcAttrMap = { + 'a': 'href', + 'audio': 'src', + 'embed': 'src', + 'iframe': 'src', + 'img': 'src', + 'video': 'src', + }; + const tagName = target.slice(0, pos); + const url = target.slice(pos + 1); + const attr = srcAttrMap[tagName]; + if ( attr === undefined ) { return; } + const elems = document.getElementsByTagName(tagName); + for ( const elem of elems ) { + if ( elem === pickerFrame ) { continue; } + const srcs = resourceURLsFromElement(elem); + if ( + (srcs.length !== 0 && srcs.includes(url) === false) || + (srcs.length === 0 && url !== 'about:blank') + ) { + continue; + } + filtersFrom(elem); + if ( + netFilterCandidates.length !== 0 || + cosmeticFilterCandidates.length !== 0 + ) { + if ( pickerBootArgs.mouse !== true ) { + elem.scrollIntoView({ + behavior: 'smooth', + block: 'center', + inline: 'center' + }); + } + showDialog({ broad: true }); + } + return; + } + + // A target was specified, but it wasn't found: abort. + quitPicker(); +}; + +/******************************************************************************/ + +// Let's have the element picker code flushed from memory when no longer +// in use: to ensure this, release all local references. + +const quitPicker = function() { + self.removeEventListener('scroll', onViewportChanged, { passive: true }); + self.removeEventListener('resize', onViewportChanged, { passive: true }); + self.removeEventListener('keydown', onKeyPressed, true); + vAPI.shutdown.remove(quitPicker); + if ( pickerFramePort ) { + pickerFramePort.close(); + pickerFramePort = null; + } + if ( pickerFrame ) { + pickerFrame.remove(); + pickerFrame = null; + } + vAPI.userStylesheet.remove(pickerCSS); + vAPI.userStylesheet.apply(); + vAPI.pickerFrame = false; + self.focus(); +}; + +vAPI.shutdown.add(quitPicker); + +/******************************************************************************/ + +const onDialogMessage = function(msg) { + switch ( msg.what ) { + case 'start': + startPicker(); + if ( pickerFramePort === null ) { break; } + if ( targetElements.length === 0 ) { + highlightElements([], true); + } + break; + case 'optimizeCandidates': + onOptimizeCandidates(msg); + break; + case 'dialogCreate': + filterToDOMInterface.queryAll(msg); + filterToDOMInterface.preview(true, true); + quitPicker(); + break; + case 'dialogSetFilter': { + const resultset = filterToDOMInterface.queryAll(msg) || []; + highlightElements(resultset.map(a => a.elem), true); + if ( msg.filter === '!' ) { break; } + pickerFramePort.postMessage({ + what: 'resultsetDetails', + count: resultset.length, + opt: resultset.length !== 0 ? resultset[0].opt : undefined, + }); + break; + } + case 'quitPicker': + filterToDOMInterface.preview(false); + quitPicker(); + break; + case 'highlightElementAtPoint': + highlightElementAtPoint(msg.mx, msg.my); + break; + case 'unhighlight': + highlightElements([]); + break; + case 'filterElementAtPoint': + filterElementAtPoint(msg.mx, msg.my, msg.broad); + break; + case 'zapElementAtPoint': + zapElementAtPoint(msg.mx, msg.my, msg.options); + if ( msg.options.highlight !== true && msg.options.stay !== true ) { + quitPicker(); + } + break; + case 'togglePreview': + filterToDOMInterface.preview(msg.state); + if ( msg.state === false ) { + highlightElements(targetElements, true); + } + break; + default: + break; + } +}; + +/******************************************************************************/ + +// epicker-ui.html will be injected in the page through an iframe, and +// is a sandboxed so as to prevent the page from interfering with its +// content and behavior. +// +// The purpose of epicker.js is to: +// - Install the element picker UI, and wait for the component to establish +// a direct communication channel. +// - Lookup candidate filters from elements at a specific position. +// - Highlight element(s) at a specific position or according to whether +// they match candidate filters; +// - Preview the result of applying a candidate filter; +// +// When the element picker is installed on a page, the only change the page +// sees is an iframe with a random attribute. The page can't see the content +// of the iframe, and cannot interfere with its style properties. However the +// page can remove the iframe. + +// The DOM filterer will not be present when cosmetic filtering is disabled. +const noCosmeticFiltering = + vAPI.domFilterer instanceof Object === false || + vAPI.noSpecificCosmeticFiltering === true; + +// https://github.com/gorhill/uBlock/issues/1529 +// In addition to inline styles, harden the element picker styles by using +// dedicated CSS rules. +const pickerCSSStyle = [ + 'background: transparent', + 'border: 0', + 'border-radius: 0', + 'box-shadow: none', + 'color-scheme: light dark', + 'display: block', + 'filter: none', + 'height: 100vh', + 'left: 0', + 'margin: 0', + 'max-height: none', + 'max-width: none', + 'min-height: unset', + 'min-width: unset', + 'opacity: 1', + 'outline: 0', + 'padding: 0', + 'pointer-events: auto', + 'position: fixed', + 'top: 0', + 'transform: none', + 'visibility: hidden', + 'width: 100%', + 'z-index: 2147483647', + '' +].join(' !important;\n'); + + +const pickerCSS = ` +:root > [${pickerUniqueId}] { + ${pickerCSSStyle} +} +:root > [${pickerUniqueId}-loaded] { + visibility: visible !important; +} +:root [${pickerUniqueId}-clickblind] { + pointer-events: none !important; +} +`; + +vAPI.userStylesheet.add(pickerCSS); +vAPI.userStylesheet.apply(); + +let pickerBootArgs; +let pickerFramePort = null; + +const bootstrap = async ( ) => { + pickerBootArgs = await vAPI.messaging.send('elementPicker', { + what: 'elementPickerArguments', + }); + if ( typeof pickerBootArgs !== 'object' ) { return; } + if ( pickerBootArgs === null ) { return; } + // Restore net filter union data if origin is the same. + const eprom = pickerBootArgs.eprom || null; + if ( eprom !== null && eprom.lastNetFilterSession === lastNetFilterSession ) { + lastNetFilterHostname = eprom.lastNetFilterHostname || ''; + lastNetFilterUnion = eprom.lastNetFilterUnion || ''; + } + const url = new URL(pickerBootArgs.pickerURL); + if ( pickerBootArgs.zap ) { + url.searchParams.set('zap', '1'); + } + return new Promise(resolve => { + const iframe = document.createElement('iframe'); + iframe.setAttribute(pickerUniqueId, ''); + document.documentElement.append(iframe); + iframe.addEventListener('load', ( ) => { + iframe.setAttribute(`${pickerUniqueId}-loaded`, ''); + const channel = new MessageChannel(); + pickerFramePort = channel.port1; + pickerFramePort.onmessage = ev => { + onDialogMessage(ev.data || {}); + }; + pickerFramePort.onmessageerror = ( ) => { + quitPicker(); + }; + iframe.contentWindow.postMessage( + { what: 'epickerStart' }, + url.href, + [ channel.port2 ] + ); + resolve(iframe); + }, { once: true }); + iframe.contentWindow.location = url.href; + }); +}; + +let pickerFrame = await bootstrap(); +if ( Boolean(pickerFrame) === false ) { + quitPicker(); +} + +/******************************************************************************/ + +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/load-3p-css.js b/src/js/scriptlets/load-3p-css.js new file mode 100644 index 0000000..bb7d542 --- /dev/null +++ b/src/js/scriptlets/load-3p-css.js @@ -0,0 +1,67 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2020-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +(( ) => { + if ( typeof vAPI !== 'object' ) { return; } + + if ( vAPI.dynamicReloadToken === undefined ) { + vAPI.dynamicReloadToken = vAPI.randomToken(); + } + + for ( const sheet of Array.from(document.styleSheets) ) { + let loaded = false; + try { + loaded = sheet.rules.length !== 0; + } catch(ex) { + } + if ( loaded ) { continue; } + const link = sheet.ownerNode || null; + if ( link === null || link.localName !== 'link' ) { continue; } + if ( link.hasAttribute(vAPI.dynamicReloadToken) ) { continue; } + const clone = link.cloneNode(true); + clone.setAttribute(vAPI.dynamicReloadToken, ''); + link.replaceWith(clone); + } +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/load-large-media-all.js b/src/js/scriptlets/load-large-media-all.js new file mode 100644 index 0000000..a44539e --- /dev/null +++ b/src/js/scriptlets/load-large-media-all.js @@ -0,0 +1,62 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +(( ) => { + +/******************************************************************************/ + +if ( + typeof vAPI !== 'object' || + vAPI.loadAllLargeMedia instanceof Function === false +) { + return; +} + +vAPI.loadAllLargeMedia(); +vAPI.loadAllLargeMedia = undefined; + +/******************************************************************************/ + +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/load-large-media-interactive.js b/src/js/scriptlets/load-large-media-interactive.js new file mode 100644 index 0000000..57198e4 --- /dev/null +++ b/src/js/scriptlets/load-large-media-interactive.js @@ -0,0 +1,299 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +(( ) => { + +/******************************************************************************/ + +// This can happen +if ( typeof vAPI !== 'object' || vAPI.loadAllLargeMedia instanceof Function ) { + return; +} + +/******************************************************************************/ + +const largeMediaElementAttribute = 'data-' + vAPI.sessionId; +const largeMediaElementSelector = + ':root audio[' + largeMediaElementAttribute + '],\n' + + ':root img[' + largeMediaElementAttribute + '],\n' + + ':root picture[' + largeMediaElementAttribute + '],\n' + + ':root video[' + largeMediaElementAttribute + ']'; + +/******************************************************************************/ + +const isMediaElement = function(elem) { + return /^(?:audio|img|picture|video)$/.test(elem.localName); +}; + +/******************************************************************************/ + +const mediaNotLoaded = function(elem) { + switch ( elem.localName ) { + case 'audio': + case 'video': { + const src = elem.src || ''; + if ( src.startsWith('blob:') ) { + elem.autoplay = false; + elem.pause(); + } + return elem.readyState === 0 || elem.error !== null; + } + case 'img': { + if ( elem.naturalWidth !== 0 || elem.naturalHeight !== 0 ) { + break; + } + const style = window.getComputedStyle(elem); + // For some reason, style can be null with Pale Moon. + return style !== null ? + style.getPropertyValue('display') !== 'none' : + elem.offsetHeight !== 0 && elem.offsetWidth !== 0; + } + default: + break; + } + return false; +}; + +/******************************************************************************/ + +// For all media resources which have failed to load, trigger a reload. + +// <audio> and <video> elements. +// https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement + +const surveyMissingMediaElements = function() { + let largeMediaElementCount = 0; + for ( const elem of document.querySelectorAll('audio,img,video') ) { + if ( mediaNotLoaded(elem) === false ) { continue; } + elem.setAttribute(largeMediaElementAttribute, ''); + largeMediaElementCount += 1; + switch ( elem.localName ) { + case 'img': { + const picture = elem.closest('picture'); + if ( picture !== null ) { + picture.setAttribute(largeMediaElementAttribute, ''); + } + } break; + default: + break; + } + } + return largeMediaElementCount; +}; + +if ( surveyMissingMediaElements() === 0 ) { return; } + +// Insert CSS to highlight blocked media elements. +if ( vAPI.largeMediaElementStyleSheet === undefined ) { + vAPI.largeMediaElementStyleSheet = [ + largeMediaElementSelector + ' {', + 'border: 2px dotted red !important;', + 'box-sizing: border-box !important;', + 'cursor: zoom-in !important;', + 'display: inline-block;', + 'filter: none !important;', + 'font-size: 1rem !important;', + 'min-height: 1em !important;', + 'min-width: 1em !important;', + 'opacity: 1 !important;', + 'outline: none !important;', + 'transform: none !important;', + 'visibility: visible !important;', + 'z-index: 2147483647', + '}', + ].join('\n'); + vAPI.userStylesheet.add(vAPI.largeMediaElementStyleSheet); + vAPI.userStylesheet.apply(); +} + +/******************************************************************************/ + +const loadMedia = async function(elem) { + const src = elem.getAttribute('src') || ''; + elem.removeAttribute('src'); + + await vAPI.messaging.send('scriptlets', { + what: 'temporarilyAllowLargeMediaElement', + }); + + if ( src !== '' ) { + elem.setAttribute('src', src); + } + elem.load(); +}; + +/******************************************************************************/ + +const loadImage = async function(elem) { + const src = elem.getAttribute('src') || ''; + elem.removeAttribute('src'); + + await vAPI.messaging.send('scriptlets', { + what: 'temporarilyAllowLargeMediaElement', + }); + + if ( src !== '' ) { + elem.setAttribute('src', src); + } +}; + +/******************************************************************************/ + +const loadMany = function(elems) { + for ( const elem of elems ) { + switch ( elem.localName ) { + case 'audio': + case 'video': + loadMedia(elem); + break; + case 'img': + loadImage(elem); + break; + default: + break; + } + } +}; + +/******************************************************************************/ + +const onMouseClick = function(ev) { + if ( ev.button !== 0 || ev.isTrusted === false ) { return; } + + const toLoad = []; + const elems = document.elementsFromPoint instanceof Function + ? document.elementsFromPoint(ev.clientX, ev.clientY) + : [ ev.target ]; + for ( const elem of elems ) { + if ( elem.matches(largeMediaElementSelector) === false ) { continue; } + elem.removeAttribute(largeMediaElementAttribute); + if ( mediaNotLoaded(elem) ) { + toLoad.push(elem); + } + } + + if ( toLoad.length === 0 ) { return; } + + loadMany(toLoad); + + ev.preventDefault(); + ev.stopPropagation(); +}; + +document.addEventListener('click', onMouseClick, true); + +/******************************************************************************/ + +const onLoadedData = function(ev) { + const media = ev.target; + if ( media.localName !== 'audio' && media.localName !== 'video' ) { + return; + } + const src = media.src; + if ( typeof src === 'string' && src.startsWith('blob:') === false ) { + return; + } + media.autoplay = false; + media.pause(); +}; + +// https://www.reddit.com/r/uBlockOrigin/comments/mxgpmc/ +// Support cases where the media source is not yet set. +for ( const media of document.querySelectorAll('audio,video') ) { + const src = media.src; + if ( + (typeof src === 'string') && + (src === '' || src.startsWith('blob:')) + ) { + media.autoplay = false; + media.pause(); + } +} + +document.addEventListener('loadeddata', onLoadedData); + +/******************************************************************************/ + +const onLoad = function(ev) { + const elem = ev.target; + if ( isMediaElement(elem) === false ) { return; } + elem.removeAttribute(largeMediaElementAttribute); +}; + +document.addEventListener('load', onLoad, true); + +/******************************************************************************/ + +const onLoadError = function(ev) { + const elem = ev.target; + if ( isMediaElement(elem) === false ) { return; } + if ( mediaNotLoaded(elem) ) { + elem.setAttribute(largeMediaElementAttribute, ''); + } +}; + +document.addEventListener('error', onLoadError, true); + +/******************************************************************************/ + +vAPI.loadAllLargeMedia = function() { + document.removeEventListener('click', onMouseClick, true); + document.removeEventListener('loadeddata', onLoadedData, true); + document.removeEventListener('load', onLoad, true); + document.removeEventListener('error', onLoadError, true); + + const toLoad = []; + for ( const elem of document.querySelectorAll(largeMediaElementSelector) ) { + elem.removeAttribute(largeMediaElementAttribute); + if ( mediaNotLoaded(elem) ) { + toLoad.push(elem); + } + } + loadMany(toLoad); +}; + +/******************************************************************************/ + +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/noscript-spoof.js b/src/js/scriptlets/noscript-spoof.js new file mode 100644 index 0000000..49e9093 --- /dev/null +++ b/src/js/scriptlets/noscript-spoof.js @@ -0,0 +1,89 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +// Code below has been imported from uMatrix and modified to fit uBO: +// https://github.com/gorhill/uMatrix/blob/3f8794dd899a05e066c24066c6c0a2515d5c60d2/src/js/contentscript.js#L464-L531 + +'use strict'; + +/******************************************************************************/ + +// https://github.com/gorhill/uMatrix/issues/232 +// Force `display` property, Firefox is still affected by the issue. + +(( ) => { + const noscripts = document.querySelectorAll('noscript'); + if ( noscripts.length === 0 ) { return; } + + const reMetaContent = /^\s*(\d+)\s*;\s*url=(?:"([^"]+)"|'([^']+)'|(.+))/i; + const reSafeURL = /^https?:\/\//; + let redirectTimer; + + const autoRefresh = function(root) { + const meta = root.querySelector('meta[http-equiv="refresh"][content]'); + if ( meta === null ) { return; } + const match = reMetaContent.exec(meta.getAttribute('content')); + if ( match === null ) { return; } + const refreshURL = (match[2] || match[3] || match[4] || '').trim(); + let url; + try { + url = new URL(refreshURL, document.baseURI); + } catch(ex) { + return; + } + if ( reSafeURL.test(url.href) === false ) { return; } + redirectTimer = setTimeout(( ) => { + location.assign(url.href); + }, + parseInt(match[1], 10) * 1000 + 1 + ); + meta.parentNode.removeChild(meta); + }; + + const morphNoscript = function(from) { + if ( /^application\/(?:xhtml\+)?xml/.test(document.contentType) ) { + const to = document.createElement('span'); + while ( from.firstChild !== null ) { + to.appendChild(from.firstChild); + } + return to; + } + const parser = new DOMParser(); + const doc = parser.parseFromString( + '<span>' + from.textContent + '</span>', + 'text/html' + ); + return document.adoptNode(doc.querySelector('span')); + }; + + for ( const noscript of noscripts ) { + const parent = noscript.parentNode; + if ( parent === null ) { continue; } + const span = morphNoscript(noscript); + span.style.setProperty('display', 'inline', 'important'); + if ( redirectTimer === undefined ) { + autoRefresh(span); + } + parent.replaceChild(span, noscript); + } +})(); + +/******************************************************************************/ diff --git a/src/js/scriptlets/should-inject-contentscript.js b/src/js/scriptlets/should-inject-contentscript.js new file mode 100644 index 0000000..b9a2658 --- /dev/null +++ b/src/js/scriptlets/should-inject-contentscript.js @@ -0,0 +1,40 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2018-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +// If content scripts are already injected, we need to respond with `false`, +// to "should inject content scripts?" +// +// https://github.com/uBlockOrigin/uBlock-issues/issues/403 +// If the content script was not bootstrapped, give it another try. + +(( ) => { + try { + let status = vAPI.uBO !== true; + if ( status === false && vAPI.bootstrap ) { + self.requestIdleCallback(( ) => vAPI && vAPI.bootstrap()); + } + return status; + } catch(ex) { + } + return true; +})(); diff --git a/src/js/scriptlets/subscriber.js b/src/js/scriptlets/subscriber.js new file mode 100644 index 0000000..ea7b209 --- /dev/null +++ b/src/js/scriptlets/subscriber.js @@ -0,0 +1,113 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global HTMLDocument */ + +'use strict'; + +/******************************************************************************/ + +// Injected into specific web pages, those which have been pre-selected +// because they are known to contains `abp:subscribe` links. + +/******************************************************************************/ + +(( ) => { +// >>>>> start of local scope + +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/464 +if ( document instanceof HTMLDocument === false ) { return; } + +// Maybe uBO has gone away meanwhile. +if ( typeof vAPI !== 'object' || vAPI === null ) { return; } + +const onMaybeSubscriptionLinkClicked = function(target) { + if ( vAPI instanceof Object === false ) { + document.removeEventListener('click', onMaybeSubscriptionLinkClicked); + return; + } + + try { + // https://github.com/uBlockOrigin/uBlock-issues/issues/763#issuecomment-691696716 + // Remove replacement patch if/when filterlists.com fixes encoded '&'. + const subscribeURL = new URL( + target.href.replace('&title=', '&title=') + ); + if ( + /^(abp|ubo):$/.test(subscribeURL.protocol) === false && + subscribeURL.hostname !== 'subscribe.adblockplus.org' + ) { + return; + } + const location = subscribeURL.searchParams.get('location') || ''; + const title = subscribeURL.searchParams.get('title') || ''; + if ( location === '' || title === '' ) { return true; } + // https://github.com/uBlockOrigin/uBlock-issues/issues/1797 + if ( /^(file|https?):\/\//.test(location) === false ) { return true; } + vAPI.messaging.send('scriptlets', { + what: 'subscribeTo', + location, + title, + }); + return true; + } catch (_) { + } +}; + +// https://github.com/easylist/EasyListHebrew/issues/89 +// Ensure trusted events only. + +document.addEventListener('click', ev => { + if ( ev.button !== 0 || ev.isTrusted === false ) { return; } + const target = ev.target.closest('a'); + if ( target instanceof HTMLAnchorElement === false ) { return; } + if ( onMaybeSubscriptionLinkClicked(target) === true ) { + ev.stopPropagation(); + ev.preventDefault(); + } +}); + +/******************************************************************************/ + +// <<<<< end of local scope +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/scriptlets/updater.js b/src/js/scriptlets/updater.js new file mode 100644 index 0000000..006b663 --- /dev/null +++ b/src/js/scriptlets/updater.js @@ -0,0 +1,118 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global HTMLDocument */ + +'use strict'; + +/******************************************************************************/ + +// Injected into specific webpages, those which have been pre-selected +// because they are known to contain: +// https://ublockorigin.github.io/uAssets/update-lists?listkeys=[...] + +/******************************************************************************/ + +(( ) => { +// >>>>> start of local scope + +/******************************************************************************/ + +if ( document instanceof HTMLDocument === false ) { return; } + +// Maybe uBO has gone away meanwhile. +if ( typeof vAPI !== 'object' || vAPI === null ) { return; } + +function updateStockLists(target) { + if ( vAPI instanceof Object === false ) { + document.removeEventListener('click', updateStockLists); + return; + } + try { + const updateURL = new URL(target.href); + if ( updateURL.hostname !== 'ublockorigin.github.io') { return; } + if ( updateURL.pathname !== '/uAssets/update-lists.html') { return; } + const listkeys = updateURL.searchParams.get('listkeys') || ''; + if ( listkeys === '' ) { return; } + let auto = true; + const manual = updateURL.searchParams.get('manual'); + if ( manual === '1' ) { + auto = false; + } else if ( /^\d{6}$/.test(`${manual}`) ) { + const year = parseInt(manual.slice(0,2)) || 0; + const month = parseInt(manual.slice(2,4)) || 0; + const day = parseInt(manual.slice(4,6)) || 0; + if ( year !== 0 && month !== 0 && day !== 0 ) { + const date = new Date(); + date.setUTCFullYear(2000 + year, month - 1, day); + date.setUTCHours(0); + const then = date.getTime() / 1000 / 3600; + const now = Date.now() / 1000 / 3600; + auto = then < (now - 48) || then > (now + 48); + } + } + vAPI.messaging.send('scriptlets', { + what: 'updateLists', + listkeys, + auto, + }); + return true; + } catch (_) { + } +} + +// https://github.com/easylist/EasyListHebrew/issues/89 +// Ensure trusted events only. + +document.addEventListener('click', ev => { + if ( ev.button !== 0 || ev.isTrusted === false ) { return; } + const target = ev.target.closest('a'); + if ( target instanceof HTMLAnchorElement === false ) { return; } + if ( updateStockLists(target) === true ) { + ev.stopPropagation(); + ev.preventDefault(); + } +}); + +/******************************************************************************/ + +// <<<<< end of local scope +})(); + + + + + + + + +/******************************************************************************* + + DO NOT: + - Remove the following code + - Add code beyond the following code + Reason: + - https://github.com/gorhill/uBlock/pull/3721 + - uBO never uses the return value from injected content scripts + +**/ + +void 0; diff --git a/src/js/settings.js b/src/js/settings.js new file mode 100644 index 0000000..deb033f --- /dev/null +++ b/src/js/settings.js @@ -0,0 +1,317 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +import { i18n$ } from './i18n.js'; +import { dom, qs$, qsa$ } from './dom.js'; +import { setAccentColor, setTheme } from './theme.js'; + +/******************************************************************************/ + +const handleImportFilePicker = function() { + const file = this.files[0]; + if ( file === undefined || file.name === '' ) { return; } + + const reportError = ( ) => { + window.alert(i18n$('aboutRestoreDataError')); + }; + + const expectedFileTypes = [ + 'text/plain', + 'application/json', + ]; + if ( expectedFileTypes.includes(file.type) === false ) { + return reportError(); + } + + const filename = file.name; + const fr = new FileReader(); + + fr.onload = function() { + let userData; + try { + userData = JSON.parse(this.result); + if ( typeof userData !== 'object' ) { + throw 'Invalid'; + } + if ( typeof userData.userSettings !== 'object' ) { + throw 'Invalid'; + } + if ( + Array.isArray(userData.whitelist) === false && + typeof userData.netWhitelist !== 'string' + ) { + throw 'Invalid'; + } + if ( + typeof userData.filterLists !== 'object' && + Array.isArray(userData.selectedFilterLists) === false + ) { + throw 'Invalid'; + } + } + catch (e) { + userData = undefined; + } + if ( userData === undefined ) { + return reportError(); + } + const time = new Date(userData.timeStamp); + const msg = i18n$('aboutRestoreDataConfirm') + .replace('{{time}}', time.toLocaleString()); + const proceed = window.confirm(msg); + if ( proceed !== true ) { return; } + vAPI.messaging.send('dashboard', { + what: 'restoreUserData', + userData, + file: filename, + }); + }; + + fr.readAsText(file); +}; + +/******************************************************************************/ + +const startImportFilePicker = function() { + const input = qs$('#restoreFilePicker'); + // Reset to empty string, this will ensure an change event is properly + // triggered if the user pick a file, even if it is the same as the last + // one picked. + input.value = ''; + input.click(); +}; + +/******************************************************************************/ + +const exportToFile = async function() { + const response = await vAPI.messaging.send('dashboard', { + what: 'backupUserData', + }); + if ( + response instanceof Object === false || + response.userData instanceof Object === false + ) { + return; + } + vAPI.download({ + 'url': 'data:text/plain;charset=utf-8,' + + encodeURIComponent(JSON.stringify(response.userData, null, ' ')), + 'filename': response.localData.lastBackupFile + }); + onLocalDataReceived(response.localData); +}; + +/******************************************************************************/ + +const onLocalDataReceived = function(details) { + let v, unit; + if ( typeof details.storageUsed === 'number' ) { + v = details.storageUsed; + if ( v < 1e3 ) { + unit = 'genericBytes'; + } else if ( v < 1e6 ) { + v /= 1e3; + unit = 'KB'; + } else if ( v < 1e9 ) { + v /= 1e6; + unit = 'MB'; + } else { + v /= 1e9; + unit = 'GB'; + } + } else { + v = '?'; + unit = ''; + } + dom.text( + '#storageUsed', + i18n$('storageUsed') + .replace('{{value}}', v.toLocaleString(undefined, { maximumSignificantDigits: 3 })) + .replace('{{unit}}', unit && i18n$(unit) || '') + ); + + const timeOptions = { + weekday: 'long', + year: 'numeric', + month: 'long', + day: 'numeric', + hour: 'numeric', + minute: 'numeric', + timeZoneName: 'short' + }; + + const lastBackupFile = details.lastBackupFile || ''; + if ( lastBackupFile !== '' ) { + const dt = new Date(details.lastBackupTime); + const text = i18n$('settingsLastBackupPrompt'); + const node = qs$('#settingsLastBackupPrompt'); + node.textContent = text + '\xA0' + dt.toLocaleString('fullwide', timeOptions); + node.style.display = ''; + } + + const lastRestoreFile = details.lastRestoreFile || ''; + if ( lastRestoreFile !== '' ) { + const dt = new Date(details.lastRestoreTime); + const text = i18n$('settingsLastRestorePrompt'); + const node = qs$('#settingsLastRestorePrompt'); + node.textContent = text + '\xA0' + dt.toLocaleString('fullwide', timeOptions); + node.style.display = ''; + } + + if ( details.cloudStorageSupported === false ) { + dom.attr('[data-setting-name="cloudStorageEnabled"]', 'disabled', ''); + } + + if ( details.privacySettingsSupported === false ) { + dom.attr('[data-setting-name="prefetchingDisabled"]', 'disabled', ''); + dom.attr('[data-setting-name="hyperlinkAuditingDisabled"]', 'disabled', ''); + dom.attr('[data-setting-name="webrtcIPAddressHidden"]', 'disabled', ''); + } +}; + +/******************************************************************************/ + +const resetUserData = function() { + const msg = i18n$('aboutResetDataConfirm'); + const proceed = window.confirm(msg); + if ( proceed !== true ) { return; } + vAPI.messaging.send('dashboard', { + what: 'resetUserData', + }); +}; + +/******************************************************************************/ + +const synchronizeDOM = function() { + dom.cl.toggle( + dom.body, + 'advancedUser', + qs$('[data-setting-name="advancedUserEnabled"]').checked === true + ); +}; + +/******************************************************************************/ + +const changeUserSettings = function(name, value) { + vAPI.messaging.send('dashboard', { + what: 'userSettings', + name, + value, + }); + + // Maybe reflect some changes immediately + switch ( name ) { + case 'uiTheme': + setTheme(value, true); + break; + case 'uiAccentCustom': + case 'uiAccentCustom0': + setAccentColor( + qs$('[data-setting-name="uiAccentCustom"]').checked, + qs$('[data-setting-name="uiAccentCustom0"]').value, + true + ); + break; + default: + break; + } +}; + +/******************************************************************************/ + +const onValueChanged = function(ev) { + const input = ev.target; + const name = dom.attr(input, 'data-setting-name'); + let value = input.value; + // Maybe sanitize value + switch ( name ) { + case 'largeMediaSize': + value = Math.min(Math.max(Math.floor(parseInt(value, 10) || 0), 0), 1000000); + break; + default: + break; + } + if ( value !== input.value ) { + input.value = value; + } + + changeUserSettings(name, value); +}; + +/******************************************************************************/ + +// TODO: use data-* to declare simple settings + +const onUserSettingsReceived = function(details) { + const checkboxes = qsa$('[data-setting-type="bool"]'); + for ( const checkbox of checkboxes ) { + const name = dom.attr(checkbox, 'data-setting-name') || ''; + if ( details[name] === undefined ) { + dom.attr(checkbox.closest('.checkbox'), 'disabled', ''); + dom.attr(checkbox, 'disabled', ''); + continue; + } + checkbox.checked = details[name] === true; + dom.on(checkbox, 'change', ( ) => { + changeUserSettings(name, checkbox.checked); + synchronizeDOM(); + }); + } + + if ( details.canLeakLocalIPAddresses === true ) { + qs$('[data-setting-name="webrtcIPAddressHidden"]') + .closest('div.li') + .style.display = ''; + } + + qsa$('[data-setting-type="value"]').forEach(function(elem) { + elem.value = details[dom.attr(elem, 'data-setting-name')]; + dom.on(elem, 'change', onValueChanged); + }); + + dom.on('#export', 'click', ( ) => { exportToFile(); }); + dom.on('#import', 'click', startImportFilePicker); + dom.on('#reset', 'click', resetUserData); + dom.on('#restoreFilePicker', 'change', handleImportFilePicker); + + synchronizeDOM(); +}; + +/******************************************************************************/ + +vAPI.messaging.send('dashboard', { what: 'userSettings' }).then(result => { + onUserSettingsReceived(result); +}); + +vAPI.messaging.send('dashboard', { what: 'getLocalData' }).then(result => { + onLocalDataReceived(result); +}); + +// https://github.com/uBlockOrigin/uBlock-issues/issues/591 +dom.on( + '[data-i18n-title="settingsAdvancedUserSettings"]', + 'click', + self.uBlockDashboard.openOrSelectPage +); + +/******************************************************************************/ diff --git a/src/js/start.js b/src/js/start.js new file mode 100644 index 0000000..5762619 --- /dev/null +++ b/src/js/start.js @@ -0,0 +1,508 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +import './vapi-common.js'; +import './vapi-background.js'; +import './vapi-background-ext.js'; + +/******************************************************************************/ + +// The following modules are loaded here until their content is better organized +import './commands.js'; +import './messaging.js'; +import './storage.js'; +import './tab.js'; +import './ublock.js'; +import './utils.js'; + +import io from './assets.js'; +import µb from './background.js'; +import { filteringBehaviorChanged } from './broadcast.js'; +import cacheStorage from './cachestorage.js'; +import { ubolog } from './console.js'; +import contextMenu from './contextmenu.js'; +import lz4Codec from './lz4.js'; +import { redirectEngine } from './redirect-engine.js'; +import staticFilteringReverseLookup from './reverselookup.js'; +import staticExtFilteringEngine from './static-ext-filtering.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import webRequest from './traffic.js'; + +import { + permanentFirewall, + sessionFirewall, + permanentSwitches, + sessionSwitches, + permanentURLFiltering, + sessionURLFiltering, +} from './filtering-engines.js'; + +/******************************************************************************/ + +vAPI.app.onShutdown = ( ) => { + staticFilteringReverseLookup.shutdown(); + io.updateStop(); + staticNetFilteringEngine.reset(); + staticExtFilteringEngine.reset(); + sessionFirewall.reset(); + permanentFirewall.reset(); + sessionURLFiltering.reset(); + permanentURLFiltering.reset(); + sessionSwitches.reset(); + permanentSwitches.reset(); +}; + +/******************************************************************************/ + +// This is called only once, when everything has been loaded in memory after +// the extension was launched. It can be used to inject content scripts +// in already opened web pages, to remove whatever nuisance could make it to +// the web pages before uBlock was ready. +// +// https://bugzilla.mozilla.org/show_bug.cgi?id=1652925#c19 +// Mind discarded tabs. + +const initializeTabs = async ( ) => { + const manifest = browser.runtime.getManifest(); + if ( manifest instanceof Object === false ) { return; } + + const toCheck = []; + const tabIds = []; + { + const checker = { file: 'js/scriptlets/should-inject-contentscript.js' }; + const tabs = await vAPI.tabs.query({ url: '<all_urls>' }); + for ( const tab of tabs ) { + if ( tab.discarded === true ) { continue; } + if ( tab.status === 'unloaded' ) { continue; } + const { id, url } = tab; + µb.tabContextManager.commit(id, url); + µb.bindTabToPageStore(id, 'tabCommitted', tab); + // https://github.com/chrisaljoudi/uBlock/issues/129 + // Find out whether content scripts need to be injected + // programmatically. This may be necessary for web pages which + // were loaded before uBO launched. + toCheck.push( + /^https?:\/\//.test(url) + ? vAPI.tabs.executeScript(id, checker) + : false + ); + tabIds.push(id); + } + } + // We do not want to block on content scripts injection + Promise.all(toCheck).then(results => { + for ( let i = 0; i < results.length; i++ ) { + const result = results[i]; + if ( result.length === 0 || result[0] !== true ) { continue; } + // Inject declarative content scripts programmatically. + for ( const contentScript of manifest.content_scripts ) { + for ( const file of contentScript.js ) { + vAPI.tabs.executeScript(tabIds[i], { + file: file, + allFrames: contentScript.all_frames, + runAt: contentScript.run_at + }); + } + } + } + }); +}; + +/******************************************************************************/ + +// To bring older versions up to date +// +// https://www.reddit.com/r/uBlockOrigin/comments/s7c9go/ +// Abort suspending network requests when uBO is merely being installed. + +const onVersionReady = lastVersion => { + if ( lastVersion === vAPI.app.version ) { return; } + + vAPI.storage.set({ + version: vAPI.app.version, + versionUpdateTime: Date.now(), + }); + + const lastVersionInt = vAPI.app.intFromVersion(lastVersion); + + // Special case: first installation + if ( lastVersionInt === 0 ) { + vAPI.net.unsuspend({ all: true, discard: true }); + return; + } + + // Since built-in resources may have changed since last version, we + // force a reload of all resources. + redirectEngine.invalidateResourcesSelfie(io); +}; + +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/226 +// Whitelist in memory. +// Whitelist parser needs PSL to be ready. +// gorhill 2014-12-15: not anymore +// +// https://github.com/uBlockOrigin/uBlock-issues/issues/1433 +// Allow admins to add their own trusted-site directives. + +const onNetWhitelistReady = (netWhitelistRaw, adminExtra) => { + if ( typeof netWhitelistRaw === 'string' ) { + netWhitelistRaw = netWhitelistRaw.split('\n'); + } + // Append admin-controlled trusted-site directives + if ( + adminExtra instanceof Object && + Array.isArray(adminExtra.trustedSiteDirectives) + ) { + for ( const directive of adminExtra.trustedSiteDirectives ) { + µb.netWhitelistDefault.push(directive); + netWhitelistRaw.push(directive); + } + } + µb.netWhitelist = µb.whitelistFromArray(netWhitelistRaw); + µb.netWhitelistModifyTime = Date.now(); +}; + +/******************************************************************************/ + +// User settings are in memory + +const onUserSettingsReady = fetched => { + // Terminate suspended state? + const tnow = Date.now() - vAPI.T0; + if ( + vAPI.Net.canSuspend() && + fetched.suspendUntilListsAreLoaded === false + ) { + vAPI.net.unsuspend({ all: true, discard: true }); + ubolog(`Unsuspend network activity listener at ${tnow} ms`); + µb.supportStats.unsuspendAfter = `${tnow} ms`; + } else if ( + vAPI.Net.canSuspend() === false && + fetched.suspendUntilListsAreLoaded + ) { + vAPI.net.suspend(); + ubolog(`Suspend network activity listener at ${tnow} ms`); + } + + // `externalLists` will be deprecated in some future, it is kept around + // for forward compatibility purpose, and should reflect the content of + // `importedLists`. + if ( Array.isArray(fetched.externalLists) ) { + fetched.externalLists = fetched.externalLists.join('\n'); + vAPI.storage.set({ externalLists: fetched.externalLists }); + } + if ( + fetched.importedLists.length === 0 && + fetched.externalLists !== '' + ) { + fetched.importedLists = + fetched.externalLists.trim().split(/[\n\r]+/); + } + + fromFetch(µb.userSettings, fetched); + + if ( µb.privacySettingsSupported ) { + vAPI.browserSettings.set({ + 'hyperlinkAuditing': !µb.userSettings.hyperlinkAuditingDisabled, + 'prefetching': !µb.userSettings.prefetchingDisabled, + 'webrtcIPAddress': !µb.userSettings.webrtcIPAddressHidden + }); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1513 + if ( + vAPI.net.canUncloakCnames && + µb.userSettings.cnameUncloakEnabled === false + ) { + vAPI.net.setOptions({ cnameUncloakEnabled: false }); + } +}; + +/******************************************************************************/ + +// https://bugzilla.mozilla.org/show_bug.cgi?id=1588916 +// Save magic format numbers into the cache storage itself. +// https://github.com/uBlockOrigin/uBlock-issues/issues/1365 +// Wait for removal of invalid cached data to be completed. + +const onCacheSettingsReady = async (fetched = {}) => { + if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { + µb.compiledFormatChanged = true; + µb.selfieIsInvalid = true; + ubolog(`Serialized format of static filter lists changed`); + } + if ( fetched.selfieMagic !== µb.systemSettings.selfieMagic ) { + µb.selfieIsInvalid = true; + ubolog(`Serialized format of selfie changed`); + } + if ( µb.selfieIsInvalid ) { + µb.selfieManager.destroy(); + cacheStorage.set(µb.systemSettings); + } +}; + +/******************************************************************************/ + +const onHiddenSettingsReady = async ( ) => { + // Maybe customize webext flavor + if ( µb.hiddenSettings.modifyWebextFlavor !== 'unset' ) { + const tokens = µb.hiddenSettings.modifyWebextFlavor.split(/\s+/); + for ( const token of tokens ) { + switch ( token[0] ) { + case '+': + vAPI.webextFlavor.soup.add(token.slice(1)); + break; + case '-': + vAPI.webextFlavor.soup.delete(token.slice(1)); + break; + default: + vAPI.webextFlavor.soup.add(token); + break; + } + } + ubolog(`Override default webext flavor with ${tokens}`); + } + + // Maybe disable WebAssembly + if ( vAPI.canWASM && µb.hiddenSettings.disableWebAssembly !== true ) { + const wasmModuleFetcher = function(path) { + return fetch(`${path}.wasm`, { mode: 'same-origin' }).then( + WebAssembly.compileStreaming + ).catch(reason => { + ubolog(reason); + }); + }; + staticNetFilteringEngine.enableWASM(wasmModuleFetcher, './js/wasm/').then(result => { + if ( result !== true ) { return; } + ubolog(`WASM modules ready ${Date.now()-vAPI.T0} ms after launch`); + }); + } + + // Maybe override default cache storage + µb.supportStats.cacheBackend = await cacheStorage.select( + µb.hiddenSettings.cacheStorageAPI + ); + ubolog(`Backend storage for cache will be ${µb.supportStats.cacheBackend}`); +}; + +/******************************************************************************/ + +const onFirstFetchReady = (fetched, adminExtra) => { + // https://github.com/uBlockOrigin/uBlock-issues/issues/507 + // Firefox-specific: somehow `fetched` is undefined under certain + // circumstances even though we asked to load with default values. + if ( fetched instanceof Object === false ) { + fetched = createDefaultProps(); + } + + // Order is important -- do not change: + fromFetch(µb.localSettings, fetched); + fromFetch(µb.restoreBackupSettings, fetched); + + permanentFirewall.fromString(fetched.dynamicFilteringString); + sessionFirewall.assign(permanentFirewall); + permanentURLFiltering.fromString(fetched.urlFilteringString); + sessionURLFiltering.assign(permanentURLFiltering); + permanentSwitches.fromString(fetched.hostnameSwitchesString); + sessionSwitches.assign(permanentSwitches); + + onNetWhitelistReady(fetched.netWhitelist, adminExtra); + onVersionReady(fetched.version); +}; + +/******************************************************************************/ + +const toFetch = (from, fetched) => { + for ( const k in from ) { + if ( from.hasOwnProperty(k) === false ) { continue; } + fetched[k] = from[k]; + } +}; + +const fromFetch = (to, fetched) => { + for ( const k in to ) { + if ( to.hasOwnProperty(k) === false ) { continue; } + if ( fetched.hasOwnProperty(k) === false ) { continue; } + to[k] = fetched[k]; + } +}; + +const createDefaultProps = ( ) => { + const fetchableProps = { + 'dynamicFilteringString': µb.dynamicFilteringDefault.join('\n'), + 'urlFilteringString': '', + 'hostnameSwitchesString': µb.hostnameSwitchesDefault.join('\n'), + 'lastRestoreFile': '', + 'lastRestoreTime': 0, + 'lastBackupFile': '', + 'lastBackupTime': 0, + 'netWhitelist': µb.netWhitelistDefault, + 'version': '0.0.0.0' + }; + toFetch(µb.localSettings, fetchableProps); + toFetch(µb.restoreBackupSettings, fetchableProps); + return fetchableProps; +}; + +/******************************************************************************/ + +(async ( ) => { +// >>>>> start of async/await scope + +try { + ubolog(`Start sequence of loading storage-based data ${Date.now()-vAPI.T0} ms after launch`); + + // https://github.com/gorhill/uBlock/issues/531 + await µb.restoreAdminSettings(); + ubolog(`Admin settings ready ${Date.now()-vAPI.T0} ms after launch`); + + await µb.loadHiddenSettings(); + await onHiddenSettingsReady(); + ubolog(`Hidden settings ready ${Date.now()-vAPI.T0} ms after launch`); + + const adminExtra = await vAPI.adminStorage.get('toAdd'); + ubolog(`Extra admin settings ready ${Date.now()-vAPI.T0} ms after launch`); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 + // Wait for onCacheSettingsReady() to be fully ready. + const [ , , lastVersion ] = await Promise.all([ + µb.loadSelectedFilterLists().then(( ) => { + ubolog(`List selection ready ${Date.now()-vAPI.T0} ms after launch`); + }), + cacheStorage.get( + { compiledMagic: 0, selfieMagic: 0 } + ).then(fetched => { + ubolog(`Cache magic numbers ready ${Date.now()-vAPI.T0} ms after launch`); + onCacheSettingsReady(fetched); + }), + vAPI.storage.get(createDefaultProps()).then(fetched => { + ubolog(`First fetch ready ${Date.now()-vAPI.T0} ms after launch`); + onFirstFetchReady(fetched, adminExtra); + return fetched.version; + }), + µb.loadUserSettings().then(fetched => { + ubolog(`User settings ready ${Date.now()-vAPI.T0} ms after launch`); + onUserSettingsReady(fetched); + }), + µb.loadPublicSuffixList().then(( ) => { + ubolog(`PSL ready ${Date.now()-vAPI.T0} ms after launch`); + }), + ]); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1547 + if ( lastVersion === '0.0.0.0' && vAPI.webextFlavor.soup.has('chromium') ) { + vAPI.app.restart(); + return; + } +} catch (ex) { + console.trace(ex); +} + +// Prime the filtering engines before first use. +staticNetFilteringEngine.prime(); + +// https://github.com/uBlockOrigin/uBlock-issues/issues/817#issuecomment-565730122 +// Still try to load filter lists regardless of whether a serious error +// occurred in the previous initialization steps. +let selfieIsValid = false; +try { + selfieIsValid = await µb.selfieManager.load(); + if ( selfieIsValid === true ) { + ubolog(`Selfie ready ${Date.now()-vAPI.T0} ms after launch`); + } +} catch (ex) { + console.trace(ex); +} +if ( selfieIsValid !== true ) { + try { + await µb.loadFilterLists(); + ubolog(`Filter lists ready ${Date.now()-vAPI.T0} ms after launch`); + } catch (ex) { + console.trace(ex); + } +} + +// Flush memory cache -- unsure whether the browser does this internally +// when loading a new extension. +filteringBehaviorChanged(); + +// Final initialization steps after all needed assets are in memory. + +// https://github.com/uBlockOrigin/uBlock-issues/issues/974 +// This can be used to defer filtering decision-making. +µb.readyToFilter = true; + +// Initialize internal state with maybe already existing tabs. +await initializeTabs(); + +// Start network observers. +webRequest.start(); + +// Ensure that the resources allocated for decompression purpose (likely +// large buffers) are garbage-collectable immediately after launch. +// Otherwise I have observed that it may take quite a while before the +// garbage collection of these resources kicks in. Relinquishing as soon +// as possible ensure minimal memory usage baseline. +lz4Codec.relinquish(); + +// https://github.com/chrisaljoudi/uBlock/issues/184 +// Check for updates not too far in the future. +io.addObserver(µb.assetObserver.bind(µb)); +µb.scheduleAssetUpdater({ + updateDelay: µb.userSettings.autoUpdate + ? µb.hiddenSettings.autoUpdateDelayAfterLaunch * 1000 + : 0 +}); + +// Force an update of the context menu according to the currently +// active tab. +contextMenu.update(); + +// https://github.com/uBlockOrigin/uBlock-issues/issues/717 +// Prevent the extension from being restarted mid-session. +browser.runtime.onUpdateAvailable.addListener(details => { + const toInt = vAPI.app.intFromVersion; + if ( + µb.hiddenSettings.extensionUpdateForceReload === true || + toInt(details.version) <= toInt(vAPI.app.version) + ) { + vAPI.app.restart(); + } +}); + +µb.supportStats.allReadyAfter = `${Date.now() - vAPI.T0} ms`; +if ( selfieIsValid ) { + µb.supportStats.allReadyAfter += ' (selfie)'; +} +ubolog(`All ready ${µb.supportStats.allReadyAfter} after launch`); + +µb.isReadyResolve(); + +// <<<<< end of async/await scope +})(); diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js new file mode 100644 index 0000000..fb677ad --- /dev/null +++ b/src/js/static-dnr-filtering.js @@ -0,0 +1,497 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import staticNetFilteringEngine from './static-net-filtering.js'; +import { LineIterator } from './text-utils.js'; +import * as sfp from './static-filtering-parser.js'; + +import { + CompiledListReader, + CompiledListWriter, +} from './static-filtering-io.js'; + +/******************************************************************************/ + +// http://www.cse.yorku.ca/~oz/hash.html#djb2 +// Must mirror content script surveyor's version + +const hashFromStr = (type, s) => { + const len = s.length; + const step = len + 7 >>> 3; + let hash = (type << 5) + type ^ len; + for ( let i = 0; i < len; i += step ) { + hash = (hash << 5) + hash ^ s.charCodeAt(i); + } + return hash & 0xFFFFFF; +}; + +/******************************************************************************/ + +// Copied from cosmetic-filter.js for the time being to avoid unwanted +// dependencies + +const rePlainSelector = /^[#.][\w\\-]+/; +const rePlainSelectorEx = /^[^#.\[(]+([#.][\w-]+)|([#.][\w-]+)$/; +const rePlainSelectorEscaped = /^[#.](?:\\[0-9A-Fa-f]+ |\\.|\w|-)+/; +const reEscapeSequence = /\\([0-9A-Fa-f]+ |.)/g; + +const keyFromSelector = selector => { + let key = ''; + let matches = rePlainSelector.exec(selector); + if ( matches ) { + key = matches[0]; + } else { + matches = rePlainSelectorEx.exec(selector); + if ( matches === null ) { return; } + key = matches[1] || matches[2]; + } + if ( key.indexOf('\\') === -1 ) { return key; } + matches = rePlainSelectorEscaped.exec(selector); + if ( matches === null ) { return; } + key = ''; + const escaped = matches[0]; + let beg = 0; + reEscapeSequence.lastIndex = 0; + for (;;) { + matches = reEscapeSequence.exec(escaped); + if ( matches === null ) { + return key + escaped.slice(beg); + } + key += escaped.slice(beg, matches.index); + beg = reEscapeSequence.lastIndex; + if ( matches[1].length === 1 ) { + key += matches[1]; + } else { + key += String.fromCharCode(parseInt(matches[1], 16)); + } + } +}; + +/******************************************************************************/ + +function addExtendedToDNR(context, parser) { + if ( parser.isExtendedFilter() === false ) { return false; } + + // Scriptlet injection + if ( parser.isScriptletFilter() ) { + if ( parser.hasOptions() === false ) { return; } + if ( context.scriptletFilters === undefined ) { + context.scriptletFilters = new Map(); + } + const exception = parser.isException(); + const args = parser.getScriptletArgs(); + const argsToken = JSON.stringify(args); + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + if ( exception ) { continue; } + let details = context.scriptletFilters.get(argsToken); + if ( details === undefined ) { + context.scriptletFilters.set(argsToken, details = { args }); + if ( context.trustedSource ) { + details.trustedSource = true; + } + } + if ( not ) { + if ( details.excludeMatches === undefined ) { + details.excludeMatches = []; + } + details.excludeMatches.push(hn); + continue; + } + if ( details.matches === undefined ) { + details.matches = []; + } + if ( details.matches.includes('*') ) { continue; } + if ( hn === '*' ) { + details.matches = [ '*' ]; + continue; + } + details.matches.push(hn); + } + return; + } + + // Response header filtering + if ( parser.isResponseheaderFilter() ) { + if ( parser.hasError() ) { return; } + if ( parser.hasOptions() === false ) { return; } + if ( parser.isException() ) { return; } + const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER); + if ( node === 0 ) { return; } + const header = parser.getNodeString(node); + if ( context.responseHeaderRules === undefined ) { + context.responseHeaderRules = []; + } + const rule = { + action: { + responseHeaders: [ + { + header, + operation: 'remove', + } + ], + type: 'modifyHeaders' + }, + condition: { + resourceTypes: [ + 'main_frame', + 'sub_frame' + ] + }, + }; + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + if ( not ) { + if ( rule.condition.excludedInitiatorDomains === undefined ) { + rule.condition.excludedInitiatorDomains = []; + } + rule.condition.excludedInitiatorDomains.push(hn); + continue; + } + if ( hn === '*' ) { + if ( rule.condition.initiatorDomains !== undefined ) { + rule.condition.initiatorDomains = undefined; + } + continue; + } + if ( rule.condition.initiatorDomains === undefined ) { + rule.condition.initiatorDomains = []; + } + rule.condition.initiatorDomains.push(hn); + } + context.responseHeaderRules.push(rule); + return; + } + + // HTML filtering + if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { + return; + } + + // Cosmetic filtering + + // Generic cosmetic filtering + if ( parser.hasOptions() === false ) { + const { compiled } = parser.result; + if ( compiled === undefined ) { return; } + if ( compiled.length <= 1 ) { return; } + if ( parser.isException() ) { + if ( context.genericCosmeticExceptions === undefined ) { + context.genericCosmeticExceptions = new Set(); + } + context.genericCosmeticExceptions.add(compiled); + return; + } + if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { return; } + const key = keyFromSelector(compiled); + if ( key === undefined ) { + if ( context.genericHighCosmeticFilters === undefined ) { + context.genericHighCosmeticFilters = new Set(); + } + context.genericHighCosmeticFilters.add(compiled); + return; + } + const type = key.charCodeAt(0); + const hash = hashFromStr(type, key.slice(1)); + if ( context.genericCosmeticFilters === undefined ) { + context.genericCosmeticFilters = new Map(); + } + let bucket = context.genericCosmeticFilters.get(hash); + if ( bucket === undefined ) { + context.genericCosmeticFilters.set(hash, bucket = []); + } + bucket.push(compiled); + return; + } + + // Specific cosmetic filtering + // https://github.com/chrisaljoudi/uBlock/issues/151 + // Negated hostname means the filter applies to all non-negated hostnames + // of same filter OR globally if there is no non-negated hostnames. + if ( context.specificCosmeticFilters === undefined ) { + context.specificCosmeticFilters = new Map(); + } + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + let { compiled, exception, raw } = parser.result; + if ( exception ) { continue; } + let rejected; + if ( compiled === undefined ) { + rejected = `Invalid filter: ${hn}##${raw}`; + } + if ( rejected ) { + compiled = rejected; + } + let details = context.specificCosmeticFilters.get(compiled); + if ( details === undefined ) { + details = {}; + if ( rejected ) { details.rejected = true; } + context.specificCosmeticFilters.set(compiled, details); + } + if ( rejected ) { continue; } + if ( not ) { + if ( details.excludeMatches === undefined ) { + details.excludeMatches = []; + } + details.excludeMatches.push(hn); + continue; + } + if ( details.matches === undefined ) { + details.matches = []; + } + if ( details.matches.includes('*') ) { continue; } + if ( hn === '*' ) { + details.matches = [ '*' ]; + continue; + } + details.matches.push(hn); + } +} + +/******************************************************************************/ + +function addToDNR(context, list) { + const env = context.env || []; + const writer = new CompiledListWriter(); + const lineIter = new LineIterator( + sfp.utils.preparser.prune(list.text, env) + ); + const parser = new sfp.AstFilterParser({ + toDNR: true, + nativeCssHas: env.includes('native_css_has'), + badTypes: [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE ], + }); + const compiler = staticNetFilteringEngine.createCompiler(); + + writer.properties.set('name', list.name); + compiler.start(writer); + + while ( lineIter.eot() === false ) { + let line = lineIter.next(); + while ( line.endsWith(' \\') ) { + if ( lineIter.peek(4) !== ' ' ) { break; } + line = line.slice(0, -2).trim() + lineIter.next().trim(); + } + + parser.parse(line); + + if ( parser.isComment() ) { + if ( line === `!#trusted on ${context.secret}` ) { + parser.trustedSource = true; + context.trustedSource = true; + } else if ( line === `!#trusted off ${context.secret}` ) { + parser.trustedSource = false; + context.trustedSource = false; + } + continue; + } + + if ( parser.isFilter() === false ) { continue; } + if ( parser.hasError() ) { + if ( parser.astError === sfp.AST_ERROR_OPTION_EXCLUDED ) { + context.invalid.add(`Incompatible with DNR: ${line}`); + } + continue; + } + + if ( parser.isExtendedFilter() ) { + addExtendedToDNR(context, parser); + continue; + } + if ( parser.isNetworkFilter() === false ) { continue; } + + if ( compiler.compile(parser, writer) ) { continue; } + + if ( compiler.error !== undefined ) { + context.invalid.add(compiler.error); + } + } + + compiler.finish(writer); + + staticNetFilteringEngine.dnrFromCompiled( + 'add', + context, + new CompiledListReader(writer.toString()) + ); +} + +/******************************************************************************/ + +function finalizeRuleset(context, network) { + const ruleset = network.ruleset; + + // Assign rule ids + const rulesetMap = new Map(); + { + let ruleId = 1; + for ( const rule of ruleset ) { + rulesetMap.set(ruleId++, rule); + } + } + // Merge rules where possible by merging arrays of a specific property. + // + // https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579 + // Do not merge rules which have errors. + const mergeRules = (rulesetMap, mergeTarget) => { + const mergeMap = new Map(); + const sorter = (_, v) => { + if ( Array.isArray(v) ) { + return typeof v[0] === 'string' ? v.sort() : v; + } + if ( v instanceof Object ) { + const sorted = {}; + for ( const kk of Object.keys(v).sort() ) { + sorted[kk] = v[kk]; + } + return sorted; + } + return v; + }; + const ruleHasher = (rule, target) => { + return JSON.stringify(rule, (k, v) => { + if ( k.startsWith('_') ) { return; } + if ( k === target ) { return; } + return sorter(k, v); + }); + }; + const extractTargetValue = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return v; } + if ( v instanceof Object ) { + const r = extractTargetValue(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + const extractTargetOwner = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return obj; } + if ( v instanceof Object ) { + const r = extractTargetOwner(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + for ( const [ id, rule ] of rulesetMap ) { + if ( rule._error !== undefined ) { continue; } + const hash = ruleHasher(rule, mergeTarget); + if ( mergeMap.has(hash) === false ) { + mergeMap.set(hash, []); + } + mergeMap.get(hash).push(id); + } + for ( const ids of mergeMap.values() ) { + if ( ids.length === 1 ) { continue; } + const leftHand = rulesetMap.get(ids[0]); + const leftHandSet = new Set( + extractTargetValue(leftHand, mergeTarget) || [] + ); + for ( let i = 1; i < ids.length; i++ ) { + const rightHandId = ids[i]; + const rightHand = rulesetMap.get(rightHandId); + const rightHandArray = extractTargetValue(rightHand, mergeTarget); + if ( rightHandArray !== undefined ) { + if ( leftHandSet.size !== 0 ) { + for ( const item of rightHandArray ) { + leftHandSet.add(item); + } + } + } else { + leftHandSet.clear(); + } + rulesetMap.delete(rightHandId); + } + const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); + if ( leftHandSet.size > 1 ) { + //if ( leftHandOwner === undefined ) { debugger; } + leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); + } else if ( leftHandSet.size === 0 ) { + if ( leftHandOwner !== undefined ) { + leftHandOwner[mergeTarget] = undefined; + } + } + } + }; + mergeRules(rulesetMap, 'resourceTypes'); + mergeRules(rulesetMap, 'initiatorDomains'); + mergeRules(rulesetMap, 'requestDomains'); + mergeRules(rulesetMap, 'removeParams'); + mergeRules(rulesetMap, 'responseHeaders'); + + // Patch id + const rulesetFinal = []; + { + let ruleId = 1; + for ( const rule of rulesetMap.values() ) { + if ( rule._error === undefined ) { + rule.id = ruleId++; + } else { + rule.id = 0; + } + rulesetFinal.push(rule); + } + for ( const invalid of context.invalid ) { + rulesetFinal.push({ _error: [ invalid ] }); + } + } + + network.ruleset = rulesetFinal; +} + +/******************************************************************************/ + +async function dnrRulesetFromRawLists(lists, options = {}) { + const context = Object.assign({}, options); + staticNetFilteringEngine.dnrFromCompiled('begin', context); + context.extensionPaths = new Map(context.extensionPaths || []); + const toLoad = []; + const toDNR = (context, list) => addToDNR(context, list); + for ( const list of lists ) { + if ( list instanceof Promise ) { + toLoad.push(list.then(list => toDNR(context, list))); + } else { + toLoad.push(toDNR(context, list)); + } + } + await Promise.all(toLoad); + const result = { + network: staticNetFilteringEngine.dnrFromCompiled('end', context), + genericCosmetic: context.genericCosmeticFilters, + genericHighCosmetic: context.genericHighCosmeticFilters, + genericCosmeticExceptions: context.genericCosmeticExceptions, + specificCosmetic: context.specificCosmeticFilters, + scriptlet: context.scriptletFilters, + }; + if ( context.responseHeaderRules ) { + result.network.ruleset.push(...context.responseHeaderRules); + } + finalizeRuleset(context, result.network); + return result; +} + +/******************************************************************************/ + +export { dnrRulesetFromRawLists }; diff --git a/src/js/static-ext-filtering-db.js b/src/js/static-ext-filtering-db.js new file mode 100644 index 0000000..64a9c8d --- /dev/null +++ b/src/js/static-ext-filtering-db.js @@ -0,0 +1,171 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +const StaticExtFilteringHostnameDB = class { + constructor(nBits, version = 0) { + this.version = version; + this.nBits = nBits; + this.strToIdMap = new Map(); + this.hostnameToSlotIdMap = new Map(); + this.regexToSlotIdMap = new Map(); + this.regexMap = new Map(); + // Array of integer pairs + this.hostnameSlots = []; + // Array of strings (selectors and pseudo-selectors) + this.strSlots = []; + this.size = 0; + this.cleanupTimer = vAPI.defer.create(( ) => { + this.strToIdMap.clear(); + }); + } + + store(hn, bits, s) { + this.size += 1; + let iStr = this.strToIdMap.get(s); + if ( iStr === undefined ) { + iStr = this.strSlots.length; + this.strSlots.push(s); + this.strToIdMap.set(s, iStr); + if ( this.cleanupTimer.ongoing() === false ) { + this.collectGarbage(true); + } + } + const strId = iStr << this.nBits | bits; + const hnIsNotRegex = hn.charCodeAt(0) !== 0x2F /* / */; + let iHn = hnIsNotRegex + ? this.hostnameToSlotIdMap.get(hn) + : this.regexToSlotIdMap.get(hn); + if ( iHn === undefined ) { + if ( hnIsNotRegex ) { + this.hostnameToSlotIdMap.set(hn, this.hostnameSlots.length); + } else { + this.regexToSlotIdMap.set(hn, this.hostnameSlots.length); + } + this.hostnameSlots.push(strId, 0); + return; + } + // Add as last item. + while ( this.hostnameSlots[iHn+1] !== 0 ) { + iHn = this.hostnameSlots[iHn+1]; + } + this.hostnameSlots[iHn+1] = this.hostnameSlots.length; + this.hostnameSlots.push(strId, 0); + } + + clear() { + this.hostnameToSlotIdMap.clear(); + this.regexToSlotIdMap.clear(); + this.hostnameSlots.length = 0; + this.strSlots.length = 0; + this.strToIdMap.clear(); + this.regexMap.clear(); + this.size = 0; + } + + collectGarbage(later = false) { + if ( later ) { + return this.cleanupTimer.onidle(5000, { timeout: 5000 }); + } + this.cleanupTimer.off(); + this.strToIdMap.clear(); + } + + // modifiers = 0: all items + // modifiers = 1: only specific items + // modifiers = 2: only generic items + // modifiers = 3: only regex-based items + // + retrieve(hostname, out, modifiers = 0) { + let hn = hostname; + if ( modifiers === 2 ) { hn = ''; } + for (;;) { + const hnSlot = this.hostnameToSlotIdMap.get(hn); + if ( hnSlot !== undefined ) { + this.retrieveFromSlot(hnSlot, out); + } + if ( hn === '' ) { break; } + const pos = hn.indexOf('.'); + if ( pos === -1 ) { + if ( modifiers === 1 ) { break; } + hn = ''; + } else { + hn = hn.slice(pos + 1); + } + } + if ( modifiers !== 0 && modifiers !== 3 ) { return; } + if ( this.regexToSlotIdMap.size === 0 ) { return; } + // TODO: consider using a combined regex to test once for whether + // iterating is worth it. + for ( const restr of this.regexToSlotIdMap.keys() ) { + let re = this.regexMap.get(restr); + if ( re === undefined ) { + this.regexMap.set(restr, (re = new RegExp(restr.slice(1,-1)))); + } + if ( re.test(hostname) === false ) { continue; } + this.retrieveFromSlot(this.regexToSlotIdMap.get(restr), out); + } + } + + retrieveFromSlot(hnSlot, out) { + if ( hnSlot === undefined ) { return; } + const mask = out.length - 1; // out.length must be power of two + do { + const strId = this.hostnameSlots[hnSlot+0]; + out[strId & mask].add(this.strSlots[strId >>> this.nBits]); + hnSlot = this.hostnameSlots[hnSlot+1]; + } while ( hnSlot !== 0 ); + } + + toSelfie() { + return { + version: this.version, + hostnameToSlotIdMap: Array.from(this.hostnameToSlotIdMap), + regexToSlotIdMap: Array.from(this.regexToSlotIdMap), + hostnameSlots: this.hostnameSlots, + strSlots: this.strSlots, + size: this.size + }; + } + + fromSelfie(selfie) { + if ( selfie === undefined ) { return; } + this.hostnameToSlotIdMap = new Map(selfie.hostnameToSlotIdMap); + // Regex-based lookup available in uBO 1.47.0 and above + if ( Array.isArray(selfie.regexToSlotIdMap) ) { + this.regexToSlotIdMap = new Map(selfie.regexToSlotIdMap); + } + this.hostnameSlots = selfie.hostnameSlots; + this.strSlots = selfie.strSlots; + this.size = selfie.size; + } +}; + +/******************************************************************************/ + +export { + StaticExtFilteringHostnameDB, +}; + +/******************************************************************************/ diff --git a/src/js/static-ext-filtering.js b/src/js/static-ext-filtering.js new file mode 100644 index 0000000..8a2905e --- /dev/null +++ b/src/js/static-ext-filtering.js @@ -0,0 +1,184 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import htmlFilteringEngine from './html-filtering.js'; +import httpheaderFilteringEngine from './httpheader-filtering.js'; +import io from './assets.js'; +import logger from './logger.js'; +import scriptletFilteringEngine from './scriptlet-filtering.js'; + +/******************************************************************************* + + All static extended filters are of the form: + + field 1: one hostname, or a list of comma-separated hostnames + field 2: `##` or `#@#` + field 3: selector + + The purpose of the static extended filtering engine is to coarse-parse and + dispatch to appropriate specialized filtering engines. There are currently + three specialized filtering engines: + + - cosmetic filtering (aka "element hiding" in Adblock Plus) + - scriptlet injection: selector starts with `script:inject` + - New shorter syntax (1.15.12): `example.com##+js(bab-defuser.js)` + - html filtering: selector starts with `^` + + Depending on the specialized filtering engine, field 1 may or may not be + optional. + + The static extended filtering engine also offers parsing capabilities which + are available to all other specialized filtering engines. For example, + cosmetic and html filtering can ask the extended filtering engine to + compile/validate selectors. + +**/ + +//-------------------------------------------------------------------------- +// Public API +//-------------------------------------------------------------------------- + +const staticExtFilteringEngine = { + get acceptedCount() { + return cosmeticFilteringEngine.acceptedCount + + scriptletFilteringEngine.acceptedCount + + httpheaderFilteringEngine.acceptedCount + + htmlFilteringEngine.acceptedCount; + }, + get discardedCount() { + return cosmeticFilteringEngine.discardedCount + + scriptletFilteringEngine.discardedCount + + httpheaderFilteringEngine.discardedCount + + htmlFilteringEngine.discardedCount; + }, +}; + +//-------------------------------------------------------------------------- +// Public methods +//-------------------------------------------------------------------------- + +staticExtFilteringEngine.reset = function() { + cosmeticFilteringEngine.reset(); + scriptletFilteringEngine.reset(); + httpheaderFilteringEngine.reset(); + htmlFilteringEngine.reset(); +}; + +staticExtFilteringEngine.freeze = function() { + cosmeticFilteringEngine.freeze(); + scriptletFilteringEngine.freeze(); + httpheaderFilteringEngine.freeze(); + htmlFilteringEngine.freeze(); +}; + +staticExtFilteringEngine.compile = function(parser, writer) { + if ( parser.isExtendedFilter() === false ) { return false; } + + if ( parser.hasError() ) { + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid extended filter in ${writer.properties.get('name') || '?'}: ${parser.raw}` + }); + return true; + } + + // Scriptlet injection + if ( parser.isScriptletFilter() ) { + scriptletFilteringEngine.compile(parser, writer); + return true; + } + + // Response header filtering + if ( parser.isResponseheaderFilter() ) { + httpheaderFilteringEngine.compile(parser, writer); + return true; + } + + // HTML filtering + // TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML + // filtering syntax. + if ( parser.isHtmlFilter() ) { + htmlFilteringEngine.compile(parser, writer); + return true; + } + + // Cosmetic filtering + if ( parser.isCosmeticFilter() ) { + cosmeticFilteringEngine.compile(parser, writer); + return true; + } + + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Unknown extended filter in ${writer.properties.get('name') || '?'}: ${parser.raw}` + }); + return true; +}; + +staticExtFilteringEngine.fromCompiledContent = function(reader, options) { + cosmeticFilteringEngine.fromCompiledContent(reader, options); + scriptletFilteringEngine.fromCompiledContent(reader, options); + httpheaderFilteringEngine.fromCompiledContent(reader, options); + htmlFilteringEngine.fromCompiledContent(reader, options); +}; + +staticExtFilteringEngine.toSelfie = function(path) { + return io.put( + `${path}/main`, + JSON.stringify({ + cosmetic: cosmeticFilteringEngine.toSelfie(), + scriptlets: scriptletFilteringEngine.toSelfie(), + httpHeaders: httpheaderFilteringEngine.toSelfie(), + html: htmlFilteringEngine.toSelfie(), + }) + ); +}; + +staticExtFilteringEngine.fromSelfie = function(path) { + return io.get(`${path}/main`).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( selfie instanceof Object === false ) { return false; } + cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); + httpheaderFilteringEngine.fromSelfie(selfie.httpHeaders); + htmlFilteringEngine.fromSelfie(selfie.html); + if ( scriptletFilteringEngine.fromSelfie(selfie.scriptlets) === false ) { + return false; + } + return true; + }); +}; + +/******************************************************************************/ + +export default staticExtFilteringEngine; + +/******************************************************************************/ diff --git a/src/js/static-filtering-io.js b/src/js/static-filtering-io.js new file mode 100644 index 0000000..3f016ab --- /dev/null +++ b/src/js/static-filtering-io.js @@ -0,0 +1,144 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://www.reddit.com/r/uBlockOrigin/comments/oq6kt5/ubo_loads_generic_filter_instead_of_specific/ +// Ensure blocks of content are sorted in ascending id order, such that the +// specific cosmetic filters will be found (and thus reported) before the +// generic ones. + +const serialize = JSON.stringify; +const unserialize = JSON.parse; + +const blockStartPrefix = '#block-start-'; // ensure no special regex characters +const blockEndPrefix = '#block-end-'; // ensure no special regex characters + +class CompiledListWriter { + constructor() { + this.blockId = undefined; + this.block = undefined; + this.blocks = new Map(); + this.properties = new Map(); + } + push(args) { + this.block.push(serialize(args)); + } + pushMany(many) { + for ( const args of many ) { + this.block.push(serialize(args)); + } + } + last() { + if ( Array.isArray(this.block) && this.block.length !== 0 ) { + return this.block[this.block.length - 1]; + } + } + select(blockId) { + if ( blockId === this.blockId ) { return; } + this.blockId = blockId; + this.block = this.blocks.get(blockId); + if ( this.block === undefined ) { + this.blocks.set(blockId, (this.block = [])); + } + return this; + } + toString() { + const result = []; + const sortedBlocks = + Array.from(this.blocks).sort((a, b) => a[0] - b[0]); + for ( const [ id, lines ] of sortedBlocks ) { + if ( lines.length === 0 ) { continue; } + result.push( + blockStartPrefix + id, + lines.join('\n'), + blockEndPrefix + id + ); + } + return result.join('\n'); + } + static serialize(arg) { + return serialize(arg); + } +} + +class CompiledListReader { + constructor(raw, blockId) { + this.block = ''; + this.len = 0; + this.offset = 0; + this.line = ''; + this.blocks = new Map(); + this.properties = new Map(); + const reBlockStart = new RegExp(`^${blockStartPrefix}([\\w:]+)\\n`, 'gm'); + let match = reBlockStart.exec(raw); + while ( match !== null ) { + const sectionId = match[1]; + const beg = match.index + match[0].length; + const end = raw.indexOf(blockEndPrefix + sectionId, beg); + this.blocks.set(sectionId, raw.slice(beg, end)); + reBlockStart.lastIndex = end; + match = reBlockStart.exec(raw); + } + if ( blockId !== undefined ) { + this.select(blockId); + } + } + next() { + if ( this.offset === this.len ) { + this.line = ''; + return false; + } + let pos = this.block.indexOf('\n', this.offset); + if ( pos !== -1 ) { + this.line = this.block.slice(this.offset, pos); + this.offset = pos + 1; + } else { + this.line = this.block.slice(this.offset); + this.offset = this.len; + } + return true; + } + select(blockId) { + this.block = this.blocks.get(blockId) || ''; + this.len = this.block.length; + this.offset = 0; + return this; + } + fingerprint() { + return this.line; + } + args() { + return unserialize(this.line); + } + static unserialize(arg) { + return unserialize(arg); + } +} + +/******************************************************************************/ + +export { + CompiledListReader, + CompiledListWriter, +}; diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js new file mode 100644 index 0000000..eb8988b --- /dev/null +++ b/src/js/static-filtering-parser.js @@ -0,0 +1,4461 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2020-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import Regex from '../lib/regexanalyzer/regex.js'; +import * as cssTree from '../lib/csstree/css-tree.js'; + +/******************************************************************************* + * + * The parser creates a simple unidirectional AST from a raw line of text. + * Each node in the AST is a sequence of numbers, so as to avoid the need to + * make frequent memory allocation to represent the AST. + * + * All the AST nodes are allocated in the same integer-only array, which + * array is reused when parsing new lines. + * + * The AST can only be walked from top to bottom, then left to right. + * + * Each node typically refer to a corresponding string slice in the source + * text. + * + * It may happens a node requires to normalize the corresponding source slice, + * in which case there will be a reference in the AST to a transformed source + * string. (For example, a domain name might contain unicode characters, in + * which case the corresponding node will contain a reference to the + * (transformed) punycoded version of the domain name.) + * + * The AST can be easily used for syntax coloring purpose, in which case it's + * just a matter of walking through all the nodes in natural order. + * + * A tree walking utility class exists for compilation and syntax coloring + * purpose. + * +**/ + +/******************************************************************************/ + +let iota = 0; + +iota = 0; +export const AST_TYPE_NONE = iota++; +export const AST_TYPE_UNKNOWN = iota++; +export const AST_TYPE_COMMENT = iota++; +export const AST_TYPE_NETWORK = iota++; +export const AST_TYPE_EXTENDED = iota++; + +iota = 0; +export const AST_TYPE_NETWORK_PATTERN_ANY = iota++; +export const AST_TYPE_NETWORK_PATTERN_HOSTNAME = iota++; +export const AST_TYPE_NETWORK_PATTERN_PLAIN = iota++; +export const AST_TYPE_NETWORK_PATTERN_REGEX = iota++; +export const AST_TYPE_NETWORK_PATTERN_GENERIC = iota++; +export const AST_TYPE_NETWORK_PATTERN_BAD = iota++; +export const AST_TYPE_EXTENDED_COSMETIC = iota++; +export const AST_TYPE_EXTENDED_SCRIPTLET = iota++; +export const AST_TYPE_EXTENDED_HTML = iota++; +export const AST_TYPE_EXTENDED_RESPONSEHEADER = iota++; +export const AST_TYPE_COMMENT_PREPARSER = iota++; + +iota = 0; +export const AST_FLAG_UNSUPPORTED = 1 << iota++; +export const AST_FLAG_IGNORE = 1 << iota++; +export const AST_FLAG_HAS_ERROR = 1 << iota++; +export const AST_FLAG_IS_EXCEPTION = 1 << iota++; +export const AST_FLAG_EXT_STRONG = 1 << iota++; +export const AST_FLAG_EXT_STYLE = 1 << iota++; +export const AST_FLAG_EXT_SCRIPTLET_ADG = 1 << iota++; +export const AST_FLAG_NET_PATTERN_LEFT_HNANCHOR = 1 << iota++; +export const AST_FLAG_NET_PATTERN_RIGHT_PATHANCHOR = 1 << iota++; +export const AST_FLAG_NET_PATTERN_LEFT_ANCHOR = 1 << iota++; +export const AST_FLAG_NET_PATTERN_RIGHT_ANCHOR = 1 << iota++; +export const AST_FLAG_HAS_OPTIONS = 1 << iota++; + +iota = 0; +export const AST_ERROR_NONE = 1 << iota++; +export const AST_ERROR_REGEX = 1 << iota++; +export const AST_ERROR_PATTERN = 1 << iota++; +export const AST_ERROR_DOMAIN_NAME = 1 << iota++; +export const AST_ERROR_OPTION_DUPLICATE = 1 << iota++; +export const AST_ERROR_OPTION_UNKNOWN = 1 << iota++; +export const AST_ERROR_OPTION_BADVALUE = 1 << iota++; +export const AST_ERROR_OPTION_EXCLUDED = 1 << iota++; +export const AST_ERROR_IF_TOKEN_UNKNOWN = 1 << iota++; +export const AST_ERROR_UNTRUSTED_SOURCE = 1 << iota++; + +iota = 0; +const NODE_RIGHT_INDEX = iota++; +const NOOP_NODE_SIZE = iota; +const NODE_TYPE_INDEX = iota++; +const NODE_DOWN_INDEX = iota++; +const NODE_BEG_INDEX = iota++; +const NODE_END_INDEX = iota++; +const NODE_FLAGS_INDEX = iota++; +const NODE_TRANSFORM_INDEX = iota++; +const FULL_NODE_SIZE = iota; + +iota = 0; +export const NODE_TYPE_NOOP = iota++; +export const NODE_TYPE_LINE_RAW = iota++; +export const NODE_TYPE_LINE_BODY = iota++; +export const NODE_TYPE_WHITESPACE = iota++; +export const NODE_TYPE_COMMENT = iota++; +export const NODE_TYPE_IGNORE = iota++; +export const NODE_TYPE_EXT_RAW = iota++; +export const NODE_TYPE_EXT_OPTIONS_ANCHOR = iota++; +export const NODE_TYPE_EXT_OPTIONS = iota++; +export const NODE_TYPE_EXT_DECORATION = iota++; +export const NODE_TYPE_EXT_PATTERN_RAW = iota++; +export const NODE_TYPE_EXT_PATTERN_COSMETIC = iota++; +export const NODE_TYPE_EXT_PATTERN_HTML = iota++; +export const NODE_TYPE_EXT_PATTERN_RESPONSEHEADER = iota++; +export const NODE_TYPE_EXT_PATTERN_SCRIPTLET = iota++; +export const NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN = iota++; +export const NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARGS = iota++; +export const NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG = iota++; +export const NODE_TYPE_NET_RAW = iota++; +export const NODE_TYPE_NET_EXCEPTION = iota++; +export const NODE_TYPE_NET_PATTERN_RAW = iota++; +export const NODE_TYPE_NET_PATTERN = iota++; +export const NODE_TYPE_NET_PATTERN_PART = iota++; +export const NODE_TYPE_NET_PATTERN_PART_SPECIAL = iota++; +export const NODE_TYPE_NET_PATTERN_PART_UNICODE = iota++; +export const NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR = iota++; +export const NODE_TYPE_NET_PATTERN_LEFT_ANCHOR = iota++; +export const NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR = iota++; +export const NODE_TYPE_NET_OPTIONS_ANCHOR = iota++; +export const NODE_TYPE_NET_OPTIONS = iota++; +export const NODE_TYPE_NET_OPTION_SEPARATOR = iota++; +export const NODE_TYPE_NET_OPTION_SENTINEL = iota++; +export const NODE_TYPE_NET_OPTION_RAW = iota++; +export const NODE_TYPE_NET_OPTION_NAME_NOT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_UNKNOWN = iota++; +export const NODE_TYPE_NET_OPTION_NAME_1P = iota++; +export const NODE_TYPE_NET_OPTION_NAME_STRICT1P = iota++; +export const NODE_TYPE_NET_OPTION_NAME_3P = iota++; +export const NODE_TYPE_NET_OPTION_NAME_STRICT3P = iota++; +export const NODE_TYPE_NET_OPTION_NAME_ALL = iota++; +export const NODE_TYPE_NET_OPTION_NAME_BADFILTER = iota++; +export const NODE_TYPE_NET_OPTION_NAME_CNAME = iota++; +export const NODE_TYPE_NET_OPTION_NAME_CSP = iota++; +export const NODE_TYPE_NET_OPTION_NAME_CSS = iota++; +export const NODE_TYPE_NET_OPTION_NAME_DENYALLOW = iota++; +export const NODE_TYPE_NET_OPTION_NAME_DOC = iota++; +export const NODE_TYPE_NET_OPTION_NAME_EHIDE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_EMPTY = iota++; +export const NODE_TYPE_NET_OPTION_NAME_FONT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_FRAME = iota++; +export const NODE_TYPE_NET_OPTION_NAME_FROM = iota++; +export const NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK = iota++; +export const NODE_TYPE_NET_OPTION_NAME_GHIDE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_HEADER = iota++; +export const NODE_TYPE_NET_OPTION_NAME_IMAGE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_IMPORTANT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_INLINEFONT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_MATCHCASE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_MEDIA = iota++; +export const NODE_TYPE_NET_OPTION_NAME_METHOD = iota++; +export const NODE_TYPE_NET_OPTION_NAME_MP4 = iota++; +export const NODE_TYPE_NET_OPTION_NAME_NOOP = iota++; +export const NODE_TYPE_NET_OPTION_NAME_OBJECT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_OTHER = iota++; +export const NODE_TYPE_NET_OPTION_NAME_PERMISSIONS = iota++; +export const NODE_TYPE_NET_OPTION_NAME_PING = iota++; +export const NODE_TYPE_NET_OPTION_NAME_POPUNDER = iota++; +export const NODE_TYPE_NET_OPTION_NAME_POPUP = iota++; +export const NODE_TYPE_NET_OPTION_NAME_REDIRECT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM = iota++; +export const NODE_TYPE_NET_OPTION_NAME_REPLACE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_SCRIPT = iota++; +export const NODE_TYPE_NET_OPTION_NAME_SHIDE = iota++; +export const NODE_TYPE_NET_OPTION_NAME_TO = iota++; +export const NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM = iota++; +export const NODE_TYPE_NET_OPTION_NAME_XHR = iota++; +export const NODE_TYPE_NET_OPTION_NAME_WEBRTC = iota++; +export const NODE_TYPE_NET_OPTION_NAME_WEBSOCKET = iota++; +export const NODE_TYPE_NET_OPTION_ASSIGN = iota++; +export const NODE_TYPE_NET_OPTION_VALUE = iota++; +export const NODE_TYPE_OPTION_VALUE_DOMAIN_LIST = iota++; +export const NODE_TYPE_OPTION_VALUE_DOMAIN_RAW = iota++; +export const NODE_TYPE_OPTION_VALUE_NOT = iota++; +export const NODE_TYPE_OPTION_VALUE_DOMAIN = iota++; +export const NODE_TYPE_OPTION_VALUE_SEPARATOR = iota++; +export const NODE_TYPE_PREPARSE_DIRECTIVE = iota++; +export const NODE_TYPE_PREPARSE_DIRECTIVE_VALUE = iota++; +export const NODE_TYPE_PREPARSE_DIRECTIVE_IF = iota++; +export const NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE = iota++; +export const NODE_TYPE_COMMENT_URL = iota++; +export const NODE_TYPE_COUNT = iota; + +iota = 0; +export const NODE_FLAG_IGNORE = 1 << iota++; +export const NODE_FLAG_ERROR = 1 << iota++; +export const NODE_FLAG_IS_NEGATED = 1 << iota++; +export const NODE_FLAG_OPTION_HAS_VALUE = 1 << iota++; +export const NODE_FLAG_PATTERN_UNTOKENIZABLE = 1 << iota++; + +export const nodeTypeFromOptionName = new Map([ + [ '', NODE_TYPE_NET_OPTION_NAME_UNKNOWN ], + [ '1p', NODE_TYPE_NET_OPTION_NAME_1P ], + /* synonym */ [ 'first-party', NODE_TYPE_NET_OPTION_NAME_1P ], + [ 'strict1p', NODE_TYPE_NET_OPTION_NAME_STRICT1P ], + [ '3p', NODE_TYPE_NET_OPTION_NAME_3P ], + /* synonym */ [ 'third-party', NODE_TYPE_NET_OPTION_NAME_3P ], + [ 'strict3p', NODE_TYPE_NET_OPTION_NAME_STRICT3P ], + [ 'all', NODE_TYPE_NET_OPTION_NAME_ALL ], + [ 'badfilter', NODE_TYPE_NET_OPTION_NAME_BADFILTER ], + [ 'cname', NODE_TYPE_NET_OPTION_NAME_CNAME ], + [ 'csp', NODE_TYPE_NET_OPTION_NAME_CSP ], + [ 'css', NODE_TYPE_NET_OPTION_NAME_CSS ], + /* synonym */ [ 'stylesheet', NODE_TYPE_NET_OPTION_NAME_CSS ], + [ 'denyallow', NODE_TYPE_NET_OPTION_NAME_DENYALLOW ], + [ 'doc', NODE_TYPE_NET_OPTION_NAME_DOC ], + /* synonym */ [ 'document', NODE_TYPE_NET_OPTION_NAME_DOC ], + [ 'ehide', NODE_TYPE_NET_OPTION_NAME_EHIDE ], + /* synonym */ [ 'elemhide', NODE_TYPE_NET_OPTION_NAME_EHIDE ], + [ 'empty', NODE_TYPE_NET_OPTION_NAME_EMPTY ], + [ 'font', NODE_TYPE_NET_OPTION_NAME_FONT ], + [ 'frame', NODE_TYPE_NET_OPTION_NAME_FRAME ], + /* synonym */ [ 'subdocument', NODE_TYPE_NET_OPTION_NAME_FRAME ], + [ 'from', NODE_TYPE_NET_OPTION_NAME_FROM ], + /* synonym */ [ 'domain', NODE_TYPE_NET_OPTION_NAME_FROM ], + [ 'genericblock', NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK ], + [ 'ghide', NODE_TYPE_NET_OPTION_NAME_GHIDE ], + /* synonym */ [ 'generichide', NODE_TYPE_NET_OPTION_NAME_GHIDE ], + [ 'header', NODE_TYPE_NET_OPTION_NAME_HEADER ], + [ 'image', NODE_TYPE_NET_OPTION_NAME_IMAGE ], + [ 'important', NODE_TYPE_NET_OPTION_NAME_IMPORTANT ], + [ 'inline-font', NODE_TYPE_NET_OPTION_NAME_INLINEFONT ], + [ 'inline-script', NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT ], + [ 'match-case', NODE_TYPE_NET_OPTION_NAME_MATCHCASE ], + [ 'media', NODE_TYPE_NET_OPTION_NAME_MEDIA ], + [ 'method', NODE_TYPE_NET_OPTION_NAME_METHOD ], + [ 'mp4', NODE_TYPE_NET_OPTION_NAME_MP4 ], + [ '_', NODE_TYPE_NET_OPTION_NAME_NOOP ], + [ 'object', NODE_TYPE_NET_OPTION_NAME_OBJECT ], + /* synonym */ [ 'object-subrequest', NODE_TYPE_NET_OPTION_NAME_OBJECT ], + [ 'other', NODE_TYPE_NET_OPTION_NAME_OTHER ], + [ 'permissions', NODE_TYPE_NET_OPTION_NAME_PERMISSIONS ], + [ 'ping', NODE_TYPE_NET_OPTION_NAME_PING ], + /* synonym */ [ 'beacon', NODE_TYPE_NET_OPTION_NAME_PING ], + [ 'popunder', NODE_TYPE_NET_OPTION_NAME_POPUNDER ], + [ 'popup', NODE_TYPE_NET_OPTION_NAME_POPUP ], + [ 'redirect', NODE_TYPE_NET_OPTION_NAME_REDIRECT ], + /* synonym */ [ 'rewrite', NODE_TYPE_NET_OPTION_NAME_REDIRECT ], + [ 'redirect-rule', NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE ], + [ 'removeparam', NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM ], + [ 'replace', NODE_TYPE_NET_OPTION_NAME_REPLACE ], + /* synonym */ [ 'queryprune', NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM ], + [ 'script', NODE_TYPE_NET_OPTION_NAME_SCRIPT ], + [ 'shide', NODE_TYPE_NET_OPTION_NAME_SHIDE ], + /* synonym */ [ 'specifichide', NODE_TYPE_NET_OPTION_NAME_SHIDE ], + [ 'to', NODE_TYPE_NET_OPTION_NAME_TO ], + [ 'uritransform', NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM ], + [ 'xhr', NODE_TYPE_NET_OPTION_NAME_XHR ], + /* synonym */ [ 'xmlhttprequest', NODE_TYPE_NET_OPTION_NAME_XHR ], + [ 'webrtc', NODE_TYPE_NET_OPTION_NAME_WEBRTC ], + [ 'websocket', NODE_TYPE_NET_OPTION_NAME_WEBSOCKET ], +]); + +export const nodeNameFromNodeType = new Map([ + [ NODE_TYPE_NOOP, 'noop' ], + [ NODE_TYPE_LINE_RAW, 'lineRaw' ], + [ NODE_TYPE_LINE_BODY, 'lineBody' ], + [ NODE_TYPE_WHITESPACE, 'whitespace' ], + [ NODE_TYPE_COMMENT, 'comment' ], + [ NODE_TYPE_IGNORE, 'ignore' ], + [ NODE_TYPE_EXT_RAW, 'extRaw' ], + [ NODE_TYPE_EXT_OPTIONS_ANCHOR, 'extOptionsAnchor' ], + [ NODE_TYPE_EXT_OPTIONS, 'extOptions' ], + [ NODE_TYPE_EXT_DECORATION, 'extDecoration' ], + [ NODE_TYPE_EXT_PATTERN_RAW, 'extPatternRaw' ], + [ NODE_TYPE_EXT_PATTERN_COSMETIC, 'extPatternCosmetic' ], + [ NODE_TYPE_EXT_PATTERN_HTML, 'extPatternHtml' ], + [ NODE_TYPE_EXT_PATTERN_RESPONSEHEADER, 'extPatternResponseheader' ], + [ NODE_TYPE_EXT_PATTERN_SCRIPTLET, 'extPatternScriptlet' ], + [ NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN, 'extPatternScriptletToken' ], + [ NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARGS, 'extPatternScriptletArgs' ], + [ NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG, 'extPatternScriptletArg' ], + [ NODE_TYPE_NET_RAW, 'netRaw' ], + [ NODE_TYPE_NET_EXCEPTION, 'netException' ], + [ NODE_TYPE_NET_PATTERN_RAW, 'netPatternRaw' ], + [ NODE_TYPE_NET_PATTERN, 'netPattern' ], + [ NODE_TYPE_NET_PATTERN_PART, 'netPatternPart' ], + [ NODE_TYPE_NET_PATTERN_PART_SPECIAL, 'netPatternPartSpecial' ], + [ NODE_TYPE_NET_PATTERN_PART_UNICODE, 'netPatternPartUnicode' ], + [ NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR, 'netPatternLeftHnanchor' ], + [ NODE_TYPE_NET_PATTERN_LEFT_ANCHOR, 'netPatternLeftAnchor' ], + [ NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR, 'netPatternRightAnchor' ], + [ NODE_TYPE_NET_OPTIONS_ANCHOR, 'netOptionsAnchor' ], + [ NODE_TYPE_NET_OPTIONS, 'netOptions' ], + [ NODE_TYPE_NET_OPTION_RAW, 'netOptionRaw' ], + [ NODE_TYPE_NET_OPTION_SEPARATOR, 'netOptionSeparator'], + [ NODE_TYPE_NET_OPTION_SENTINEL, 'netOptionSentinel' ], + [ NODE_TYPE_NET_OPTION_NAME_NOT, 'netOptionNameNot'], + [ NODE_TYPE_NET_OPTION_ASSIGN, 'netOptionAssign' ], + [ NODE_TYPE_NET_OPTION_VALUE, 'netOptionValue' ], + [ NODE_TYPE_OPTION_VALUE_DOMAIN_LIST, 'netOptionValueDomainList' ], + [ NODE_TYPE_OPTION_VALUE_DOMAIN_RAW, 'netOptionValueDomainRaw' ], + [ NODE_TYPE_OPTION_VALUE_NOT, 'netOptionValueNot' ], + [ NODE_TYPE_OPTION_VALUE_DOMAIN, 'netOptionValueDomain' ], + [ NODE_TYPE_OPTION_VALUE_SEPARATOR, 'netOptionsValueSeparator' ], +]); +{ + for ( const [ name, type ] of nodeTypeFromOptionName ) { + nodeNameFromNodeType.set(type, name); + } +} + +/******************************************************************************/ + +// Precomputed AST layouts for most common filters. + +const astTemplates = { + // ||example.com^ + netHnAnchoredHostnameAscii: { + flags: AST_FLAG_NET_PATTERN_LEFT_HNANCHOR | + AST_FLAG_NET_PATTERN_RIGHT_PATHANCHOR, + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_PATTERN_RAW, + beg: 0, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR, + beg: 0, + end: 2, + }, { + type: NODE_TYPE_NET_PATTERN, + beg: 2, + end: -1, + register: true, + }, { + type: NODE_TYPE_NET_PATTERN_PART_SPECIAL, + beg: -1, + end: 0, + }], + }], + }], + }, + // ||example.com^$third-party + net3pHnAnchoredHostnameAscii: { + flags: AST_FLAG_NET_PATTERN_LEFT_HNANCHOR | + AST_FLAG_NET_PATTERN_RIGHT_PATHANCHOR | + AST_FLAG_HAS_OPTIONS, + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_PATTERN_RAW, + beg: 0, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR, + beg: 0, + end: 2, + }, { + type: NODE_TYPE_NET_PATTERN, + beg: 2, + end: -13, + register: true, + }, { + type: NODE_TYPE_NET_PATTERN_PART_SPECIAL, + beg: -13, + end: -12, + }], + }, { + type: NODE_TYPE_NET_OPTIONS_ANCHOR, + beg: -12, + end: -11, + }, { + type: NODE_TYPE_NET_OPTIONS, + beg: -11, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_NET_OPTION_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_OPTION_NAME_3P, + beg: 0, + end: 0, + register: true, + }], + }], + }], + }], + }, + // ||example.com/path/to/resource + netHnAnchoredPlainAscii: { + flags: AST_FLAG_NET_PATTERN_LEFT_HNANCHOR, + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_PATTERN_RAW, + beg: 0, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR, + beg: 0, + end: 2, + }, { + type: NODE_TYPE_NET_PATTERN, + beg: 2, + end: 0, + register: true, + }], + }], + }], + }, + // example.com + // -resource. + netPlainAscii: { + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_PATTERN_RAW, + beg: 0, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_NET_PATTERN, + beg: 0, + end: 0, + register: true, + }], + }], + }], + }, + // 127.0.0.1 example.com + netHosts1: { + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_PATTERN_RAW, + beg: 0, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_IGNORE, + beg: 0, + end: 10, + }, { + type: NODE_TYPE_NET_PATTERN, + beg: 10, + end: 0, + register: true, + }], + }], + }], + }, + // 0.0.0.0 example.com + netHosts2: { + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_NET_PATTERN_RAW, + beg: 0, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_IGNORE, + beg: 0, + end: 8, + }, { + type: NODE_TYPE_NET_PATTERN, + beg: 8, + end: 0, + register: true, + }], + }], + }], + }, + // ##.ads-container + extPlainGenericSelector: { + type: NODE_TYPE_LINE_BODY, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_EXT_RAW, + beg: 0, + end: 0, + children: [{ + type: NODE_TYPE_EXT_OPTIONS_ANCHOR, + beg: 0, + end: 2, + register: true, + }, { + type: NODE_TYPE_EXT_PATTERN_RAW, + beg: 2, + end: 0, + register: true, + children: [{ + type: NODE_TYPE_EXT_PATTERN_COSMETIC, + beg: 0, + end: 0, + }], + }], + }], + }, +}; + +/******************************************************************************/ + +export const removableHTTPHeaders = new Set([ + 'location', + 'refresh', + 'report-to', + 'set-cookie', +]); + +export const preparserIfTokens = new Set([ + 'ext_ublock', + 'ext_ubol', + 'ext_devbuild', + 'env_chromium', + 'env_edge', + 'env_firefox', + 'env_legacy', + 'env_mobile', + 'env_mv3', + 'env_safari', + 'cap_html_filtering', + 'cap_user_stylesheet', + 'false', + 'ext_abp', + 'adguard', + 'adguard_app_android', + 'adguard_app_ios', + 'adguard_app_mac', + 'adguard_app_windows', + 'adguard_ext_android_cb', + 'adguard_ext_chromium', + 'adguard_ext_edge', + 'adguard_ext_firefox', + 'adguard_ext_opera', + 'adguard_ext_safari', +]); + +/******************************************************************************/ + +const exCharCodeAt = (s, i) => { + const pos = i >= 0 ? i : s.length + i; + return pos >= 0 ? s.charCodeAt(pos) : -1; +}; + +/******************************************************************************/ + +class ArgListParser { + constructor(separatorChar = ',', mustQuote = false) { + this.separatorChar = this.actualSeparatorChar = separatorChar; + this.separatorCode = this.actualSeparatorCode = separatorChar.charCodeAt(0); + this.mustQuote = mustQuote; + this.quoteBeg = 0; this.quoteEnd = 0; + this.argBeg = 0; this.argEnd = 0; + this.separatorBeg = 0; this.separatorEnd = 0; + this.transform = false; + this.failed = false; + this.reWhitespaceStart = /^\s+/; + this.reWhitespaceEnd = /\s+$/; + this.reOddTrailingEscape = /(?:^|[^\\])(?:\\\\)*\\$/; + this.reTrailingEscapeChars = /\\+$/; + } + nextArg(pattern, beg = 0) { + const len = pattern.length; + this.quoteBeg = beg + this.leftWhitespaceCount(pattern.slice(beg)); + this.failed = false; + const qc = pattern.charCodeAt(this.quoteBeg); + if ( qc === 0x22 /* " */ || qc === 0x27 /* ' */ || qc === 0x60 /* ` */ ) { + this.indexOfNextArgSeparator(pattern, qc); + if ( this.argEnd !== len ) { + this.quoteEnd = this.argEnd + 1; + this.separatorBeg = this.separatorEnd = this.quoteEnd; + this.separatorEnd += this.leftWhitespaceCount(pattern.slice(this.quoteEnd)); + if ( this.separatorEnd === len ) { return this; } + if ( pattern.charCodeAt(this.separatorEnd) === this.separatorCode ) { + this.separatorEnd += 1; + return this; + } + } + } + this.indexOfNextArgSeparator(pattern, this.separatorCode); + this.separatorBeg = this.separatorEnd = this.argEnd; + if ( this.separatorBeg < len ) { + this.separatorEnd += 1; + } + this.argEnd -= this.rightWhitespaceCount(pattern.slice(0, this.separatorBeg)); + this.quoteEnd = this.argEnd; + if ( this.mustQuote ) { + this.failed = true; + } + return this; + } + normalizeArg(s, char = '') { + if ( char === '' ) { char = this.actualSeparatorChar; } + let out = ''; + let pos = 0; + while ( (pos = s.lastIndexOf(char)) !== -1 ) { + out = s.slice(pos) + out; + s = s.slice(0, pos); + const match = this.reTrailingEscapeChars.exec(s); + if ( match === null ) { continue; } + const tail = (match[0].length & 1) !== 0 + ? match[0].slice(0, -1) + : match[0]; + out = tail + out; + s = s.slice(0, -match[0].length); + } + if ( out === '' ) { return s; } + return s + out; + } + leftWhitespaceCount(s) { + const match = this.reWhitespaceStart.exec(s); + return match === null ? 0 : match[0].length; + } + rightWhitespaceCount(s) { + const match = this.reWhitespaceEnd.exec(s); + return match === null ? 0 : match[0].length; + } + indexOfNextArgSeparator(pattern, separatorCode) { + this.argBeg = this.argEnd = separatorCode !== this.separatorCode + ? this.quoteBeg + 1 + : this.quoteBeg; + this.transform = false; + if ( separatorCode !== this.actualSeparatorCode ) { + this.actualSeparatorCode = separatorCode; + this.actualSeparatorChar = String.fromCharCode(separatorCode); + } + while ( this.argEnd < pattern.length ) { + const pos = pattern.indexOf(this.actualSeparatorChar, this.argEnd); + if ( pos === -1 ) { + return (this.argEnd = pattern.length); + } + if ( this.reOddTrailingEscape.test(pattern.slice(0, pos)) === false ) { + return (this.argEnd = pos); + } + this.transform = true; + this.argEnd = pos + 1; + } + } +} + +/******************************************************************************/ + +class AstWalker { + constructor(parser, from = 0) { + this.parser = parser; + this.stack = []; + this.reset(from); + } + get depth() { + return this.stackPtr; + } + reset(from = 0) { + this.nodes = this.parser.nodes; + this.stackPtr = 0; + return (this.current = from || this.parser.rootNode); + } + next() { + const current = this.current; + if ( current === 0 ) { return 0; } + const down = this.nodes[current+NODE_DOWN_INDEX]; + if ( down !== 0 ) { + this.stack[this.stackPtr++] = this.current; + return (this.current = down); + } + const right = this.nodes[current+NODE_RIGHT_INDEX]; + if ( right !== 0 && this.stackPtr !== 0 ) { + return (this.current = right); + } + while ( this.stackPtr !== 0 ) { + const parent = this.stack[--this.stackPtr]; + const right = this.nodes[parent+NODE_RIGHT_INDEX]; + if ( right !== 0 ) { + return (this.current = right); + } + } + return (this.current = 0); + } + right() { + const current = this.current; + if ( current === 0 ) { return 0; } + const right = this.nodes[current+NODE_RIGHT_INDEX]; + if ( right !== 0 && this.stackPtr !== 0 ) { + return (this.current = right); + } + while ( this.stackPtr !== 0 ) { + const parent = this.stack[--this.stackPtr]; + const right = this.nodes[parent+NODE_RIGHT_INDEX]; + if ( right !== 0 ) { + return (this.current = right); + } + } + return (this.current = 0); + } + until(which) { + let node = this.next(); + while ( node !== 0 ) { + if ( this.nodes[node+NODE_TYPE_INDEX] === which ) { return node; } + node = this.next(); + } + return 0; + } + canGoDown() { + return this.nodes[this.current+NODE_DOWN_INDEX] !== 0; + } + dispose() { + this.parser.walkerJunkyard.push(this); + } +} + +/******************************************************************************/ + +class DomainListIterator { + constructor(parser, root) { + this.parser = parser; + this.walker = parser.getWalker(); + this.value = undefined; + this.item = { hn: '', not: false, bad: false }; + this.reuse(root); + } + next() { + if ( this.done ) { return this.value; } + let node = this.walker.current; + let ready = false; + while ( node !== 0 ) { + switch ( this.parser.getNodeType(node) ) { + case NODE_TYPE_OPTION_VALUE_DOMAIN_RAW: + this.item.hn = ''; + this.item.not = false; + this.item.bad = this.parser.getNodeFlags(node, NODE_FLAG_ERROR) !== 0; + break; + case NODE_TYPE_OPTION_VALUE_NOT: + this.item.not = true; + break; + case NODE_TYPE_OPTION_VALUE_DOMAIN: + this.item.hn = this.parser.getNodeTransform(node); + this.value = this.item; + ready = true; + break; + default: + break; + } + node = this.walker.next(); + if ( ready ) { return this; } + } + return this.stop(); + } + reuse(root) { + this.walker.reset(root); + this.done = false; + return this; + } + stop() { + this.done = true; + this.value = undefined; + this.parser.domainListIteratorJunkyard.push(this); + return this; + } + [Symbol.iterator]() { + return this; + } +} + +/******************************************************************************/ + +export class AstFilterParser { + constructor(options = {}) { + this.raw = ''; + this.rawEnd = 0; + this.nodes = new Uint32Array(16384); + this.nodePoolPtr = FULL_NODE_SIZE; + this.nodePoolEnd = this.nodes.length; + this.astTransforms = [ null ]; + this.astTransformPtr = 1; + this.rootNode = 0; + this.astType = AST_TYPE_NONE; + this.astTypeFlavor = AST_TYPE_NONE; + this.astFlags = 0; + this.astError = 0; + this.nodeTypeRegister = []; + this.nodeTypeRegisterPtr = 0; + this.nodeTypeLookupTable = new Uint32Array(NODE_TYPE_COUNT); + this.punycoder = new URL('https://ublock0.invalid/'); + this.domainListIteratorJunkyard = []; + this.walkerJunkyard = []; + this.hasWhitespace = false; + this.hasUnicode = false; + this.hasUppercase = false; + // Options + this.options = options; + this.interactive = options.interactive || false; + this.badTypes = new Set(options.badTypes || []); + this.maxTokenLength = options.maxTokenLength || 7; + // TODO: rethink this + this.result = { exception: false, raw: '', compiled: '', error: undefined }; + this.selectorCompiler = new ExtSelectorCompiler(options); + // Regexes + this.reWhitespaceStart = /^\s+/; + this.reWhitespaceEnd = /\s+$/; + this.reCommentLine = /^(?:!|#\s|####|\[adblock)/i; + this.reExtAnchor = /(#@?(?:\$\?|\$|%|\?)?#).{1,2}/; + this.reInlineComment = /(?:\s+#).*?$/; + this.reNetException = /^@@/; + this.reNetAnchor = /(?:)\$[^,\w~]/; + this.reHnAnchoredPlainAscii = /^\|\|[0-9a-z%&,\-.\/:;=?_]+$/; + this.reHnAnchoredHostnameAscii = /^\|\|(?:[\da-z][\da-z_-]*\.)*[\da-z_-]*[\da-z]\^$/; + this.reHnAnchoredHostnameUnicode = /^\|\|(?:[\p{L}\p{N}][\p{L}\p{N}\u{2d}]*\.)*[\p{L}\p{N}\u{2d}]*[\p{L}\p{N}]\^$/u; + this.reHn3pAnchoredHostnameAscii = /^\|\|(?:[\da-z][\da-z_-]*\.)*[\da-z_-]*[\da-z]\^\$third-party$/; + this.rePlainAscii = /^[0-9a-z%&\-.\/:;=?_]{2,}$/; + this.reNetHosts1 = /^127\.0\.0\.1 (?:[\da-z][\da-z_-]*\.)+[\da-z-]*[a-z]$/; + this.reNetHosts2 = /^0\.0\.0\.0 (?:[\da-z][\da-z_-]*\.)+[\da-z-]*[a-z]$/; + this.rePlainGenericCosmetic = /^##[.#][A-Za-z_][\w-]*$/; + this.reHostnameAscii = /^(?:[\da-z][\da-z_-]*\.)*[\da-z][\da-z-]*[\da-z]$/; + this.rePlainEntity = /^(?:[\da-z][\da-z_-]*\.)+\*$/; + this.reHostsSink = /^[\w%.:\[\]-]+\s+/; + this.reHostsRedirect = /(?:0\.0\.0\.0|broadcasthost|local|localhost(?:\.localdomain)?|ip6-\w+)(?:[^\w.-]|$)/; + this.reNetOptionComma = /,(?:~?[13a-z-]+(?:=.*?)?|_+)(?:,|$)/; + this.rePointlessLeftAnchor = /^\|\|?\*+/; + this.reIsTokenChar = /^[%0-9A-Za-z]/; + this.rePointlessLeadingWildcards = /^(\*+)[^%0-9A-Za-z\u{a0}-\u{10FFFF}]/u; + this.rePointlessTrailingSeparator = /\*(\^\**)$/; + this.rePointlessTrailingWildcards = /(?:[^%0-9A-Za-z]|[%0-9A-Za-z]{7,})(\*+)$/; + this.reHasWhitespaceChar = /\s/; + this.reHasUppercaseChar = /[A-Z]/; + this.reHasUnicodeChar = /[^\x00-\x7F]/; + this.reUnicodeChars = /\P{ASCII}/gu; + this.reBadHostnameChars = /[\x00-\x24\x26-\x29\x2b\x2c\x2f\x3b-\x40\x5c\x5e\x60\x7b-\x7f]/; + this.reIsEntity = /^[^*]+\.\*$/; + this.rePreparseDirectiveIf = /^!#if /; + this.rePreparseDirectiveAny = /^!#(?:else|endif|if |include )/; + this.reURL = /\bhttps?:\/\/\S+/; + this.reHasPatternSpecialChars = /[\*\^]/; + this.rePatternAllSpecialChars = /[\*\^]+|[^\x00-\x7f]+/g; + // https://github.com/uBlockOrigin/uBlock-issues/issues/1146 + // From https://codemirror.net/doc/manual.html#option_specialChars + this.reHasInvalidChar = /[\x00-\x1F\x7F-\x9F\xAD\u061C\u200B-\u200F\u2028\u2029\uFEFF\uFFF9-\uFFFC]/; + this.reHostnamePatternPart = /^[^\x00-\x24\x26-\x29\x2B\x2C\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\x7F]+/; + this.reHostnameLabel = /[^.]+/g; + this.reResponseheaderPattern = /^\^responseheader\(.*\)$/; + this.rePatternScriptletJsonArgs = /^\{.*\}$/; + this.reGoodRegexToken = /[^\x01%0-9A-Za-z][%0-9A-Za-z]{7,}|[^\x01%0-9A-Za-z][%0-9A-Za-z]{1,6}[^\x01%0-9A-Za-z]/; + this.reBadCSP = /(?:=|;)\s*report-(?:to|uri)\b/; + this.reNoopOption = /^_+$/; + this.scriptletArgListParser = new ArgListParser(','); + } + + finish() { + this.selectorCompiler.finish(); + } + + parse(raw) { + this.raw = raw; + this.rawEnd = raw.length; + this.nodePoolPtr = FULL_NODE_SIZE; + this.nodeTypeRegisterPtr = 0; + this.astTransformPtr = 1; + this.astType = AST_TYPE_NONE; + this.astTypeFlavor = AST_TYPE_NONE; + this.astFlags = 0; + this.astError = 0; + this.rootNode = this.allocTypedNode(NODE_TYPE_LINE_RAW, 0, this.rawEnd); + if ( this.rawEnd === 0 ) { return; } + + // Fast-track very common simple filters using pre-computed AST layouts + // to skip parsing and validation. + const c1st = this.raw.charCodeAt(0); + const clast = exCharCodeAt(this.raw, -1); + if ( c1st === 0x7C /* | */ ) { + if ( + clast === 0x5E /* ^ */ && + this.reHnAnchoredHostnameAscii.test(this.raw) + ) { + // ||example.com^ + this.astType = AST_TYPE_NETWORK; + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + const node = this.astFromTemplate(this.rootNode, + astTemplates.netHnAnchoredHostnameAscii + ); + this.linkDown(this.rootNode, node); + return; + } + if ( + this.raw.endsWith('$third-party') && + this.reHn3pAnchoredHostnameAscii.test(this.raw) + ) { + // ||example.com^$third-party + this.astType = AST_TYPE_NETWORK; + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + const node = this.astFromTemplate(this.rootNode, + astTemplates.net3pHnAnchoredHostnameAscii + ); + this.linkDown(this.rootNode, node); + return; + } + if ( this.reHnAnchoredPlainAscii.test(this.raw) ) { + // ||example.com/path/to/resource + this.astType = AST_TYPE_NETWORK; + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_PLAIN; + const node = this.astFromTemplate(this.rootNode, + astTemplates.netHnAnchoredPlainAscii + ); + this.linkDown(this.rootNode, node); + return; + } + } else if ( c1st === 0x23 /* # */ ) { + if ( this.rePlainGenericCosmetic.test(this.raw) ) { + // ##.ads-container + this.astType = AST_TYPE_EXTENDED; + this.astTypeFlavor = AST_TYPE_EXTENDED_COSMETIC; + const node = this.astFromTemplate(this.rootNode, + astTemplates.extPlainGenericSelector + ); + this.linkDown(this.rootNode, node); + this.result.exception = false; + this.result.raw = this.raw.slice(2); + this.result.compiled = this.raw.slice(2); + return; + } + } else if ( c1st === 0x31 /* 1 */ ) { + if ( this.reNetHosts1.test(this.raw) ) { + // 127.0.0.1 example.com + this.astType = AST_TYPE_NETWORK; + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + const node = this.astFromTemplate(this.rootNode, + astTemplates.netHosts1 + ); + this.linkDown(this.rootNode, node); + return; + } + } else if ( c1st === 0x30 /* 0 */ ) { + if ( this.reNetHosts2.test(this.raw) ) { + // 0.0.0.0 example.com + this.astType = AST_TYPE_NETWORK; + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + const node = this.astFromTemplate(this.rootNode, + astTemplates.netHosts2 + ); + this.linkDown(this.rootNode, node); + return; + } + } else if ( + (c1st !== 0x2F /* / */ || clast !== 0x2F /* / */) && + (this.rePlainAscii.test(this.raw)) + ) { + // example.com + // -resource. + this.astType = AST_TYPE_NETWORK; + this.astTypeFlavor = this.reHostnameAscii.test(this.raw) + ? AST_TYPE_NETWORK_PATTERN_HOSTNAME + : AST_TYPE_NETWORK_PATTERN_PLAIN; + const node = this.astFromTemplate(this.rootNode, + astTemplates.netPlainAscii + ); + this.linkDown(this.rootNode, node); + return; + } + + // All else: full parsing and validation. + this.hasWhitespace = this.reHasWhitespaceChar.test(raw); + this.linkDown(this.rootNode, this.parseRaw(this.rootNode)); + } + + astFromTemplate(parent, template) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const beg = template.beg + (template.beg >= 0 ? parentBeg : parentEnd); + const end = template.end + (template.end <= 0 ? parentEnd : parentBeg); + const node = this.allocTypedNode(template.type, beg, end); + if ( template.register ) { + this.addNodeToRegister(template.type, node); + } + if ( template.flags ) { + this.addFlags(template.flags); + } + if ( template.nodeFlags ) { + this.addNodeFlags(node, template.nodeFlags); + } + const children = template.children; + if ( children === undefined ) { return node; } + const head = this.astFromTemplate(node, children[0]); + this.linkDown(node, head); + const n = children.length; + if ( n === 1 ) { return node; } + let prev = head; + for ( let i = 1; i < n; i++ ) { + prev = this.linkRight(prev, this.astFromTemplate(node, children[i])); + } + return node; + } + + getType() { + return this.astType; + } + + isComment() { + return this.astType === AST_TYPE_COMMENT; + } + + isFilter() { + return this.isNetworkFilter() || this.isExtendedFilter(); + } + + isNetworkFilter() { + return this.astType === AST_TYPE_NETWORK; + } + + isExtendedFilter() { + return this.astType === AST_TYPE_EXTENDED; + } + + isCosmeticFilter() { + return this.astType === AST_TYPE_EXTENDED && + this.astTypeFlavor === AST_TYPE_EXTENDED_COSMETIC; + } + + isScriptletFilter() { + return this.astType === AST_TYPE_EXTENDED && + this.astTypeFlavor === AST_TYPE_EXTENDED_SCRIPTLET; + } + + isHtmlFilter() { + return this.astType === AST_TYPE_EXTENDED && + this.astTypeFlavor === AST_TYPE_EXTENDED_HTML; + } + + isResponseheaderFilter() { + return this.astType === AST_TYPE_EXTENDED && + this.astTypeFlavor === AST_TYPE_EXTENDED_RESPONSEHEADER; + } + + getFlags(flags = 0xFFFFFFFF) { + return this.astFlags & flags; + } + + addFlags(flags) { + this.astFlags |= flags; + } + + parseRaw(parent) { + const head = this.allocHeadNode(); + let prev = head, next = 0; + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const l1 = this.hasWhitespace + ? this.leftWhitespaceCount(this.getNodeString(parent)) + : 0; + if ( l1 !== 0 ) { + next = this.allocTypedNode( + NODE_TYPE_WHITESPACE, + parentBeg, + parentBeg + l1 + ); + prev = this.linkRight(prev, next); + if ( l1 === parentEnd ) { return this.throwHeadNode(head); } + } + const r0 = this.hasWhitespace + ? parentEnd - this.rightWhitespaceCount(this.getNodeString(parent)) + : parentEnd; + if ( r0 !== l1 ) { + next = this.allocTypedNode( + NODE_TYPE_LINE_BODY, + parentBeg + l1, + parentBeg + r0 + ); + this.linkDown(next, this.parseFilter(next)); + prev = this.linkRight(prev, next); + } + if ( r0 !== parentEnd ) { + next = this.allocTypedNode( + NODE_TYPE_WHITESPACE, + parentBeg + r0, + parentEnd + ); + this.linkRight(prev, next); + } + return this.throwHeadNode(head); + } + + parseFilter(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const parentStr = this.getNodeString(parent); + + // A comment? + if ( this.reCommentLine.test(parentStr) ) { + const head = this.allocTypedNode(NODE_TYPE_COMMENT, parentBeg, parentEnd); + this.astType = AST_TYPE_COMMENT; + if ( this.interactive ) { + this.linkDown(head, this.parseComment(head)); + } + return head; + } + + // An extended filter? (or rarely, a comment) + if ( this.reExtAnchor.test(parentStr) ) { + const match = this.reExtAnchor.exec(parentStr); + const matchLen = match[1].length; + const head = this.allocTypedNode(NODE_TYPE_EXT_RAW, parentBeg, parentEnd); + this.linkDown(head, this.parseExt(head, parentBeg + match.index, matchLen)); + return head; + } else if ( parentStr.charCodeAt(0) === 0x23 /* # */ ) { + const head = this.allocTypedNode(NODE_TYPE_COMMENT, parentBeg, parentEnd); + this.astType = AST_TYPE_COMMENT; + return head; + } + + // Good to know in advance to avoid costly tests later on + this.hasUppercase = this.reHasUppercaseChar.test(parentStr); + this.hasUnicode = this.reHasUnicodeChar.test(parentStr); + + // A network filter (probably) + this.astType = AST_TYPE_NETWORK; + + // Parse inline comment if any + let tail = 0, tailStart = parentEnd; + if ( this.hasWhitespace && this.reInlineComment.test(parentStr) ) { + const match = this.reInlineComment.exec(parentStr); + tailStart = parentBeg + match.index; + tail = this.allocTypedNode(NODE_TYPE_COMMENT, tailStart, parentEnd); + } + + const head = this.allocTypedNode(NODE_TYPE_NET_RAW, parentBeg, tailStart); + if ( this.linkDown(head, this.parseNet(head)) === 0 ) { + this.astType = AST_TYPE_UNKNOWN; + this.addFlags(AST_FLAG_UNSUPPORTED | AST_FLAG_HAS_ERROR); + } + if ( tail !== 0 ) { + this.linkRight(head, tail); + } + return head; + } + + parseComment(parent) { + const parentStr = this.getNodeString(parent); + if ( this.rePreparseDirectiveAny.test(parentStr) ) { + this.astTypeFlavor = AST_TYPE_COMMENT_PREPARSER; + return this.parsePreparseDirective(parent, parentStr); + } + if ( this.reURL.test(parentStr) === false ) { return 0; } + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const match = this.reURL.exec(parentStr); + const urlBeg = parentBeg + match.index; + const urlEnd = urlBeg + match[0].length; + const head = this.allocTypedNode(NODE_TYPE_COMMENT, parentBeg, urlBeg); + let next = this.allocTypedNode(NODE_TYPE_COMMENT_URL, urlBeg, urlEnd); + let prev = this.linkRight(head, next); + if ( urlEnd !== parentEnd ) { + next = this.allocTypedNode(NODE_TYPE_COMMENT, urlEnd, parentEnd); + this.linkRight(prev, next); + } + return head; + } + + parsePreparseDirective(parent, s) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const match = this.rePreparseDirectiveAny.exec(s); + const directiveEnd = parentBeg + match[0].length; + const head = this.allocTypedNode( + NODE_TYPE_PREPARSE_DIRECTIVE, + parentBeg, + directiveEnd + ); + if ( directiveEnd !== parentEnd ) { + const type = s.startsWith('!#if ') + ? NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE + : NODE_TYPE_PREPARSE_DIRECTIVE_VALUE; + const next = this.allocTypedNode(type, directiveEnd, parentEnd); + this.addNodeToRegister(type, next); + this.linkRight(head, next); + if ( type === NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE ) { + const rawToken = this.getNodeString(next).trim(); + if ( utils.preparser.evaluateExpr(rawToken) === undefined ) { + this.addNodeFlags(next, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + this.astError = AST_ERROR_IF_TOKEN_UNKNOWN; + } + } + } + return head; + } + + // Very common, look into fast-tracking such plain pattern: + // /^[^!#\$\*\^][^#\$\*\^]*[^\$\*\|]$/ + parseNet(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const parentStr = this.getNodeString(parent); + const head = this.allocHeadNode(); + let patternBeg = parentBeg; + let prev = head, next = 0, tail = 0; + if ( this.reNetException.test(parentStr) ) { + this.addFlags(AST_FLAG_IS_EXCEPTION); + next = this.allocTypedNode(NODE_TYPE_NET_EXCEPTION, parentBeg, parentBeg+2); + prev = this.linkRight(prev, next); + patternBeg += 2; + } + let anchorBeg = this.indexOfNetAnchor(parentStr, patternBeg); + if ( anchorBeg === -1 ) { return 0; } + anchorBeg += parentBeg; + if ( anchorBeg !== parentEnd ) { + tail = this.allocTypedNode( + NODE_TYPE_NET_OPTIONS_ANCHOR, + anchorBeg, + anchorBeg + 1 + ); + next = this.allocTypedNode( + NODE_TYPE_NET_OPTIONS, + anchorBeg + 1, + parentEnd + ); + this.addFlags(AST_FLAG_HAS_OPTIONS); + this.addNodeToRegister(NODE_TYPE_NET_OPTIONS, next); + this.linkDown(next, this.parseNetOptions(next)); + this.linkRight(tail, next); + } + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN_RAW, + patternBeg, + anchorBeg + ); + this.addNodeToRegister(NODE_TYPE_NET_PATTERN_RAW, next); + this.linkDown(next, this.parseNetPattern(next)); + prev = this.linkRight(prev, next); + if ( tail !== 0 ) { + this.linkRight(prev, tail); + } + if ( this.astType === AST_TYPE_NETWORK ) { + this.validateNet(); + } + return this.throwHeadNode(head); + } + + validateNet() { + const isException = this.isException(); + let bad = false, realBad = false; + let abstractTypeCount = 0; + let behaviorTypeCount = 0; + let docTypeCount = 0; + let modifierType = 0; + let requestTypeCount = 0; + let unredirectableTypeCount = 0; + for ( let i = 0, n = this.nodeTypeRegisterPtr; i < n; i++ ) { + const type = this.nodeTypeRegister[i]; + const targetNode = this.nodeTypeLookupTable[type]; + if ( targetNode === 0 ) { continue; } + if ( this.badTypes.has(type) ) { + this.addNodeFlags(NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + this.astError = AST_ERROR_OPTION_EXCLUDED; + } + const flags = this.getNodeFlags(targetNode); + if ( (flags & NODE_FLAG_ERROR) !== 0 ) { continue; } + const isNegated = (flags & NODE_FLAG_IS_NEGATED) !== 0; + const hasValue = (flags & NODE_FLAG_OPTION_HAS_VALUE) !== 0; + bad = false; realBad = false; + switch ( type ) { + case NODE_TYPE_NET_OPTION_NAME_ALL: + realBad = isNegated || hasValue || modifierType !== 0; + break; + case NODE_TYPE_NET_OPTION_NAME_1P: + case NODE_TYPE_NET_OPTION_NAME_3P: + realBad = hasValue; + break; + case NODE_TYPE_NET_OPTION_NAME_BADFILTER: + case NODE_TYPE_NET_OPTION_NAME_NOOP: + realBad = isNegated || hasValue; + break; + case NODE_TYPE_NET_OPTION_NAME_CSS: + case NODE_TYPE_NET_OPTION_NAME_FONT: + case NODE_TYPE_NET_OPTION_NAME_IMAGE: + case NODE_TYPE_NET_OPTION_NAME_MEDIA: + case NODE_TYPE_NET_OPTION_NAME_OBJECT: + case NODE_TYPE_NET_OPTION_NAME_OTHER: + case NODE_TYPE_NET_OPTION_NAME_SCRIPT: + case NODE_TYPE_NET_OPTION_NAME_XHR: + realBad = hasValue; + if ( realBad ) { break; } + requestTypeCount += 1; + break; + case NODE_TYPE_NET_OPTION_NAME_CNAME: + realBad = isException === false || isNegated || hasValue; + if ( realBad ) { break; } + modifierType = type; + break; + case NODE_TYPE_NET_OPTION_NAME_CSP: + realBad = (hasValue || isException) === false || + modifierType !== 0 || + this.reBadCSP.test( + this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_CSP) + ); + if ( realBad ) { break; } + modifierType = type; + break; + case NODE_TYPE_NET_OPTION_NAME_DENYALLOW: + realBad = isNegated || hasValue === false || + this.getBranchFromType(NODE_TYPE_NET_OPTION_NAME_FROM) === 0; + break; + case NODE_TYPE_NET_OPTION_NAME_DOC: + case NODE_TYPE_NET_OPTION_NAME_FRAME: + realBad = hasValue; + if ( realBad ) { break; } + docTypeCount += 1; + break; + case NODE_TYPE_NET_OPTION_NAME_EHIDE: + case NODE_TYPE_NET_OPTION_NAME_GHIDE: + case NODE_TYPE_NET_OPTION_NAME_SHIDE: + realBad = isNegated || hasValue || modifierType !== 0; + if ( realBad ) { break; } + behaviorTypeCount += 1; + unredirectableTypeCount += 1; + break; + case NODE_TYPE_NET_OPTION_NAME_EMPTY: + case NODE_TYPE_NET_OPTION_NAME_MP4: + realBad = isNegated || hasValue || modifierType !== 0; + if ( realBad ) { break; } + modifierType = type; + break; + case NODE_TYPE_NET_OPTION_NAME_FROM: + case NODE_TYPE_NET_OPTION_NAME_METHOD: + case NODE_TYPE_NET_OPTION_NAME_TO: + realBad = isNegated || hasValue === false; + break; + case NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK: + bad = true; + realBad = isException === false || isNegated || hasValue; + break; + case NODE_TYPE_NET_OPTION_NAME_HEADER: + realBad = isNegated || hasValue === false; + break; + case NODE_TYPE_NET_OPTION_NAME_IMPORTANT: + realBad = isException || isNegated || hasValue; + break; + case NODE_TYPE_NET_OPTION_NAME_INLINEFONT: + case NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT: + realBad = hasValue; + if ( realBad ) { break; } + modifierType = type; + unredirectableTypeCount += 1; + break; + case NODE_TYPE_NET_OPTION_NAME_MATCHCASE: + realBad = this.isRegexPattern() === false; + break; + case NODE_TYPE_NET_OPTION_NAME_PERMISSIONS: + realBad = modifierType !== 0 || (hasValue || isException) === false; + if ( realBad ) { break; } + modifierType = type; + break; + case NODE_TYPE_NET_OPTION_NAME_PING: + case NODE_TYPE_NET_OPTION_NAME_WEBSOCKET: + realBad = hasValue; + if ( realBad ) { break; } + requestTypeCount += 1; + unredirectableTypeCount += 1; + break; + case NODE_TYPE_NET_OPTION_NAME_POPUNDER: + case NODE_TYPE_NET_OPTION_NAME_POPUP: + realBad = hasValue; + if ( realBad ) { break; } + abstractTypeCount += 1; + unredirectableTypeCount += 1; + break; + case NODE_TYPE_NET_OPTION_NAME_REDIRECT: + case NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + case NODE_TYPE_NET_OPTION_NAME_REPLACE: + case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + realBad = isNegated || (isException || hasValue) === false || + modifierType !== 0; + if ( realBad ) { break; } + modifierType = type; + break; + case NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: + realBad = isNegated || modifierType !== 0; + if ( realBad ) { break; } + modifierType = type; + break; + case NODE_TYPE_NET_OPTION_NAME_STRICT1P: + case NODE_TYPE_NET_OPTION_NAME_STRICT3P: + realBad = isNegated || hasValue; + break; + case NODE_TYPE_NET_OPTION_NAME_UNKNOWN: + this.astError = AST_ERROR_OPTION_UNKNOWN; + realBad = true; + break; + case NODE_TYPE_NET_OPTION_NAME_WEBRTC: + realBad = true; + break; + case NODE_TYPE_NET_PATTERN_RAW: + realBad = this.hasOptions() === false && + this.getNetPattern().length <= 1; + break; + default: + break; + } + if ( bad || realBad ) { + this.addNodeFlags(targetNode, NODE_FLAG_ERROR); + } + if ( realBad ) { + this.addFlags(AST_FLAG_HAS_ERROR); + } + } + switch ( modifierType ) { + case NODE_TYPE_NET_OPTION_NAME_CNAME: + realBad = abstractTypeCount || behaviorTypeCount || requestTypeCount; + break; + case NODE_TYPE_NET_OPTION_NAME_CSP: + case NODE_TYPE_NET_OPTION_NAME_PERMISSIONS: + realBad = abstractTypeCount || behaviorTypeCount || requestTypeCount; + break; + case NODE_TYPE_NET_OPTION_NAME_INLINEFONT: + case NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT: + realBad = behaviorTypeCount; + break; + case NODE_TYPE_NET_OPTION_NAME_EMPTY: + realBad = abstractTypeCount || behaviorTypeCount; + break; + case NODE_TYPE_NET_OPTION_NAME_MEDIA: + case NODE_TYPE_NET_OPTION_NAME_MP4: + realBad = abstractTypeCount || behaviorTypeCount || docTypeCount || requestTypeCount; + break; + case NODE_TYPE_NET_OPTION_NAME_REDIRECT: + case NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: { + realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount; + break; + } + case NODE_TYPE_NET_OPTION_NAME_REPLACE: { + realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount; + if ( realBad ) { break; } + if ( isException !== true && this.options.trustedSource !== true ) { + this.astError = AST_ERROR_UNTRUSTED_SOURCE; + realBad = true; + break; + } + const value = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_REPLACE); + if ( parseReplaceValue(value) === undefined ) { + this.astError = AST_ERROR_OPTION_BADVALUE; + realBad = true; + } + break; + } + case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount; + if ( realBad ) { break; } + if ( isException !== true && this.options.trustedSource !== true ) { + this.astError = AST_ERROR_UNTRUSTED_SOURCE; + realBad = true; + break; + } + const value = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM); + if ( parseReplaceValue(value) === undefined ) { + this.astError = AST_ERROR_OPTION_BADVALUE; + realBad = true; + } + break; + case NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: + realBad = abstractTypeCount || behaviorTypeCount; + break; + default: + break; + } + if ( realBad ) { + const targetNode = this.getBranchFromType(modifierType); + this.addNodeFlags(targetNode, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + } + + indexOfNetAnchor(s, start = 0) { + const end = s.length; + if ( end === start ) { return end; } + let j = s.lastIndexOf('$'); + if ( j === -1 ) { return end; } + if ( (j+1) === end ) { return end; } + for (;;) { + const before = s.charCodeAt(j-1); + if ( j !== start && before === 0x24 /* $ */ ) { return -1; } + const after = s.charCodeAt(j+1); + if ( + after !== 0x29 /* ) */ && + after !== 0x2F /* / */ && + after !== 0x7C /* | */ && + before !== 0x5C /* \ */ + ) { + return j; + } + if ( j <= start ) { break; } + j = s.lastIndexOf('$', j-1); + if ( j === -1 ) { break; } + } + return end; + } + + parseNetPattern(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + + // Empty pattern + if ( parentEnd === parentBeg ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_ANY; + const node = this.allocTypedNode( + NODE_TYPE_NET_PATTERN, + parentBeg, + parentEnd + ); + this.addNodeToRegister(NODE_TYPE_NET_PATTERN, node); + this.setNodeTransform(node, '*'); + return node; + } + + const head = this.allocHeadNode(); + let prev = head, next = 0, tail = 0; + let pattern = this.getNodeString(parent); + const hasWildcard = pattern.includes('*'); + const c1st = pattern.charCodeAt(0); + const c2nd = pattern.charCodeAt(1) || 0; + const clast = exCharCodeAt(pattern, -1); + + // Common case: Easylist syntax-based hostname + if ( + hasWildcard === false && + c1st === 0x7C /* | */ && c2nd === 0x7C /* | */ && + clast === 0x5E /* ^ */ && + this.isAdblockHostnamePattern(pattern) + ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + this.addFlags( + AST_FLAG_NET_PATTERN_LEFT_HNANCHOR | + AST_FLAG_NET_PATTERN_RIGHT_PATHANCHOR + ); + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR, + parentBeg, + parentBeg + 2 + ); + prev = this.linkRight(prev, next); + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN, + parentBeg + 2, + parentEnd - 1 + ); + pattern = pattern.slice(2, -1); + const normal = this.hasUnicode + ? this.normalizeHostnameValue(pattern) + : pattern; + if ( normal !== undefined && normal !== pattern ) { + this.setNodeTransform(next, normal); + } + this.addNodeToRegister(NODE_TYPE_NET_PATTERN, next); + prev = this.linkRight(prev, next); + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN_PART_SPECIAL, + parentEnd - 1, + parentEnd + ); + this.linkRight(prev, next); + return this.throwHeadNode(head); + } + + let patternBeg = parentBeg; + let patternEnd = parentEnd; + + // Hosts file entry? + if ( + this.hasWhitespace && + this.isException() === false && + this.hasOptions() === false && + this.reHostsSink.test(pattern) + ) { + const match = this.reHostsSink.exec(pattern); + patternBeg += match[0].length; + pattern = pattern.slice(patternBeg); + next = this.allocTypedNode(NODE_TYPE_IGNORE, parentBeg, patternBeg); + prev = this.linkRight(prev, next); + if ( + this.reHostsRedirect.test(pattern) || + this.reHostnameAscii.test(pattern) === false + ) { + this.astType = AST_TYPE_NONE; + this.addFlags(AST_FLAG_IGNORE); + next = this.allocTypedNode(NODE_TYPE_IGNORE, patternBeg, parentEnd); + prev = this.linkRight(prev, next); + return this.throwHeadNode(head); + } + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + this.addFlags( + AST_FLAG_NET_PATTERN_LEFT_HNANCHOR | + AST_FLAG_NET_PATTERN_RIGHT_PATHANCHOR + ); + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN, + patternBeg, + parentEnd + ); + this.addNodeToRegister(NODE_TYPE_NET_PATTERN, next); + this.linkRight(prev, next); + return this.throwHeadNode(head); + } + + // Regex? + if ( + c1st === 0x2F /* / */ && clast === 0x2F /* / */ && + pattern.length > 2 + ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_REGEX; + const normal = this.normalizeRegexPattern(pattern); + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN, + patternBeg, + patternEnd + ); + this.addNodeToRegister(NODE_TYPE_NET_PATTERN, next); + if ( normal !== '' ) { + if ( normal !== pattern ) { + this.setNodeTransform(next, normal); + } + if ( this.interactive ) { + const tokenizable = utils.regex.toTokenizableStr(normal); + if ( this.reGoodRegexToken.test(tokenizable) === false ) { + this.addNodeFlags(next, NODE_FLAG_PATTERN_UNTOKENIZABLE); + } + } + } else { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_BAD; + this.astError = AST_ERROR_REGEX; + this.addFlags(AST_FLAG_HAS_ERROR); + this.addNodeFlags(next, NODE_FLAG_ERROR); + } + this.linkRight(prev, next); + return this.throwHeadNode(head); + } + + // Left anchor + if ( c1st === 0x7C /* '|' */ ) { + if ( c2nd === 0x7C /* '|' */ ) { + const type = this.isTokenCharCode(pattern.charCodeAt(2) || 0) + ? NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR + : NODE_TYPE_IGNORE; + next = this.allocTypedNode(type, patternBeg, patternBeg+2); + if ( type === NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR ) { + this.addFlags(AST_FLAG_NET_PATTERN_LEFT_HNANCHOR); + } + patternBeg += 2; + pattern = pattern.slice(2); + } else { + const type = this.isTokenCharCode(c2nd) + ? NODE_TYPE_NET_PATTERN_LEFT_ANCHOR + : NODE_TYPE_IGNORE; + next = this.allocTypedNode(type, patternBeg, patternBeg+1); + if ( type === NODE_TYPE_NET_PATTERN_LEFT_ANCHOR ) { + this.addFlags(AST_FLAG_NET_PATTERN_LEFT_ANCHOR); + } + patternBeg += 1; + pattern = pattern.slice(1); + } + prev = this.linkRight(prev, next); + if ( patternBeg === patternEnd ) { + this.addNodeFlags(next, NODE_FLAG_IGNORE); + } + } + + // Right anchor + if ( exCharCodeAt(pattern, -1) === 0x7C /* | */ ) { + const type = exCharCodeAt(pattern, -2) !== 0x2A /* * */ + ? NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR + : NODE_TYPE_IGNORE; + tail = this.allocTypedNode(type, patternEnd-1, patternEnd); + if ( type === NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR ) { + this.addFlags(AST_FLAG_NET_PATTERN_RIGHT_ANCHOR); + } + patternEnd -= 1; + pattern = pattern.slice(0, -1); + if ( patternEnd === patternBeg ) { + this.addNodeFlags(tail, NODE_FLAG_IGNORE); + } + } + + // Ignore pointless leading wildcards + if ( hasWildcard && this.rePointlessLeadingWildcards.test(pattern) ) { + const match = this.rePointlessLeadingWildcards.exec(pattern); + const ignoreLen = match[1].length; + next = this.allocTypedNode( + NODE_TYPE_IGNORE, + patternBeg, + patternBeg + ignoreLen + ); + prev = this.linkRight(prev, next); + patternBeg += ignoreLen; + pattern = pattern.slice(ignoreLen); + } + + // Ignore pointless trailing separators + if ( this.rePointlessTrailingSeparator.test(pattern) ) { + const match = this.rePointlessTrailingSeparator.exec(pattern); + const ignoreLen = match[1].length; + next = this.allocTypedNode( + NODE_TYPE_IGNORE, + patternEnd - ignoreLen, + patternEnd + ); + patternEnd -= ignoreLen; + pattern = pattern.slice(0, -ignoreLen); + if ( tail !== 0 ) { this.linkRight(next, tail); } + tail = next; + } + + // Ignore pointless trailing wildcards. Exception: when removing the + // trailing wildcard make the pattern look like a regex. + if ( hasWildcard && this.rePointlessTrailingWildcards.test(pattern) ) { + const match = this.rePointlessTrailingWildcards.exec(pattern); + const ignoreLen = match[1].length; + const needWildcard = pattern.charCodeAt(0) === 0x2F && + exCharCodeAt(pattern, -ignoreLen-1) === 0x2F; + const goodWildcardBeg = patternEnd - ignoreLen; + const badWildcardBeg = goodWildcardBeg + (needWildcard ? 1 : 0); + if ( badWildcardBeg !== patternEnd ) { + next = this.allocTypedNode( + NODE_TYPE_IGNORE, + badWildcardBeg, + patternEnd + ); + if ( tail !== 0 ) {this.linkRight(next, tail); } + tail = next; + } + if ( goodWildcardBeg !== badWildcardBeg ) { + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN_PART_SPECIAL, + goodWildcardBeg, + badWildcardBeg + ); + if ( tail !== 0 ) { this.linkRight(next, tail); } + tail = next; + } + patternEnd -= ignoreLen; + pattern = pattern.slice(0, -ignoreLen); + } + + const patternHasWhitespace = this.hasWhitespace && + this.reHasWhitespaceChar.test(pattern); + const needNormalization = this.needPatternNormalization(pattern); + const normal = needNormalization + ? this.normalizePattern(pattern) + : pattern; + next = this.allocTypedNode(NODE_TYPE_NET_PATTERN, patternBeg, patternEnd); + if ( patternHasWhitespace || normal === undefined ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_BAD; + this.addFlags(AST_FLAG_HAS_ERROR); + this.astError = AST_ERROR_PATTERN; + this.addNodeFlags(next, NODE_FLAG_ERROR); + } else if ( normal === '*' ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_ANY; + } else if ( this.reHostnameAscii.test(normal) ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_HOSTNAME; + } else if ( this.reHasPatternSpecialChars.test(normal) ) { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_GENERIC; + } else { + this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_PLAIN; + } + this.addNodeToRegister(NODE_TYPE_NET_PATTERN, next); + if ( needNormalization && normal !== undefined ) { + this.setNodeTransform(next, normal); + } + if ( this.interactive ) { + this.linkDown(next, this.parsePatternParts(next, pattern)); + } + prev = this.linkRight(prev, next); + + if ( tail !== 0 ) { + this.linkRight(prev, tail); + } + return this.throwHeadNode(head); + } + + isAdblockHostnamePattern(pattern) { + if ( this.hasUnicode ) { + return this.reHnAnchoredHostnameUnicode.test(pattern); + } + return this.reHnAnchoredHostnameAscii.test(pattern); + } + + parsePatternParts(parent, pattern) { + if ( pattern.length === 0 ) { return 0; } + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const matches = pattern.matchAll(this.rePatternAllSpecialChars); + const head = this.allocHeadNode(); + let prev = head, next = 0; + let plainPartBeg = 0; + for ( const match of matches ) { + const plainPartEnd = match.index; + if ( plainPartEnd !== plainPartBeg ) { + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN_PART, + parentBeg + plainPartBeg, + parentBeg + plainPartEnd + ); + prev = this.linkRight(prev, next); + } + plainPartBeg = plainPartEnd + match[0].length; + const type = match[0].charCodeAt(0) < 0x80 + ? NODE_TYPE_NET_PATTERN_PART_SPECIAL + : NODE_TYPE_NET_PATTERN_PART_UNICODE; + next = this.allocTypedNode( + type, + parentBeg + plainPartEnd, + parentBeg + plainPartBeg + ); + prev = this.linkRight(prev, next); + } + if ( plainPartBeg !== pattern.length ) { + next = this.allocTypedNode( + NODE_TYPE_NET_PATTERN_PART, + parentBeg + plainPartBeg, + parentBeg + pattern.length + ); + this.linkRight(prev, next); + } + return this.throwHeadNode(head); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1118#issuecomment-650730158 + // Be ready to deal with non-punycode-able Unicode characters. + // https://github.com/uBlockOrigin/uBlock-issues/issues/772 + // Encode Unicode characters beyond the hostname part. + // Prepend with '*' character to prevent the browser API from refusing to + // punycode -- this occurs when the extracted label starts with a dash. + needPatternNormalization(pattern) { + return pattern.length === 0 || this.hasUppercase || this.hasUnicode; + } + + normalizePattern(pattern) { + if ( pattern.length === 0 ) { return '*'; } + if ( this.reHasInvalidChar.test(pattern) ) { return; } + let normal = pattern.toLowerCase(); + if ( this.hasUnicode === false ) { return normal; } + // Punycode hostname part of the pattern. + if ( this.reHostnamePatternPart.test(normal) ) { + const match = this.reHostnamePatternPart.exec(normal); + const hn = match[0].replace(this.reHostnameLabel, s => { + if ( this.reHasUnicodeChar.test(s) === false ) { return s; } + if ( s.charCodeAt(0) === 0x2D /* - */ ) { s = '*' + s; } + return this.normalizeHostnameValue(s, 0b0001) || s; + }); + normal = hn + normal.slice(match.index + match[0].length); + } + if ( this.reHasUnicodeChar.test(normal) === false ) { return normal; } + // Percent-encode remaining Unicode characters. + try { + normal = normal.replace(this.reUnicodeChars, s => + encodeURIComponent(s).toLowerCase() + ); + } catch (ex) { + return; + } + return normal; + } + + getNetPattern() { + const node = this.nodeTypeLookupTable[NODE_TYPE_NET_PATTERN]; + return this.getNodeTransform(node); + } + + isAnyPattern() { + return this.astTypeFlavor === AST_TYPE_NETWORK_PATTERN_ANY; + } + + isHostnamePattern() { + return this.astTypeFlavor === AST_TYPE_NETWORK_PATTERN_HOSTNAME; + } + + isRegexPattern() { + return this.astTypeFlavor === AST_TYPE_NETWORK_PATTERN_REGEX; + } + + isPlainPattern() { + return this.astTypeFlavor === AST_TYPE_NETWORK_PATTERN_PLAIN; + } + + isGenericPattern() { + return this.astTypeFlavor === AST_TYPE_NETWORK_PATTERN_GENERIC; + } + + isBadPattern() { + return this.astTypeFlavor === AST_TYPE_NETWORK_PATTERN_BAD; + } + + parseNetOptions(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + if ( parentEnd === parentBeg ) { return 0; } + const s = this.getNodeString(parent); + const optionsEnd = s.length; + const head = this.allocHeadNode(); + let prev = head, next = 0; + let optionBeg = 0, optionEnd = 0; + let emptyOption = false, badComma = false; + while ( optionBeg !== optionsEnd ) { + optionEnd = this.endOfNetOption(s, optionBeg); + next = this.allocTypedNode( + NODE_TYPE_NET_OPTION_RAW, + parentBeg + optionBeg, + parentBeg + optionEnd + ); + emptyOption = optionEnd === optionBeg; + this.linkDown(next, this.parseNetOption(next)); + prev = this.linkRight(prev, next); + if ( optionEnd === optionsEnd ) { break; } + optionBeg = optionEnd + 1; + next = this.allocTypedNode( + NODE_TYPE_NET_OPTION_SEPARATOR, + parentBeg + optionEnd, + parentBeg + optionBeg + ); + badComma = optionBeg === optionsEnd; + prev = this.linkRight(prev, next); + if ( emptyOption || badComma ) { + this.addNodeFlags(next, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + } + this.linkRight(prev, + this.allocSentinelNode(NODE_TYPE_NET_OPTION_SENTINEL, parentEnd) + ); + return this.throwHeadNode(head); + } + + endOfNetOption(s, beg) { + const match = this.reNetOptionComma.exec(s.slice(beg)); + return match !== null ? beg + match.index : s.length; + } + + parseNetOption(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const s = this.getNodeString(parent); + const optionEnd = s.length; + const head = this.allocHeadNode(); + let prev = head, next = 0; + let nameBeg = 0; + if ( s.charCodeAt(0) === 0x7E ) { + this.addNodeFlags(parent, NODE_FLAG_IS_NEGATED); + next = this.allocTypedNode( + NODE_TYPE_NET_OPTION_NAME_NOT, + parentBeg, + parentBeg+1 + ); + prev = this.linkRight(prev, next); + nameBeg += 1; + } + const equalPos = s.indexOf('='); + const nameEnd = equalPos !== -1 ? equalPos : s.length; + const name = s.slice(nameBeg, nameEnd); + let nodeOptionType = nodeTypeFromOptionName.get(name); + if ( nodeOptionType === undefined ) { + nodeOptionType = this.reNoopOption.test(name) + ? NODE_TYPE_NET_OPTION_NAME_NOOP + : NODE_TYPE_NET_OPTION_NAME_UNKNOWN; + } + next = this.allocTypedNode( + nodeOptionType, + parentBeg + nameBeg, + parentBeg + nameEnd + ); + if ( + nodeOptionType !== NODE_TYPE_NET_OPTION_NAME_NOOP && + this.getBranchFromType(nodeOptionType) !== 0 + ) { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + this.astError = AST_ERROR_OPTION_DUPLICATE; + } else { + this.addNodeToRegister(nodeOptionType, parent); + } + prev = this.linkRight(prev, next); + if ( equalPos === -1 ) { + return this.throwHeadNode(head); + } + const valueBeg = equalPos + 1; + next = this.allocTypedNode( + NODE_TYPE_NET_OPTION_ASSIGN, + parentBeg + equalPos, + parentBeg + valueBeg + ); + prev = this.linkRight(prev, next); + if ( (equalPos+1) === optionEnd ) { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + return this.throwHeadNode(head); + } + this.addNodeFlags(parent, NODE_FLAG_OPTION_HAS_VALUE); + next = this.allocTypedNode( + NODE_TYPE_NET_OPTION_VALUE, + parentBeg + valueBeg, + parentBeg + optionEnd + ); + switch ( nodeOptionType ) { + case NODE_TYPE_NET_OPTION_NAME_DENYALLOW: + this.linkDown(next, this.parseDomainList(next, '|'), 0b00000); + break; + case NODE_TYPE_NET_OPTION_NAME_FROM: + case NODE_TYPE_NET_OPTION_NAME_TO: + this.linkDown(next, this.parseDomainList(next, '|', 0b11010)); + break; + default: + break; + } + this.linkRight(prev, next); + return this.throwHeadNode(head); + } + + getNetOptionValue(type) { + if ( this.nodeTypeRegister.includes(type) === false ) { return ''; } + const optionNode = this.nodeTypeLookupTable[type]; + if ( optionNode === 0 ) { return ''; } + const valueNode = this.findDescendantByType(optionNode, NODE_TYPE_NET_OPTION_VALUE); + if ( valueNode === 0 ) { return ''; } + return this.getNodeTransform(valueNode); + } + + parseDomainList(parent, separator, mode = 0b00000) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const containerNode = this.allocTypedNode( + NODE_TYPE_OPTION_VALUE_DOMAIN_LIST, + parentBeg, + parentEnd + ); + if ( parentEnd === parentBeg ) { return containerNode; } + const separatorCode = separator.charCodeAt(0); + const listNode = this.allocHeadNode(); + let prev = listNode; + let domainNode = 0; + let separatorNode = 0; + const s = this.getNodeString(parent); + const listEnd = s.length; + let beg = 0, end = 0, c = 0; + while ( beg < listEnd ) { + c = s.charCodeAt(beg); + if ( c === 0x7E /* ~ */ ) { + c = s.charCodeAt(beg+1) || 0; + } + if ( c !== 0x2F /* / */ ) { + end = s.indexOf(separator, beg); + } else { + end = s.indexOf('/', beg+1); + end = s.indexOf(separator, end !== -1 ? end+1 : beg); + } + if ( end === -1 ) { end = listEnd; } + if ( end !== beg ) { + domainNode = this.allocTypedNode( + NODE_TYPE_OPTION_VALUE_DOMAIN_RAW, + parentBeg + beg, + parentBeg + end + ); + this.linkDown(domainNode, this.parseDomain(domainNode, mode)); + prev = this.linkRight(prev, domainNode); + } else { + domainNode = 0; + if ( separatorNode !== 0 ) { + this.addNodeFlags(separatorNode, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + } + if ( s.charCodeAt(end) === separatorCode ) { + beg = end; + end += 1; + separatorNode = this.allocTypedNode( + NODE_TYPE_OPTION_VALUE_SEPARATOR, + parentBeg + beg, + parentBeg + end + ); + prev = this.linkRight(prev, separatorNode); + if ( domainNode === 0 ) { + this.addNodeFlags(separatorNode, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + } else { + separatorNode = 0; + } + beg = end; + } + // Dangling separator node + if ( separatorNode !== 0 ) { + this.addNodeFlags(separatorNode, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + this.linkDown(containerNode, this.throwHeadNode(listNode)); + return containerNode; + } + + parseDomain(parent, mode = 0b0000) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + let head = 0, next = 0; + let beg = parentBeg; + const c = this.charCodeAt(beg); + if ( c === 0x7E /* ~ */ ) { + this.addNodeFlags(parent, NODE_FLAG_IS_NEGATED); + head = this.allocTypedNode(NODE_TYPE_OPTION_VALUE_NOT, beg, beg + 1); + if ( (mode & 0b1000) === 0 ) { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + } + beg += 1; + } + if ( beg !== parentEnd ) { + next = this.allocTypedNode(NODE_TYPE_OPTION_VALUE_DOMAIN, beg, parentEnd); + const hn = this.normalizeDomainValue(this.getNodeString(next), mode); + if ( hn !== undefined ) { + if ( hn !== '' ) { + this.setNodeTransform(next, hn); + } else { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + this.astError = AST_ERROR_DOMAIN_NAME; + } + } + if ( head === 0 ) { + head = next; + } else { + this.linkRight(head, next); + } + } else { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + return head; + } + + // mode bits: + // 0b00001: can use wildcard at any position + // 0b00010: can use entity-based hostnames + // 0b00100: can use single wildcard + // 0b01000: can be negated + // 0b10000: can be a regex + normalizeDomainValue(s, modeBits) { + if ( (modeBits & 0b10000) === 0 || + s.length <= 2 || + s.charCodeAt(0) !== 0x2F /* / */ || + exCharCodeAt(s, -1) !== 0x2F /* / */ + ) { + return this.normalizeHostnameValue(s, modeBits); + } + const source = this.normalizeRegexPattern(s); + if ( source === '' ) { return ''; } + return `/${source}/`; + } + + parseExt(parent, anchorBeg, anchorLen) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const head = this.allocHeadNode(); + let prev = head, next = 0; + this.astType = AST_TYPE_EXTENDED; + this.addFlags(this.extFlagsFromAnchor(anchorBeg)); + if ( anchorBeg > parentBeg ) { + next = this.allocTypedNode( + NODE_TYPE_EXT_OPTIONS, + parentBeg, + anchorBeg + ); + this.addFlags(AST_FLAG_HAS_OPTIONS); + this.addNodeToRegister(NODE_TYPE_EXT_OPTIONS, next); + this.linkDown(next, this.parseDomainList(next, ',', 0b11110)); + prev = this.linkRight(prev, next); + } + next = this.allocTypedNode( + NODE_TYPE_EXT_OPTIONS_ANCHOR, + anchorBeg, + anchorBeg + anchorLen + ); + this.addNodeToRegister(NODE_TYPE_EXT_OPTIONS_ANCHOR, next); + prev = this.linkRight(prev, next); + next = this.allocTypedNode( + NODE_TYPE_EXT_PATTERN_RAW, + anchorBeg + anchorLen, + parentEnd + ); + this.addNodeToRegister(NODE_TYPE_EXT_PATTERN_RAW, next); + const down = this.parseExtPattern(next); + if ( down !== 0 ) { + this.linkDown(next, down); + } else { + this.addNodeFlags(next, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + this.linkRight(prev, next); + this.validateExt(); + return this.throwHeadNode(head); + } + + extFlagsFromAnchor(anchorBeg) { + let c = this.charCodeAt(anchorBeg+1) ; + if ( c === 0x23 /* # */ ) { return 0; } + if ( c === 0x25 /* % */ ) { return AST_FLAG_EXT_SCRIPTLET_ADG; } + if ( c === 0x3F /* ? */ ) { return AST_FLAG_EXT_STRONG; } + if ( c === 0x24 /* $ */ ) { + c = this.charCodeAt(anchorBeg+2); + if ( c === 0x23 /* # */ ) { return AST_FLAG_EXT_STYLE; } + if ( c === 0x3F /* ? */ ) { + return AST_FLAG_EXT_STYLE | AST_FLAG_EXT_STRONG; + } + } + if ( c === 0x40 /* @ */ ) { + return AST_FLAG_IS_EXCEPTION | this.extFlagsFromAnchor(anchorBeg+1); + } + return AST_FLAG_UNSUPPORTED | AST_FLAG_HAS_ERROR; + } + + validateExt() { + const isException = this.isException(); + let realBad = false; + for ( let i = 0, n = this.nodeTypeRegisterPtr; i < n; i++ ) { + const type = this.nodeTypeRegister[i]; + const targetNode = this.nodeTypeLookupTable[type]; + if ( targetNode === 0 ) { continue; } + const flags = this.getNodeFlags(targetNode); + if ( (flags & NODE_FLAG_ERROR) !== 0 ) { continue; } + realBad = false; + switch ( type ) { + case NODE_TYPE_EXT_PATTERN_RESPONSEHEADER: { + const pattern = this.getNodeString(targetNode); + realBad = + pattern !== '' && removableHTTPHeaders.has(pattern) === false || + pattern === '' && isException === false; + break; + } + case NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN: { + if ( this.interactive !== true ) { break; } + if ( isException ) { break; } + const { trustedSource, trustedScriptletTokens } = this.options; + if ( trustedScriptletTokens instanceof Set === false ) { break; } + const token = this.getNodeString(targetNode); + if ( trustedScriptletTokens.has(token) && trustedSource !== true ) { + this.astError = AST_ERROR_UNTRUSTED_SOURCE; + realBad = true; + } + break; + } + default: + break; + } + if ( realBad ) { + this.addNodeFlags(targetNode, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + } + } + + parseExtPattern(parent) { + const c = this.charCodeAt(this.nodes[parent+NODE_BEG_INDEX]); + // ##+js(...) + if ( c === 0x2B /* + */ ) { + const s = this.getNodeString(parent); + if ( /^\+js\(.*\)$/.exec(s) !== null ) { + this.astTypeFlavor = AST_TYPE_EXTENDED_SCRIPTLET; + return this.parseExtPatternScriptlet(parent); + } + } + // #%#//scriptlet(...) + if ( this.getFlags(AST_FLAG_EXT_SCRIPTLET_ADG) ) { + const s = this.getNodeString(parent); + if ( /^\/\/scriptlet\(.*\)$/.exec(s) !== null ) { + this.astTypeFlavor = AST_TYPE_EXTENDED_SCRIPTLET; + return this.parseExtPatternScriptlet(parent); + } + return 0; + } + // ##^... | ##^responseheader(...) + if ( c === 0x5E /* ^ */ ) { + const s = this.getNodeString(parent); + if ( this.reResponseheaderPattern.test(s) ) { + this.astTypeFlavor = AST_TYPE_EXTENDED_RESPONSEHEADER; + return this.parseExtPatternResponseheader(parent); + } + this.astTypeFlavor = AST_TYPE_EXTENDED_HTML; + return this.parseExtPatternHtml(parent); + } + // ##... + this.astTypeFlavor = AST_TYPE_EXTENDED_COSMETIC; + return this.parseExtPatternCosmetic(parent); + } + + parseExtPatternScriptlet(parent) { + const beg = this.nodes[parent+NODE_BEG_INDEX]; + const end = this.nodes[parent+NODE_END_INDEX]; + const s = this.getNodeString(parent); + const rawArg0 = beg + (s.startsWith('+js') ? 4 : 12); + const rawArg1 = end - 1; + const head = this.allocTypedNode(NODE_TYPE_EXT_DECORATION, beg, rawArg0); + let prev = head, next = 0; + next = this.allocTypedNode(NODE_TYPE_EXT_PATTERN_SCRIPTLET, rawArg0, rawArg1); + this.addNodeToRegister(NODE_TYPE_EXT_PATTERN_SCRIPTLET, next); + this.linkDown(next, this.parseExtPatternScriptletArgs(next)); + prev = this.linkRight(prev, next); + next = this.allocTypedNode(NODE_TYPE_EXT_DECORATION, rawArg1, end); + this.linkRight(prev, next); + return head; + } + + parseExtPatternScriptletArgs(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + if ( parentEnd === parentBeg ) { return 0; } + const head = this.allocHeadNode(); + let prev = head, next = 0; + const s = this.getNodeString(parent); + const argsEnd = s.length; + // token + this.scriptletArgListParser.mustQuote = + this.getFlags(AST_FLAG_EXT_SCRIPTLET_ADG) !== 0; + const details = this.scriptletArgListParser.nextArg(s, 0); + if ( details.argBeg > 0 ) { + next = this.allocTypedNode( + NODE_TYPE_EXT_DECORATION, + parentBeg, + parentBeg + details.argBeg + ); + prev = this.linkRight(prev, next); + } + const token = s.slice(details.argBeg, details.argEnd); + const tokenEnd = details.argEnd - (token.endsWith('.js') ? 3 : 0); + next = this.allocTypedNode( + NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN, + parentBeg + details.argBeg, + parentBeg + tokenEnd + ); + this.addNodeToRegister(NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN, next); + if ( details.failed ) { + this.addNodeFlags(next, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + prev = this.linkRight(prev, next); + if ( tokenEnd < details.argEnd ) { + next = this.allocTypedNode( + NODE_TYPE_IGNORE, + parentBeg + tokenEnd, + parentBeg + details.argEnd + ); + prev = this.linkRight(prev, next); + } + if ( details.quoteEnd < argsEnd ) { + next = this.allocTypedNode( + NODE_TYPE_EXT_DECORATION, + parentBeg + details.argEnd, + parentBeg + details.separatorEnd + ); + prev = this.linkRight(prev, next); + } + // all args + next = this.allocTypedNode( + NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARGS, + parentBeg + details.separatorEnd, + parentBeg + argsEnd + ); + this.linkDown(next, this.parseExtPatternScriptletArglist(next)); + prev = this.linkRight(prev, next); + return this.throwHeadNode(head); + } + + parseExtPatternScriptletArglist(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + if ( parentEnd === parentBeg ) { return 0; } + const s = this.getNodeString(parent); + const argsEnd = s.length; + const head = this.allocHeadNode(); + let prev = head, next = 0; + let decorationBeg = 0; + let i = 0; + for (;;) { + const details = this.scriptletArgListParser.nextArg(s, i); + if ( decorationBeg < details.argBeg ) { + next = this.allocTypedNode( + NODE_TYPE_EXT_DECORATION, + parentBeg + decorationBeg, + parentBeg + details.argBeg + ); + prev = this.linkRight(prev, next); + } + if ( i === argsEnd ) { break; } + next = this.allocTypedNode( + NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG, + parentBeg + details.argBeg, + parentBeg + details.argEnd + ); + if ( details.transform ) { + const arg = s.slice(details.argBeg, details.argEnd); + this.setNodeTransform(next, + this.scriptletArgListParser.normalizeArg(arg) + ); + } + prev = this.linkRight(prev, next); + if ( details.failed ) { + this.addNodeFlags(next, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + decorationBeg = details.argEnd; + i = details.separatorEnd; + } + return this.throwHeadNode(head); + } + + getScriptletArgs() { + const args = []; + if ( this.isScriptletFilter() === false ) { return args; } + const root = this.getBranchFromType(NODE_TYPE_EXT_PATTERN_SCRIPTLET); + const walker = this.getWalker(root); + for ( let node = walker.next(); node !== 0; node = walker.next() ) { + switch ( this.getNodeType(node) ) { + case NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN: + case NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG: + args.push(this.getNodeTransform(node)); + break; + default: + break; + } + } + walker.dispose(); + return args; + } + + parseExtPatternResponseheader(parent) { + const beg = this.nodes[parent+NODE_BEG_INDEX]; + const end = this.nodes[parent+NODE_END_INDEX]; + const s = this.getNodeString(parent); + const rawArg0 = beg + 16; + const rawArg1 = end - 1; + const head = this.allocTypedNode(NODE_TYPE_EXT_DECORATION, beg, rawArg0); + let prev = head, next = 0; + const trimmedArg0 = rawArg0 + this.leftWhitespaceCount(s); + const trimmedArg1 = rawArg1 - this.rightWhitespaceCount(s); + if ( trimmedArg0 !== rawArg0 ) { + next = this.allocTypedNode(NODE_TYPE_WHITESPACE, rawArg0, trimmedArg0); + prev = this.linkRight(prev, next); + } + next = this.allocTypedNode(NODE_TYPE_EXT_PATTERN_RESPONSEHEADER, rawArg0, rawArg1); + this.addNodeToRegister(NODE_TYPE_EXT_PATTERN_RESPONSEHEADER, next); + if ( rawArg1 === rawArg0 && this.isException() === false ) { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + prev = this.linkRight(prev, next); + if ( trimmedArg1 !== rawArg1 ) { + next = this.allocTypedNode(NODE_TYPE_WHITESPACE, trimmedArg1, rawArg1); + prev = this.linkRight(prev, next); + } + next = this.allocTypedNode(NODE_TYPE_EXT_DECORATION, rawArg1, end); + this.linkRight(prev, next); + return head; + } + + parseExtPatternHtml(parent) { + const beg = this.nodes[parent+NODE_BEG_INDEX]; + const end = this.nodes[parent+NODE_END_INDEX]; + const head = this.allocTypedNode(NODE_TYPE_EXT_DECORATION, beg, beg + 1); + let prev = head, next = 0; + next = this.allocTypedNode(NODE_TYPE_EXT_PATTERN_HTML, beg + 1, end); + this.linkRight(prev, next); + if ( (this.hasOptions() || this.isException()) === false ) { + this.addNodeFlags(parent, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + return head; + } + this.result.exception = this.isException(); + this.result.raw = this.getNodeString(next); + this.result.compiled = undefined; + const success = this.selectorCompiler.compile( + this.result.raw, + this.result, { + asProcedural: this.getFlags(AST_FLAG_EXT_STRONG) !== 0 + } + ); + if ( success !== true ) { + this.addNodeFlags(next, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + return head; + } + + parseExtPatternCosmetic(parent) { + const parentBeg = this.nodes[parent+NODE_BEG_INDEX]; + const parentEnd = this.nodes[parent+NODE_END_INDEX]; + const head = this.allocTypedNode( + NODE_TYPE_EXT_PATTERN_COSMETIC, + parentBeg, + parentEnd + ); + this.result.exception = this.isException(); + this.result.raw = this.getNodeString(head); + this.result.compiled = undefined; + const success = this.selectorCompiler.compile( + this.result.raw, + this.result, { + asProcedural: this.getFlags(AST_FLAG_EXT_STRONG) !== 0, + adgStyleSyntax: this.getFlags(AST_FLAG_EXT_STYLE) !== 0, + } + ); + if ( success !== true ) { + this.addNodeFlags(head, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } + return head; + } + + hasError() { + return (this.astFlags & AST_FLAG_HAS_ERROR) !== 0; + } + + isUnsupported() { + return (this.astFlags & AST_FLAG_UNSUPPORTED) !== 0; + } + + hasOptions() { + return (this.astFlags & AST_FLAG_HAS_OPTIONS) !== 0; + } + + isNegatedOption(type) { + const node = this.nodeTypeLookupTable[type]; + const flags = this.nodes[node+NODE_FLAGS_INDEX]; + return (flags & NODE_FLAG_IS_NEGATED) !== 0; + } + + isException() { + return (this.astFlags & AST_FLAG_IS_EXCEPTION) !== 0; + } + + isLeftHnAnchored() { + return (this.astFlags & AST_FLAG_NET_PATTERN_LEFT_HNANCHOR) !== 0; + } + + isLeftAnchored() { + return (this.astFlags & AST_FLAG_NET_PATTERN_LEFT_ANCHOR) !== 0; + } + + isRightAnchored() { + return (this.astFlags & AST_FLAG_NET_PATTERN_RIGHT_ANCHOR) !== 0; + } + + linkRight(prev, next) { + return (this.nodes[prev+NODE_RIGHT_INDEX] = next); + } + + linkDown(node, down) { + return (this.nodes[node+NODE_DOWN_INDEX] = down); + } + + makeChain(nodes) { + for ( let i = 1; i < nodes.length; i++ ) { + this.nodes[nodes[i-1]+NODE_RIGHT_INDEX] = nodes[i]; + } + return nodes[0]; + } + + allocHeadNode() { + const node = this.nodePoolPtr; + this.nodePoolPtr += NOOP_NODE_SIZE; + if ( this.nodePoolPtr > this.nodePoolEnd ) { + this.growNodePool(this.nodePoolPtr); + } + this.nodes[node+NODE_RIGHT_INDEX] = 0; + return node; + } + + throwHeadNode(head) { + return this.nodes[head+NODE_RIGHT_INDEX]; + } + + allocTypedNode(type, beg, end) { + const node = this.nodePoolPtr; + this.nodePoolPtr += FULL_NODE_SIZE; + if ( this.nodePoolPtr > this.nodePoolEnd ) { + this.growNodePool(this.nodePoolPtr); + } + this.nodes[node+NODE_RIGHT_INDEX] = 0; + this.nodes[node+NODE_TYPE_INDEX] = type; + this.nodes[node+NODE_DOWN_INDEX] = 0; + this.nodes[node+NODE_BEG_INDEX] = beg; + this.nodes[node+NODE_END_INDEX] = end; + this.nodes[node+NODE_TRANSFORM_INDEX] = 0; + this.nodes[node+NODE_FLAGS_INDEX] = 0; + return node; + } + + allocSentinelNode(type, beg) { + return this.allocTypedNode(type, beg, beg); + } + + growNodePool(min) { + const oldSize = this.nodes.length; + const newSize = (min + 16383) & ~16383; + if ( newSize === oldSize ) { return; } + const newArray = new Uint32Array(newSize); + newArray.set(this.nodes); + this.nodes = newArray; + this.nodePoolEnd = newSize; + } + + getNodeTypes() { + return this.nodeTypeRegister.slice(0, this.nodeTypeRegisterPtr); + } + + getNodeType(node) { + return node !== 0 ? this.nodes[node+NODE_TYPE_INDEX] : 0; + } + + getNodeFlags(node, flags = 0xFFFFFFFF) { + return this.nodes[node+NODE_FLAGS_INDEX] & flags; + } + + setNodeFlags(node, flags) { + this.nodes[node+NODE_FLAGS_INDEX] = flags; + } + + addNodeFlags(node, flags) { + if ( node === 0 ) { return; } + this.nodes[node+NODE_FLAGS_INDEX] |= flags; + } + + removeNodeFlags(node, flags) { + this.nodes[node+NODE_FLAGS_INDEX] &= ~flags; + } + + addNodeToRegister(type, node) { + this.nodeTypeRegister[this.nodeTypeRegisterPtr++] = type; + this.nodeTypeLookupTable[type] = node; + } + + getBranchFromType(type) { + const ptr = this.nodeTypeRegisterPtr; + if ( ptr === 0 ) { return 0; } + return this.nodeTypeRegister.lastIndexOf(type, ptr-1) !== -1 + ? this.nodeTypeLookupTable[type] + : 0; + } + + nodeIsEmptyString(node) { + return this.nodes[node+NODE_END_INDEX] === + this.nodes[node+NODE_BEG_INDEX]; + } + + getNodeString(node) { + const beg = this.nodes[node+NODE_BEG_INDEX]; + const end = this.nodes[node+NODE_END_INDEX]; + if ( end === beg ) { return ''; } + if ( beg === 0 && end === this.rawEnd ) { + return this.raw; + } + return this.raw.slice(beg, end); + } + + getNodeStringBeg(node) { + return this.nodes[node+NODE_BEG_INDEX]; + } + + getNodeStringEnd(node) { + return this.nodes[node+NODE_END_INDEX]; + } + + getNodeStringLen(node) { + if ( node === 0 ) { return ''; } + return this.nodes[node+NODE_END_INDEX] - this.nodes[node+NODE_BEG_INDEX]; + } + + isNodeTransformed(node) { + return this.nodes[node+NODE_TRANSFORM_INDEX] !== 0; + } + + getNodeTransform(node) { + if ( node === 0 ) { return ''; } + const slot = this.nodes[node+NODE_TRANSFORM_INDEX]; + return slot !== 0 ? this.astTransforms[slot] : this.getNodeString(node); + } + + setNodeTransform(node, value) { + const slot = this.astTransformPtr++; + this.astTransforms[slot] = value; + this.nodes[node+NODE_TRANSFORM_INDEX] = slot; + } + + getTypeString(type) { + const node = this.getBranchFromType(type); + if ( node === 0 ) { return; } + return this.getNodeString(node); + } + + leftWhitespaceCount(s) { + const match = this.reWhitespaceStart.exec(s); + return match === null ? 0 : match[0].length; + } + + rightWhitespaceCount(s) { + const match = this.reWhitespaceEnd.exec(s); + return match === null ? 0 : match[0].length; + } + + nextCommaInCommaSeparatedListString(s, start) { + const n = s.length; + if ( n === 0 ) { return -1; } + const ilastchar = n - 1; + let i = start; + while ( i < n ) { + const c = s.charCodeAt(i); + if ( c === 0x2C /* ',' */ ) { return i + 1; } + if ( c === 0x5C /* '\\' */ ) { + if ( i < ilastchar ) { i += 1; } + } + } + return -1; + } + + endOfLiteralRegex(s, start) { + const n = s.length; + if ( n === 0 ) { return -1; } + const ilastchar = n - 1; + let i = start + 1; + while ( i < n ) { + const c = s.charCodeAt(i); + if ( c === 0x2F /* '/' */ ) { return i + 1; } + if ( c === 0x5C /* '\\' */ ) { + if ( i < ilastchar ) { i += 1; } + } + i += 1; + } + return -1; + } + + charCodeAt(pos) { + return pos < this.rawEnd ? this.raw.charCodeAt(pos) : -1; + } + + isTokenCharCode(c) { + return c === 0x25 || + c >= 0x30 && c <= 0x39 || + c >= 0x41 && c <= 0x5A || + c >= 0x61 && c <= 0x7A; + } + + // Ultimately, let the browser API do the hostname normalization, after + // making some other trivial checks. + // + // mode bits: + // 0b00001: can use wildcard at any position + // 0b00010: can use entity-based hostnames + // 0b00100: can use single wildcard + // 0b01000: can be negated + // + // returns: + // undefined: no normalization needed, use original hostname + // empty string: hostname is invalid + // non-empty string: normalized hostname + normalizeHostnameValue(s, modeBits = 0b00000) { + if ( this.reHostnameAscii.test(s) ) { return; } + if ( this.reBadHostnameChars.test(s) ) { return ''; } + let hn = s; + const hasWildcard = hn.includes('*'); + if ( hasWildcard ) { + if ( modeBits === 0 ) { return ''; } + if ( hn.length === 1 ) { + if ( (modeBits & 0b0100) === 0 ) { return ''; } + return; + } + if ( (modeBits & 0b0010) !== 0 ) { + if ( this.rePlainEntity.test(hn) ) { return; } + if ( this.reIsEntity.test(hn) === false ) { return ''; } + } else if ( (modeBits & 0b0001) === 0 ) { + return ''; + } + hn = hn.replace(/\*/g, '__asterisk__'); + } + this.punycoder.hostname = '_'; + try { + this.punycoder.hostname = hn; + hn = this.punycoder.hostname; + } catch (_) { + return ''; + } + if ( hn === '_' || hn === '' ) { return ''; } + if ( hasWildcard ) { + hn = this.punycoder.hostname.replace(/__asterisk__/g, '*'); + } + if ( + (modeBits & 0b0001) === 0 && ( + hn.charCodeAt(0) === 0x2E /* . */ || + exCharCodeAt(hn, -1) === 0x2E /* . */ + ) + ) { + return ''; + } + return hn; + } + + normalizeRegexPattern(s) { + try { + const source = /^\/.+\/$/.test(s) ? s.slice(1,-1) : s; + const regex = new RegExp(source); + return regex.source; + } catch (ex) { + this.normalizeRegexPattern.message = ex.toString(); + } + return ''; + } + + getDomainListIterator(root) { + const iter = this.domainListIteratorJunkyard.length !== 0 + ? this.domainListIteratorJunkyard.pop().reuse(root) + : new DomainListIterator(this, root); + return root !== 0 ? iter : iter.stop(); + } + + getNetFilterFromOptionIterator() { + return this.getDomainListIterator( + this.getBranchFromType(NODE_TYPE_NET_OPTION_NAME_FROM) + ); + } + + getNetFilterToOptionIterator() { + return this.getDomainListIterator( + this.getBranchFromType(NODE_TYPE_NET_OPTION_NAME_TO) + ); + } + + getNetFilterDenyallowOptionIterator() { + return this.getDomainListIterator( + this.getBranchFromType(NODE_TYPE_NET_OPTION_NAME_DENYALLOW) + ); + } + + getExtFilterDomainIterator() { + return this.getDomainListIterator( + this.getBranchFromType(NODE_TYPE_EXT_OPTIONS) + ); + } + + getWalker(from) { + if ( this.walkerJunkyard.length === 0 ) { + return new AstWalker(this, from); + } + const walker = this.walkerJunkyard.pop(); + walker.reset(from); + return walker; + } + + findDescendantByType(from, type) { + const walker = this.getWalker(from); + let node = walker.next(); + while ( node !== 0 ) { + if ( this.getNodeType(node) === type ) { return node; } + node = walker.next(); + } + return 0; + } + + dump() { + if ( this.astType === AST_TYPE_COMMENT ) { return; } + const walker = this.getWalker(); + for ( let node = walker.reset(); node !== 0; node = walker.next() ) { + const type = this.nodes[node+NODE_TYPE_INDEX]; + const value = this.getNodeString(node); + const name = nodeNameFromNodeType.get(type) || `${type}`; + const bits = this.getNodeFlags(node).toString(2).padStart(4, '0'); + const indent = ' '.repeat(walker.depth); + console.log(`${indent}type=${name} "${value}" 0b${bits}`); + if ( this.isNodeTransformed(node) ) { + console.log(`${indent} transform="${this.getNodeTransform(node)}`); + } + } + } +} + +/******************************************************************************/ + +export function parseRedirectValue(arg) { + let token = arg.trim(); + let priority = 0; + const asDataURI = token.charCodeAt(0) === 0x25 /* '%' */; + if ( asDataURI ) { token = token.slice(1); } + const match = /:-?\d+$/.exec(token); + if ( match !== null ) { + priority = parseInt(token.slice(match.index + 1), 10); + token = token.slice(0, match.index); + } + return { token, priority, asDataURI }; +} + +export function parseQueryPruneValue(arg) { + let s = arg.trim(); + if ( s === '' ) { return { all: true }; } + const out = { }; + out.not = s.charCodeAt(0) === 0x7E /* '~' */; + if ( out.not ) { + s = s.slice(1); + } + const match = /^\/(.+)\/(i)?$/.exec(s); + if ( match !== null ) { + try { + out.re = new RegExp(match[1], match[2] || ''); + } + catch(ex) { + out.bad = true; + } + return out; + } + // TODO: remove once no longer used in filter lists + if ( s.startsWith('|') ) { + try { + out.re = new RegExp('^' + s.slice(1), 'i'); + } catch(ex) { + out.bad = true; + } + return out; + } + // Multiple values not supported (because very inefficient) + if ( s.includes('|') ) { + out.bad = true; + return out; + } + out.name = s; + return out; +} + +export function parseHeaderValue(arg) { + let s = arg.trim(); + const out = { }; + let pos = s.indexOf(':'); + if ( pos === -1 ) { pos = s.length; } + out.name = s.slice(0, pos); + out.bad = out.name === ''; + s = s.slice(pos + 1); + out.not = s.charCodeAt(0) === 0x7E /* '~' */; + if ( out.not ) { s = s.slice(1); } + out.value = s; + const match = /^\/(.+)\/(i)?$/.exec(s); + if ( match !== null ) { + try { + out.re = new RegExp(match[1], match[2] || ''); + } + catch(ex) { + out.bad = true; + } + } + return out; +} + + +// https://adguard.com/kb/general/ad-filtering/create-own-filters/#replace-modifier + +export function parseReplaceValue(s) { + if ( s.charCodeAt(0) !== 0x2F /* / */ ) { return; } + const parser = new ArgListParser('/'); + parser.nextArg(s, 1); + let pattern = s.slice(parser.argBeg, parser.argEnd); + if ( parser.transform ) { + pattern = parser.normalizeArg(pattern); + } + if ( pattern === '' ) { return; } + pattern = parser.normalizeArg(pattern, '$'); + pattern = parser.normalizeArg(pattern, ','); + parser.nextArg(s, parser.separatorEnd); + let replacement = s.slice(parser.argBeg, parser.argEnd); + if ( parser.separatorEnd === parser.separatorBeg ) { return; } + if ( parser.transform ) { + replacement = parser.normalizeArg(replacement); + } + replacement = parser.normalizeArg(replacement, '$'); + replacement = parser.normalizeArg(replacement, ','); + const flags = s.slice(parser.separatorEnd); + try { + return { re: new RegExp(pattern, flags), replacement }; + } catch(_) { + } +} + +/******************************************************************************/ + +export const netOptionTokenDescriptors = new Map([ + [ '1p', { canNegate: true } ], + /* synonym */ [ 'first-party', { canNegate: true } ], + [ 'strict1p', { } ], + [ '3p', { canNegate: true } ], + /* synonym */ [ 'third-party', { canNegate: true } ], + [ 'strict3p', { } ], + [ 'all', { } ], + [ 'badfilter', { } ], + [ 'cname', { allowOnly: true } ], + [ 'csp', { mustAssign: true } ], + [ 'css', { canNegate: true } ], + /* synonym */ [ 'stylesheet', { canNegate: true } ], + [ 'denyallow', { mustAssign: true } ], + [ 'doc', { canNegate: true } ], + /* synonym */ [ 'document', { canNegate: true } ], + [ 'ehide', { } ], + /* synonym */ [ 'elemhide', { } ], + [ 'empty', { blockOnly: true } ], + [ 'frame', { canNegate: true } ], + /* synonym */ [ 'subdocument', { canNegate: true } ], + [ 'from', { mustAssign: true } ], + /* synonym */ [ 'domain', { mustAssign: true } ], + [ 'font', { canNegate: true } ], + [ 'genericblock', { } ], + [ 'ghide', { } ], + /* synonym */ [ 'generichide', { } ], + [ 'header', { mustAssign: true } ], + [ 'image', { canNegate: true } ], + [ 'important', { blockOnly: true } ], + [ 'inline-font', { canNegate: true } ], + [ 'inline-script', { canNegate: true } ], + [ 'match-case', { } ], + [ 'media', { canNegate: true } ], + [ 'method', { mustAssign: true } ], + [ 'mp4', { blockOnly: true } ], + [ '_', { } ], + [ 'object', { canNegate: true } ], + /* synonym */ [ 'object-subrequest', { canNegate: true } ], + [ 'other', { canNegate: true } ], + [ 'permissions', { mustAssign: true } ], + [ 'ping', { canNegate: true } ], + /* synonym */ [ 'beacon', { canNegate: true } ], + [ 'popunder', { } ], + [ 'popup', { canNegate: true } ], + [ 'redirect', { mustAssign: true } ], + /* synonym */ [ 'rewrite', { mustAssign: true } ], + [ 'redirect-rule', { mustAssign: true } ], + [ 'removeparam', { } ], + [ 'replace', { mustAssign: true } ], + /* synonym */ [ 'queryprune', { } ], + [ 'script', { canNegate: true } ], + [ 'shide', { } ], + /* synonym */ [ 'specifichide', { } ], + [ 'to', { mustAssign: true } ], + [ 'uritransform', { mustAssign: true } ], + [ 'xhr', { canNegate: true } ], + /* synonym */ [ 'xmlhttprequest', { canNegate: true } ], + [ 'webrtc', { } ], + [ 'websocket', { canNegate: true } ], +]); + +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/1004 +// Detect and report invalid CSS selectors. + +// Discard new ABP's `-abp-properties` directive until it is +// implemented (if ever). Unlikely, see: +// https://github.com/gorhill/uBlock/issues/1752 + +// https://github.com/gorhill/uBlock/issues/2624 +// Convert Adguard's `-ext-has='...'` into uBO's `:has(...)`. + +// https://github.com/uBlockOrigin/uBlock-issues/issues/89 +// Do not discard unknown pseudo-elements. + +class ExtSelectorCompiler { + constructor(instanceOptions) { + this.reParseRegexLiteral = /^\/(.+)\/([imu]+)?$/; + + // Use a regex for most common CSS selectors known to be valid in any + // context. + const cssIdentifier = '[A-Za-z_][\\w-]*'; + const cssClassOrId = `[.#]${cssIdentifier}`; + const cssAttribute = `\\[${cssIdentifier}(?:[*^$]?="[^"\\]\\\\]+")?\\]`; + const cssSimple = + '(?:' + + `${cssIdentifier}(?:${cssClassOrId})*(?:${cssAttribute})*` + '|' + + `${cssClassOrId}(?:${cssClassOrId})*(?:${cssAttribute})*` + '|' + + `${cssAttribute}(?:${cssAttribute})*` + + ')'; + const cssCombinator = '(?: | [+>~] )'; + this.reCommonSelector = new RegExp( + `^${cssSimple}(?:${cssCombinator}${cssSimple})*$` + ); + // Resulting regex literal: + // /^(?:[A-Za-z_][\w-]*(?:[.#][A-Za-z_][\w-]*)*(?:\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\])*|[.#][A-Za-z_][\w-]*(?:[.#][A-Za-z_][\w-]*)*(?:\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\])*|\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\](?:\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\])*)(?:(?:\s+|\s*[>+~]\s*)(?:[A-Za-z_][\w-]*(?:[.#][A-Za-z_][\w-]*)*(?:\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\])*|[.#][A-Za-z_][\w-]*(?:[.#][A-Za-z_][\w-]*)*(?:\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\])*|\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\](?:\[[A-Za-z_][\w-]*(?:[*^$]?="[^"\]\\]+")?\])*))*$/ + + this.reEatBackslashes = /\\([()])/g; + this.reEscapeRegex = /[.*+?^${}()|[\]\\]/g; + // https://developer.mozilla.org/en-US/docs/Web/CSS/Pseudo-classes + this.knownPseudoClasses = new Set([ + 'active', 'any-link', 'autofill', + 'blank', + 'checked', 'current', + 'default', 'defined', 'dir', 'disabled', + 'empty', 'enabled', + 'first', 'first-child', 'first-of-type', 'fullscreen', 'future', 'focus', 'focus-visible', 'focus-within', + 'has', 'host', 'host-context', 'hover', + 'indeterminate', 'in-range', 'invalid', 'is', + 'lang', 'last-child', 'last-of-type', 'left', 'link', 'local-link', + 'modal', + 'not', 'nth-child', 'nth-col', 'nth-last-child', 'nth-last-col', 'nth-last-of-type', 'nth-of-type', + 'only-child', 'only-of-type', 'optional', 'out-of-range', + 'past', 'picture-in-picture', 'placeholder-shown', 'paused', 'playing', + 'read-only', 'read-write', 'required', 'right', 'root', + 'scope', 'state', 'target', 'target-within', + 'user-invalid', 'valid', 'visited', + 'where', + ]); + this.knownPseudoClassesWithArgs = new Set([ + 'dir', + 'has', 'host-context', + 'is', + 'lang', + 'not', 'nth-child', 'nth-col', 'nth-last-child', 'nth-last-col', 'nth-last-of-type', 'nth-of-type', + 'state', + 'where', + ]); + // https://developer.mozilla.org/en-US/docs/Web/CSS/Pseudo-elements + this.knownPseudoElements = new Set([ + 'after', + 'backdrop', 'before', + 'cue', 'cue-region', + 'first-letter', 'first-line', 'file-selector-button', + 'grammar-error', 'marker', + 'part', 'placeholder', + 'selection', 'slotted', 'spelling-error', + 'target-text', + ]); + this.knownPseudoElementsWithArgs = new Set([ + 'part', + 'slotted', + ]); + // https://github.com/gorhill/uBlock/issues/2793 + this.normalizedOperators = new Map([ + [ '-abp-has', 'has' ], + [ '-abp-contains', 'has-text' ], + [ 'contains', 'has-text' ], + [ 'nth-ancestor', 'upward' ], + [ 'watch-attrs', 'watch-attr' ], + ]); + this.actionOperators = new Set([ + ':remove', + ':style', + ]); + this.proceduralOperatorNames = new Set([ + 'has-text', + 'if', + 'if-not', + 'matches-attr', + 'matches-css', + 'matches-css-after', + 'matches-css-before', + 'matches-media', + 'matches-path', + 'min-text-length', + 'others', + 'upward', + 'watch-attr', + 'xpath', + ]); + this.maybeProceduralOperatorNames = new Set([ + 'has', + 'not', + ]); + this.proceduralActionNames = new Set([ + 'remove', + 'remove-attr', + 'remove-class', + 'style', + ]); + this.normalizedExtendedSyntaxOperators = new Map([ + [ 'contains', 'has-text' ], + [ 'has', 'has' ], + ]); + this.reIsRelativeSelector = /^\s*[+>~]/; + this.reExtendedSyntax = /\[-(?:abp|ext)-[a-z-]+=(['"])(?:.+?)(?:\1)\]/; + this.reExtendedSyntaxReplacer = /\[-(?:abp|ext)-([a-z-]+)=(['"])(.+?)\2\]/g; + this.abpProceduralOpReplacer = /:-abp-(?:[a-z]+)\(/g; + this.nativeCssHas = instanceOptions.nativeCssHas === true; + // https://www.w3.org/TR/css-syntax-3/#typedef-ident-token + this.reInvalidIdentifier = /^\d/; + this.error = undefined; + } + + // CSSTree library holds onto last string parsed, and this is problematic + // when the string is a slice of a huge parent string (typically a whole + // filter list), it causes the huge parent string to stay in memory. + // Asking CSSTree to parse an empty string resolves this issue. + finish() { + cssTree.parse(''); + } + + compile(raw, out, compileOptions = {}) { + this.asProcedural = compileOptions.asProcedural === true; + + // https://github.com/gorhill/uBlock/issues/952 + // Find out whether we are dealing with an Adguard-specific cosmetic + // filter, and if so, translate it if supported, or discard it if not + // supported. + // We have an Adguard/ABP cosmetic filter if and only if the + // character is `$`, `%` or `?`, otherwise it's not a cosmetic + // filter. + // Adguard's style injection: translate to uBO's format. + if ( compileOptions.adgStyleSyntax === true ) { + raw = this.translateAdguardCSSInjectionFilter(raw); + if ( raw === '' ) { return false; } + } + + // Normalize AdGuard's attribute-based procedural operators. + // Normalize ABP's procedural operator names + if ( this.asProcedural ) { + if ( this.reExtendedSyntax.test(raw) ) { + raw = raw.replace(this.reExtendedSyntaxReplacer, (a, a1, a2, a3) => { + const op = this.normalizedExtendedSyntaxOperators.get(a1); + if ( op === undefined ) { return a; } + return `:${op}(${a3})`; + }); + } else { + let asProcedural = false; + raw = raw.replace(this.abpProceduralOpReplacer, match => { + if ( match === ':-abp-contains(' ) { return ':has-text('; } + if ( match === ':-abp-has(' ) { return ':has('; } + asProcedural = true; + return match; + }); + this.asProcedural = asProcedural; + } + } + + // Relative selectors not allowed at top level. + if ( this.reIsRelativeSelector.test(raw) ) { return false; } + + if ( this.reCommonSelector.test(raw) ) { + out.compiled = raw; + return true; + } + + this.error = undefined; + out.compiled = this.compileSelector(raw); + if ( out.compiled === undefined ) { + out.error = this.error; + return false; + } + + if ( out.compiled instanceof Object ) { + out.compiled.raw = raw; + out.compiled = JSON.stringify(out.compiled); + } + return true; + } + + compileSelector(raw) { + const parts = this.astFromRaw(raw, 'selectorList'); + if ( parts === undefined ) { return; } + if ( this.astHasType(parts, 'Error') ) { return; } + if ( this.astHasType(parts, 'Selector') === false ) { return; } + if ( this.astIsValidSelectorList(parts) === false ) { return; } + if ( + this.astHasType(parts, 'ProceduralSelector') === false && + this.astHasType(parts, 'ActionSelector') === false + ) { + return this.astSerialize(parts); + } + const r = this.astCompile(parts); + if ( this.isCssable(r) ) { + r.cssable = true; + } + return r; + } + + isCssable(r) { + if ( r instanceof Object === false ) { return false; } + if ( Array.isArray(r.action) && r.action[0] !== 'style' ) { return false; } + if ( Array.isArray(r.tasks) === false ) { return true; } + if ( r.tasks[0][0] === 'matches-media' ) { + if ( r.tasks.length === 1 ) { return true; } + if ( r.tasks.length === 2 ) { + if ( r.selector !== '' ) { return false; } + if ( r.tasks[1][0] === 'spath' ) { return true; } + } + } + return false; + } + + astFromRaw(raw, type) { + let ast; + try { + ast = cssTree.parse(raw, { + context: type, + parseValue: false, + }); + } catch(reason) { + const lines = [ reason.message ]; + const extra = reason.sourceFragment().split('\n'); + if ( extra.length !== 0 ) { lines.push(''); } + const match = /^[^|]+\|/.exec(extra[0]); + const beg = match !== null ? match[0].length : 0; + lines.push(...extra.map(a => a.slice(beg))); + this.error = lines.join('\n'); + return; + } + const parts = []; + this.astFlatten(ast, parts); + return parts; + } + + astFlatten(data, out) { + const head = data.children && data.children.head; + let args; + switch ( data.type ) { + case 'AttributeSelector': + case 'ClassSelector': + case 'Combinator': + case 'IdSelector': + case 'MediaFeature': + case 'Nth': + case 'Raw': + case 'TypeSelector': + out.push({ data }); + break; + case 'Declaration': + if ( data.value ) { + this.astFlatten(data.value, args = []); + } + out.push({ data, args }); + args = undefined; + break; + case 'DeclarationList': + case 'Identifier': + case 'MediaQueryList': + case 'Selector': + case 'SelectorList': + args = out; + out.push({ data }); + break; + case 'MediaQuery': + case 'PseudoClassSelector': + case 'PseudoElementSelector': + if ( head ) { args = []; } + out.push({ data, args }); + break; + case 'Value': + args = out; + break; + default: + break; + } + if ( head ) { + if ( args ) { + this.astFlatten(head.data, args); + } + let next = head.next; + while ( next ) { + this.astFlatten(next.data, args); + next = next.next; + } + } + if ( data.type !== 'PseudoClassSelector' ) { return; } + if ( data.name.startsWith('-abp-') && this.asProcedural === false ) { + this.error = `${data.name} requires '#?#' separator syntax`; + return; + } + // Post-analysis, mind: + // - https://w3c.github.io/csswg-drafts/selectors-4/#has-pseudo + // - https://w3c.github.io/csswg-drafts/selectors-4/#negation + data.name = this.normalizedOperators.get(data.name) || data.name; + if ( this.proceduralOperatorNames.has(data.name) ) { + data.type = 'ProceduralSelector'; + } else if ( this.proceduralActionNames.has(data.name) ) { + data.type = 'ActionSelector'; + } else if ( data.name.startsWith('-abp-') ) { + data.type = 'Error'; + this.error = `${data.name} is not supported`; + return; + } + if ( this.maybeProceduralOperatorNames.has(data.name) === false ) { + return; + } + if ( this.astHasType(args, 'ActionSelector') ) { + data.type = 'Error'; + this.error = 'invalid use of action operator'; + return; + } + if ( this.astHasType(args, 'ProceduralSelector') ) { + data.type = 'ProceduralSelector'; + return; + } + switch ( data.name ) { + case 'has': + if ( + this.asProcedural || + this.nativeCssHas !== true || + this.astHasName(args, 'has') + ) { + data.type = 'ProceduralSelector'; + } else if ( this.astHasType(args, 'PseudoElementSelector') ) { + data.type = 'Error'; + } + break; + case 'not': { + if ( this.astHasType(args, 'Combinator', 0) === false ) { break; } + if ( this.astIsValidSelectorList(args) !== true ) { + data.type = 'Error'; + } + break; + } + default: + break; + } + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/2300 + // Unquoted attribute values are parsed as Identifier instead of String. + astSerializePart(part) { + const out = []; + const { data } = part; + switch ( data.type ) { + case 'AttributeSelector': { + const name = data.name.name; + if ( this.reInvalidIdentifier.test(name) ) { return; } + if ( data.matcher === null ) { + out.push(`[${name}]`); + break; + } + let value = data.value.value; + if ( typeof value !== 'string' ) { + value = data.value.name; + } + value = value.replace(/["\\]/g, '\\$&'); + let flags = ''; + if ( typeof data.flags === 'string' ) { + if ( /^(is?|si?)$/.test(data.flags) === false ) { return; } + flags = ` ${data.flags}`; + } + out.push(`[${name}${data.matcher}"${value}"${flags}]`); + break; + } + case 'ClassSelector': + if ( this.reInvalidIdentifier.test(data.name) ) { return; } + out.push(`.${data.name}`); + break; + case 'Combinator': + out.push(data.name); + break; + case 'Identifier': + if ( this.reInvalidIdentifier.test(data.name) ) { return; } + out.push(data.name); + break; + case 'IdSelector': + if ( this.reInvalidIdentifier.test(data.name) ) { return; } + out.push(`#${data.name}`); + break; + case 'Nth': { + if ( data.selector !== null ) { return; } + if ( data.nth.type === 'AnPlusB' ) { + const a = parseInt(data.nth.a, 10) || null; + const b = parseInt(data.nth.b, 10) || null; + if ( a !== null ) { + out.push(`${a}n`); + if ( b === null ) { break; } + if ( b < 0 ) { + out.push(`${b}`); + } else { + out.push(`+${b}`); + } + } else if ( b !== null ) { + out.push(`${b}`); + } + } else if ( data.nth.type === 'Identifier' ) { + out.push(data.nth.name); + } + break; + } + case 'PseudoElementSelector': { + const hasArgs = Array.isArray(part.args); + if ( data.name.charCodeAt(0) !== 0x2D /* '-' */ ) { + if ( this.knownPseudoElements.has(data.name) === false ) { return; } + if ( this.knownPseudoElementsWithArgs.has(data.name) && hasArgs === false ) { return; } + } + out.push(`::${data.name}`); + if ( hasArgs ) { + const arg = this.astSerialize(part.args); + if ( typeof arg !== 'string' ) { return; } + out.push(`(${arg})`); + } + break; + } + case 'PseudoClassSelector': { + const hasArgs = Array.isArray(part.args); + if ( data.name.charCodeAt(0) !== 0x2D /* '-' */ ) { + if ( this.knownPseudoClasses.has(data.name) === false ) { return; } + if ( this.knownPseudoClassesWithArgs.has(data.name) && hasArgs === false ) { return; } + } + out.push(`:${data.name}`); + if ( hasArgs ) { + const arg = this.astSerialize(part.args); + if ( typeof arg !== 'string' ) { return; } + out.push(`(${arg.trim()})`); + } + break; + } + case 'Raw': + out.push(data.value); + break; + case 'TypeSelector': + if ( this.reInvalidIdentifier.test(data.name) ) { return; } + out.push(data.name); + break; + default: + break; + } + return out.join(''); + } + + astSerialize(parts, plainCSS = true) { + const out = []; + for ( const part of parts ) { + const { data } = part; + switch ( data.type ) { + case 'AttributeSelector': + case 'ClassSelector': + case 'Identifier': + case 'IdSelector': + case 'Nth': + case 'PseudoClassSelector': + case 'PseudoElementSelector': { + const s = this.astSerializePart(part); + if ( s === undefined ) { return; } + out.push(s); + break; + } + case 'Combinator': { + const s = this.astSerializePart(part); + if ( s === undefined ) { return; } + if ( out.length !== 0 ) { out.push(' '); } + if ( s !== ' ' ) { out.push(s, ' '); } + break; + } + case 'TypeSelector': { + const s = this.astSerializePart(part); + if ( s === undefined ) { return; } + if ( s === '*' && out.length !== 0 ) { + const before = out[out.length-1]; + if ( before.endsWith(' ') === false ) { return; } + } + out.push(s); + break; + } + case 'Raw': + if ( plainCSS ) { return; } + out.push(this.astSerializePart(part)); + break; + case 'Selector': + if ( out.length !== 0 ) { out.push(', '); } + break; + case 'SelectorList': + break; + default: + return; + } + } + return out.join(''); + } + + astCompile(parts, details = {}) { + if ( Array.isArray(parts) === false ) { return; } + if ( parts.length === 0 ) { return; } + if ( parts[0].data.type !== 'SelectorList' ) { return; } + const out = { selector: '' }; + const prelude = []; + const tasks = []; + let startOfSelector = true; + for ( const part of parts ) { + if ( out.action !== undefined ) { return; } + const { data } = part; + switch ( data.type ) { + case 'ActionSelector': { + if ( details.noaction ) { return; } + if ( prelude.length !== 0 ) { + if ( tasks.length === 0 ) { + out.selector = prelude.join(''); + } else { + tasks.push(this.createSpathTask(prelude.join(''))); + } + prelude.length = 0; + } + const args = this.compileArgumentAst(data.name, part.args); + if ( args === undefined ) { return; } + out.action = [ data.name, args ]; + break; + } + case 'AttributeSelector': + case 'ClassSelector': + case 'IdSelector': + case 'PseudoClassSelector': + case 'PseudoElementSelector': + case 'TypeSelector': { + const s = this.astSerializePart(part); + if ( s === undefined ) { return; } + prelude.push(s); + startOfSelector = false; + break; + } + case 'Combinator': { + const s = this.astSerializePart(part); + if ( s === undefined ) { return; } + if ( startOfSelector === false || prelude.length !== 0 ) { + prelude.push(' '); + } + if ( s !== ' ' ) { prelude.push(s, ' '); } + startOfSelector = false; + break; + } + case 'ProceduralSelector': { + if ( prelude.length !== 0 ) { + let spath = prelude.join(''); + prelude.length = 0; + if ( spath.endsWith(' ') ) { spath += '*'; } + if ( tasks.length === 0 ) { + out.selector = spath; + } else { + tasks.push(this.createSpathTask(spath)); + } + } + const args = this.compileArgumentAst(data.name, part.args); + if ( args === undefined ) { return; } + tasks.push([ data.name, args ]); + startOfSelector = false; + break; + } + case 'Selector': + if ( prelude.length !== 0 ) { + prelude.push(', '); + } + startOfSelector = true; + break; + case 'SelectorList': + startOfSelector = true; + break; + default: + return; + } + } + if ( tasks.length === 0 && out.action === undefined ) { + if ( prelude.length === 0 ) { return; } + return prelude.join('').trim(); + } + if ( prelude.length !== 0 ) { + tasks.push(this.createSpathTask(prelude.join(''))); + } + if ( tasks.length !== 0 ) { + out.tasks = tasks; + } + return out; + } + + astHasType(parts, type, depth = 0x7FFFFFFF) { + if ( Array.isArray(parts) === false ) { return false; } + for ( const part of parts ) { + if ( part.data.type === type ) { return true; } + if ( + Array.isArray(part.args) && + depth !== 0 && + this.astHasType(part.args, type, depth-1) + ) { + return true; + } + } + return false; + } + + astHasName(parts, name) { + if ( Array.isArray(parts) === false ) { return false; } + for ( const part of parts ) { + if ( part.data.name === name ) { return true; } + if ( Array.isArray(part.args) && this.astHasName(part.args, name) ) { + return true; + } + } + return false; + } + + astSelectorsFromSelectorList(args) { + if ( Array.isArray(args) === false ) { return; } + if ( args.length < 3 ) { return; } + if ( args[0].data instanceof Object === false ) { return; } + if ( args[0].data.type !== 'SelectorList' ) { return; } + if ( args[1].data instanceof Object === false ) { return; } + if ( args[1].data.type !== 'Selector' ) { return; } + const out = []; + let beg = 1, end = 0, i = 2; + for (;;) { + if ( i < args.length ) { + const type = args[i].data instanceof Object && args[i].data.type; + if ( type === 'Selector' ) { + end = i; + } + } else { + end = args.length; + } + if ( end !== 0 ) { + const components = args.slice(beg+1, end); + if ( components.length === 0 ) { return; } + out.push(components); + if ( end === args.length ) { break; } + beg = end; end = 0; + } + if ( i === args.length ) { break; } + i += 1; + } + return out; + } + + astIsValidSelector(components) { + const len = components.length; + if ( len === 0 ) { return false; } + if ( components[0].data.type === 'Combinator' ) { return false; } + if ( len === 1 ) { return true; } + if ( components[len-1].data.type === 'Combinator' ) { return false; } + return true; + } + + astIsValidSelectorList(args) { + const selectors = this.astSelectorsFromSelectorList(args); + if ( Array.isArray(selectors) === false || selectors.length === 0 ) { + return false; + } + for ( const selector of selectors ) { + if ( this.astIsValidSelector(selector) !== true ) { return false; } + } + return true; + } + + translateAdguardCSSInjectionFilter(suffix) { + const matches = /^(.*)\s*\{([^}]+)\}\s*$/.exec(suffix); + if ( matches === null ) { return ''; } + const selector = matches[1].trim(); + const style = matches[2].trim(); + // Special style directive `remove: true` is converted into a + // `:remove()` operator. + if ( /^\s*remove:\s*true[; ]*$/.test(style) ) { + return `${selector}:remove()`; + } + // For some reasons, many of Adguard's plain cosmetic filters are + // "disguised" as style-based cosmetic filters: convert such filters + // to plain cosmetic filters. + return /display\s*:\s*none\s*!important;?$/.test(style) + ? selector + : `${selector}:style(${style})`; + } + + createSpathTask(selector) { + return [ 'spath', selector ]; + } + + compileArgumentAst(operator, parts) { + switch ( operator ) { + case 'has': { + let r = this.astCompile(parts, { noaction: true }); + if ( typeof r === 'string' ) { + r = { selector: r.replace(/^\s*:scope\s*/, '') }; + } + return r; + } + case 'not': { + return this.astCompile(parts, { noaction: true }); + } + default: + break; + } + if ( Array.isArray(parts) === false || parts.length === 0 ) { return; } + const arg = this.astSerialize(parts, false); + if ( arg === undefined ) { return; } + switch ( operator ) { + case 'has-text': + return this.compileText(arg); + case 'if': + return this.compileSelector(arg); + case 'if-not': + return this.compileSelector(arg); + case 'matches-attr': + return this.compileMatchAttrArgument(arg); + case 'matches-css': + return this.compileCSSDeclaration(arg); + case 'matches-css-after': + return this.compileCSSDeclaration(`after, ${arg}`); + case 'matches-css-before': + return this.compileCSSDeclaration(`before, ${arg}`); + case 'matches-media': + return this.compileMediaQuery(arg); + case 'matches-path': + return this.compileText(arg); + case 'min-text-length': + return this.compileInteger(arg); + case 'others': + return this.compileNoArgument(arg); + case 'remove': + return this.compileNoArgument(arg); + case 'remove-attr': + return this.compileText(arg); + case 'remove-class': + return this.compileText(arg); + case 'style': + return this.compileStyleProperties(arg); + case 'upward': + return this.compileUpwardArgument(arg); + case 'watch-attr': + return this.compileAttrList(arg); + case 'xpath': + return this.compileXpathExpression(arg); + default: + break; + } + } + + isBadRegex(s) { + try { + void new RegExp(s); + } catch (ex) { + this.isBadRegex.message = ex.toString(); + return true; + } + return false; + } + + unquoteString(s) { + const end = s.length; + if ( end === 0 ) { + return { s: '', end }; + } + if ( /^['"]/.test(s) === false ) { + return { s, i: end }; + } + const quote = s.charCodeAt(0); + const out = []; + let i = 1, c = 0; + for (;;) { + c = s.charCodeAt(i); + if ( c === quote ) { + i += 1; + break; + } + if ( c === 0x5C /* '\\' */ ) { + i += 1; + if ( i === end ) { break; } + c = s.charCodeAt(i); + if ( c !== 0x5C && c !== quote ) { + out.push(0x5C); + } + } + out.push(c); + i += 1; + if ( i === end ) { break; } + } + return { s: String.fromCharCode(...out), i }; + } + + compileMatchAttrArgument(s) { + if ( s === '' ) { return; } + let attr = '', value = ''; + let r = this.unquoteString(s); + if ( r.i === s.length ) { + const pos = r.s.indexOf('='); + if ( pos === -1 ) { + attr = r.s; + } else { + attr = r.s.slice(0, pos); + value = r.s.slice(pos+1); + } + } else { + attr = r.s; + if ( s.charCodeAt(r.i) !== 0x3D ) { return; } + value = s.slice(r.i+1); + } + if ( attr === '' ) { return; } + if ( value.length !== 0 ) { + r = this.unquoteString(value); + if ( r.i !== value.length ) { return; } + value = r.s; + } + return { attr, value }; + } + + // Remove potentially present quotes before processing. + compileText(s) { + if ( s === '' ) { + this.error = 'argument missing'; + return; + } + const r = this.unquoteString(s); + if ( r.i !== s.length ) { return; } + return r.s; + } + + compileCSSDeclaration(s) { + let pseudo; { + const match = /^[a-z-]+,/.exec(s); + if ( match !== null ) { + pseudo = match[0].slice(0, -1); + s = s.slice(match[0].length).trim(); + } + } + const pos = s.indexOf(':'); + if ( pos === -1 ) { return; } + const name = s.slice(0, pos).trim(); + const value = s.slice(pos + 1).trim(); + const match = this.reParseRegexLiteral.exec(value); + let regexDetails; + if ( match !== null ) { + regexDetails = match[1]; + if ( this.isBadRegex(regexDetails) ) { return; } + if ( match[2] ) { + regexDetails = [ regexDetails, match[2] ]; + } + } else { + regexDetails = '^' + value.replace(this.reEscapeRegex, '\\$&') + '$'; + } + return { name, pseudo, value: regexDetails }; + } + + compileInteger(s, min = 0, max = 0x7FFFFFFF) { + if ( /^\d+$/.test(s) === false ) { return; } + const n = parseInt(s, 10); + if ( n < min || n >= max ) { return; } + return n; + } + + compileMediaQuery(s) { + const parts = this.astFromRaw(s, 'mediaQueryList'); + if ( parts === undefined ) { return; } + if ( this.astHasType(parts, 'Raw') ) { return; } + if ( this.astHasType(parts, 'MediaQuery') === false ) { return; } + // TODO: normalize by serializing resulting AST + return s; + } + + compileUpwardArgument(s) { + const i = this.compileInteger(s, 1, 256); + if ( i !== undefined ) { return i; } + const parts = this.astFromRaw(s, 'selectorList' ); + if ( this.astIsValidSelectorList(parts) !== true ) { return; } + if ( this.astHasType(parts, 'ProceduralSelector') ) { return; } + if ( this.astHasType(parts, 'ActionSelector') ) { return; } + if ( this.astHasType(parts, 'Error') ) { return; } + return s; + } + + compileNoArgument(s) { + if ( s === '' ) { return s; } + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/668 + // https://github.com/uBlockOrigin/uBlock-issues/issues/1693 + // https://github.com/uBlockOrigin/uBlock-issues/issues/1811 + // Forbid instances of: + // - `image-set(` + // - `url(` + // - any instance of `//` + // - backslashes `\` + // - opening comment `/*` + compileStyleProperties(s) { + if ( /image-set\(|url\(|\/\s*\/|\\|\/\*/i.test(s) ) { return; } + const parts = this.astFromRaw(s, 'declarationList'); + if ( parts === undefined ) { return; } + if ( this.astHasType(parts, 'Declaration') === false ) { return; } + return s; + } + + compileAttrList(s) { + if ( s === '' ) { return s; } + const attrs = s.split('\s*,\s*'); + const out = []; + for ( const attr of attrs ) { + if ( attr !== '' ) { + out.push(attr); + } + } + return out; + } + + compileXpathExpression(s) { + const r = this.unquoteString(s); + if ( r.i !== s.length ) { return; } + try { + globalThis.document.createExpression(r.s, null); + } catch (e) { + return; + } + return r.s; + } +} + +// bit 0: can be used as auto-completion hint +// bit 1: can not be used in HTML filtering +// +export const proceduralOperatorTokens = new Map([ + [ '-abp-contains', 0b00 ], + [ '-abp-has', 0b00, ], + [ 'contains', 0b00, ], + [ 'has', 0b01 ], + [ 'has-text', 0b01 ], + [ 'if', 0b00 ], + [ 'if-not', 0b00 ], + [ 'matches-attr', 0b11 ], + [ 'matches-css', 0b11 ], + [ 'matches-media', 0b11 ], + [ 'matches-path', 0b11 ], + [ 'min-text-length', 0b01 ], + [ 'not', 0b01 ], + [ 'nth-ancestor', 0b00 ], + [ 'others', 0b11 ], + [ 'remove', 0b11 ], + [ 'remove-attr', 0b11 ], + [ 'remove-class', 0b11 ], + [ 'style', 0b11 ], + [ 'upward', 0b01 ], + [ 'watch-attr', 0b11 ], + [ 'watch-attrs', 0b00 ], + [ 'xpath', 0b01 ], +]); + +/******************************************************************************/ + +export const utils = (( ) => { + + // Depends on: + // https://github.com/foo123/RegexAnalyzer + const regexAnalyzer = Regex && Regex.Analyzer || null; + + class regex { + static firstCharCodeClass(s) { + return /^[\x01\x03%0-9A-Za-z]/.test(s) ? 1 : 0; + } + + static lastCharCodeClass(s) { + return /[\x01\x03%0-9A-Za-z]$/.test(s) ? 1 : 0; + } + + static tokenizableStrFromNode(node) { + switch ( node.type ) { + case 1: /* T_SEQUENCE, 'Sequence' */ { + let s = ''; + for ( let i = 0; i < node.val.length; i++ ) { + s += this.tokenizableStrFromNode(node.val[i]); + } + return s; + } + case 2: /* T_ALTERNATION, 'Alternation' */ + case 8: /* T_CHARGROUP, 'CharacterGroup' */ { + if ( node.flags.NegativeMatch ) { return '\x01'; } + let firstChar = 0; + let lastChar = 0; + for ( let i = 0; i < node.val.length; i++ ) { + const s = this.tokenizableStrFromNode(node.val[i]); + if ( firstChar === 0 && this.firstCharCodeClass(s) === 1 ) { + firstChar = 1; + } + if ( lastChar === 0 && this.lastCharCodeClass(s) === 1 ) { + lastChar = 1; + } + if ( firstChar === 1 && lastChar === 1 ) { break; } + } + return String.fromCharCode(firstChar, lastChar); + } + case 4: /* T_GROUP, 'Group' */ { + if ( + node.flags.NegativeLookAhead === 1 || + node.flags.NegativeLookBehind === 1 + ) { + return ''; + } + return this.tokenizableStrFromNode(node.val); + } + case 16: /* T_QUANTIFIER, 'Quantifier' */ { + if ( node.flags.max === 0 ) { return ''; } + const s = this.tokenizableStrFromNode(node.val); + const first = this.firstCharCodeClass(s); + const last = this.lastCharCodeClass(s); + if ( node.flags.min !== 0 ) { + return String.fromCharCode(first, last); + } + return String.fromCharCode(first+2, last+2); + } + case 64: /* T_HEXCHAR, 'HexChar' */ { + if ( + node.flags.Code === '01' || + node.flags.Code === '02' || + node.flags.Code === '03' + ) { + return '\x00'; + } + return node.flags.Char; + } + case 128: /* T_SPECIAL, 'Special' */ { + const flags = node.flags; + if ( + flags.EndCharGroup === 1 || // dangling `]` + flags.EndGroup === 1 || // dangling `)` + flags.EndRepeats === 1 // dangling `}` + ) { + throw new Error('Unmatched bracket'); + } + return flags.MatchEnd === 1 || + flags.MatchStart === 1 || + flags.MatchWordBoundary === 1 + ? '\x00' + : '\x01'; + } + case 256: /* T_CHARS, 'Characters' */ { + for ( let i = 0; i < node.val.length; i++ ) { + if ( this.firstCharCodeClass(node.val[i]) === 1 ) { + return '\x01'; + } + } + return '\x00'; + } + // Ranges are assumed to always involve token-related characters. + case 512: /* T_CHARRANGE, 'CharacterRange' */ { + return '\x01'; + } + case 1024: /* T_STRING, 'String' */ { + return node.val; + } + case 2048: /* T_COMMENT, 'Comment' */ { + return ''; + } + default: + break; + } + return '\x01'; + } + + static isValid(reStr) { + try { + void new RegExp(reStr); + if ( regexAnalyzer !== null ) { + void this.tokenizableStrFromNode( + regexAnalyzer(reStr, false).tree() + ); + } + } catch(ex) { + return false; + } + return true; + } + + static isRE2(reStr) { + if ( regexAnalyzer === null ) { return true; } + let tree; + try { + tree = regexAnalyzer(reStr, false).tree(); + } catch(ex) { + return; + } + const isRE2 = node => { + if ( node instanceof Object === false ) { return true; } + if ( node.flags instanceof Object ) { + if ( node.flags.LookAhead === 1 ) { return false; } + if ( node.flags.NegativeLookAhead === 1 ) { return false; } + if ( node.flags.LookBehind === 1 ) { return false; } + if ( node.flags.NegativeLookBehind === 1 ) { return false; } + } + if ( Array.isArray(node.val) ) { + for ( const entry of node.val ) { + if ( isRE2(entry) === false ) { return false; } + } + } + if ( node.val instanceof Object ) { + return isRE2(node.val); + } + return true; + }; + return isRE2(tree); + } + + static toTokenizableStr(reStr) { + if ( regexAnalyzer === null ) { return ''; } + let s = ''; + try { + s = this.tokenizableStrFromNode( + regexAnalyzer(reStr, false).tree() + ); + } catch(ex) { + } + // Process optional sequences + const reOptional = /[\x02\x03]+/; + for (;;) { + const match = reOptional.exec(s); + if ( match === null ) { break; } + const left = s.slice(0, match.index); + const middle = match[0]; + const right = s.slice(match.index + middle.length); + s = left; + s += this.firstCharCodeClass(right) === 1 || + this.firstCharCodeClass(middle) === 1 + ? '\x01' + : '\x00'; + s += this.lastCharCodeClass(left) === 1 || + this.lastCharCodeClass(middle) === 1 + ? '\x01' + : '\x00'; + s += right; + } + return s; + } + } + + const preparserTokens = new Map([ + [ 'ext_ublock', 'ublock' ], + [ 'ext_ubol', 'ubol' ], + [ 'ext_devbuild', 'devbuild' ], + [ 'env_chromium', 'chromium' ], + [ 'env_edge', 'edge' ], + [ 'env_firefox', 'firefox' ], + [ 'env_legacy', 'legacy' ], + [ 'env_mobile', 'mobile' ], + [ 'env_mv3', 'mv3' ], + [ 'env_safari', 'safari' ], + [ 'cap_html_filtering', 'html_filtering' ], + [ 'cap_user_stylesheet', 'user_stylesheet' ], + [ 'false', 'false' ], + // Hoping ABP-only list maintainers can at least make use of it to + // help non-ABP content blockers better deal with filters benefiting + // only ABP. + [ 'ext_abp', 'false' ], + // Compatibility with other blockers + // https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#adguard-specific + [ 'adguard', 'adguard' ], + [ 'adguard_app_android', 'false' ], + [ 'adguard_app_ios', 'false' ], + [ 'adguard_app_mac', 'false' ], + [ 'adguard_app_windows', 'false' ], + [ 'adguard_ext_android_cb', 'false' ], + [ 'adguard_ext_chromium', 'chromium' ], + [ 'adguard_ext_edge', 'edge' ], + [ 'adguard_ext_firefox', 'firefox' ], + [ 'adguard_ext_opera', 'chromium' ], + [ 'adguard_ext_safari', 'false' ], + ]); + + const toURL = url => { + try { + return new URL(url.trim()); + } catch (ex) { + } + }; + + // Useful reference: + // https://adguard.com/kb/general/ad-filtering/create-own-filters/#conditions-directive + + class preparser { + static evaluateExprToken(token, env = []) { + const not = token.charCodeAt(0) === 0x21 /* ! */; + if ( not ) { token = token.slice(1); } + const state = preparserTokens.get(token); + if ( state === undefined ) { return; } + return state === 'false' && not || env.includes(state) !== not; + } + + static evaluateExpr(expr, env = []) { + if ( expr.startsWith('(') && expr.endsWith(')') ) { + expr = expr.slice(1, -1); + } + const matches = Array.from(expr.matchAll(/(?:(?:&&|\|\|)\s+)?\S+/g)); + if ( matches.length === 0 ) { return; } + if ( matches[0][0].startsWith('|') || matches[0][0].startsWith('&') ) { return; } + let result = this.evaluateExprToken(matches[0][0], env); + for ( let i = 1; i < matches.length; i++ ) { + const parts = matches[i][0].split(/ +/); + if ( parts.length !== 2 ) { return; } + const state = this.evaluateExprToken(parts[1], env); + if ( state === undefined ) { return; } + if ( parts[0] === '||' ) { + result = result || state; + } else if ( parts[0] === '&&' ) { + result = result && state; + } else { + return; + } + } + return result; + } + + // This method returns an array of indices, corresponding to position in + // the content string which should alternatively be parsed and discarded. + static splitter(content, env = []) { + const reIf = /^!#(if|else|endif)\b([^\n]*)(?:[\n\r]+|$)/gm; + const stack = []; + const parts = [ 0 ]; + let discard = false; + + const shouldDiscard = ( ) => stack.some(v => v); + + const begif = (startDiscard, match) => { + if ( discard === false && startDiscard ) { + parts.push(match.index); + discard = true; + } + stack.push(startDiscard); + }; + + const endif = match => { + stack.pop(); + const stopDiscard = shouldDiscard() === false; + if ( discard && stopDiscard ) { + parts.push(match.index + match[0].length); + discard = false; + } + }; + + for (;;) { + const match = reIf.exec(content); + if ( match === null ) { break; } + + switch ( match[1] ) { + case 'if': { + const startDiscard = this.evaluateExpr(match[2].trim(), env) === false; + begif(startDiscard, match); + break; + } + case 'else': { + if ( stack.length === 0 ) { break; } + const startDiscard = stack[stack.length-1] === false; + endif(match); + begif(startDiscard, match); + break; + } + case 'endif': { + endif(match); + break; + } + default: + break; + } + } + + parts.push(content.length); + return parts; + } + + static expandIncludes(parts, env = []) { + const out = []; + const reInclude = /^!#include +(\S+)[^\n\r]*(?:[\n\r]+|$)/gm; + for ( const part of parts ) { + if ( typeof part === 'string' ) { + out.push(part); + continue; + } + if ( part instanceof Object === false ) { continue; } + const content = part.content; + const slices = this.splitter(content, env); + for ( let i = 0, n = slices.length - 1; i < n; i++ ) { + const slice = content.slice(slices[i+0], slices[i+1]); + if ( (i & 1) !== 0 ) { + out.push(slice); + continue; + } + let lastIndex = 0; + for (;;) { + const match = reInclude.exec(slice); + if ( match === null ) { break; } + if ( toURL(match[1]) !== undefined ) { continue; } + if ( match[1].indexOf('..') !== -1 ) { continue; } + // Compute nested list path relative to parent list path + const pos = part.url.lastIndexOf('/'); + if ( pos === -1 ) { continue; } + const subURL = part.url.slice(0, pos + 1) + match[1].trim(); + out.push( + slice.slice(lastIndex, match.index + match[0].length), + `! >>>>>>>> ${subURL}\n`, + { url: subURL }, + `! <<<<<<<< ${subURL}\n` + ); + lastIndex = reInclude.lastIndex; + } + out.push(lastIndex === 0 ? slice : slice.slice(lastIndex)); + } + } + return out; + } + + static prune(content, env) { + const parts = this.splitter(content, env); + const out = []; + for ( let i = 0, n = parts.length - 1; i < n; i += 2 ) { + const beg = parts[i+0]; + const end = parts[i+1]; + out.push(content.slice(beg, end)); + } + return out.join('\n'); + } + + static getHints() { + const out = []; + const vals = new Set(); + for ( const [ key, val ] of preparserTokens ) { + if ( vals.has(val) ) { continue; } + vals.add(val); + out.push(key); + } + return out; + } + + static getTokens(env) { + const out = new Map(); + for ( const [ key, val ] of preparserTokens ) { + out.set(key, val !== 'false' && env.includes(val)); + } + return Array.from(out); + } + } + + return { + preparser, + regex, + }; +})(); + +/******************************************************************************/ diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js new file mode 100644 index 0000000..d1e9a70 --- /dev/null +++ b/src/js/static-net-filtering.js @@ -0,0 +1,5651 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals vAPI */ + +'use strict'; + +/******************************************************************************/ + +import { queueTask, dropTask } from './tasks.js'; +import BidiTrieContainer from './biditrie.js'; +import HNTrieContainer from './hntrie.js'; +import { sparseBase64 } from './base64-custom.js'; +import { CompiledListReader } from './static-filtering-io.js'; +import * as sfp from './static-filtering-parser.js'; + +import { + domainFromHostname, + hostnameFromNetworkURL, +} from './uri-utils.js'; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility +// +// This import would be best done dynamically, but since dynamic imports are +// not supported by older browsers, for now a static import is necessary. +import { FilteringContext } from './filtering-context.js'; + +/******************************************************************************/ + +// Access to a key-val store is optional and useful only for optimal +// initialization at module load time. Probably could re-arrange code +// to export an init() function with optimization parameters which would +// need to be called by module clients. For now, I want modularizing with +// minimal amount of changes. + +const keyvalStore = typeof vAPI !== 'undefined' + ? vAPI.localStorage + : { getItem() { return null; }, setItem() {}, removeItem() {} }; + +/******************************************************************************/ + +// 0fedcba9876543210 +// ||||||| | || | +// ||||||| | || | +// ||||||| | || | +// ||||||| | || | +// ||||||| | || +---- bit 0- 1: block=0, allow=1, block important=2 +// ||||||| | |+------ bit 2: unused +// ||||||| | +------- bit 3- 4: party [0-3] +// ||||||| +--------- bit 5- 9: type [0-31] +// ||||||+-------------- bit 10: headers-based filters +// |||||+--------------- bit 11: redirect filters +// ||||+---------------- bit 12: removeparam filters +// |||+----------------- bit 13: csp filters +// ||+------------------ bit 14: permissions filters +// |+------------------- bit 15: uritransform filters +// +-------------------- bit 16: replace filters +// TODO: bit 11-16 can be converted into 3-bit value, as these options are not +// meant to be combined. + +const RealmBitsMask = 0b00000000111; +const ActionBitsMask = 0b00000000011; +const TypeBitsMask = 0b01111100000; +const TypeBitsOffset = 5; + +const BLOCK_REALM = 0b00000000000000000; +const ALLOW_REALM = 0b00000000000000001; +const IMPORTANT_REALM = 0b00000000000000010; +const BLOCKIMPORTANT_REALM = BLOCK_REALM | IMPORTANT_REALM; +const ANYPARTY_REALM = 0b00000000000000000; +const FIRSTPARTY_REALM = 0b00000000000001000; +const THIRDPARTY_REALM = 0b00000000000010000; +const ALLPARTIES_REALM = FIRSTPARTY_REALM | THIRDPARTY_REALM; +const HEADERS_REALM = 0b00000010000000000; +const REDIRECT_REALM = 0b00000100000000000; +const REMOVEPARAM_REALM = 0b00001000000000000; +const CSP_REALM = 0b00010000000000000; +const PERMISSIONS_REALM = 0b00100000000000000; +const URLTRANSFORM_REALM = 0b01000000000000000; +const REPLACE_REALM = 0b10000000000000000; +const MODIFY_REALMS = REDIRECT_REALM | CSP_REALM | + REMOVEPARAM_REALM | PERMISSIONS_REALM | + URLTRANSFORM_REALM | REPLACE_REALM; + +const typeNameToTypeValue = { + 'no_type': 0 << TypeBitsOffset, + 'stylesheet': 1 << TypeBitsOffset, + 'image': 2 << TypeBitsOffset, + 'object': 3 << TypeBitsOffset, + 'object_subrequest': 3 << TypeBitsOffset, + 'script': 4 << TypeBitsOffset, + 'fetch': 5 << TypeBitsOffset, + 'xmlhttprequest': 5 << TypeBitsOffset, + 'sub_frame': 6 << TypeBitsOffset, + 'font': 7 << TypeBitsOffset, + 'media': 8 << TypeBitsOffset, + 'websocket': 9 << TypeBitsOffset, + 'beacon': 10 << TypeBitsOffset, + 'ping': 10 << TypeBitsOffset, + 'other': 11 << TypeBitsOffset, + 'popup': 12 << TypeBitsOffset, // start of behavioral filtering + 'popunder': 13 << TypeBitsOffset, + 'main_frame': 14 << TypeBitsOffset, // start of 1p behavioral filtering + 'generichide': 15 << TypeBitsOffset, + 'specifichide': 16 << TypeBitsOffset, + 'inline-font': 17 << TypeBitsOffset, + 'inline-script': 18 << TypeBitsOffset, + 'cname': 19 << TypeBitsOffset, + 'webrtc': 20 << TypeBitsOffset, + 'unsupported': 21 << TypeBitsOffset, +}; + +const otherTypeBitValue = typeNameToTypeValue.other; + +const bitFromType = type => + 1 << ((typeNameToTypeValue[type] >>> TypeBitsOffset) - 1); + +// All network request types to bitmap +// bring origin to 0 (from TypeBitsOffset -- see typeNameToTypeValue) +// left-shift 1 by the above-calculated value +// subtract 1 to set all type bits +const allNetworkTypesBits = + (1 << (otherTypeBitValue >>> TypeBitsOffset)) - 1; + +const allTypesBits = + allNetworkTypesBits | + 1 << (typeNameToTypeValue['popup'] >>> TypeBitsOffset) - 1 | + 1 << (typeNameToTypeValue['main_frame'] >>> TypeBitsOffset) - 1 | + 1 << (typeNameToTypeValue['inline-font'] >>> TypeBitsOffset) - 1 | + 1 << (typeNameToTypeValue['inline-script'] >>> TypeBitsOffset) - 1; +const unsupportedTypeBit = + 1 << (typeNameToTypeValue['unsupported'] >>> TypeBitsOffset) - 1; + +const typeValueToTypeName = [ + '', + 'stylesheet', + 'image', + 'object', + 'script', + 'xhr', + 'frame', + 'font', + 'media', + 'websocket', + 'ping', + 'other', + 'popup', + 'popunder', + 'document', + 'generichide', + 'specifichide', + 'inline-font', + 'inline-script', + 'cname', + '', + '', + 'webrtc', + 'unsupported', +]; + +const typeValueToDNRTypeName = [ + '', + 'stylesheet', + 'image', + 'object', + 'script', + 'xmlhttprequest', + 'sub_frame', + 'font', + 'media', + 'websocket', + 'ping', + 'other', +]; + +// Do not change order. Compiled filter lists rely on this order being +// consistent across sessions. +const MODIFIER_TYPE_REDIRECT = 1; +const MODIFIER_TYPE_REDIRECTRULE = 2; +const MODIFIER_TYPE_REMOVEPARAM = 3; +const MODIFIER_TYPE_CSP = 4; +const MODIFIER_TYPE_PERMISSIONS = 5; +const MODIFIER_TYPE_URLTRANSFORM = 6; +const MODIFIER_TYPE_REPLACE = 7; + +const modifierBitsFromType = new Map([ + [ MODIFIER_TYPE_REDIRECT, REDIRECT_REALM ], + [ MODIFIER_TYPE_REDIRECTRULE, REDIRECT_REALM ], + [ MODIFIER_TYPE_REMOVEPARAM, REMOVEPARAM_REALM ], + [ MODIFIER_TYPE_CSP, CSP_REALM ], + [ MODIFIER_TYPE_PERMISSIONS, PERMISSIONS_REALM ], + [ MODIFIER_TYPE_URLTRANSFORM, URLTRANSFORM_REALM ], + [ MODIFIER_TYPE_REPLACE, REPLACE_REALM ], +]); + +const modifierTypeFromName = new Map([ + [ 'redirect', MODIFIER_TYPE_REDIRECT ], + [ 'redirect-rule', MODIFIER_TYPE_REDIRECTRULE ], + [ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ], + [ 'csp', MODIFIER_TYPE_CSP ], + [ 'permissions', MODIFIER_TYPE_PERMISSIONS ], + [ 'uritransform', MODIFIER_TYPE_URLTRANSFORM ], + [ 'replace', MODIFIER_TYPE_REPLACE ], +]); + +const modifierNameFromType = new Map([ + [ MODIFIER_TYPE_REDIRECT, 'redirect' ], + [ MODIFIER_TYPE_REDIRECTRULE, 'redirect-rule' ], + [ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ], + [ MODIFIER_TYPE_CSP, 'csp' ], + [ MODIFIER_TYPE_PERMISSIONS, 'permissions' ], + [ MODIFIER_TYPE_URLTRANSFORM, 'uritransform' ], + [ MODIFIER_TYPE_REPLACE, 'replace' ], +]); + +//const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111; + +const MAX_TOKEN_LENGTH = 7; + +// Four upper bits of token hash are reserved for built-in predefined +// token hashes, which should never end up being used when tokenizing +// any arbitrary string. +const NO_TOKEN_HASH = 0x50000000; +const DOT_TOKEN_HASH = 0x10000000; +const ANY_TOKEN_HASH = 0x20000000; +const ANY_HTTPS_TOKEN_HASH = 0x30000000; +const ANY_HTTP_TOKEN_HASH = 0x40000000; +const EMPTY_TOKEN_HASH = 0xF0000000; +const INVALID_TOKEN_HASH = 0xFFFFFFFF; + +/******************************************************************************/ + +// See the following as short-lived registers, used during evaluation. They are +// valid until the next evaluation. + +let $requestMethodBit = 0; +let $requestTypeValue = 0; +let $requestURL = ''; +let $requestURLRaw = ''; +let $requestHostname = ''; +let $docHostname = ''; +let $docDomain = ''; +let $tokenBeg = 0; +let $patternMatchLeft = 0; +let $patternMatchRight = 0; +let $isBlockImportant = false; + +const $docEntity = { + entity: '', + last: '', + compute() { + if ( this.last !== $docHostname ) { + this.last = $docHostname; + const pos = $docDomain.indexOf('.'); + this.entity = pos !== -1 + ? `${$docHostname.slice(0, pos - $docDomain.length)}.*` + : ''; + } + return this.entity; + }, +}; + +const $requestEntity = { + entity: '', + last: '', + compute() { + if ( this.last !== $requestHostname ) { + this.last = $requestHostname; + const requestDomain = domainFromHostname($requestHostname); + const pos = requestDomain.indexOf('.'); + this.entity = pos !== -1 + ? `${$requestHostname.slice(0, pos - requestDomain.length)}.*` + : ''; + } + return this.entity; + }, +}; + +const $httpHeaders = { + init(headers) { + this.headers = headers; + this.parsed.clear(); + }, + reset() { + this.headers = []; + this.parsed.clear(); + }, + lookup(name) { + if ( this.parsed.size === 0 ) { + for ( let i = 0, n = this.headers.length; i < n; i++ ) { + const { name, value } = this.headers[i]; + this.parsed.set(name, value); + } + } + return this.parsed.get(name); + }, + headers: [], + parsed: new Map(), +}; + +/******************************************************************************/ + +// Local helpers + +const restrSeparator = '(?:[^%.0-9a-z_-]|$)'; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions +const reEscape = /[.*+?^${}()|[\]\\]/g; + +// Convert a plain string (devoid of special characters) into a regex. +const restrFromPlainPattern = s => s.replace(reEscape, '\\$&'); + +const restrFromGenericPattern = function(s, anchor = 0) { + let reStr = s.replace(restrFromGenericPattern.rePlainChars, '\\$&') + .replace(restrFromGenericPattern.reSeparators, restrSeparator) + .replace(restrFromGenericPattern.reDanglingAsterisks, '') + .replace(restrFromGenericPattern.reAsterisks, '\\S*?'); + if ( anchor & 0b100 ) { + reStr = ( + reStr.startsWith('\\.') ? + restrFromGenericPattern.restrHostnameAnchor2 : + restrFromGenericPattern.restrHostnameAnchor1 + ) + reStr; + } else if ( anchor & 0b010 ) { + reStr = '^' + reStr; + } + if ( anchor & 0b001 ) { + reStr += '$'; + } + return reStr; +}; +restrFromGenericPattern.rePlainChars = /[.+?${}()|[\]\\]/g; +restrFromGenericPattern.reSeparators = /\^/g; +restrFromGenericPattern.reDanglingAsterisks = /^\*+|\*+$/g; +restrFromGenericPattern.reAsterisks = /\*+/g; +restrFromGenericPattern.restrHostnameAnchor1 = '^[a-z-]+://(?:[^/?#]+\\.)?'; +restrFromGenericPattern.restrHostnameAnchor2 = '^[a-z-]+://(?:[^/?#]+)?'; + +/******************************************************************************/ + +class LogData { + constructor(categoryBits, tokenHash, iunit) { + this.result = 0; + this.source = 'static'; + this.tokenHash = tokenHash; + if ( iunit === 0 ) { + this.raw = this.regex = ''; + return; + } + this.result = (categoryBits & ALLOW_REALM) === 0 ? 1 : 2; + const pattern = []; + const regex = []; + const options = []; + const denyallow = []; + const fromDomains = []; + const toDomains = []; + const logData = { + pattern, + regex, + denyallow, + fromDomains, + toDomains, + options, + isRegex: false, + }; + filterLogData(iunit, logData); + if ( (categoryBits & THIRDPARTY_REALM) !== 0 ) { + logData.options.unshift('3p'); + } else if ( (categoryBits & FIRSTPARTY_REALM) !== 0 ) { + logData.options.unshift('1p'); + } + const type = categoryBits & TypeBitsMask; + if ( type !== 0 ) { + logData.options.unshift(typeValueToTypeName[type >>> TypeBitsOffset]); + } + let raw = logData.pattern.join(''); + if ( + logData.isRegex === false && + raw.charCodeAt(0) === 0x2F /* '/' */ && + raw.charCodeAt(raw.length - 1) === 0x2F /* '/' */ + ) { + raw += '*'; + } + if ( (categoryBits & ALLOW_REALM) !== 0 ) { + raw = '@@' + raw; + } + if ( denyallow.length !== 0 ) { + options.push(`denyallow=${denyallow.join('|')}`); + } + if ( fromDomains.length !== 0 ) { + options.push(`from=${fromDomains.join('|')}`); + } + if ( toDomains.length !== 0 ) { + options.push(`to=${toDomains.join('|')}`); + } + if ( options.length !== 0 ) { + raw += '$' + options.join(','); + } + this.raw = raw; + this.regex = logData.regex.join(''); + } + isUntokenized() { + return this.tokenHash === NO_TOKEN_HASH; + } + isPureHostname() { + return this.tokenHash === DOT_TOKEN_HASH; + } +} + +/******************************************************************************/ + +const charClassMap = new Uint32Array(128); +const CHAR_CLASS_SEPARATOR = 0b00000001; + +{ + const reSeparators = /[^\w%.-]/; + for ( let i = 0; i < 128; i++ ) { + if ( reSeparators.test(String.fromCharCode(i)) ) { + charClassMap[i] |= CHAR_CLASS_SEPARATOR; + } + } +} + +const isSeparatorChar = c => (charClassMap[c] & CHAR_CLASS_SEPARATOR) !== 0; + +/******************************************************************************/ + +const FILTER_DATA_PAGE_SIZE = 65536; + +const roundToFilterDataPageSize = + len => (len + FILTER_DATA_PAGE_SIZE-1) & ~(FILTER_DATA_PAGE_SIZE-1); + +let filterData = new Int32Array(FILTER_DATA_PAGE_SIZE * 5); +let filterDataWritePtr = 2; +const filterDataGrow = len => { + if ( len <= filterData.length ) { return; } + const newLen = roundToFilterDataPageSize(len); + const newBuf = new Int32Array(newLen); + newBuf.set(filterData); + filterData = newBuf; +}; +const filterDataShrink = ( ) => { + const newLen = Math.max( + roundToFilterDataPageSize(filterDataWritePtr), + FILTER_DATA_PAGE_SIZE + ); + if ( newLen >= filterData.length ) { return; } + const newBuf = new Int32Array(newLen); + newBuf.set(filterData.subarray(0, filterDataWritePtr)); + filterData = newBuf; +}; +const filterDataAlloc = (...args) => { + const len = args.length; + const idata = filterDataAllocLen(len); + for ( let i = 0; i < len; i++ ) { + filterData[idata+i] = args[i]; + } + return idata; +}; +const filterDataAllocLen = len => { + const idata = filterDataWritePtr; + filterDataWritePtr += len; + if ( filterDataWritePtr > filterData.length ) { + filterDataGrow(filterDataWritePtr); + } + return idata; +}; +const filterSequenceAdd = (a, b) => { + const iseq = filterDataAllocLen(2); + filterData[iseq+0] = a; + filterData[iseq+1] = b; + return iseq; +}; +const filterDataReset = ( ) => { + filterData.fill(0); + filterDataWritePtr = 2; +}; +const filterDataToSelfie = ( ) => { + return JSON.stringify(Array.from(filterData.subarray(0, filterDataWritePtr))); +}; +const filterDataFromSelfie = selfie => { + if ( typeof selfie !== 'string' || selfie === '' ) { return false; } + const data = JSON.parse(selfie); + if ( Array.isArray(data) === false ) { return false; } + filterDataGrow(data.length); + filterDataWritePtr = data.length; + filterData.set(data); + filterDataShrink(); + return true; +}; + +const filterRefs = [ null ]; +let filterRefsWritePtr = 1; +const filterRefAdd = ref => { + const i = filterRefsWritePtr; + filterRefs[i] = ref; + filterRefsWritePtr += 1; + return i; +}; +const filterRefsReset = ( ) => { + filterRefs.fill(null); + filterRefsWritePtr = 1; +}; +const filterRefsToSelfie = ( ) => { + const refs = []; + for ( let i = 0; i < filterRefsWritePtr; i++ ) { + const v = filterRefs[i]; + if ( v instanceof RegExp ) { + refs.push({ t: 1, s: v.source, f: v.flags }); + continue; + } + if ( Array.isArray(v) ) { + refs.push({ t: 2, v }); + continue; + } + if ( typeof v !== 'object' || v === null ) { + refs.push({ t: 0, v }); + continue; + } + const out = Object.create(null); + for ( const prop of Object.keys(v) ) { + const value = v[prop]; + out[prop] = prop.startsWith('$') + ? (typeof value === 'string' ? '' : null) + : value; + } + refs.push({ t: 3, v: out }); + } + return JSON.stringify(refs); +}; +const filterRefsFromSelfie = selfie => { + if ( typeof selfie !== 'string' || selfie === '' ) { return false; } + const refs = JSON.parse(selfie); + if ( Array.isArray(refs) === false ) { return false; } + for ( let i = 0; i < refs.length; i++ ) { + const v = refs[i]; + switch ( v.t ) { + case 0: + case 2: + case 3: + filterRefs[i] = v.v; + break; + case 1: + filterRefs[i] = new RegExp(v.s, v.f); + break; + default: + throw new Error('Unknown filter reference!'); + } + } + filterRefsWritePtr = refs.length; + return true; +}; + +/******************************************************************************/ + +const origHNTrieContainer = new HNTrieContainer(); +const destHNTrieContainer = new HNTrieContainer(); + +/******************************************************************************/ + +const bidiTrieMatchExtra = (l, r, ix) => { + for (;;) { + $patternMatchLeft = l; + $patternMatchRight = r; + const iu = filterData[ix+0]; + if ( filterMatch(iu) ) { return iu; } + ix = filterData[ix+1]; + if ( ix === 0 ) { break; } + } + return 0; +}; + +const bidiTrie = new BidiTrieContainer(bidiTrieMatchExtra); + +const bidiTriePrime = ( ) => { + bidiTrie.reset(keyvalStore.getItem('SNFE.bidiTrie')); +}; + +const bidiTrieOptimize = (shrink = false) => { + keyvalStore.setItem('SNFE.bidiTrie', bidiTrie.optimize(shrink)); +}; + +/******************************************************************************* + + Each filter class will register itself in the map. + + IMPORTANT: any change which modifies the mapping will have to be + reflected with µBlock.systemSettings.compiledMagic. + +*/ + +const filterClasses = []; +const filterArgsToUnit = new Map(); +let filterClassIdGenerator = 0; + +const registerFilterClass = fc => { + const fid = filterClassIdGenerator++; + fc.fid = fid; + fc.fidstr = `${fid}`; + filterClasses[fid] = fc; +}; + +const filterFromCompiled = args => { + const fc = filterClasses[args[0]]; + const keygen = fc.keyFromArgs; + if ( keygen === undefined ) { + return fc.fromCompiled(args); + } + const key = `${fc.fidstr} ${(keygen(args) || '')}`; + let idata = filterArgsToUnit.get(key); + if ( idata !== undefined ) { return idata; } + idata = fc.fromCompiled(args); + filterArgsToUnit.set(key, idata); + return idata; +}; + +const filterGetClass = idata => { + return filterClasses[filterData[idata+0]]; +}; + +const filterMatch = idata => filterClasses[filterData[idata+0]].match(idata); + +const filterHasOriginHit = idata => { + const fc = filterClasses[filterData[idata+0]]; + return fc.hasOriginHit !== undefined && fc.hasOriginHit(idata); +}; + +const filterGetDomainOpt = (idata, out) => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.getDomainOpt === undefined ) { return; } + const fromOpt = fc.getDomainOpt(idata); + if ( out === undefined ) { return fromOpt; } + out.push(fromOpt); +}; + +const filterGetRegexPattern = (idata, out) => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.hasRegexPattern === undefined ) { return; } + const reStr = fc.getRegexPattern(idata); + if ( out === undefined ) { return reStr; } + out.push(reStr); +}; + +const filterIsBidiTrieable = idata => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.isBidiTrieable === undefined ) { return false; } + return fc.isBidiTrieable(idata) === true; +}; + +const filterToBidiTrie = idata => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.toBidiTrie === undefined ) { return; } + return fc.toBidiTrie(idata); +}; + +const filterMatchAndFetchModifiers = (idata, env) => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.matchAndFetchModifiers === undefined ) { return; } + return fc.matchAndFetchModifiers(idata, env); +}; + +const filterGetModifierType = idata => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.getModifierType === undefined ) { return; } + return fc.getModifierType(idata); +}; + +const filterLogData = (idata, details) => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.logData === undefined ) { return; } + fc.logData(idata, details); +}; + +const filterDumpInfo = (idata) => { + const fc = filterGetClass(idata); + if ( fc.dumpInfo === undefined ) { return; } + return fc.dumpInfo(idata); +}; + +const dnrRuleFromCompiled = (args, rule) => { + const fc = filterClasses[args[0]]; + if ( fc.dnrFromCompiled === undefined ) { return false; } + fc.dnrFromCompiled(args, rule); + return true; +}; + +const dnrAddRuleError = (rule, msg) => { + rule._error = rule._error || []; + rule._error.push(msg); +}; + +const dnrAddRuleWarning = (rule, msg) => { + rule._warning = rule._warning || []; + rule._warning.push(msg); +}; + +/******************************************************************************* + + Filter classes + + Pattern: + FilterPatternAny + FilterPatternPlain + FilterPatternPlain1 + FilterPatternPlainX + FilterPatternGeneric + FilterRegex + FilterPlainTrie + FilterHostnameDict + + Pattern modifiers: + FilterAnchorHnLeft + FilterAnchorHn + FilterAnchorRight + FilterAnchorLeft + FilterTrailingSeparator + + Context, immediate: + FilterOriginHit + FilterOriginMiss + FilterOriginEntityMiss + FilterOriginEntityHit + FilterOriginHitSet + FilterOriginMissSet + FilterJustOrigin + FilterHTTPJustOrigin + FilterHTTPSJustOrigin + + Other options: + FilterDenyAllow + FilterImportant + FilterNotType + FilterStrictParty + FilterModifier + + Collection: + FilterCollection + FilterCompositeAll + FilterBucket + FilterBucketIf + FilterBucketIfOriginHits + FilterBucketIfRegexHits + FilterDomainHitAny + + A single filter can be made of many parts, in which case FilterCompositeAll + is used to hold all the parts, and where all the parts must be a match in + order for the filter to be a match. + +**/ + +/******************************************************************************/ + +class FilterPatternAny { + static match() { + return true; + } + + static compile() { + return [ FilterPatternAny.fid ]; + } + + static fromCompiled(args) { + return filterDataAlloc(args[0]); + } + + static keyFromArgs() { + } + + static logData(idata, details) { + details.pattern.push('*'); + details.regex.push('^'); + } +} + +registerFilterClass(FilterPatternAny); + +/******************************************************************************/ + +class FilterImportant { + static match() { + return ($isBlockImportant = true); + } + + static compile() { + return [ FilterImportant.fid ]; + } + + static fromCompiled(args) { + return filterDataAlloc(args[0]); + } + + static dnrFromCompiled(args, rule) { + rule.priority = (rule.priority || 1) + 10; + } + + static keyFromArgs() { + } + + static logData(idata, details) { + details.options.unshift('important'); + } +} + +registerFilterClass(FilterImportant); + +/******************************************************************************/ + +class FilterPatternPlain { + static isBidiTrieable(idata) { + return filterData[idata+2] <= 255; + } + + static toBidiTrie(idata) { + return { + i: filterData[idata+1], + n: filterData[idata+2], + itok: filterData[idata+3], + }; + } + + static match(idata) { + const left = $tokenBeg; + const n = filterData[idata+2]; + if ( + bidiTrie.startsWith( + left, + bidiTrie.haystackLen, + filterData[idata+1], + n + ) === 0 + ) { + return false; + } + $patternMatchLeft = left; + $patternMatchRight = left + n; + return true; + } + + static compile(details) { + const { tokenBeg } = details; + if ( tokenBeg === 0 ) { + return [ FilterPatternPlain.fid, details.pattern, 0 ]; + } + if ( tokenBeg === 1 ) { + return [ FilterPatternPlain1.fid, details.pattern, 1 ]; + } + return [ FilterPatternPlainX.fid, details.pattern, tokenBeg ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(4); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = bidiTrie.storeString(args[1]); // i + filterData[idata+2] = args[1].length; // n + filterData[idata+3] = args[2]; // tokenBeg + return idata; + } + + static dnrFromCompiled(args, rule) { + if ( rule.condition === undefined ) { + rule.condition = {}; + } else if ( rule.condition.urlFilter !== undefined ) { + dnrAddRuleError(rule, `urlFilter already defined: ${rule.condition.urlFilter}`); + } + rule.condition.urlFilter = args[1]; + } + + static logData(idata, details) { + const s = bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + details.pattern.push(s); + details.regex.push(restrFromPlainPattern(s)); + // https://github.com/gorhill/uBlock/issues/3037 + // Make sure the logger reflects accurately internal match, taking + // into account MAX_TOKEN_LENGTH. + if ( /^[0-9a-z%]{1,6}$/i.exec(s.slice(filterData[idata+3])) !== null ) { + details.regex.push('(?![0-9A-Za-z%])'); + } + } + + static dumpInfo(idata) { + const pattern = bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + return `${pattern} ${filterData[idata+3]}`; + } +} + +FilterPatternPlain.isPatternPlain = true; + +registerFilterClass(FilterPatternPlain); + + +class FilterPatternPlain1 extends FilterPatternPlain { + static match(idata) { + const left = $tokenBeg - 1; + const n = filterData[idata+2]; + if ( + bidiTrie.startsWith( + left, + bidiTrie.haystackLen, + filterData[idata+1], + n + ) === 0 + ) { + return false; + } + $patternMatchLeft = left; + $patternMatchRight = left + n; + return true; + } +} + +registerFilterClass(FilterPatternPlain1); + + +class FilterPatternPlainX extends FilterPatternPlain { + static match(idata) { + const left = $tokenBeg - filterData[idata+3]; + const n = filterData[idata+2]; + if ( + bidiTrie.startsWith( + left, + bidiTrie.haystackLen, + filterData[idata+1], + n + ) === 0 + ) { + return false; + } + $patternMatchLeft = left; + $patternMatchRight = left + n; + return true; + } +} + +registerFilterClass(FilterPatternPlainX); + +/******************************************************************************/ + +class FilterPatternGeneric { + static hasRegexPattern() { + return true; + } + + static getRegexPattern(idata) { + return restrFromGenericPattern( + bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ), + filterData[idata+3] + ); + } + + static match(idata) { + const refs = filterRefs[filterData[idata+4]]; + if ( refs.$re === null ) { + refs.$re = new RegExp(this.getRegexPattern(idata)); + } + return refs.$re.test($requestURL); + } + + static compile(details) { + const out = [ + FilterPatternGeneric.fid, + details.pattern, + details.anchor, + ]; + details.anchor = 0; + return out; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(5); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = bidiTrie.storeString(args[1]); // i + filterData[idata+2] = args[1].length; // n + filterData[idata+3] = args[2]; // anchor + filterData[idata+4] = filterRefAdd({ $re: null }); + return idata; + } + + static dnrFromCompiled(args, rule) { + if ( rule.condition === undefined ) { + rule.condition = {}; + } else if ( rule.condition.urlFilter !== undefined ) { + dnrAddRuleError(rule, `urlFilter already defined: ${rule.condition.urlFilter}`); + } + let pattern = args[1]; + if ( args[2] & 0b100 ) { + if ( pattern.startsWith('.') ) { + pattern = `*${pattern}`; + } + pattern = `||${pattern}`; + } else if ( args[2] & 0b010 ) { + pattern = `|${pattern}`; + } + if ( args[2] & 0b001 ) { + pattern += '|'; + } + rule.condition.urlFilter = pattern; + } + + static keyFromArgs(args) { + return `${args[1]}\t${args[2]}`; + } + + static logData(idata, details) { + details.pattern.length = 0; + const anchor = filterData[idata+3]; + if ( (anchor & 0b100) !== 0 ) { + details.pattern.push('||'); + } else if ( (anchor & 0b010) !== 0 ) { + details.pattern.push('|'); + } + const s = bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + details.pattern.push(s); + if ( (anchor & 0b001) !== 0 ) { + details.pattern.push('|'); + } + details.regex.length = 0; + details.regex.push(restrFromGenericPattern(s, anchor & ~0b100)); + } + + static dumpInfo(idata) { + return bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + } +} + +FilterPatternGeneric.isSlow = true; + +registerFilterClass(FilterPatternGeneric); + +/******************************************************************************/ + +class FilterAnchorHnLeft { + static match(idata) { + const len = $requestHostname.length; + const haystackCodes = bidiTrie.haystack; + let lastBeg = filterData[idata+2]; + let lastEnd = filterData[idata+3]; + if ( + len !== filterData[idata+1] || + lastBeg === -1 || + haystackCodes[lastBeg-3] !== 0x3A /* ':' */ || + haystackCodes[lastBeg-2] !== 0x2F /* '/' */ || + haystackCodes[lastBeg-1] !== 0x2F /* '/' */ + ) { + lastBeg = len !== 0 ? haystackCodes.indexOf(0x3A) : -1; + if ( lastBeg !== -1 ) { + if ( + lastBeg >= bidiTrie.haystackLen || + haystackCodes[lastBeg+1] !== 0x2F || + haystackCodes[lastBeg+2] !== 0x2F + ) { + lastBeg = -1; + } + } + if ( lastBeg !== -1 ) { + lastBeg += 3; + lastEnd = lastBeg + len; + } else { + lastEnd = -1; + } + filterData[idata+1] = len; + filterData[idata+2] = lastBeg; + filterData[idata+3] = lastEnd; + } + const left = $patternMatchLeft; + return left < lastEnd && ( + left === lastBeg || + left > lastBeg && haystackCodes[left-1] === 0x2E /* '.' */ + ); + } + + static compile() { + return [ FilterAnchorHnLeft.fid ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(4); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = 0; // lastLen + filterData[idata+2] = -1; // lastBeg + filterData[idata+3] = -1; // lastEnd + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.condition.urlFilter = `||${rule.condition.urlFilter}`; + } + + static keyFromArgs() { + } + + static logData(idata, details) { + details.pattern.unshift('||'); + } +} + +registerFilterClass(FilterAnchorHnLeft); + +/******************************************************************************/ + +class FilterAnchorHn extends FilterAnchorHnLeft { + static match(idata) { + return super.match(idata) && filterData[idata+3] === $patternMatchRight; + } + + static compile() { + return [ FilterAnchorHn.fid ]; + } + + static dnrFromCompiled(args, rule) { + rule.condition.requestDomains = [ rule.condition.urlFilter ]; + rule.condition.urlFilter = undefined; + } + + static keyFromArgs() { + } + + static logData(idata, details) { + super.logData(idata, details); + details.pattern.push('^'); + details.regex.push('\\.?', restrSeparator); + } +} + +registerFilterClass(FilterAnchorHn); + +/******************************************************************************/ + +class FilterAnchorLeft { + static match() { + return $patternMatchLeft === 0; + } + + static compile() { + return [ FilterAnchorLeft.fid ]; + } + + static fromCompiled(args) { + return filterDataAlloc(args[0]); + } + + static dnrFromCompiled(args, rule) { + rule.condition.urlFilter = `|${rule.condition.urlFilter}`; + } + + static keyFromArgs() { + } + + static logData(idata, details) { + details.pattern.unshift('|'); + details.regex.unshift('^'); + } +} + +registerFilterClass(FilterAnchorLeft); + +/******************************************************************************/ + +class FilterAnchorRight { + static match() { + return $patternMatchRight === $requestURL.length; + } + + static compile() { + return [ FilterAnchorRight.fid ]; + } + + static fromCompiled(args) { + return filterDataAlloc(args[0]); + } + + static dnrFromCompiled(args, rule) { + rule.condition.urlFilter = `${rule.condition.urlFilter}|`; + } + + static keyFromArgs() { + } + + static logData(idata, details) { + details.pattern.push('|'); + details.regex.push('$'); + } +} + +registerFilterClass(FilterAnchorRight); + +/******************************************************************************/ + +class FilterTrailingSeparator { + static match() { + if ( $patternMatchRight === $requestURL.length ) { return true; } + if ( isSeparatorChar(bidiTrie.haystack[$patternMatchRight]) ) { + $patternMatchRight += 1; + return true; + } + return false; + } + + static compile() { + return [ FilterTrailingSeparator.fid ]; + } + + static fromCompiled(args) { + return filterDataAlloc(args[0]); + } + + static dnrFromCompiled(args, rule) { + rule.condition.urlFilter = `${rule.condition.urlFilter}^`; + } + + static keyFromArgs() { + } + + static logData(idata, details) { + details.pattern.push('^'); + details.regex.push(restrSeparator); + } +} + +registerFilterClass(FilterTrailingSeparator); + +/******************************************************************************/ + +class FilterRegex { + static hasRegexPattern() { + return true; + } + + static getRegexPattern(idata) { + return bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + } + + static match(idata) { + const refs = filterRefs[filterData[idata+4]]; + if ( refs.$re === null ) { + refs.$re = new RegExp( + this.getRegexPattern(idata), + filterData[idata+3] === 0 ? 'i' : '' + ); + } + if ( refs.$re.test($requestURLRaw) === false ) { return false; } + $patternMatchLeft = $requestURLRaw.search(refs.$re); + return true; + } + + static compile(details) { + return [ + FilterRegex.fid, + details.pattern, + details.patternMatchCase ? 1 : 0 + ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(5); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = bidiTrie.storeString(args[1]); // i + filterData[idata+2] = args[1].length; // n + filterData[idata+3] = args[2]; // match-case + filterData[idata+4] = filterRefAdd({ $re: null }); + return idata; + } + + static dnrFromCompiled(args, rule) { + if ( rule.condition === undefined ) { + rule.condition = {}; + } + if ( sfp.utils.regex.isRE2(args[1]) === false ) { + dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`); + } + rule.condition.regexFilter = args[1]; + if ( args[2] === 1 ) { + rule.condition.isUrlFilterCaseSensitive = true; + } + } + + static keyFromArgs(args) { + return `${args[1]}\t${args[2]}`; + } + + static logData(idata, details) { + const s = bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + details.pattern.push('/', s, '/'); + details.regex.push(s); + details.isRegex = true; + if ( filterData[idata+3] !== 0 ) { + details.options.push('match-case'); + } + } + + static dumpInfo(idata) { + return [ + '/', + bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ), + '/', + filterData[idata+3] !== 0 ? ' (match-case)' : '', + ].join(''); + } +} + +FilterRegex.isSlow = true; + +registerFilterClass(FilterRegex); + +/******************************************************************************/ + +class FilterMethod { + static match(idata) { + if ( $requestMethodBit === 0 ) { return false; } + const methodBits = filterData[idata+1]; + const notMethodBits = filterData[idata+2]; + return (methodBits !== 0 && ($requestMethodBit & methodBits) !== 0) || + (notMethodBits !== 0 && ($requestMethodBit & notMethodBits) === 0); + } + + static compile(details) { + return [ FilterMethod.fid, details.methodBits, details.notMethodBits ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(3); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = args[1]; // methodBits + filterData[idata+2] = args[2]; // notMethodBits + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.condition = rule.condition || {}; + const rc = rule.condition; + let methodBits = args[1]; + let notMethodBits = args[2]; + if ( methodBits !== 0 && rc.requestMethods === undefined ) { + rc.requestMethods = []; + } + if ( notMethodBits !== 0 && rc.excludedRequestMethods === undefined ) { + rc.excludedRequestMethods = []; + } + for ( let i = 1; methodBits !== 0 || notMethodBits !== 0; i++ ) { + const bit = 1 << i; + const methodName = FilteringContext.getMethodName(bit); + if ( (methodBits & bit) !== 0 ) { + methodBits &= ~bit; + rc.requestMethods.push(methodName); + } else if ( (notMethodBits & bit) !== 0 ) { + notMethodBits &= ~bit; + rc.excludedRequestMethods.push(methodName); + } + } + } + + static keyFromArgs(args) { + return `${args[1]} ${args[2]}`; + } + + static logData(idata, details) { + const methods = []; + let methodBits = filterData[idata+1]; + let notMethodBits = filterData[idata+2]; + for ( let i = 0; methodBits !== 0 || notMethodBits !== 0; i++ ) { + const bit = 1 << i; + const methodName = FilteringContext.getMethodName(bit); + if ( (methodBits & bit) !== 0 ) { + methodBits &= ~bit; + methods.push(methodName); + } else if ( (notMethodBits & bit) !== 0 ) { + notMethodBits &= ~bit; + methods.push(`~${methodName}`); + } + } + details.options.push(`method=${methods.join('|')}`); + } + + static dumpInfo(idata) { + return `0b${filterData[idata+1].toString(2)} 0b${filterData[idata+2].toString(2)}`; + } +} + +registerFilterClass(FilterMethod); + +/******************************************************************************/ + +// stylesheet: 1 => bit 0 +// image: 2 => bit 1 +// object: 3 => bit 2 +// script: 4 => bit 3 +// ... + +class FilterNotType { + static match(idata) { + return $requestTypeValue !== 0 && + (filterData[idata+1] & (1 << ($requestTypeValue - 1))) === 0; + } + + static compile(details) { + return [ FilterNotType.fid, details.notTypeBits ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(2); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = args[1]; // notTypeBits + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.condition = rule.condition || {}; + const rc = rule.condition; + if ( rc.excludedResourceTypes === undefined ) { + rc.excludedResourceTypes = [ 'main_frame' ]; + } + let bits = args[1]; + for ( let i = 1; bits !== 0 && i < typeValueToDNRTypeName.length; i++ ) { + const bit = 1 << (i - 1); + if ( (bits & bit) === 0 ) { continue; } + bits &= ~bit; + const type = typeValueToDNRTypeName[i]; + if ( type === undefined ) { continue; } + if ( rc.excludedResourceTypes.includes(type) ) { continue; } + rc.excludedResourceTypes.push(type); + } + } + + static keyFromArgs(args) { + return `${args[1]}`; + } + + static logData(idata, details) { + let bits = filterData[idata+1]; + for ( let i = 1; bits !== 0 && i < typeValueToTypeName.length; i++ ) { + const bit = 1 << (i - 1); + if ( (bits & bit) === 0 ) { continue; } + bits &= ~bit; + details.options.push(`~${typeValueToTypeName[i]}`); + } + } + + static dumpInfo(idata) { + return `0b${filterData[idata+1].toString(2)}`; + } +} + +registerFilterClass(FilterNotType); + +/******************************************************************************/ + +// A helper class to parse `domain=` option. + +class DomainOptIterator { + constructor(domainOpt) { + this.reset(domainOpt); + } + reset(domainOpt) { + this.domainOpt = domainOpt; + this.i = 0; + this.value = undefined; + this.done = false; + return this; + } + next() { + if ( this.i === -1 ) { + this.domainOpt = ''; + this.value = undefined; + this.done = true; + return this; + } + const pos = this.domainOpt.indexOf('|', this.i); + if ( pos !== -1 ) { + this.value = this.domainOpt.slice(this.i, pos); + this.i = pos + 1; + } else { + this.value = this.domainOpt.slice(this.i); + this.i = -1; + } + return this; + } + [Symbol.iterator]() { + return this; + } +} + +// A helper instance to reuse throughout +const domainOptIterator = new DomainOptIterator(''); + +/******************************************************************************/ + +// The optimal class is picked according to the content of the `from=` +// filter option. +const compileDomainOpt = (ctors, iterable, prepend, units) => { + const hostnameHits = []; + const hostnameMisses = []; + const entityHits = []; + const entityMisses = []; + const regexHits = []; + const regexMisses = []; + for ( const s of iterable ) { + const len = s.length; + const beg = len > 1 && s.charCodeAt(0) === 0x7E /* '~' */ ? 1 : 0; + if ( len <= beg ) { continue; } + if ( s.charCodeAt(beg) === 0x2F /* / */ ) { + if ( beg === 0 ) { regexHits.push(s); continue; } + regexMisses.push(s.slice(1)); continue; + } + if ( s.endsWith('.*') === false ) { + if ( beg === 0 ) { hostnameHits.push(s); continue; } + hostnameMisses.push(s.slice(1)); continue; + } + if ( beg === 0 ) { entityHits.push(s); continue; } + entityMisses.push(s.slice(1)); continue; + } + const toTrie = []; + let trieWhich = 0b00; + if ( hostnameHits.length > 1 ) { + toTrie.push(...hostnameHits); + hostnameHits.length = 0; + trieWhich |= 0b01; + } + if ( entityHits.length > 1 ) { + toTrie.push(...entityHits); + entityHits.length = 0; + trieWhich |= 0b10; + } + const compiledHit = []; + if ( toTrie.length !== 0 ) { + compiledHit.push( + ctors[2].compile(toTrie.sort(), trieWhich) + ); + } + for ( const hn of hostnameHits ) { + compiledHit.push(ctors[0].compile(hn)); + } + for ( const hn of entityHits ) { + compiledHit.push(ctors[1].compile(hn)); + } + for ( const hn of regexHits ) { + compiledHit.push(ctors[3].compile(hn)); + } + if ( compiledHit.length > 1 ) { + compiledHit[0] = FilterDomainHitAny.compile(compiledHit.slice()); + compiledHit.length = 1; + } + toTrie.length = trieWhich = 0; + if ( hostnameMisses.length > 1 ) { + toTrie.push(...hostnameMisses); + hostnameMisses.length = 0; + trieWhich |= 0b01; + } + if ( entityMisses.length > 1 ) { + toTrie.push(...entityMisses); + entityMisses.length = 0; + trieWhich |= 0b10; + } + const compiledMiss = []; + if ( toTrie.length !== 0 ) { + compiledMiss.push( + ctors[6].compile(toTrie.sort(), trieWhich) + ); + } + for ( const hn of hostnameMisses ) { + compiledMiss.push(ctors[4].compile(hn)); + } + for ( const hn of entityMisses ) { + compiledMiss.push(ctors[5].compile(hn)); + } + for ( const hn of regexMisses ) { + compiledMiss.push(ctors[7].compile(hn)); + } + if ( prepend ) { + if ( compiledHit.length !== 0 ) { + units.unshift(compiledHit[0]); + } + if ( compiledMiss.length !== 0 ) { + units.unshift(...compiledMiss); + } + } else { + if ( compiledMiss.length !== 0 ) { + units.push(...compiledMiss); + } + if ( compiledHit.length !== 0 ) { + units.push(compiledHit[0]); + } + } +}; + +/******************************************************************************/ + +class FilterDomainHit { + static getDomainOpt(idata) { + return this.hntrieContainer.extractHostname( + filterData[idata+1], + filterData[idata+2] + ); + } + + static match(idata) { + return this.hntrieContainer.matchesHostname( + this.getMatchTarget(), + filterData[idata+1], + filterData[idata+2] + ); + } + + static compile(hostname) { + return [ this.fid, hostname ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(3); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = this.hntrieContainer.storeHostname(args[1]); // i + filterData[idata+2] = args[1].length; // n + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.condition = rule.condition || {}; + const prop = this.dnrConditionName; + if ( rule.condition[prop] === undefined ) { + rule.condition[prop] = []; + } + rule.condition[prop].push(args[1]); + } + + static dumpInfo(idata) { + return this.getDomainOpt(idata); + } +} + +/******************************************************************************/ + +class FilterDomainHitSet { + static getDomainOpt(idata) { + return this.hntrieContainer.extractDomainOpt( + filterData[idata+1], + filterData[idata+2] + ); + } + + static getTrieCount(idata) { + const itrie = filterData[idata+4]; + if ( itrie === 0 ) { return 0; } + return Array.from( + this.hntrieContainer.trieIterator(filterData[idata+4]) + ).length; + } + + static getLastResult(idata) { + return filterData[idata+5]; + } + + static getMatchedHostname(idata) { + const lastResult = filterData[idata+5]; + if ( lastResult === -1 ) { return ''; } + return this.getMatchTarget(lastResult >>> 8).slice(lastResult & 0xFF); + } + + static match(idata) { + const refs = filterRefs[filterData[idata+6]]; + const docHostname = this.getMatchTarget(0b01); + if ( docHostname === refs.$last ) { + return filterData[idata+5] !== -1; + } + refs.$last = docHostname; + const which = filterData[idata+3]; + const itrie = filterData[idata+4] || this.toTrie(idata); + if ( itrie === 0 ) { return false; } + if ( (which & 0b01) !== 0 ) { + const pos = this.hntrieContainer + .setNeedle(docHostname) + .matches(itrie); + if ( pos !== -1 ) { + filterData[idata+5] = 0b01 << 8 | pos; + return true; + } + } + if ( (which & 0b10) !== 0 ) { + const pos = this.hntrieContainer + .setNeedle(this.getMatchTarget(0b10)) + .matches(itrie); + if ( pos !== -1 ) { + filterData[idata+5] = 0b10 << 8 | pos; + return true; + } + } + filterData[idata+5] = -1; + return false; + } + + static add(idata, hn) { + this.hntrieContainer.setNeedle(hn).add(filterData[idata+4]); + filterData[idata+3] |= hn.charCodeAt(hn.length - 1) !== 0x2A /* '*' */ + ? 0b01 + : 0b10; + filterData[idata+5] = -1; + filterRefs[filterData[idata+6]].$last = ''; + } + + static create(fid = -1) { + const idata = filterDataAllocLen(7); + filterData[idata+0] = fid !== -1 ? fid : this.fid; + filterData[idata+1] = 0; + filterData[idata+2] = 0; + filterData[idata+3] = 0; + filterData[idata+4] = this.hntrieContainer.createTrie(); + filterData[idata+5] = -1; // $lastResult + filterData[idata+6] = filterRefAdd({ $last: '' }); + return idata; + } + + static compile(hostnames, which) { + const stringified = Array.isArray(hostnames) + ? hostnames.join('|') + : hostnames; + return [ this.fid, stringified, which ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(7); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = this.hntrieContainer.storeDomainOpt(args[1]); + filterData[idata+2] = args[1].length; + filterData[idata+3] = args[2]; // which + filterData[idata+4] = 0; // itrie + filterData[idata+5] = -1; // $lastResult + filterData[idata+6] = filterRefAdd({ $last: '' }); + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.condition = rule.condition || {}; + const prop = this.dnrConditionName; + if ( rule.condition[prop] === undefined ) { + rule.condition[prop] = []; + } + rule.condition[prop].push(...args[1].split('|')); + } + + static toTrie(idata) { + if ( filterData[idata+2] === 0 ) { return 0; } + const itrie = filterData[idata+4] = + this.hntrieContainer.createTrieFromStoredDomainOpt( + filterData[idata+1], + filterData[idata+2] + ); + return itrie; + } + + static keyFromArgs(args) { + return args[1]; + } + + static dumpInfo(idata) { + return `0b${filterData[idata+3].toString(2)} ${this.getDomainOpt(idata)}`; + } +} + +/******************************************************************************/ + +class FilterDomainRegexHit { + static getDomainOpt(idata) { + const ref = filterRefs[filterData[idata+1]]; + return ref.restr; + } + + static match(idata) { + const ref = filterRefs[filterData[idata+1]]; + if ( ref.$re === null ) { + ref.$re = new RegExp(ref.restr.slice(1,-1)); + } + return ref.$re.test(this.getMatchTarget()); + } + + static compile(restr) { + return [ this.fid, restr ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(2); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = filterRefAdd({ restr: args[1], $re: null }); + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.condition = rule.condition || {}; + const prop = this.dnrConditionName; + if ( rule.condition[prop] === undefined ) { + rule.condition[prop] = []; + } + rule.condition[prop].push(args[1]); + } + + static dumpInfo(idata) { + return this.getDomainOpt(idata); + } +} + +/******************************************************************************/ + +// Implement the following filter option: +// - domain= +// - from= + +class FilterFromDomainHit extends FilterDomainHit { + static hasOriginHit() { + return true; + } + + static getMatchTarget() { + return $docHostname; + } + + static get dnrConditionName() { + return 'initiatorDomains'; + } + + static logData(idata, details) { + details.fromDomains.push(this.getDomainOpt(idata)); + } +} +Object.defineProperty(FilterFromDomainHit, 'hntrieContainer', { + value: origHNTrieContainer +}); + +class FilterFromDomainMiss extends FilterFromDomainHit { + static hasOriginHit() { + return false; + } + + static get dnrConditionName() { + return 'excludedInitiatorDomains'; + } + + static match(idata) { + return super.match(idata) === false; + } + + static logData(idata, details) { + details.fromDomains.push(`~${this.getDomainOpt(idata)}`); + } +} + +class FilterFromEntityHit extends FilterFromDomainHit { + static getMatchTarget() { + return $docEntity.compute(); + } +} + +class FilterFromEntityMiss extends FilterFromDomainMiss { + static getMatchTarget() { + return $docEntity.compute(); + } +} + +class FilterFromDomainHitSet extends FilterDomainHitSet { + static hasOriginHit() { + return true; + } + + static getMatchTarget(which) { + return (which & 0b01) !== 0 + ? $docHostname + : $docEntity.compute(); + } + + static get dnrConditionName() { + return 'initiatorDomains'; + } + + static logData(idata, details) { + details.fromDomains.push(this.getDomainOpt(idata)); + } +} +Object.defineProperty(FilterFromDomainHitSet, 'hntrieContainer', { + value: origHNTrieContainer +}); + +class FilterFromDomainMissSet extends FilterFromDomainHitSet { + static hasOriginHit() { + return false; + } + + static match(idata) { + return super.match(idata) === false; + } + + static get dnrConditionName() { + return 'excludedInitiatorDomains'; + } + + static logData(idata, details) { + details.fromDomains.push('~' + this.getDomainOpt(idata).replace(/\|/g, '|~')); + } +} + +class FilterFromRegexHit extends FilterDomainRegexHit { + static getMatchTarget() { + return $docHostname; + } + + static get dnrConditionName() { + return 'initiatorDomains'; + } + + static logData(idata, details) { + details.fromDomains.push(`${this.getDomainOpt(idata)}`); + } +} + +class FilterFromRegexMiss extends FilterFromRegexHit { + static match(idata) { + return super.match(idata) === false; + } + + static get dnrConditionName() { + return 'excludedInitiatorDomains'; + } + + static logData(idata, details) { + details.fromDomains.push(`~${this.getDomainOpt(idata)}`); + } +} + +registerFilterClass(FilterFromDomainHit); +registerFilterClass(FilterFromDomainMiss); +registerFilterClass(FilterFromEntityHit); +registerFilterClass(FilterFromEntityMiss); +registerFilterClass(FilterFromDomainHitSet); +registerFilterClass(FilterFromDomainMissSet); +registerFilterClass(FilterFromRegexHit); +registerFilterClass(FilterFromRegexMiss); + +const fromOptClasses = [ + FilterFromDomainHit, + FilterFromEntityHit, + FilterFromDomainHitSet, + FilterFromRegexHit, + FilterFromDomainMiss, + FilterFromEntityMiss, + FilterFromDomainMissSet, + FilterFromRegexMiss, +]; + +const compileFromDomainOpt = (...args) => { + return compileDomainOpt(fromOptClasses, ...args); +}; + +/******************************************************************************/ + +// Implement the following filter option: +// - to= + +class FilterToDomainHit extends FilterDomainHit { + static getMatchTarget() { + return $requestHostname; + } + + static get dnrConditionName() { + return 'requestDomains'; + } + + static logData(idata, details) { + details.toDomains.push(this.getDomainOpt(idata)); + } +} +Object.defineProperty(FilterToDomainHit, 'hntrieContainer', { + value: destHNTrieContainer +}); + +class FilterToDomainMiss extends FilterToDomainHit { + static get dnrConditionName() { + return 'excludedRequestDomains'; + } + + static match(idata) { + return super.match(idata) === false; + } + + static logData(idata, details) { + details.toDomains.push(`~${this.getDomainOpt(idata)}`); + } +} + +class FilterToEntityHit extends FilterToDomainHit { + static getMatchTarget() { + return $requestEntity.compute(); + } +} + +class FilterToEntityMiss extends FilterToDomainMiss { + static getMatchTarget() { + return $requestEntity.compute(); + } +} + +class FilterToDomainHitSet extends FilterDomainHitSet { + static getMatchTarget(which) { + return (which & 0b01) !== 0 + ? $requestHostname + : $requestEntity.compute(); + } + + static get dnrConditionName() { + return 'requestDomains'; + } + + static logData(idata, details) { + details.toDomains.push(this.getDomainOpt(idata)); + } +} +Object.defineProperty(FilterToDomainHitSet, 'hntrieContainer', { + value: destHNTrieContainer +}); + +class FilterToDomainMissSet extends FilterToDomainHitSet { + static match(idata) { + return super.match(idata) === false; + } + + static get dnrConditionName() { + return 'excludedRequestDomains'; + } + + static logData(idata, details) { + details.toDomains.push('~' + this.getDomainOpt(idata).replace(/\|/g, '|~')); + } +} + +class FilterToRegexHit extends FilterDomainRegexHit { + static getMatchTarget() { + return $requestHostname; + } + + static get dnrConditionName() { + return 'requestDomains'; + } + + static logData(idata, details) { + details.toDomains.push(`${this.getDomainOpt(idata)}`); + } +} + +class FilterToRegexMiss extends FilterToRegexHit { + static match(idata) { + return super.match(idata) === false; + } + + static get dnrConditionName() { + return 'excludedRequestDomains'; + } + + static logData(idata, details) { + details.toDomains.push(`~${this.getDomainOpt(idata)}`); + } +} + +registerFilterClass(FilterToDomainHit); +registerFilterClass(FilterToDomainMiss); +registerFilterClass(FilterToEntityHit); +registerFilterClass(FilterToEntityMiss); +registerFilterClass(FilterToDomainHitSet); +registerFilterClass(FilterToDomainMissSet); +registerFilterClass(FilterToRegexHit); +registerFilterClass(FilterToRegexMiss); + +const toOptClasses = [ + FilterToDomainHit, + FilterToEntityHit, + FilterToDomainHitSet, + FilterToRegexHit, + FilterToDomainMiss, + FilterToEntityMiss, + FilterToDomainMissSet, + FilterToRegexMiss, +]; + +const compileToDomainOpt = (...args) => { + return compileDomainOpt(toOptClasses, ...args); +}; + +/******************************************************************************/ + +class FilterDenyAllow extends FilterToDomainMissSet { + static compile(details) { + return super.compile(details.denyallowOpt, 0b01); + } + + static logData(idata, details) { + details.denyallow.push(this.getDomainOpt(idata)); + } +} + +registerFilterClass(FilterDenyAllow); + +/******************************************************************************/ + +class FilterModifier { + static getModifierType(idata) { + return filterData[idata+2]; + } + + static match() { + return true; + } + + static matchAndFetchModifiers(idata, env) { + if ( this.getModifierType(idata) !== env.type ) { return; } + env.results.push(new FilterModifierResult(idata, env)); + } + + static compile(details) { + return [ + FilterModifier.fid, + details.action, + details.modifyType, + details.modifyValue || '', + ]; + } + + static fromCompiled(args) { + const idata = filterDataAllocLen(4); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = args[1]; // actionBits + filterData[idata+2] = args[2]; // type + filterData[idata+3] = filterRefAdd({ + value: args[3], + $cache: null, + }); + return idata; + } + + static dnrFromCompiled(args, rule) { + rule.__modifierAction = args[1]; + rule.__modifierType = modifierNameFromType.get(args[2]); + rule.__modifierValue = args[3]; + } + + static keyFromArgs(args) { + return `${args[1]}\t${args[2]}\t${args[3]}`; + } + + static logData(idata, details) { + let opt = modifierNameFromType.get(filterData[idata+2]); + const refs = filterRefs[filterData[idata+3]]; + if ( refs.value !== '' ) { + opt += `=${refs.value}`; + } + details.options.push(opt); + } + + static dumpInfo(idata) { + const s = modifierNameFromType.get(filterData[idata+2]); + const refs = filterRefs[filterData[idata+3]]; + if ( refs.value === '' ) { return s; } + return `${s}=${refs.value}`; + } +} + +registerFilterClass(FilterModifier); + +// Helper class for storing instances of FilterModifier which were found to +// be a match. + +class FilterModifierResult { + constructor(imodifierunit, env) { + this.imodifierunit = imodifierunit; + this.refs = filterRefs[filterData[imodifierunit+3]]; + this.ireportedunit = env.iunit; + this.th = env.th; + this.bits = (env.bits & ~RealmBitsMask) | filterData[imodifierunit+1]; + } + + get result() { + return (this.bits & ALLOW_REALM) === 0 ? 1 : 2; + } + + get value() { + return this.refs.value; + } + + get cache() { + return this.refs.$cache; + } + + set cache(a) { + this.refs.$cache = a; + } + + logData() { + const r = new LogData(this.bits, this.th, this.ireportedunit); + r.result = this.result; + r.modifier = true; + return r; + } +} + +/******************************************************************************/ + +class FilterCollection { + static getCount(idata) { + let n = 0; + this.forEach(idata, ( ) => { n += 1; }); + return n; + } + + static forEach(idata, fn) { + let i = filterData[idata+1]; + if ( i === 0 ) { return; } + do { + const iunit = filterData[i+0]; + const r = fn(iunit); + if ( r !== undefined ) { return r; } + i = filterData[i+1]; + } while ( i !== 0 ); + } + + static unshift(idata, iunit) { + filterData[idata+1] = filterSequenceAdd(iunit, filterData[idata+1]); + } + + static shift(idata) { + filterData[idata+1] = filterData[filterData[idata+1]+1]; + } + + static create(fid = -1) { + return filterDataAlloc( + fid !== -1 ? fid : FilterCollection.fid, + 0 + ); + } + + static compile(fc, fdata) { + return [ fc.fid, fdata ]; + } + + static fromCompiled(args) { + const units = args[1]; + const n = units.length; + let iunit, inext = 0; + let i = n; + while ( i-- ) { + iunit = filterFromCompiled(units[i]); + inext = filterSequenceAdd(iunit, inext); + } + const idata = filterDataAllocLen(2); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = inext; // i + return idata; + } + + static dnrFromCompiled(args, rule) { + for ( const unit of args[1] ) { + dnrRuleFromCompiled(unit, rule); + } + } + + static logData(idata, details) { + this.forEach(idata, iunit => { + filterLogData(iunit, details); + }); + } + + static dumpInfo(idata) { + return this.getCount(idata); + } +} + +registerFilterClass(FilterCollection); + +/******************************************************************************/ + +class FilterDomainHitAny extends FilterCollection { + static getDomainOpt(idata) { + const domainOpts = []; + this.forEach(idata, iunit => { + if ( filterHasOriginHit(iunit) !== true ) { return; } + filterGetDomainOpt(iunit, domainOpts); + }); + return domainOpts.join('|'); + } + + static hasOriginHit(idata) { + this.forEach(idata, iunit => { + if ( filterHasOriginHit(iunit) ) { return true; } + }); + return false; + } + + static match(idata) { + let i = filterData[idata+1]; + while ( i !== 0 ) { + if ( filterMatch(filterData[i+0]) ) { return true; } + i = filterData[i+1]; + } + return false; + } + + static compile(fdata) { + return super.compile(FilterDomainHitAny, fdata); + } + + static fromCompiled(args) { + return super.fromCompiled(args); + } +} + +registerFilterClass(FilterDomainHitAny); + +/******************************************************************************/ + +class FilterCompositeAll extends FilterCollection { + // FilterPatternPlain is assumed to be first filter in sequence. This can + // be revisited if needed. + static isBidiTrieable(idata) { + return filterIsBidiTrieable(filterData[filterData[idata+1]+0]); + } + + static toBidiTrie(idata) { + const iseq = filterData[idata+1]; + const details = filterToBidiTrie(filterData[iseq+0]); + this.shift(idata); + return details; + } + + static getDomainOpt(idata) { + return this.forEach(idata, iunit => { + if ( filterHasOriginHit(iunit) !== true ) { return; } + return filterGetDomainOpt(iunit); + }); + } + + static hasOriginHit(idata) { + return this.forEach(idata, iunit => { + if ( filterHasOriginHit(iunit) === true ) { return true; } + }) || false; + } + + static hasRegexPattern(idata) { + return this.forEach(idata, iunit => { + const fc = filterGetClass(iunit); + if ( fc.hasRegexPattern === undefined ) { return; } + if ( fc.hasRegexPattern(iunit) === true ) { return true; } + }) || false; + } + + static getRegexPattern(idata) { + return this.forEach(idata, iunit => { + const fc = filterGetClass(iunit); + if ( fc.getRegexPattern === undefined ) { return; } + return fc.getRegexPattern(iunit); + }); + } + + static match(idata) { + let i = filterData[idata+1]; + while ( i !== 0 ) { + if ( filterMatch(filterData[i+0]) !== true ) { + return false; + } + i = filterData[i+1]; + } + return true; + } + + // IMPORTANT: the modifier filter unit is assumed to be ALWAYS the + // first unit in the sequence. This requirement ensures that we do + // not have to traverse the sequence to find the modifier filter + // unit. + static getModifierType(idata) { + const iseq = filterData[idata+1]; + const iunit = filterData[iseq+0]; + return filterGetModifierType(iunit); + } + + static matchAndFetchModifiers(idata, env) { + const iseq = filterData[idata+1]; + const iunit = filterData[iseq+0]; + if ( + filterGetModifierType(iunit) === env.type && + this.match(idata) + ) { + filterMatchAndFetchModifiers(iunit, env); + } + } + + static compile(fdata) { + return super.compile(FilterCompositeAll, fdata); + } + + static fromCompiled(args) { + return super.fromCompiled(args); + } +} + +registerFilterClass(FilterCompositeAll); + +/******************************************************************************/ + +// Dictionary of hostnames + +class FilterHostnameDict { + static getCount(idata) { + const itrie = filterData[idata+1]; + if ( itrie !== 0 ) { + return Array.from(destHNTrieContainer.trieIterator(itrie)).length; + } + return filterRefs[filterData[idata+3]].length; + } + + static match(idata) { + const itrie = filterData[idata+1] || this.optimize(idata); + return ( + filterData[idata+2] = destHNTrieContainer + .setNeedle($requestHostname) + .matches(itrie) + ) !== -1; + } + + static add(idata, hn) { + const itrie = filterData[idata+1]; + if ( itrie === 0 ) { + filterRefs[filterData[idata+3]].push(hn); + } else { + destHNTrieContainer.setNeedle(hn).add(itrie); + } + } + + static optimize(idata) { + const itrie = filterData[idata+1]; + if ( itrie !== 0 ) { return itrie; } + const hostnames = filterRefs[filterData[idata+3]]; + filterData[idata+1] = destHNTrieContainer.createTrieFromIterable(hostnames); + filterRefs[filterData[idata+3]] = null; + return filterData[idata+1]; + } + + static create() { + const idata = filterDataAllocLen(4); + filterData[idata+0] = FilterHostnameDict.fid; // fid + filterData[idata+1] = 0; // itrie + filterData[idata+2] = -1; // lastResult + filterData[idata+3] = filterRefAdd([]); // []: hostnames + return idata; + } + + static logData(idata, details) { + const hostname = $requestHostname.slice(filterData[idata+2]); + details.pattern.push('||', hostname, '^'); + details.regex.push( + restrFromPlainPattern(hostname), + '\\.?', + restrSeparator + ); + } + + static dumpInfo(idata) { + return this.getCount(idata); + } +} + +registerFilterClass(FilterHostnameDict); + +/******************************************************************************/ + +// Dictionary of hostnames for filters which only purpose is to match +// the document origin. + +class FilterJustOrigin extends FilterFromDomainHitSet { + static create(fid = -1) { + return super.create(fid !== -1 ? fid : FilterJustOrigin.fid); + } + + static logPattern(idata, details) { + details.pattern.push('*'); + details.regex.push('^'); + } + + static logData(idata, details) { + this.logPattern(idata, details); + details.fromDomains.push(this.getMatchedHostname(idata)); + } + + static dumpInfo(idata) { + return this.getTrieCount(idata); + } +} + +registerFilterClass(FilterJustOrigin); + +/******************************************************************************/ + +class FilterHTTPSJustOrigin extends FilterJustOrigin { + static match(idata) { + return $requestURL.startsWith('https://') && super.match(idata); + } + + static create() { + return super.create(FilterHTTPSJustOrigin.fid); + } + + static logPattern(idata, details) { + details.pattern.push('|https://'); + details.regex.push('^https://'); + } +} + +registerFilterClass(FilterHTTPSJustOrigin); + +/******************************************************************************/ + +class FilterHTTPJustOrigin extends FilterJustOrigin { + static match(idata) { + return $requestURL.startsWith('http://') && super.match(idata); + } + + static create() { + return super.create(FilterHTTPJustOrigin.fid); + } + + static logPattern(idata, details) { + details.pattern.push('|http://'); + details.regex.push('^http://'); + } +} + +registerFilterClass(FilterHTTPJustOrigin); + +/******************************************************************************/ + +class FilterPlainTrie { + static match(idata) { + if ( bidiTrie.matches(filterData[idata+1], $tokenBeg) !== 0 ) { + filterData[idata+2] = bidiTrie.$iu; + return true; + } + return false; + } + + static create() { + const idata = filterDataAllocLen(3); + filterData[idata+0] = FilterPlainTrie.fid; // fid + filterData[idata+1] = bidiTrie.createTrie(); // itrie + filterData[idata+2] = 0; // matchedUnit + return idata; + } + + static addUnitToTrie(idata, iunit) { + const trieDetails = filterToBidiTrie(iunit); + const itrie = filterData[idata+1]; + const id = bidiTrie.add( + itrie, + trieDetails.i, + trieDetails.n, + trieDetails.itok + ); + // No point storing a pattern with conditions if the bidi-trie already + // contain a pattern with no conditions. + const ix = bidiTrie.getExtra(id); + if ( ix === 1 ) { return; } + // If the newly stored pattern has no condition, short-circuit existing + // ones since they will always be short-circuited by the condition-less + // pattern. + const fc = filterGetClass(iunit); + if ( fc.isPatternPlain ) { + bidiTrie.setExtra(id, 1); + return; + } + // FilterCompositeAll is assumed here, i.e. with conditions. + if ( fc === FilterCompositeAll && fc.getCount(iunit) === 1 ) { + iunit = filterData[filterData[iunit+1]+0]; + } + bidiTrie.setExtra(id, filterSequenceAdd(iunit, ix)); + } + + static logData(idata, details) { + const s = $requestURL.slice(bidiTrie.$l, bidiTrie.$r); + details.pattern.push(s); + details.regex.push(restrFromPlainPattern(s)); + if ( filterData[idata+2] !== -1 ) { + filterLogData(filterData[idata+2], details); + } + } + + static dumpInfo(idata) { + return `${Array.from(bidiTrie.trieIterator(filterData[idata+1])).length}`; + } +} + +registerFilterClass(FilterPlainTrie); + +/******************************************************************************/ + +class FilterBucket extends FilterCollection { + static getCount(idata) { + return filterData[idata+2]; + } + + static forEach(idata, fn) { + return super.forEach(filterData[idata+1], fn); + } + + static match(idata) { + const icollection = filterData[idata+1]; + let iseq = filterData[icollection+1]; + while ( iseq !== 0 ) { + const iunit = filterData[iseq+0]; + if ( filterMatch(iunit) ) { + filterData[idata+3] = iunit; + return true; + } + iseq = filterData[iseq+1]; + } + return false; + } + + static matchAndFetchModifiers(idata, env) { + const icollection = filterData[idata+1]; + let iseq = filterData[icollection+1]; + while ( iseq !== 0 ) { + const iunit = filterData[iseq+0]; + env.iunit = iunit; + filterMatchAndFetchModifiers(iunit, env); + iseq = filterData[iseq+1]; + } + } + + static unshift(idata, iunit) { + super.unshift(filterData[idata+1], iunit); + filterData[idata+2] += 1; + } + + static shift(idata) { + super.shift(filterData[idata+1]); + filterData[idata+2] -= 1; + } + + static create() { + const idata = filterDataAllocLen(4); + filterData[idata+0] = FilterBucket.fid; // fid + filterData[idata+1] = FilterCollection.create(); // icollection + filterData[idata+2] = 0; // n + filterData[idata+3] = 0; // $matchedUnit + return idata; + } + + static logData(idata, details) { + filterLogData(filterData[idata+3], details); + } + + static optimize(idata, optimizeBits = 0b11) { + if ( (optimizeBits & 0b01) !== 0 ) { + if ( filterData[idata+2] >= 3 ) { + const iplaintrie = this.optimizePatternTests(idata); + if ( iplaintrie !== 0 ) { + const icollection = filterData[idata+1]; + const i = filterData[icollection+1]; + if ( i === 0 ) { return iplaintrie; } + this.unshift(idata, iplaintrie); + } + } + } + if ( (optimizeBits & 0b10) !== 0 ) { + if ( filterData[idata+2] >= 5 ) { + const ioptimized = this.optimizeMatch( + idata, + FilterBucketIfOriginHits, + 5 + ); + if ( ioptimized !== 0 ) { + const icollection = filterData[idata+1]; + const i = filterData[icollection+1]; + if ( i === 0 ) { return ioptimized; } + this.unshift(idata, ioptimized); + } + } + if ( filterData[idata+2] >= 5 ) { + const ioptimized = this.optimizeMatch( + idata, + FilterBucketIfRegexHits, + 5 + ); + if ( ioptimized !== 0 ) { + const icollection = filterData[idata+1]; + const i = filterData[icollection+1]; + if ( i === 0 ) { return ioptimized; } + this.unshift(idata, ioptimized); + } + } + } + return 0; + } + + static optimizePatternTests(idata) { + const isrccollection = filterData[idata+1]; + let n = 0; + let iseq = filterData[isrccollection+1]; + do { + if ( filterIsBidiTrieable(filterData[iseq+0]) ) { n += 1; } + iseq = filterData[iseq+1]; + } while ( iseq !== 0 && n < 3 ); + if ( n < 3 ) { return 0; } + const iplaintrie = FilterPlainTrie.create(); + iseq = filterData[isrccollection+1]; + let iprev = 0; + for (;;) { + const iunit = filterData[iseq+0]; + const inext = filterData[iseq+1]; + if ( filterIsBidiTrieable(iunit) ) { + FilterPlainTrie.addUnitToTrie(iplaintrie, iunit); + if ( iprev !== 0 ) { + filterData[iprev+1] = inext; + } else { + filterData[isrccollection+1] = inext; + } + filterData[idata+2] -= 1; + } else { + iprev = iseq; + } + if ( inext === 0 ) { break; } + iseq = inext; + } + return iplaintrie; + } + + static optimizeMatch(idata, fc, min) { + const isrccollection = filterData[idata+1]; + const candidates = []; + this.forEach(idata, iunit => { + if ( fc.canCoalesce(iunit) === false ) { return; } + candidates.push(iunit); + }); + if ( candidates.length < min ) { return 0; } + const idesbucket = FilterBucket.create(); + const idescollection = filterData[idesbucket+1]; + let coalesced; + let isrcseq = filterData[isrccollection+1]; + let iprev = 0; + for (;;) { + const iunit = filterData[isrcseq+0]; + const inext = filterData[isrcseq+1]; + if ( candidates.includes(iunit) ) { + coalesced = fc.coalesce(iunit, coalesced); + // move the sequence slot to new bucket + filterData[isrcseq+1] = filterData[idescollection+1]; + filterData[idescollection+1] = isrcseq; + filterData[idesbucket+2] += 1; + if ( iprev !== 0 ) { + filterData[iprev+1] = inext; + } else { + filterData[isrccollection+1] = inext; + } + filterData[idata+2] -= 1; + } else { + iprev = isrcseq; + } + if ( inext === 0 ) { break; } + isrcseq = inext; + } + return fc.create(coalesced, idesbucket); + } + + static dumpInfo(idata) { + return this.getCount(idata); + } +} + +registerFilterClass(FilterBucket); + +/******************************************************************************/ + +// Filter bucket objects which have a pre-test method before being treated +// as a plain filter bucket -- the pre-test method should be fast as it is +// used to avoid having to iterate through the content of the filter bucket. + +class FilterBucketIf extends FilterBucket { + static getCount(idata) { + return super.getCount(filterData[idata+1]); + } + + static forEach(idata, fn) { + return super.forEach(filterData[idata+1], fn); + } + + static match(idata) { + return this.preTest(idata) && super.match(filterData[idata+1]); + } + + static matchAndFetchModifiers(idata, env) { + if ( this.preTest(idata) ) { + super.matchAndFetchModifiers(filterData[idata+1], env); + } + } + + static create(fid, ibucket, itest) { + const idata = filterDataAllocLen(3); + filterData[idata+0] = fid; + filterData[idata+1] = ibucket; + filterData[idata+2] = itest; + return idata; + } + + static logData(idata, details) { + filterLogData(filterData[idata+1], details); + } +} + +registerFilterClass(FilterBucketIf); + +/******************************************************************************/ + +class FilterBucketIfOriginHits extends FilterBucketIf { + static preTest(idata) { + return filterMatch(filterData[idata+2]); + } + + static canCoalesce(iunit) { + return filterHasOriginHit(iunit); + } + + static coalesce(iunit, coalesced) { + if ( coalesced === undefined ) { + coalesced = new Set(); + } + const domainOpt = filterGetDomainOpt(iunit); + if ( domainOpt.includes('|') ) { + for ( const hn of domainOptIterator.reset(domainOpt) ) { + coalesced.add(hn); + } + } else { + coalesced.add(domainOpt); + } + return coalesced; + } + + static create(coalesced, ibucket) { + const units = []; + compileFromDomainOpt(coalesced, false, units); + const ihittest = filterFromCompiled(units[0]); + const ipretest = super.create( + FilterBucketIfOriginHits.fid, + ibucket, + ihittest + ); + return ipretest; + } +} + +registerFilterClass(FilterBucketIfOriginHits); + +/******************************************************************************/ + +class FilterBucketIfRegexHits extends FilterBucketIf { + static preTest(idata) { + return filterRefs[filterData[idata+2]].test($requestURLRaw); + } + + static canCoalesce(iunit) { + const fc = filterGetClass(iunit); + if ( fc.hasRegexPattern === undefined ) { return false; } + if ( fc.hasRegexPattern(iunit) !== true ) { return false; } + return true; + } + + static coalesce(iunit, coalesced) { + if ( coalesced === undefined ) { + coalesced = new Set(); + } + coalesced.add(filterGetRegexPattern(iunit)); + return coalesced; + } + + static create(coalesced, ibucket) { + const reString = Array.from(coalesced).join('|'); + return super.create( + FilterBucketIfRegexHits.fid, + ibucket, + filterRefAdd(new RegExp(reString, 'i')) + ); + } + + static dumpInfo(idata) { + return filterRefs[filterData[idata+2]].source; + } +} + +registerFilterClass(FilterBucketIfRegexHits); + +/******************************************************************************/ + +class FilterStrictParty { + // TODO: disregard `www.`? + static match(idata) { + return ($requestHostname === $docHostname) === (filterData[idata+1] === 0); + } + + static compile(details) { + return [ + FilterStrictParty.fid, + details.strictParty > 0 ? 0 : 1 + ]; + } + + static fromCompiled(args) { + return filterDataAlloc( + args[0], // fid + args[1] + ); + } + + static dnrFromCompiled(args, rule) { + const partyness = args[1] === 0 ? 1 : 3; + dnrAddRuleError(rule, `FilterStrictParty: Strict partyness strict${partyness}p not supported`); + } + + static keyFromArgs(args) { + return `${args[1]}`; + } + + static logData(idata, details) { + details.options.push( + filterData[idata+1] === 0 ? 'strict1p' : 'strict3p' + ); + } +} + +registerFilterClass(FilterStrictParty); + +/******************************************************************************/ + +class FilterOnHeaders { + static match(idata) { + const refs = filterRefs[filterData[idata+1]]; + if ( refs.$parsed === null ) { + refs.$parsed = sfp.parseHeaderValue(refs.headerOpt); + } + const { bad, name, not, re, value } = refs.$parsed; + if ( bad ) { return false; } + const headerValue = $httpHeaders.lookup(name); + if ( headerValue === undefined ) { return false; } + if ( value === '' ) { return true; } + return re === undefined + ? (headerValue === value) !== not + : re.test(headerValue) !== not; + } + + static compile(details) { + return [ FilterOnHeaders.fid, details.headerOpt ]; + } + + static fromCompiled(args) { + return filterDataAlloc( + args[0], // fid + filterRefAdd({ + headerOpt: args[1], + $parsed: null, + }) + ); + } + + static logData(idata, details) { + const irefs = filterData[idata+1]; + const headerOpt = filterRefs[irefs].headerOpt; + let opt = 'header'; + if ( headerOpt !== '' ) { + opt += `=${headerOpt}`; + } + details.options.push(opt); + } +} + +registerFilterClass(FilterOnHeaders); + +/******************************************************************************/ +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/2630 +// Slice input URL into a list of safe-integer token values, instead of a list +// of substrings. The assumption is that with dealing only with numeric +// values, less underlying memory allocations, and also as a consequence +// less work for the garbage collector down the road. +// Another assumption is that using a numeric-based key value for Map() is +// more efficient than string-based key value (but that is something I would +// have to benchmark). +// Benchmark for string-based tokens vs. safe-integer token values: +// https://gorhill.github.io/obj-vs-set-vs-map/tokenize-to-str-vs-to-int.html + +// http://www.cse.yorku.ca/~oz/hash.html#djb2 +// Use above algorithm to generate token hash. + +const urlTokenizer = new (class { + constructor() { + this._chars = '0123456789%abcdefghijklmnopqrstuvwxyz'; + this._validTokenChars = new Uint8Array(128); + for ( let i = 0, n = this._chars.length; i < n; i++ ) { + this._validTokenChars[this._chars.charCodeAt(i)] = i + 1; + } + + this._urlIn = ''; + this._urlOut = ''; + this._tokenized = false; + this._hasQuery = 0; + // https://www.reddit.com/r/uBlockOrigin/comments/dzw57l/ + // Remember: 1 token needs two slots + this._tokens = new Uint32Array(2064); + + this.knownTokens = new Uint8Array(65536); + this.resetKnownTokens(); + } + + setURL(url) { + if ( url !== this._urlIn ) { + this._urlIn = url; + this._urlOut = url.toLowerCase(); + this._hasQuery = 0; + this._tokenized = false; + } + return this._urlOut; + } + + resetKnownTokens() { + this.knownTokens.fill(0); + this.addKnownToken(DOT_TOKEN_HASH); + this.addKnownToken(ANY_TOKEN_HASH); + this.addKnownToken(ANY_HTTPS_TOKEN_HASH); + this.addKnownToken(ANY_HTTP_TOKEN_HASH); + this.addKnownToken(NO_TOKEN_HASH); + } + + addKnownToken(th) { + this.knownTokens[th & 0xFFFF] = 1; + } + + // Tokenize on demand. + getTokens(encodeInto) { + if ( this._tokenized ) { return this._tokens; } + let i = this._tokenize(encodeInto); + this._tokens[i+0] = ANY_TOKEN_HASH; + this._tokens[i+1] = 0; + i += 2; + if ( this._urlOut.startsWith('https://') ) { + this._tokens[i+0] = ANY_HTTPS_TOKEN_HASH; + this._tokens[i+1] = 0; + i += 2; + } else if ( this._urlOut.startsWith('http://') ) { + this._tokens[i+0] = ANY_HTTP_TOKEN_HASH; + this._tokens[i+1] = 0; + i += 2; + } + this._tokens[i+0] = NO_TOKEN_HASH; + this._tokens[i+1] = 0; + this._tokens[i+2] = INVALID_TOKEN_HASH; + this._tokenized = true; + return this._tokens; + } + + hasQuery() { + if ( this._hasQuery === 0 ) { + const i = this._urlOut.indexOf('?'); + this._hasQuery = i !== -1 ? i + 1 : -1; + } + return this._hasQuery > 0; + } + + // http://www.cse.yorku.ca/~oz/hash.html#djb2 + + tokenHashFromString(s) { + const l = s.length; + if ( l === 0 ) { return EMPTY_TOKEN_HASH; } + const vtc = this._validTokenChars; + let th = vtc[s.charCodeAt(0)]; + for ( let i = 1; i !== 7 /* MAX_TOKEN_LENGTH */ && i !== l; i++ ) { + th = (th << 5) + th ^ vtc[s.charCodeAt(i)]; + } + return th & 0xFFFFFFF; + } + + stringFromTokenHash(th) { + if ( th === 0 ) { return ''; } + return th.toString(16); + } + + toSelfie() { + return sparseBase64.encode( + this.knownTokens.buffer, + this.knownTokens.byteLength + ); + } + + fromSelfie(selfie) { + return sparseBase64.decode(selfie, this.knownTokens.buffer); + } + + // https://github.com/chrisaljoudi/uBlock/issues/1118 + // We limit to a maximum number of tokens. + + _tokenize(encodeInto) { + const tokens = this._tokens; + let url = this._urlOut; + let l = url.length; + if ( l === 0 ) { return 0; } + if ( l > 2048 ) { + url = url.slice(0, 2048); + l = 2048; + } + encodeInto.haystackLen = l; + let j = 0; + let hasq = -1; + mainLoop: { + const knownTokens = this.knownTokens; + const vtc = this._validTokenChars; + const charCodes = encodeInto.haystack; + let i = 0, n = 0, ti = 0, th = 0; + for (;;) { + for (;;) { + if ( i === l ) { break mainLoop; } + const cc = url.charCodeAt(i); + charCodes[i] = cc; + i += 1; + th = vtc[cc]; + if ( th !== 0 ) { break; } + if ( cc === 0x3F /* '?' */ ) { hasq = i; } + } + ti = i - 1; n = 1; + for (;;) { + if ( i === l ) { break; } + const cc = url.charCodeAt(i); + charCodes[i] = cc; + i += 1; + const v = vtc[cc]; + if ( v === 0 ) { + if ( cc === 0x3F /* '?' */ ) { hasq = i; } + break; + } + if ( n === 7 /* MAX_TOKEN_LENGTH */ ) { continue; } + th = (th << 5) + th ^ v; + n += 1; + } + if ( knownTokens[th & 0xFFFF] !== 0 ) { + tokens[j+0] = th & 0xFFFFFFF; + tokens[j+1] = ti; + j += 2; + } + } + } + this._hasQuery = hasq; + return j; + } +})(); + +/******************************************************************************/ +/******************************************************************************/ + +class FilterCompiler { + constructor(other = undefined) { + if ( other !== undefined ) { + return Object.assign(this, other); + } + this.reToken = /[%0-9A-Za-z]+/g; + this.fromDomainOptList = []; + this.toDomainOptList = []; + this.tokenIdToNormalizedType = new Map([ + [ sfp.NODE_TYPE_NET_OPTION_NAME_CNAME, bitFromType('cname') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_CSS, bitFromType('stylesheet') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_DOC, bitFromType('main_frame') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_FONT, bitFromType('font') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_FRAME, bitFromType('sub_frame') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK, bitFromType('unsupported') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE, bitFromType('generichide') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE, bitFromType('image') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT, bitFromType('inline-font') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT, bitFromType('inline-script') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA, bitFromType('media') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT, bitFromType('object') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_OTHER, bitFromType('other') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_PING, bitFromType('ping') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER, bitFromType('popunder') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_POPUP, bitFromType('popup') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT, bitFromType('script') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE, bitFromType('specifichide') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_XHR, bitFromType('xmlhttprequest') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC, bitFromType('unsupported') ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET, bitFromType('websocket') ], + ]); + this.modifierIdToNormalizedId = new Map([ + [ sfp.NODE_TYPE_NET_OPTION_NAME_CSP, MODIFIER_TYPE_CSP ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_PERMISSIONS, MODIFIER_TYPE_PERMISSIONS ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT, MODIFIER_TYPE_REDIRECT ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE, MODIFIER_TYPE_REDIRECTRULE ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM, MODIFIER_TYPE_URLTRANSFORM ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE, MODIFIER_TYPE_REPLACE ], + ]); + // These top 100 "bad tokens" are collated using the "miss" histogram + // from tokenHistograms(). The "score" is their occurrence among the + // 200K+ URLs used in the benchmark and executed against default + // filter lists. + this.badTokens = new Map([ + [ 'https',123617 ], + [ 'com',76987 ], + [ 'js',43620 ], + [ 'www',33129 ], + [ 'jpg',32221 ], + [ 'images',31812 ], + [ 'css',19715 ], + [ 'png',19140 ], + [ 'static',15724 ], + [ 'net',15239 ], + [ 'de',13155 ], + [ 'img',11109 ], + [ 'assets',10746 ], + [ 'min',7807 ], + [ 'cdn',7568 ], + [ 'content',6900 ], + [ 'wp',6444 ], + [ 'fonts',6095 ], + [ 'svg',5976 ], + [ 'http',5813 ], + [ 'ssl',5735 ], + [ 'amazon',5440 ], + [ 'ru',5427 ], + [ 'fr',5199 ], + [ 'facebook',5178 ], + [ 'en',5146 ], + [ 'image',5028 ], + [ 'html',4837 ], + [ 'media',4833 ], + [ 'co',4783 ], + [ 'php',3972 ], + [ '2019',3943 ], + [ 'org',3924 ], + [ 'jquery',3531 ], + [ '02',3438 ], + [ 'api',3382 ], + [ 'gif',3350 ], + [ 'eu',3322 ], + [ 'prod',3289 ], + [ 'woff2',3200 ], + [ 'logo',3194 ], + [ 'themes',3107 ], + [ 'icon',3048 ], + [ 'google',3026 ], + [ 'v1',3019 ], + [ 'uploads',2963 ], + [ 'googleapis',2860 ], + [ 'v3',2816 ], + [ 'tv',2762 ], + [ 'icons',2748 ], + [ 'core',2601 ], + [ 'gstatic',2581 ], + [ 'ac',2509 ], + [ 'utag',2466 ], + [ 'id',2459 ], + [ 'ver',2448 ], + [ 'rsrc',2387 ], + [ 'files',2361 ], + [ 'uk',2357 ], + [ 'us',2271 ], + [ 'pl',2262 ], + [ 'common',2205 ], + [ 'public',2076 ], + [ '01',2016 ], + [ 'na',1957 ], + [ 'v2',1954 ], + [ '12',1914 ], + [ 'thumb',1895 ], + [ 'web',1853 ], + [ 'ui',1841 ], + [ 'default',1825 ], + [ 'main',1737 ], + [ 'false',1715 ], + [ '2018',1697 ], + [ 'embed',1639 ], + [ 'player',1634 ], + [ 'dist',1599 ], + [ 'woff',1593 ], + [ 'global',1593 ], + [ 'json',1572 ], + [ '11',1566 ], + [ '600',1559 ], + [ 'app',1556 ], + [ 'styles',1533 ], + [ 'plugins',1526 ], + [ '274',1512 ], + [ 'random',1505 ], + [ 'sites',1505 ], + [ 'imasdk',1501 ], + [ 'bridge3',1501 ], + [ 'news',1496 ], + [ 'width',1494 ], + [ 'thumbs',1485 ], + [ 'ttf',1470 ], + [ 'ajax',1463 ], + [ 'user',1454 ], + [ 'scripts',1446 ], + [ 'twitter',1440 ], + [ 'crop',1431 ], + [ 'new',1412], + ]); + this.reset(); + } + + reset() { + this.action = BLOCK_REALM; + // anchor: bit vector + // 0000 (0x0): no anchoring + // 0001 (0x1): anchored to the end of the URL. + // 0010 (0x2): anchored to the start of the URL. + // 0011 (0x3): anchored to the start and end of the URL. + // 0100 (0x4): anchored to the hostname of the URL. + // 0101 (0x5): anchored to the hostname and end of the URL. + this.anchor = 0; + this.badFilter = false; + this.error = undefined; + this.modifyType = undefined; + this.modifyValue = undefined; + this.pattern = ''; + this.patternMatchCase = false; + this.party = ANYPARTY_REALM; + this.optionUnitBits = 0; + this.fromDomainOpt = ''; + this.toDomainOpt = ''; + this.denyallowOpt = ''; + this.headerOpt = undefined; + this.isPureHostname = false; + this.isGeneric = false; + this.isRegex = false; + this.strictParty = 0; + this.token = '*'; + this.tokenHash = NO_TOKEN_HASH; + this.tokenBeg = 0; + this.typeBits = 0; + this.notTypeBits = 0; + this.methodBits = 0; + this.notMethodBits = 0; + this.wildcardPos = -1; + this.caretPos = -1; + return this; + } + + start(/* writer */) { + } + + finish(/* writer */) { + } + + clone() { + return new FilterCompiler(this); + } + + normalizeRegexSource(s) { + try { + const re = new RegExp(s); + return re.source; + } catch (ex) { + } + return ''; + } + + processMethodOption(value) { + for ( const method of value.split('|') ) { + if ( method.charCodeAt(0) === 0x7E /* '~' */ ) { + const bit = FilteringContext.getMethod(method.slice(1)) || 0; + if ( bit === 0 ) { continue; } + this.notMethodBits |= bit; + } else { + const bit = FilteringContext.getMethod(method) || 0; + if ( bit === 0 ) { continue; } + this.methodBits |= bit; + } + } + this.methodBits &= ~this.notMethodBits; + } + + // https://github.com/chrisaljoudi/uBlock/issues/589 + // Be ready to handle multiple negated types + + processTypeOption(id, not) { + if ( id !== -1 ) { + const typeBit = this.tokenIdToNormalizedType.get(id); + if ( not ) { + this.notTypeBits |= typeBit; + } else { + this.typeBits |= typeBit; + } + return; + } + // `all` option + if ( not ) { + this.notTypeBits |= allTypesBits; + } else { + this.typeBits |= allTypesBits; + } + } + + processPartyOption(firstParty, not) { + if ( not ) { + firstParty = !firstParty; + } + this.party |= firstParty ? FIRSTPARTY_REALM : THIRDPARTY_REALM; + } + + processHostnameList(iter, out = []) { + let i = 0; + for ( const { hn, not, bad } of iter ) { + if ( bad ) { return ''; } + out[i] = not ? `~${hn}` : hn; + i += 1; + } + out.length = i; + return i === 1 ? out[0] : out.join('|'); + } + + processModifierOption(modifier, value) { + if ( this.modifyType !== undefined ) { return false; } + const normalized = this.modifierIdToNormalizedId.get(modifier); + if ( normalized === undefined ) { return false; } + this.modifyType = normalized; + this.modifyValue = value || ''; + return true; + } + + processCspOption(value) { + this.modifyType = MODIFIER_TYPE_CSP; + this.modifyValue = value || ''; + this.optionUnitBits |= MODIFY_BIT; + return true; + } + + processOptionWithValue(parser, id) { + switch ( id ) { + case sfp.NODE_TYPE_NET_OPTION_NAME_CSP: + if ( this.processCspOption(parser.getNetOptionValue(id)) === false ) { return false; } + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW: + this.denyallowOpt = this.processHostnameList( + parser.getNetFilterDenyallowOptionIterator(), + ); + if ( this.denyallowOpt === '' ) { return false; } + this.optionUnitBits |= DENYALLOW_BIT; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_FROM: + this.fromDomainOpt = this.processHostnameList( + parser.getNetFilterFromOptionIterator(), + this.fromDomainOptList + ); + if ( this.fromDomainOpt === '' ) { return false; } + this.optionUnitBits |= FROM_BIT; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER: { + this.headerOpt = parser.getNetOptionValue(id) || ''; + this.optionUnitBits |= HEADER_BIT; + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD: + this.processMethodOption(parser.getNetOptionValue(id)); + this.optionUnitBits |= METHOD_BIT; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_PERMISSIONS: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: + case sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE: + case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) { + return false; + } + this.optionUnitBits |= MODIFY_BIT; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT: { + const actualId = this.action === ALLOW_REALM + ? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE + : id; + if ( this.processModifierOption(actualId, parser.getNetOptionValue(id)) === false ) { + return false; + } + this.optionUnitBits |= MODIFY_BIT; + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_TO: + this.toDomainOpt = this.processHostnameList( + parser.getNetFilterToOptionIterator(), + this.toDomainOptList + ); + if ( this.toDomainOpt === '' ) { return false; } + this.optionUnitBits |= TO_BIT; + break; + default: + break; + } + return true; + } + + process(parser) { + // important! + this.reset(); + + if ( parser.hasError() ) { + return this.FILTER_INVALID; + } + + if ( parser.isException() ) { + this.action = ALLOW_REALM; + } + + if ( parser.isLeftHnAnchored() ) { + this.anchor |= 0b100; + } else if ( parser.isLeftAnchored() ) { + this.anchor |= 0b010; + } + if ( parser.isRightAnchored() ) { + this.anchor |= 0b001; + } + + this.pattern = parser.getNetPattern(); + if ( parser.isHostnamePattern() ) { + this.isPureHostname = true; + } else if ( parser.isGenericPattern() ) { + this.isGeneric = true; + } else if ( parser.isRegexPattern() ) { + this.isRegex = true; + } + + for ( const type of parser.getNodeTypes() ) { + switch ( type ) { + case sfp.NODE_TYPE_NET_OPTION_NAME_1P: + this.processPartyOption(true, parser.isNegatedOption(type)); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P: + this.strictParty = this.strictParty === -1 ? 0 : 1; + this.optionUnitBits |= STRICT_PARTY_BIT; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_3P: + this.processPartyOption(false, parser.isNegatedOption(type)); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P: + this.strictParty = this.strictParty === 1 ? 0 : -1; + this.optionUnitBits |= STRICT_PARTY_BIT; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_ALL: + this.processTypeOption(-1); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER: + this.badFilter = true; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME: + case sfp.NODE_TYPE_NET_OPTION_NAME_CSS: + case sfp.NODE_TYPE_NET_OPTION_NAME_DOC: + case sfp.NODE_TYPE_NET_OPTION_NAME_FONT: + case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME: + case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK: + case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE: + case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT: + case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT: + case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA: + case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT: + case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER: + case sfp.NODE_TYPE_NET_OPTION_NAME_PING: + case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER: + case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP: + case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT: + case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_XHR: + case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC: + case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET: + this.processTypeOption(type, parser.isNegatedOption(type)); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_CSP: + case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW: + case sfp.NODE_TYPE_NET_OPTION_NAME_FROM: + case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER: + case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD: + case sfp.NODE_TYPE_NET_OPTION_NAME_PERMISSIONS: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: + case sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE: + case sfp.NODE_TYPE_NET_OPTION_NAME_TO: + case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + if ( this.processOptionWithValue(parser, type) === false ) { + return this.FILTER_INVALID; + } + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE: { + const not = parser.isNegatedOption(type); + this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE, not); + this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE, not); + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_EMPTY: { + const id = this.action === ALLOW_REALM + ? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE + : sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT; + if ( this.processModifierOption(id, 'empty') === false ) { + return this.FILTER_INVALID; + } + this.optionUnitBits |= MODIFY_BIT; + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT: + this.optionUnitBits |= IMPORTANT_BIT; + this.action = BLOCKIMPORTANT_REALM; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE: + this.patternMatchCase = true; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_MP4: { + const id = this.action === ALLOW_REALM + ? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE + : sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT; + if ( this.processModifierOption(id, 'noopmp4-1s') === false ) { + return this.FILTER_INVALID; + } + this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA, false); + this.optionUnitBits |= MODIFY_BIT; + break; + } + default: + break; + } + } + + if ( this.party === ALLPARTIES_REALM ) { + this.party = ANYPARTY_REALM; + } + + // Negated network types? Toggle on all network type bits. + // Negated non-network types can only toggle themselves. + // + // https://github.com/gorhill/uBlock/issues/2385 + // Toggle on all network types if: + // - at least one network type is negated; or + // - no network type is present -- i.e. all network types are + // implicitly toggled on + if ( this.notTypeBits !== 0 ) { + if ( (this.typeBits && allNetworkTypesBits) === allNetworkTypesBits ) { + this.typeBits &= ~this.notTypeBits | allNetworkTypesBits; + } else { + this.typeBits &= ~this.notTypeBits; + } + this.optionUnitBits |= NOT_TYPE_BIT; + } + + // CSP/permissions options implicitly apply only to + // document/subdocument. + if ( + this.modifyType === MODIFIER_TYPE_CSP || + this.modifyType === MODIFIER_TYPE_PERMISSIONS + ) { + if ( this.typeBits === 0 ) { + this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_DOC, false); + this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_FRAME, false); + } + } + + // https://github.com/gorhill/uBlock/issues/2283 + // Abort if type is only for unsupported types, otherwise + // toggle off `unsupported` bit. + if ( this.typeBits & unsupportedTypeBit ) { + this.typeBits &= ~unsupportedTypeBit; + if ( this.typeBits === 0 ) { return this.FILTER_UNSUPPORTED; } + } + + // Plain hostname? (from HOSTS file) + if ( this.isPureHostname && parser.hasOptions() === false ) { + this.anchor |= 0b100; + return this.FILTER_OK; + } + + // regex? + if ( this.isRegex ) { + return this.FILTER_OK; + } + + if ( this.isGeneric ) { + this.wildcardPos = this.pattern.indexOf('*'); + this.caretPos = this.pattern.indexOf('^'); + } + + if ( this.pattern.length > 1024 ) { + return this.FILTER_UNSUPPORTED; + } + + return this.FILTER_OK; + } + + // Given a string, find a good token. Tokens which are too generic, + // i.e. very common with a high probability of ending up as a miss, + // are not good. Avoid if possible. This has a significant positive + // impact on performance. + // + // For pattern-less removeparam filters, try to derive a pattern from + // the removeparam value. + + makeToken() { + if ( this.pattern === '*' ) { + if ( this.modifyType !== MODIFIER_TYPE_REMOVEPARAM ) { return; } + return this.extractTokenFromQuerypruneValue(); + } + if ( this.isRegex ) { + return this.extractTokenFromRegex(this.pattern); + } + this.extractTokenFromPattern(this.pattern); + } + + // Note: a one-char token is better than a documented bad token. + extractTokenFromPattern(pattern) { + this.reToken.lastIndex = 0; + let bestMatch = null; + let bestBadness = 0x7FFFFFFF; + for (;;) { + const match = this.reToken.exec(pattern); + if ( match === null ) { break; } + const token = match[0]; + const badness = token.length > 1 ? this.badTokens.get(token) || 0 : 1; + if ( badness >= bestBadness ) { continue; } + if ( match.index > 0 ) { + const c = pattern.charCodeAt(match.index - 1); + if ( c === 0x2A /* '*' */ ) { continue; } + } + if ( token.length < MAX_TOKEN_LENGTH ) { + const lastIndex = this.reToken.lastIndex; + if ( lastIndex < pattern.length ) { + const c = pattern.charCodeAt(lastIndex); + if ( c === 0x2A /* '*' */ ) { continue; } + } + } + bestMatch = match; + if ( badness === 0 ) { break; } + bestBadness = badness; + } + if ( bestMatch !== null ) { + this.token = bestMatch[0]; + this.tokenHash = urlTokenizer.tokenHashFromString(this.token); + this.tokenBeg = bestMatch.index; + } + } + + // https://github.com/gorhill/uBlock/issues/2781 + // For efficiency purpose, try to extract a token from a regex-based + // filter. + // https://github.com/uBlockOrigin/uBlock-issues/issues/1145#issuecomment-657036902 + // Mind `\b` directives: `/\bads\b/` should result in token being `ads`, + // not `bads`. + extractTokenFromRegex(pattern) { + pattern = sfp.utils.regex.toTokenizableStr(pattern); + this.reToken.lastIndex = 0; + let bestToken; + let bestBadness = 0x7FFFFFFF; + for (;;) { + const matches = this.reToken.exec(pattern); + if ( matches === null ) { break; } + const { 0: token, index } = matches; + if ( index === 0 || pattern.charAt(index - 1) === '\x01' ) { + continue; + } + const { lastIndex } = this.reToken; + if ( + token.length < MAX_TOKEN_LENGTH && ( + lastIndex === pattern.length || + pattern.charAt(lastIndex) === '\x01' + ) + ) { + continue; + } + const badness = token.length > 1 + ? this.badTokens.get(token) || 0 + : 1; + if ( badness < bestBadness ) { + bestToken = token; + if ( badness === 0 ) { break; } + bestBadness = badness; + } + } + if ( bestToken !== undefined ) { + this.token = bestToken.toLowerCase(); + this.tokenHash = urlTokenizer.tokenHashFromString(this.token); + } + } + + // https://github.com/uBlockOrigin/uAssets/discussions/14683#discussioncomment-3559284 + // If the removeparam value is a regex, unescape escaped commas + extractTokenFromQuerypruneValue() { + const pattern = this.modifyValue; + if ( pattern === '*' || pattern.charCodeAt(0) === 0x7E /* '~' */ ) { + return; + } + const match = /^\/(.+)\/i?$/.exec(pattern); + if ( match !== null ) { + return this.extractTokenFromRegex( + match[1].replace(/(\{\d*)\\,/, '$1,') + ); + } + if ( pattern.startsWith('|') ) { + return this.extractTokenFromRegex('\\b' + pattern.slice(1)); + } + this.extractTokenFromPattern(pattern.toLowerCase()); + } + + hasNoOptionUnits() { + return this.optionUnitBits === 0; + } + + isJustOrigin() { + if ( this.optionUnitBits !== FROM_BIT ) { return false; } + if ( this.isRegex ) { return false; } + if ( /[\/~]/.test(this.fromDomainOpt) ) { return false; } + if ( this.pattern === '*' ) { return true; } + if ( this.anchor !== 0b010 ) { return false; } + if ( /^(?:http[s*]?:(?:\/\/)?)$/.test(this.pattern) ) { return true; } + return false; + } + + domainIsEntity(s) { + const l = s.length; + return l > 2 && + s.charCodeAt(l-1) === 0x2A /* '*' */ && + s.charCodeAt(l-2) === 0x2E /* '.' */; + } + + compile(parser, writer) { + const r = this.process(parser); + + // Ignore non-static network filters + if ( r === this.FILTER_INVALID ) { return false; } + + // Ignore filters with unsupported options + if ( r === this.FILTER_UNSUPPORTED ) { + const who = writer.properties.get('name') || '?'; + this.error = `Invalid network filter in ${who}: ${parser.raw}`; + return false; + } + + writer.select( + this.badFilter + ? 'NETWORK_FILTERS:BAD' + : 'NETWORK_FILTERS:GOOD' + ); + + // Reminder: + // `redirect=` is a combination of a `redirect-rule` filter and a + // block filter. + if ( this.modifyType === MODIFIER_TYPE_REDIRECT ) { + this.modifyType = MODIFIER_TYPE_REDIRECTRULE; + // Do not generate block rule when compiling to DNR ruleset + if ( parser.options.toDNR !== true ) { + const parsedBlock = this.clone(); + parsedBlock.modifyType = undefined; + parsedBlock.optionUnitBits &= ~MODIFY_BIT; + parsedBlock.compileToFilter(writer); + } + } + + this.compileToFilter(writer); + + return true; + } + + compileToFilter(writer) { + // Pure hostnames, use more efficient dictionary lookup + if ( this.isPureHostname && this.hasNoOptionUnits() ) { + this.tokenHash = DOT_TOKEN_HASH; + this.compileToAtomicFilter(this.pattern, writer); + return; + } + + this.makeToken(); + + // Special pattern/option cases: + // - `*$domain=...` + // - `|http://$domain=...` + // - `|https://$domain=...` + // The semantic of "just-origin" filters is that contrary to normal + // filters, the original filter is split into as many filters as there + // are entries in the `domain=` option. + if ( this.isJustOrigin() ) { + if ( this.pattern === '*' || this.pattern.startsWith('http*') ) { + this.tokenHash = ANY_TOKEN_HASH; + } else if /* 'https:' */ ( this.pattern.startsWith('https') ) { + this.tokenHash = ANY_HTTPS_TOKEN_HASH; + } else /* 'http:' */ { + this.tokenHash = ANY_HTTP_TOKEN_HASH; + } + for ( const hn of this.fromDomainOptList ) { + this.compileToAtomicFilter(hn, writer); + } + return; + } + + const units = []; + + // Pattern + const patternClass = this.compilePattern(units); + + // Anchor: must never appear before pattern unit + if ( (this.anchor & 0b100) !== 0 ) { + if ( this.isPureHostname ) { + units.push(FilterAnchorHn.compile()); + } else { + units.push(FilterAnchorHnLeft.compile()); + } + } else if ( (this.anchor & 0b010) !== 0 ) { + units.push(FilterAnchorLeft.compile()); + } + if ( (this.anchor & 0b001) !== 0 ) { + units.push(FilterAnchorRight.compile()); + } + + // Method(s) + if ( this.methodBits !== 0 || this.notMethodBits !== 0 ) { + units.push(FilterMethod.compile(this)); + } + + // Not types + if ( this.notTypeBits !== 0 ) { + units.push(FilterNotType.compile(this)); + } + + // Strict partiness + if ( this.strictParty !== 0 ) { + units.push(FilterStrictParty.compile(this)); + } + + // Origin + if ( this.fromDomainOpt !== '' ) { + compileFromDomainOpt( + this.fromDomainOptList, + units.length !== 0 && patternClass.isSlow === true, + units + ); + } + + // Destination + if ( this.toDomainOpt !== '' ) { + compileToDomainOpt( + this.toDomainOptList, + units.length !== 0 && patternClass.isSlow === true, + units + ); + } + + // Deny-allow + if ( this.denyallowOpt !== '' ) { + units.push(FilterDenyAllow.compile(this)); + } + + // Header + if ( this.headerOpt !== undefined ) { + units.push(FilterOnHeaders.compile(this)); + this.action |= HEADERS_REALM; + } + + // Important + // + // IMPORTANT: must always appear at the end of the sequence, so as to + // ensure $isBlockImportant is set only for matching filters. + if ( (this.optionUnitBits & IMPORTANT_BIT) !== 0 ) { + units.push(FilterImportant.compile()); + } + + // Modifier + // + // IMPORTANT: the modifier unit MUST always appear first in a sequence + if ( this.modifyType !== undefined ) { + units.unshift(FilterModifier.compile(this)); + this.action = (this.action & ~ActionBitsMask) | + modifierBitsFromType.get(this.modifyType); + } + + this.compileToAtomicFilter( + units.length === 1 + ? units[0] + : FilterCompositeAll.compile(units), + writer + ); + } + + compilePattern(units) { + if ( this.isRegex ) { + units.push(FilterRegex.compile(this)); + return FilterRegex; + } + if ( this.pattern === '*' ) { + units.push(FilterPatternAny.compile()); + return FilterPatternAny; + } + if ( this.tokenHash === NO_TOKEN_HASH ) { + units.push(FilterPatternGeneric.compile(this)); + return FilterPatternGeneric; + } + if ( this.wildcardPos === -1 ) { + if ( this.caretPos === -1 ) { + units.push(FilterPatternPlain.compile(this)); + return FilterPatternPlain; + } + if ( this.caretPos === (this.pattern.length - 1) ) { + this.pattern = this.pattern.slice(0, -1); + units.push(FilterPatternPlain.compile(this)); + units.push(FilterTrailingSeparator.compile()); + return FilterPatternPlain; + } + } + units.push(FilterPatternGeneric.compile(this)); + return FilterPatternGeneric; + } + + compileToAtomicFilter(fdata, writer) { + const catBits = this.action | this.party; + let { typeBits } = this; + + // Typeless + if ( typeBits === 0 ) { + writer.push([ catBits, this.tokenHash, fdata ]); + return; + } + // If all network types are set, create a typeless filter. Excluded + // network types are tested at match time, se we act as if they are + // set. + if ( (typeBits & allNetworkTypesBits) === allNetworkTypesBits ) { + writer.push([ catBits, this.tokenHash, fdata ]); + typeBits &= ~allNetworkTypesBits; + if ( typeBits === 0 ) { return; } + } + // One filter per specific types + let bitOffset = 1; + do { + if ( typeBits & 1 ) { + writer.push([ + catBits | (bitOffset << TypeBitsOffset), + this.tokenHash, + fdata + ]); + } + bitOffset += 1; + typeBits >>>= 1; + } while ( typeBits !== 0 ); + } +} + +// These are to quickly test whether a filter is composite +const FROM_BIT = 0b000000001; +const TO_BIT = 0b000000010; +const DENYALLOW_BIT = 0b000000100; +const HEADER_BIT = 0b000001000; +const STRICT_PARTY_BIT = 0b000010000; +const MODIFY_BIT = 0b000100000; +const NOT_TYPE_BIT = 0b001000000; +const IMPORTANT_BIT = 0b010000000; +const METHOD_BIT = 0b100000000; + +FilterCompiler.prototype.FILTER_OK = 0; +FilterCompiler.prototype.FILTER_INVALID = 1; +FilterCompiler.prototype.FILTER_UNSUPPORTED = 2; + +/******************************************************************************/ +/******************************************************************************/ + +const FilterContainer = function() { + this.compilerVersion = '10'; + this.selfieVersion = '10'; + + this.MAX_TOKEN_LENGTH = MAX_TOKEN_LENGTH; + this.optimizeTaskId = undefined; + // As long as CategoryCount is reasonably low, we will use an array to + // store buckets using category bits as index. If ever CategoryCount + // becomes too large, we can just go back to using a Map. + this.bitsToBucket = new Map(); + this.goodFilters = new Set(); + this.badFilters = new Set(); + this.unitsToOptimize = []; + this.reset(); +}; + +/******************************************************************************/ + +FilterContainer.prototype.prime = function() { + origHNTrieContainer.reset( + keyvalStore.getItem('SNFE.origHNTrieContainer.trieDetails') + ); + destHNTrieContainer.reset( + keyvalStore.getItem('SNFE.destHNTrieContainer.trieDetails') + ); + bidiTriePrime(); +}; + +/******************************************************************************/ + +FilterContainer.prototype.reset = function() { + this.processedFilterCount = 0; + this.acceptedCount = 0; + this.discardedCount = 0; + this.goodFilters.clear(); + this.badFilters.clear(); + this.unitsToOptimize.length = 0; + this.bitsToBucket.clear(); + + urlTokenizer.resetKnownTokens(); + + filterDataReset(); + filterRefsReset(); + origHNTrieContainer.reset(); + destHNTrieContainer.reset(); + bidiTrie.reset(); + filterArgsToUnit.clear(); + + // Cancel potentially pending optimization run. + if ( this.optimizeTaskId !== undefined ) { + dropTask(this.optimizeTaskId); + this.optimizeTaskId = undefined; + } + + this.notReady = false; + + // Runtime registers + this.$catBits = 0; + this.$tokenHash = 0; + this.$filterUnit = 0; +}; + +/******************************************************************************/ + +FilterContainer.prototype.freeze = function() { + const unserialize = CompiledListReader.unserialize; + + for ( const line of this.goodFilters ) { + if ( this.badFilters.has(line) ) { + this.discardedCount += 1; + continue; + } + + const args = unserialize(line); + + const bits = args[0]; + const bucket = this.bitsToBucket.get(bits) || (new Map()); + if ( bucket.size === 0 ) { + this.bitsToBucket.set(bits, bucket); + } + + const tokenHash = args[1]; + const fdata = args[2]; + + let iunit = bucket.get(tokenHash) || 0; + + if ( tokenHash === DOT_TOKEN_HASH ) { + if ( iunit === 0 ) { + iunit = FilterHostnameDict.create(); + bucket.set(DOT_TOKEN_HASH, iunit); + this.unitsToOptimize.push({ bits, tokenHash }); + } + FilterHostnameDict.add(iunit, fdata); + continue; + } + + if ( tokenHash === ANY_TOKEN_HASH ) { + if ( iunit === 0 ) { + iunit = FilterJustOrigin.create(); + bucket.set(ANY_TOKEN_HASH, iunit); + } + FilterJustOrigin.add(iunit, fdata); + continue; + } + + if ( tokenHash === ANY_HTTPS_TOKEN_HASH ) { + if ( iunit === 0 ) { + iunit = FilterHTTPSJustOrigin.create(); + bucket.set(ANY_HTTPS_TOKEN_HASH, iunit); + } + FilterHTTPSJustOrigin.add(iunit, fdata); + continue; + } + + if ( tokenHash === ANY_HTTP_TOKEN_HASH ) { + if ( iunit === 0 ) { + iunit = FilterHTTPJustOrigin.create(); + bucket.set(ANY_HTTP_TOKEN_HASH, iunit); + } + FilterHTTPJustOrigin.add(iunit, fdata); + continue; + } + + urlTokenizer.addKnownToken(tokenHash); + + this.addFilterUnit(bits, tokenHash, filterFromCompiled(fdata)); + + // Add block-important filters to the block realm, so as to avoid + // to unconditionally match against the block-important realm for + // every network request. Block-important filters are quite rare so + // the block-important realm should be checked when and only when + // there is a matched exception filter, which important filters are + // meant to override. + if ( (bits & ActionBitsMask) === BLOCKIMPORTANT_REALM ) { + this.addFilterUnit( + bits & ~IMPORTANT_REALM, + tokenHash, + filterFromCompiled(fdata) + ); + } + } + + this.badFilters.clear(); + this.goodFilters.clear(); + filterArgsToUnit.clear(); + + this.notReady = false; + + // Optimizing is not critical for the static network filtering engine to + // work properly, so defer this until later to allow for reduced delay to + // readiness when no valid selfie is available. + if ( this.optimizeTaskId !== undefined ) { return; } + + this.optimizeTaskId = queueTask(( ) => { + this.optimizeTaskId = undefined; + this.optimize(30); + }, 2000); +}; + +/******************************************************************************/ + +FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { + if ( op === 'begin' ) { + Object.assign(context, { + good: new Set(), + bad: new Set(), + invalid: new Set(), + filterCount: 0, + acceptedFilterCount: 0, + rejectedFilterCount: 0, + }); + return; + } + + if ( op === 'add' ) { + const reader = args[0]; + reader.select('NETWORK_FILTERS:GOOD'); + while ( reader.next() ) { + context.filterCount += 1; + if ( context.good.has(reader.line) === false ) { + context.good.add(reader.line); + } + } + reader.select('NETWORK_FILTERS:BAD'); + while ( reader.next() ) { + context.bad.add(reader.line); + } + return; + } + + if ( op !== 'end' ) { return; } + + const { good, bad } = context; + const unserialize = CompiledListReader.unserialize; + const buckets = new Map(); + + for ( const line of good ) { + if ( bad.has(line) ) { + context.rejectedFilterCount += 1; + continue; + } + context.acceptedFilterCount += 1; + + const args = unserialize(line); + const bits = args[0]; + const tokenHash = args[1]; + const fdata = args[2]; + + if ( buckets.has(bits) === false ) { + buckets.set(bits, new Map()); + } + const bucket = buckets.get(bits); + + switch ( tokenHash ) { + case DOT_TOKEN_HASH: { + if ( bucket.has(DOT_TOKEN_HASH) === false ) { + bucket.set(DOT_TOKEN_HASH, [{ + condition: { + requestDomains: [] + } + }]); + } + const rule = bucket.get(DOT_TOKEN_HASH)[0]; + rule.condition.requestDomains.push(fdata); + break; + } + case ANY_TOKEN_HASH: { + if ( bucket.has(ANY_TOKEN_HASH) === false ) { + bucket.set(ANY_TOKEN_HASH, [{ + condition: { + initiatorDomains: [] + } + }]); + } + const rule = bucket.get(ANY_TOKEN_HASH)[0]; + rule.condition.initiatorDomains.push(fdata); + break; + } + case ANY_HTTPS_TOKEN_HASH: { + if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) { + bucket.set(ANY_HTTPS_TOKEN_HASH, [{ + condition: { + urlFilter: '|https://', + initiatorDomains: [] + } + }]); + } + const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0]; + rule.condition.initiatorDomains.push(fdata); + break; + } + case ANY_HTTP_TOKEN_HASH: { + if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) { + bucket.set(ANY_HTTP_TOKEN_HASH, [{ + condition: { + urlFilter: '|http://', + initiatorDomains: [] + } + }]); + } + const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0]; + rule.condition.initiatorDomains.push(fdata); + break; + } + default: { + if ( bucket.has(EMPTY_TOKEN_HASH) === false ) { + bucket.set(EMPTY_TOKEN_HASH, []); + } + const rule = {}; + dnrRuleFromCompiled(fdata, rule); + bucket.get(EMPTY_TOKEN_HASH).push(rule); + break; + } + } + } + + const realms = new Map([ + [ BLOCK_REALM, 'block' ], + [ ALLOW_REALM, 'allow' ], + [ REDIRECT_REALM, 'redirect' ], + [ REMOVEPARAM_REALM, 'removeparam' ], + [ CSP_REALM, 'csp' ], + [ PERMISSIONS_REALM, 'permissions' ], + [ URLTRANSFORM_REALM, 'uritransform' ], + ]); + const partyness = new Map([ + [ ANYPARTY_REALM, '' ], + [ FIRSTPARTY_REALM, 'firstParty' ], + [ THIRDPARTY_REALM, 'thirdParty' ], + ]); + const types = new Set([ + 'no_type', + 'stylesheet', + 'image', + 'object', + 'script', + 'xmlhttprequest', + 'sub_frame', + 'main_frame', + 'font', + 'media', + 'websocket', + 'ping', + 'other', + ]); + const ruleset = []; + for ( const [ realmBits, realmName ] of realms ) { + for ( const [ partyBits, partyName ] of partyness ) { + for ( const typeName in typeNameToTypeValue ) { + if ( types.has(typeName) === false ) { continue; } + const typeBits = typeNameToTypeValue[typeName]; + const bits = realmBits | partyBits | typeBits; + const bucket = buckets.get(bits); + if ( bucket === undefined ) { continue; } + for ( const rules of bucket.values() ) { + for ( const rule of rules ) { + rule.action = rule.action || {}; + rule.action.type = realmName; + if ( partyName !== '' ) { + rule.condition = rule.condition || {}; + rule.condition.domainType = partyName; + } + if ( typeName !== 'no_type' ) { + rule.condition = rule.condition || {}; + rule.condition.resourceTypes = [ typeName ]; + } + ruleset.push(rule); + } + } + } + } + } + + // Collect generichide filters + const generichideExclusions = []; + { + const bucket = buckets.get(ALLOW_REALM | typeNameToTypeValue['generichide']); + if ( bucket ) { + for ( const rules of bucket.values() ) { + for ( const rule of rules ) { + if ( rule.condition === undefined ) { continue; } + if ( rule.condition.initiatorDomains ) { + generichideExclusions.push(...rule.condition.initiatorDomains); + } else if ( rule.condition.requestDomains ) { + generichideExclusions.push(...rule.condition.requestDomains); + } + } + } + } + } + + // Detect and attempt salvage of rules with entity-based hostnames and/or + // regex-based domains. + const isUnsupportedDomain = hn => hn.endsWith('.*') || hn.startsWith('/'); + for ( const rule of ruleset ) { + if ( rule.condition === undefined ) { continue; } + for ( const prop of [ 'Initiator', 'Request' ] ) { + const hitProp = `${prop.toLowerCase()}Domains`; + if ( Array.isArray(rule.condition[hitProp]) ) { + if ( rule.condition[hitProp].some(hn => isUnsupportedDomain(hn)) ) { + const domains = rule.condition[hitProp].filter( + hn => isUnsupportedDomain(hn) === false + ); + if ( domains.length === 0 ) { + dnrAddRuleError(rule, `Can't salvage rule with unsupported domain= option: ${rule.condition[hitProp].join('|')}`); + } else { + dnrAddRuleWarning(rule, `Salvaged rule by ignoring ${rule.condition[hitProp].length - domains.length} unsupported domain= option: ${rule.condition[hitProp].join('|')}`); + rule.condition[hitProp] = domains; + } + } + } + const missProp = `excluded${prop}Domains`; + if ( Array.isArray(rule.condition[missProp]) ) { + if ( rule.condition[missProp].some(hn => isUnsupportedDomain(hn)) ) { + const domains = rule.condition[missProp].filter( + hn => isUnsupportedDomain(hn) === false + ); + rule.condition[missProp] = + domains.length !== 0 + ? domains + : undefined; + } + } + } + } + + // Patch modifier filters + for ( const rule of ruleset ) { + if ( rule.__modifierType === undefined ) { continue; } + switch ( rule.__modifierType ) { + case 'csp': + rule.action.type = 'modifyHeaders'; + rule.action.responseHeaders = [{ + header: 'content-security-policy', + operation: 'append', + value: rule.__modifierValue, + }]; + if ( rule.__modifierAction === ALLOW_REALM ) { + dnrAddRuleError(rule, `Unsupported csp exception: ${rule.__modifierValue}`); + } + break; + case 'permissions': + rule.action.type = 'modifyHeaders'; + rule.action.responseHeaders = [{ + header: 'permissions-policy', + operation: 'append', + value: rule.__modifierValue.split('|').join(', '), + }]; + if ( rule.__modifierAction === ALLOW_REALM ) { + dnrAddRuleError(rule, `Unsupported permissions exception: ${rule.__modifierValue}`); + } + break; + case 'redirect-rule': { + let priority = rule.priority || 1; + let token = rule.__modifierValue; + if ( token !== '' ) { + const match = /:(\d+)$/.exec(token); + if ( match !== null ) { + priority += parseInt(match[1], 10); + token = token.slice(0, match.index); + } + } + const resource = context.extensionPaths.get(token); + if ( rule.__modifierValue !== '' && resource === undefined ) { + dnrAddRuleError(rule, `Unpatchable redirect filter: ${rule.__modifierValue}`); + } + if ( rule.__modifierAction !== ALLOW_REALM ) { + const extensionPath = resource || token; + rule.action.type = 'redirect'; + rule.action.redirect = { extensionPath }; + rule.priority = priority + 1; + } else { + rule.action.type = 'block'; + rule.priority = priority + 2; + } + break; + } + case 'removeparam': + rule.action.type = 'redirect'; + if ( rule.__modifierValue === '|' ) { + rule.__modifierValue = ''; + } + if ( rule.__modifierValue !== '' ) { + rule.action.redirect = { + transform: { + queryTransform: { + removeParams: [ rule.__modifierValue ] + } + } + }; + if ( /^~?\/.+\/$/.test(rule.__modifierValue) ) { + dnrAddRuleError(rule, `Unsupported regex-based removeParam: ${rule.__modifierValue}`); + } + } else { + rule.action.redirect = { + transform: { + query: '' + } + }; + } + if ( rule.condition === undefined ) { + rule.condition = { + }; + } + if ( rule.condition.resourceTypes === undefined ) { + rule.condition.resourceTypes = [ + 'main_frame', + 'sub_frame', + 'xmlhttprequest', + ]; + } + if ( rule.__modifierAction === ALLOW_REALM ) { + dnrAddRuleError(rule, `Unsupported removeparam exception: ${rule.__modifierValue}`); + } + break; + case 'uritransform': { + const path = rule.__modifierValue; + let priority = rule.priority || 1; + if ( rule.__modifierAction !== ALLOW_REALM ) { + const transform = { path }; + rule.action.type = 'redirect'; + rule.action.redirect = { transform }; + rule.priority = priority + 1; + } else { + rule.action.type = 'block'; + rule.priority = priority + 2; + } + break; + } + default: + dnrAddRuleError(rule, `Unsupported modifier ${rule.__modifierType}`); + break; + } + } + + return { + ruleset, + filterCount: context.filterCount, + acceptedFilterCount: context.acceptedFilterCount, + rejectedFilterCount: context.rejectedFilterCount, + generichideExclusions: Array.from(new Set(generichideExclusions)), + }; +}; + +/******************************************************************************/ + +FilterContainer.prototype.addFilterUnit = function( + bits, + tokenHash, + inewunit +) { + const bucket = this.bitsToBucket.get(bits) || (new Map()); + if ( bucket.size === 0 ) { + this.bitsToBucket.set(bits, bucket); + } + const istoredunit = bucket.get(tokenHash) || 0; + if ( istoredunit === 0 ) { + bucket.set(tokenHash, inewunit); + return; + } + if ( filterData[istoredunit+0] === FilterBucket.fid ) { + FilterBucket.unshift(istoredunit, inewunit); + return; + } + const ibucketunit = FilterBucket.create(); + FilterBucket.unshift(ibucketunit, istoredunit); + FilterBucket.unshift(ibucketunit, inewunit); + bucket.set(tokenHash, ibucketunit); + this.unitsToOptimize.push({ bits, tokenHash }); +}; + +/******************************************************************************/ + +FilterContainer.prototype.optimize = function(throttle = 0) { + if ( this.optimizeTaskId !== undefined ) { + dropTask(this.optimizeTaskId); + this.optimizeTaskId = undefined; + } + + const later = throttle => { + this.optimizeTaskId = queueTask(( ) => { + this.optimizeTaskId = undefined; + this.optimize(throttle); + }, 1000); + }; + + const t0 = Date.now(); + while ( this.unitsToOptimize.length !== 0 ) { + const { bits, tokenHash } = this.unitsToOptimize.pop(); + const bucket = this.bitsToBucket.get(bits); + const iunit = bucket.get(tokenHash); + const fc = filterGetClass(iunit); + switch ( fc ) { + case FilterHostnameDict: + FilterHostnameDict.optimize(iunit); + break; + case FilterBucket: { + const optimizeBits = + (tokenHash === NO_TOKEN_HASH) || (bits & MODIFY_REALMS) !== 0 + ? 0b10 + : 0b01; + const inewunit = FilterBucket.optimize(iunit, optimizeBits); + if ( inewunit !== 0 ) { + bucket.set(tokenHash, inewunit); + } + break; + } + default: + break; + } + if ( throttle > 0 && (Date.now() - t0) > 40 ) { + return later(throttle - 1); + } + } + + filterArgsToUnit.clear(); + + // Here we do not optimize origHNTrieContainer because many origin-related + // tries are instantiated on demand. + keyvalStore.setItem( + 'SNFE.destHNTrieContainer.trieDetails', + destHNTrieContainer.optimize() + ); + bidiTrieOptimize(); + filterDataShrink(); +}; + +/******************************************************************************/ + +FilterContainer.prototype.toSelfie = async function(storage, path) { + if ( typeof storage !== 'object' || storage === null ) { return; } + if ( typeof storage.put !== 'function' ) { return; } + + bidiTrieOptimize(true); + keyvalStore.setItem( + 'SNFE.origHNTrieContainer.trieDetails', + origHNTrieContainer.optimize() + ); + + return Promise.all([ + storage.put( + `${path}/destHNTrieContainer`, + destHNTrieContainer.serialize(sparseBase64) + ), + storage.put( + `${path}/origHNTrieContainer`, + origHNTrieContainer.serialize(sparseBase64) + ), + storage.put( + `${path}/bidiTrie`, + bidiTrie.serialize(sparseBase64) + ), + storage.put( + `${path}/filterData`, + filterDataToSelfie() + ), + storage.put( + `${path}/filterRefs`, + filterRefsToSelfie() + ), + storage.put( + `${path}/main`, + JSON.stringify({ + version: this.selfieVersion, + processedFilterCount: this.processedFilterCount, + acceptedCount: this.acceptedCount, + discardedCount: this.discardedCount, + bitsToBucket: Array.from(this.bitsToBucket).map(kv => { + kv[1] = Array.from(kv[1]); + return kv; + }), + urlTokenizer: urlTokenizer.toSelfie(), + }) + ) + ]); +}; + +FilterContainer.prototype.serialize = async function() { + const selfie = []; + const storage = { + put(name, data) { + selfie.push([ name, data ]); + } + }; + await this.toSelfie(storage, ''); + return JSON.stringify(selfie); +}; + +/******************************************************************************/ + +FilterContainer.prototype.fromSelfie = async function(storage, path) { + if ( typeof storage !== 'object' || storage === null ) { return; } + if ( typeof storage.get !== 'function' ) { return; } + + this.reset(); + + this.notReady = true; + + const results = await Promise.all([ + storage.get(`${path}/main`), + storage.get(`${path}/destHNTrieContainer`).then(details => + destHNTrieContainer.unserialize(details.content, sparseBase64) + ), + storage.get(`${path}/origHNTrieContainer`).then(details => + origHNTrieContainer.unserialize(details.content, sparseBase64) + ), + storage.get(`${path}/bidiTrie`).then(details => + bidiTrie.unserialize(details.content, sparseBase64) + ), + storage.get(`${path}/filterData`).then(details => + filterDataFromSelfie(details.content) + ), + storage.get(`${path}/filterRefs`).then(details => + filterRefsFromSelfie(details.content) + ), + ]); + + if ( results.slice(1).every(v => v === true) === false ) { return false; } + + const details = results[0]; + if ( typeof details !== 'object' || details === null ) { return false; } + if ( typeof details.content !== 'string' ) { return false; } + if ( details.content === '' ) { return false; } + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( typeof selfie !== 'object' || selfie === null ) { return false; } + if ( selfie.version !== this.selfieVersion ) { return false; } + this.processedFilterCount = selfie.processedFilterCount; + this.acceptedCount = selfie.acceptedCount; + this.discardedCount = selfie.discardedCount; + this.bitsToBucket = new Map(selfie.bitsToBucket.map(kv => { + kv[1] = new Map(kv[1]); + return kv; + })); + urlTokenizer.fromSelfie(selfie.urlTokenizer); + + // If this point is never reached, it means the internal state is + // unreliable, and the caller is then responsible for resetting the + // engine and populate properly, in which case the `notReady` barrier + // will be properly reset. + + this.notReady = false; + + return true; +}; + +FilterContainer.prototype.unserialize = async function(s) { + const selfie = new Map(JSON.parse(s)); + const storage = { + async get(name) { + return { content: selfie.get(name) }; + } + }; + return this.fromSelfie(storage, ''); +}; + +/******************************************************************************/ + +FilterContainer.prototype.createCompiler = function() { + return new FilterCompiler(); +}; + +/******************************************************************************/ + +FilterContainer.prototype.fromCompiled = function(reader) { + reader.select('NETWORK_FILTERS:GOOD'); + while ( reader.next() ) { + this.acceptedCount += 1; + if ( this.goodFilters.has(reader.line) ) { + this.discardedCount += 1; + } else { + this.goodFilters.add(reader.line); + } + } + + reader.select('NETWORK_FILTERS:BAD'); + while ( reader.next() ) { + this.badFilters.add(reader.line); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.matchAndFetchModifiers = function( + fctxt, + modifierName +) { + if ( this.notReady ) { return; } + + const typeBits = typeNameToTypeValue[fctxt.type] || otherTypeBitValue; + + $requestURL = urlTokenizer.setURL(fctxt.url); + $requestURLRaw = fctxt.url; + $docHostname = fctxt.getDocHostname(); + $docDomain = fctxt.getDocDomain(); + $requestHostname = fctxt.getHostname(); + $requestMethodBit = fctxt.method || 0; + $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + + const modifierType = modifierTypeFromName.get(modifierName); + const modifierBits = modifierBitsFromType.get(modifierType); + + const partyBits = fctxt.is3rdPartyToDoc() ? THIRDPARTY_REALM : FIRSTPARTY_REALM; + + const catBits00 = modifierBits; + const catBits01 = modifierBits | typeBits; + const catBits10 = modifierBits | partyBits; + const catBits11 = modifierBits | typeBits | partyBits; + + const bucket00 = this.bitsToBucket.get(catBits00); + const bucket01 = typeBits !== 0 + ? this.bitsToBucket.get(catBits01) + : undefined; + const bucket10 = partyBits !== 0 + ? this.bitsToBucket.get(catBits10) + : undefined; + const bucket11 = typeBits !== 0 && partyBits !== 0 + ? this.bitsToBucket.get(catBits11) + : undefined; + + if ( + bucket00 === undefined && bucket01 === undefined && + bucket10 === undefined && bucket11 === undefined + ) { + return; + } + + const results = []; + const env = { + type: modifierType || 0, + bits: 0, + th: 0, + iunit: 0, + results, + }; + + const tokenHashes = urlTokenizer.getTokens(bidiTrie); + let i = 0; + let th = 0, iunit = 0; + for (;;) { + th = tokenHashes[i]; + if ( th === INVALID_TOKEN_HASH ) { break; } + env.th = th; + $tokenBeg = tokenHashes[i+1]; + if ( + (bucket00 !== undefined) && + (iunit = bucket00.get(th) || 0) !== 0 + ) { + env.bits = catBits00; env.iunit = iunit; + filterMatchAndFetchModifiers(iunit, env); + } + if ( + (bucket01 !== undefined) && + (iunit = bucket01.get(th) || 0) !== 0 + ) { + env.bits = catBits01; env.iunit = iunit; + filterMatchAndFetchModifiers(iunit, env); + } + if ( + (bucket10 !== undefined) && + (iunit = bucket10.get(th) || 0) !== 0 + ) { + env.bits = catBits10; env.iunit = iunit; + filterMatchAndFetchModifiers(iunit, env); + } + if ( + (bucket11 !== undefined) && + (iunit = bucket11.get(th) || 0) !== 0 + ) { + env.bits = catBits11; env.iunit = iunit; + filterMatchAndFetchModifiers(iunit, env); + } + i += 2; + } + + if ( results.length === 0 ) { return; } + + // One single result is expected to be a common occurrence, and in such + // case there is no need to process exception vs. block, block important + // occurrences. + if ( results.length === 1 ) { + const result = results[0]; + if ( (result.bits & ALLOW_REALM) !== 0 ) { return; } + return [ result ]; + } + + const toAddImportant = new Map(); + const toAdd = new Map(); + const toRemove = new Map(); + + for ( const result of results ) { + const actionBits = result.bits & ActionBitsMask; + const modifyValue = result.value; + if ( actionBits === BLOCKIMPORTANT_REALM ) { + toAddImportant.set(modifyValue, result); + } else if ( actionBits === BLOCK_REALM ) { + toAdd.set(modifyValue, result); + } else { + toRemove.set(modifyValue, result); + } + } + if ( toAddImportant.size === 0 && toAdd.size === 0 ) { return; } + + // Remove entries overridden by important block filters. + if ( toAddImportant.size !== 0 ) { + for ( const key of toAddImportant.keys() ) { + toAdd.delete(key); + toRemove.delete(key); + } + } + + // Exception filters + // + // Remove excepted block filters and unused exception filters. + // + // Special case, except-all: + // - Except-all applies only if there is at least one normal block filters. + // - Except-all does not apply to important block filters. + if ( toRemove.size !== 0 ) { + if ( toRemove.has('') === false ) { + for ( const key of toRemove.keys() ) { + if ( toAdd.has(key) ) { + toAdd.delete(key); + } else { + toRemove.delete(key); + } + } + } + else if ( toAdd.size !== 0 ) { + toAdd.clear(); + if ( toRemove.size !== 1 ) { + const entry = toRemove.get(''); + toRemove.clear(); + toRemove.set('', entry); + } + } else { + toRemove.clear(); + } + } + + if ( + toAdd.size === 0 && + toAddImportant.size === 0 && + toRemove.size === 0 + ) { + return; + } + + const out = Array.from(toAdd.values()); + if ( toAddImportant.size !== 0 ) { + out.push(...toAddImportant.values()); + } + if ( toRemove.size !== 0 ) { + out.push(...toRemove.values()); + } + return out; +}; + +/******************************************************************************/ + +FilterContainer.prototype.realmMatchString = function( + realmBits, + typeBits, + partyBits +) { + if ( this.notReady ) { return false; } + + const exactType = typeBits & 0x80000000; + typeBits &= 0x7FFFFFFF; + + const catBits00 = realmBits; + const catBits01 = realmBits | typeBits; + const catBits10 = realmBits | partyBits; + const catBits11 = realmBits | typeBits | partyBits; + + const bucket00 = exactType === 0 + ? this.bitsToBucket.get(catBits00) + : undefined; + const bucket01 = exactType !== 0 || typeBits !== 0 + ? this.bitsToBucket.get(catBits01) + : undefined; + const bucket10 = exactType === 0 && partyBits !== 0 + ? this.bitsToBucket.get(catBits10) + : undefined; + const bucket11 = (exactType !== 0 || typeBits !== 0) && partyBits !== 0 + ? this.bitsToBucket.get(catBits11) + : undefined; + + if ( + bucket00 === undefined && bucket01 === undefined && + bucket10 === undefined && bucket11 === undefined + ) { + return false; + } + + let catBits = 0, iunit = 0; + + // Pure hostname-based filters + let tokenHash = DOT_TOKEN_HASH; + if ( + (bucket00 !== undefined) && + (iunit = bucket00.get(DOT_TOKEN_HASH) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits00; + } else if ( + (bucket01 !== undefined) && + (iunit = bucket01.get(DOT_TOKEN_HASH) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits01; + } else if ( + (bucket10 !== undefined) && + (iunit = bucket10.get(DOT_TOKEN_HASH) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits10; + } else if ( + (bucket11 !== undefined) && + (iunit = bucket11.get(DOT_TOKEN_HASH) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits11; + } + // Pattern-based filters + else { + const tokenHashes = urlTokenizer.getTokens(bidiTrie); + let i = 0; + for (;;) { + tokenHash = tokenHashes[i]; + if ( tokenHash === INVALID_TOKEN_HASH ) { return false; } + $tokenBeg = tokenHashes[i+1]; + if ( + (bucket00 !== undefined) && + (iunit = bucket00.get(tokenHash) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits00; + break; + } + if ( + (bucket01 !== undefined) && + (iunit = bucket01.get(tokenHash) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits01; + break; + } + if ( + (bucket10 !== undefined) && + (iunit = bucket10.get(tokenHash) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits10; + break; + } + if ( + (bucket11 !== undefined) && + (iunit = bucket11.get(tokenHash) || 0) !== 0 && + (filterMatch(iunit) === true) + ) { + catBits = catBits11; + break; + } + i += 2; + } + } + + this.$catBits = catBits; + this.$tokenHash = tokenHash; + this.$filterUnit = iunit; + return true; +}; + +/******************************************************************************/ + +// Specialized handler + +// https://github.com/gorhill/uBlock/issues/1477 +// Special case: blocking-generichide filter ALWAYS exists, it is implicit -- +// thus we always first check for exception filters, then for important block +// filter if and only if there was a hit on an exception filter. +// https://github.com/gorhill/uBlock/issues/2103 +// User may want to override `generichide` exception filters. +// https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ +// Add support for `specifichide`. + +FilterContainer.prototype.matchRequestReverse = function(type, url) { + const typeBits = typeNameToTypeValue[type] | 0x80000000; + + // Prime tokenizer: we get a normalized URL in return. + $requestURL = urlTokenizer.setURL(url); + $requestURLRaw = url; + $requestMethodBit = 0; + $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $isBlockImportant = false; + this.$filterUnit = 0; + + // These registers will be used by various filters + $docHostname = $requestHostname = hostnameFromNetworkURL(url); + $docDomain = domainFromHostname($docHostname); + + // Exception filters + if ( this.realmMatchString(ALLOW_REALM, typeBits, FIRSTPARTY_REALM) ) { + // Important block filters. + if ( this.realmMatchString(BLOCKIMPORTANT_REALM, typeBits, FIRSTPARTY_REALM) ) { + return 1; + } + return 2; + } + return 0; + +}; + +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/116 +// Some type of requests are exceptional, they need custom handling, +// not the generic handling. +// https://github.com/chrisaljoudi/uBlock/issues/519 +// Use exact type match for anything beyond `other`. Also, be prepared to +// support unknown types. +// https://github.com/uBlockOrigin/uBlock-issues/issues/1501 +// Add support to evaluate allow realm before block realm. + +/** + * Matches a URL string using filtering context. + * @param {FilteringContext} fctxt - The filtering context + * @param {integer} [modifier=0] - A bit vector modifying the behavior of the + * matching algorithm: + * Bit 0: match exact type. + * Bit 1: lookup allow realm regardless of whether there was a match in + * block realm. + * + * @returns {integer} 0=no match, 1=block, 2=allow (exception) + */ +FilterContainer.prototype.matchRequest = function(fctxt, modifiers = 0) { + let typeBits = typeNameToTypeValue[fctxt.type]; + if ( modifiers === 0 ) { + if ( typeBits === undefined ) { + typeBits = otherTypeBitValue; + } else if ( typeBits === 0 || typeBits > otherTypeBitValue ) { + modifiers |= 0b0001; + } + } + if ( (modifiers & 0b0001) !== 0 ) { + if ( typeBits === undefined ) { return 0; } + typeBits |= 0x80000000; + } + + const partyBits = fctxt.is3rdPartyToDoc() ? THIRDPARTY_REALM : FIRSTPARTY_REALM; + + // Prime tokenizer: we get a normalized URL in return. + $requestURL = urlTokenizer.setURL(fctxt.url); + $requestURLRaw = fctxt.url; + this.$filterUnit = 0; + + // These registers will be used by various filters + $docHostname = fctxt.getDocHostname(); + $docDomain = fctxt.getDocDomain(); + $requestHostname = fctxt.getHostname(); + $requestMethodBit = fctxt.method || 0; + $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $isBlockImportant = false; + + // Evaluate block realm before allow realm, and allow realm before + // block-important realm, i.e. by order of likelihood of a match. + const r = this.realmMatchString(BLOCK_REALM, typeBits, partyBits); + if ( r || (modifiers & 0b0010) !== 0 ) { + if ( $isBlockImportant ) { return 1; } + if ( this.realmMatchString(ALLOW_REALM, typeBits, partyBits) ) { + if ( this.realmMatchString(BLOCKIMPORTANT_REALM, typeBits, partyBits) ) { + return 1; + } + return 2; + } + if ( r ) { return 1; } + } + return 0; +}; + +/******************************************************************************/ + +FilterContainer.prototype.matchHeaders = function(fctxt, headers) { + const typeBits = typeNameToTypeValue[fctxt.type] || otherTypeBitValue; + const partyBits = fctxt.is3rdPartyToDoc() ? THIRDPARTY_REALM : FIRSTPARTY_REALM; + + // Prime tokenizer: we get a normalized URL in return. + $requestURL = urlTokenizer.setURL(fctxt.url); + $requestURLRaw = fctxt.url; + this.$filterUnit = 0; + + // These registers will be used by various filters + $docHostname = fctxt.getDocHostname(); + $docDomain = fctxt.getDocDomain(); + $requestHostname = fctxt.getHostname(); + $requestMethodBit = fctxt.method || 0; + $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $httpHeaders.init(headers); + + let r = 0; + if ( this.realmMatchString(HEADERS_REALM | BLOCK_REALM, typeBits, partyBits) ) { + r = 1; + } + if ( r !== 0 && $isBlockImportant !== true ) { + if ( this.realmMatchString(HEADERS_REALM | ALLOW_REALM, typeBits, partyBits) ) { + r = 2; + if ( this.realmMatchString(HEADERS_REALM | BLOCKIMPORTANT_REALM, typeBits, partyBits) ) { + r = 1; + } + } + } + + $httpHeaders.reset(); + + return r; +}; + +/******************************************************************************/ + +FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) { + const directives = this.matchAndFetchModifiers(fctxt, 'redirect-rule'); + // No directive is the most common occurrence. + if ( directives === undefined ) { return; } + const highest = directives.length - 1; + // More than a single directive means more work. + if ( highest !== 0 ) { + directives.sort((a, b) => compareRedirectRequests(redirectEngine, a, b)); + } + // Redirect to highest-ranked directive + const directive = directives[highest]; + if ( (directive.bits & ALLOW_REALM) !== 0 ) { return directives; } + const { token } = parseRedirectRequestValue(directive); + fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token); + if ( fctxt.redirectURL === undefined ) { return; } + return directives; +}; + +FilterContainer.prototype.transformRequest = function(fctxt) { + const directives = this.matchAndFetchModifiers(fctxt, 'uritransform'); + if ( directives === undefined ) { return; } + const directive = directives[directives.length-1]; + if ( (directive.bits & ALLOW_REALM) !== 0 ) { return directives; } + if ( directive.refs instanceof Object === false ) { return; } + const { refs } = directive; + if ( refs.$cache === null ) { + refs.$cache = sfp.parseReplaceValue(refs.value); + } + const cache = refs.$cache; + if ( cache === undefined ) { return; } + const redirectURL = new URL(fctxt.url); + const before = redirectURL.pathname + redirectURL.search; + if ( cache.re.test(before) !== true ) { return; } + const after = before.replace(cache.re, cache.replacement); + if ( after === before ) { return; } + const searchPos = after.includes('?') && after.indexOf('?') || after.length; + redirectURL.pathname = after.slice(0, searchPos); + redirectURL.search = after.slice(searchPos); + fctxt.redirectURL = redirectURL.href; + return directives; +}; + +function parseRedirectRequestValue(directive) { + if ( directive.cache === null ) { + directive.cache = sfp.parseRedirectValue(directive.value); + } + return directive.cache; +} + +function compareRedirectRequests(redirectEngine, a, b) { + const { token: atok, priority: aint, bits: abits } = + parseRedirectRequestValue(a); + if ( redirectEngine.hasToken(atok) === false ) { return -1; } + const { token: btok, priority: bint, bits: bbits } = + parseRedirectRequestValue(b); + if ( redirectEngine.hasToken(btok) === false ) { return 1; } + if ( abits !== bbits ) { + if ( (abits & IMPORTANT_REALM) !== 0 ) { return 1; } + if ( (bbits & IMPORTANT_REALM) !== 0 ) { return -1; } + if ( (abits & ALLOW_REALM) !== 0 ) { return -1; } + if ( (bbits & ALLOW_REALM) !== 0 ) { return 1; } + } + return aint - bint; +} + +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/1626 +// Do not redirect when the number of query parameters does not change. + +FilterContainer.prototype.filterQuery = function(fctxt) { + const directives = this.matchAndFetchModifiers(fctxt, 'removeparam'); + if ( directives === undefined ) { return; } + const url = fctxt.url; + const qpos = url.indexOf('?'); + if ( qpos === -1 ) { return; } + let hpos = url.indexOf('#', qpos + 1); + if ( hpos === -1 ) { hpos = url.length; } + const params = new Map(); + const query = url.slice(qpos + 1, hpos); + for ( let i = 0; i < query.length; ) { + let pos = query.indexOf('&', i); + if ( pos === -1 ) { pos = query.length; } + const kv = query.slice(i, pos); + i = pos + 1; + pos = kv.indexOf('='); + if ( pos !== -1 ) { + params.set(kv.slice(0, pos), kv.slice(pos + 1)); + } else { + params.set(kv, ''); + } + } + const inParamCount = params.size; + const out = []; + for ( const directive of directives ) { + if ( params.size === 0 ) { break; } + const isException = (directive.bits & ALLOW_REALM) !== 0; + if ( isException && directive.value === '' ) { + out.push(directive); + break; + } + const { all, bad, name, not, re } = parseQueryPruneValue(directive); + if ( bad ) { continue; } + if ( all ) { + if ( isException === false ) { params.clear(); } + out.push(directive); + break; + } + if ( name !== undefined ) { + const value = params.get(name); + if ( not === false ) { + if ( value !== undefined ) { + if ( isException === false ) { params.delete(name); } + out.push(directive); + } + continue; + } + if ( value !== undefined ) { params.delete(name); } + if ( params.size !== 0 ) { + if ( isException === false ) { params.clear(); } + out.push(directive); + } + if ( value !== undefined ) { params.set(name, value); } + continue; + } + if ( re === undefined ) { continue; } + let filtered = false; + for ( const [ key, raw ] of params ) { + let value = raw; + try { value = decodeURIComponent(value); } + catch(ex) { } + if ( re.test(`${key}=${value}`) === not ) { continue; } + if ( isException === false ) { params.delete(key); } + filtered = true; + } + if ( filtered ) { + out.push(directive); + } + } + if ( out.length === 0 ) { return; } + if ( params.size !== inParamCount ) { + fctxt.redirectURL = url.slice(0, qpos); + if ( params.size !== 0 ) { + fctxt.redirectURL += '?' + Array.from(params).map(a => + a[1] === '' ? a[0] : `${a[0]}=${a[1]}` + ).join('&'); + } + if ( hpos !== url.length ) { + fctxt.redirectURL += url.slice(hpos); + } + } + return out; +}; + +function parseQueryPruneValue(directive) { + if ( directive.cache === null ) { + directive.cache = + sfp.parseQueryPruneValue(directive.value); + } + return directive.cache; +} + +/******************************************************************************/ + +FilterContainer.prototype.hasQuery = function(fctxt) { + urlTokenizer.setURL(fctxt.url); + return urlTokenizer.hasQuery(); +}; + +/******************************************************************************/ + +FilterContainer.prototype.toLogData = function() { + if ( this.$filterUnit !== 0 ) { + return new LogData(this.$catBits, this.$tokenHash, this.$filterUnit); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.isBlockImportant = function() { + return this.$filterUnit !== 0 && $isBlockImportant; +}; + +/******************************************************************************/ + +FilterContainer.prototype.getFilterCount = function() { + return this.acceptedCount - this.discardedCount; +}; + +/******************************************************************************/ + +FilterContainer.prototype.enableWASM = function(wasmModuleFetcher, path) { + return Promise.all([ + bidiTrie.enableWASM(wasmModuleFetcher, path), + origHNTrieContainer.enableWASM(wasmModuleFetcher, path), + destHNTrieContainer.enableWASM(wasmModuleFetcher, path), + ]).then(results => { + return results.every(a => a === true); + }); +}; + +/******************************************************************************/ + +FilterContainer.prototype.test = async function(docURL, type, url) { + const fctxt = new FilteringContext(); + fctxt.setDocOriginFromURL(docURL); + fctxt.setType(type); + fctxt.setURL(url); + const r = this.matchRequest(fctxt); + console.info(`${r}`); + if ( r !== 0 ) { + console.info(this.toLogData()); + } +}; + +/******************************************************************************/ + +FilterContainer.prototype.bucketHistogram = function() { + const results = []; + for ( const [ bits, bucket ] of this.bitsToBucket ) { + for ( const [ th, iunit ] of bucket ) { + const token = urlTokenizer.stringFromTokenHash(th); + const fc = filterGetClass(iunit); + const count = fc.getCount !== undefined ? fc.getCount(iunit) : 1; + results.push({ bits: bits.toString(16), token, count, f: fc.name }); + } + } + results.sort((a, b) => { + return b.count - a.count; + }); + console.info(results); +}; + +/******************************************************************************/ + +// Dump the internal state of the filtering engine to the console. +// Useful to make development decisions and investigate issues. + +FilterContainer.prototype.dump = function() { + const thConstants = new Map([ + [ NO_TOKEN_HASH, 'NO_TOKEN_HASH' ], + [ DOT_TOKEN_HASH, 'DOT_TOKEN_HASH' ], + [ ANY_TOKEN_HASH, 'ANY_TOKEN_HASH' ], + [ ANY_HTTPS_TOKEN_HASH, 'ANY_HTTPS_TOKEN_HASH' ], + [ ANY_HTTP_TOKEN_HASH, 'ANY_HTTP_TOKEN_HASH' ], + [ EMPTY_TOKEN_HASH, 'EMPTY_TOKEN_HASH' ], + ]); + + const out = []; + + const toOutput = (depth, line) => { + out.push(`${' '.repeat(depth*2)}${line}`); + }; + + const dumpUnit = (idata, depth = 0) => { + const fc = filterGetClass(idata); + fcCounts.set(fc.name, (fcCounts.get(fc.name) || 0) + 1); + const info = filterDumpInfo(idata) || ''; + toOutput(depth, info !== '' ? `${fc.name}: ${info}` : fc.name); + switch ( fc ) { + case FilterBucket: + case FilterCompositeAll: + case FilterDomainHitAny: { + fc.forEach(idata, i => { + dumpUnit(i, depth+1); + }); + break; + } + case FilterBucketIfOriginHits: { + dumpUnit(filterData[idata+2], depth+1); + dumpUnit(filterData[idata+1], depth+1); + break; + } + case FilterBucketIfRegexHits: { + dumpUnit(filterData[idata+1], depth+1); + break; + } + case FilterPlainTrie: { + for ( const details of bidiTrie.trieIterator(filterData[idata+1]) ) { + toOutput(depth+1, details.pattern); + let ix = details.iextra; + if ( ix === 1 ) { continue; } + for (;;) { + if ( ix === 0 ) { break; } + dumpUnit(filterData[ix+0], depth+2); + ix = filterData[ix+1]; + } + } + break; + } + default: + break; + } + }; + + const fcCounts = new Map(); + const thCounts = new Set(); + + const realms = new Map([ + [ BLOCK_REALM, 'block' ], + [ BLOCKIMPORTANT_REALM, 'block-important' ], + [ ALLOW_REALM, 'unblock' ], + [ REDIRECT_REALM, 'redirect' ], + [ REMOVEPARAM_REALM, 'removeparam' ], + [ CSP_REALM, 'csp' ], + [ PERMISSIONS_REALM, 'permissions' ], + [ URLTRANSFORM_REALM, 'uritransform' ], + [ REPLACE_REALM, 'replace' ], + ]); + const partyness = new Map([ + [ ANYPARTY_REALM, 'any-party' ], + [ FIRSTPARTY_REALM, '1st-party' ], + [ THIRDPARTY_REALM, '3rd-party' ], + ]); + for ( const [ realmBits, realmName ] of realms ) { + toOutput(1, `+ realm: ${realmName}`); + for ( const [ partyBits, partyName ] of partyness ) { + toOutput(2, `+ party: ${partyName}`); + const processedTypeBits = new Set(); + for ( const typeName in typeNameToTypeValue ) { + const typeBits = typeNameToTypeValue[typeName]; + if ( processedTypeBits.has(typeBits) ) { continue; } + processedTypeBits.add(typeBits); + const bits = realmBits | partyBits | typeBits; + const bucket = this.bitsToBucket.get(bits); + if ( bucket === undefined ) { continue; } + const thCount = bucket.size; + toOutput(3, `+ type: ${typeName} (${thCount})`); + for ( const [ th, iunit ] of bucket) { + thCounts.add(th); + const ths = thConstants.has(th) + ? thConstants.get(th) + : `0x${th.toString(16)}`; + toOutput(4, `+ th: ${ths}`); + dumpUnit(iunit, 5); + } + } + } + } + + const knownTokens = + urlTokenizer.knownTokens + .reduce((a, b) => b !== 0 ? a+1 : a, 0); + + out.unshift([ + 'Static Network Filtering Engine internals:', + ` Distinct token hashes: ${thCounts.size.toLocaleString('en')}`, + ` Known-token sieve (Uint8Array): ${knownTokens.toLocaleString('en')} out of 65,536`, + ` Filter data (Int32Array): ${filterDataWritePtr.toLocaleString('en')}`, + ` Filter refs (JS array): ${filterRefsWritePtr.toLocaleString('en')}`, + ' Origin trie container:', + origHNTrieContainer.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'), + ' Request trie container:', + destHNTrieContainer.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'), + ' Pattern trie container:', + bidiTrie.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'), + '+ Filter class stats:', + Array.from(fcCounts) + .sort((a, b) => b[1] - a[1]) + .map(a => ` ${a[0]}: ${a[1].toLocaleString('en')}`) + .join('\n'), + '+ Filter tree:', + ].join('\n')); + return out.join('\n'); +}; + +/******************************************************************************/ + +const staticNetFilteringEngine = new FilterContainer(); + +export default staticNetFilteringEngine; diff --git a/src/js/storage.js b/src/js/storage.js new file mode 100644 index 0000000..151717c --- /dev/null +++ b/src/js/storage.js @@ -0,0 +1,1703 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; +import punycode from '../lib/punycode.js'; + +import io from './assets.js'; +import { broadcast, filteringBehaviorChanged, onBroadcast } from './broadcast.js'; +import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import logger from './logger.js'; +import lz4Codec from './lz4.js'; +import staticExtFilteringEngine from './static-ext-filtering.js'; +import staticFilteringReverseLookup from './reverselookup.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import µb from './background.js'; +import { hostnameFromURI } from './uri-utils.js'; +import { i18n, i18n$ } from './i18n.js'; +import { redirectEngine } from './redirect-engine.js'; +import { sparseBase64 } from './base64-custom.js'; +import { ubolog, ubologSet } from './console.js'; +import * as sfp from './static-filtering-parser.js'; + +import { + permanentFirewall, + permanentSwitches, + permanentURLFiltering, +} from './filtering-engines.js'; + +import { + CompiledListReader, + CompiledListWriter, +} from './static-filtering-io.js'; + +import { + LineIterator, + orphanizeString, +} from './text-utils.js'; + +/******************************************************************************/ + +µb.getBytesInUse = async function() { + const promises = []; + let bytesInUse; + + // Not all platforms implement this method. + promises.push( + vAPI.storage.getBytesInUse instanceof Function + ? vAPI.storage.getBytesInUse(null) + : undefined + ); + + if ( + navigator.storage instanceof Object && + navigator.storage.estimate instanceof Function + ) { + promises.push(navigator.storage.estimate()); + } + + const results = await Promise.all(promises); + + const processCount = count => { + if ( typeof count !== 'number' ) { return; } + if ( bytesInUse === undefined ) { bytesInUse = 0; } + bytesInUse += count; + return bytesInUse; + }; + + processCount(results[0]); + if ( results.length > 1 && results[1] instanceof Object ) { + processCount(results[1].usage); + } + µb.storageUsed = bytesInUse; + return bytesInUse; +}; + +/******************************************************************************/ + +{ + let localSettingsLastSaved = Date.now(); + + const shouldSave = ( ) => { + if ( µb.localSettingsLastModified > localSettingsLastSaved ) { + µb.saveLocalSettings(); + } + saveTimer.on(saveDelay); + }; + + const saveTimer = vAPI.defer.create(shouldSave); + const saveDelay = { sec: 23 }; + + saveTimer.onidle(saveDelay); + + µb.saveLocalSettings = function() { + localSettingsLastSaved = Date.now(); + return vAPI.storage.set(this.localSettings); + }; +} + +/******************************************************************************/ + +µb.loadUserSettings = async function() { + const usDefault = this.userSettingsDefault; + + const results = await Promise.all([ + vAPI.storage.get(Object.assign(usDefault)), + vAPI.adminStorage.get('userSettings'), + ]); + + const usUser = results[0] instanceof Object && results[0] || + Object.assign(usDefault); + + if ( Array.isArray(results[1]) ) { + const adminSettings = results[1]; + for ( const entry of adminSettings ) { + if ( entry.length < 1 ) { continue; } + const name = entry[0]; + if ( usDefault.hasOwnProperty(name) === false ) { continue; } + const value = entry.length < 2 + ? usDefault[name] + : this.settingValueFromString(usDefault, name, entry[1]); + if ( value === undefined ) { continue; } + usUser[name] = usDefault[name] = value; + } + } + + return usUser; +}; + +µb.saveUserSettings = function() { + // `externalLists` will be deprecated in some future, it is kept around + // for forward compatibility purpose, and should reflect the content of + // `importedLists`. + // + // https://github.com/uBlockOrigin/uBlock-issues/issues/1803 + // Do this before computing modified settings. + this.userSettings.externalLists = + this.userSettings.importedLists.join('\n'); + + const toSave = this.getModifiedSettings( + this.userSettings, + this.userSettingsDefault + ); + + const toRemove = []; + for ( const key in this.userSettings ) { + if ( this.userSettings.hasOwnProperty(key) === false ) { continue; } + if ( toSave.hasOwnProperty(key) ) { continue; } + toRemove.push(key); + } + if ( toRemove.length !== 0 ) { + vAPI.storage.remove(toRemove); + } + vAPI.storage.set(toSave); +}; + +/******************************************************************************/ + +// Admin hidden settings have precedence over user hidden settings. + +µb.loadHiddenSettings = async function() { + const hsDefault = this.hiddenSettingsDefault; + const hsAdmin = this.hiddenSettingsAdmin; + const hsUser = this.hiddenSettings; + + const results = await Promise.all([ + vAPI.adminStorage.get([ + 'advancedSettings', + 'disableDashboard', + 'disabledPopupPanelParts', + ]), + vAPI.storage.get('hiddenSettings'), + ]); + + if ( results[0] instanceof Object ) { + const { + advancedSettings, + disableDashboard, + disabledPopupPanelParts + } = results[0]; + if ( Array.isArray(advancedSettings) ) { + for ( const entry of advancedSettings ) { + if ( entry.length < 1 ) { continue; } + const name = entry[0]; + if ( hsDefault.hasOwnProperty(name) === false ) { continue; } + const value = entry.length < 2 + ? hsDefault[name] + : this.hiddenSettingValueFromString(name, entry[1]); + if ( value === undefined ) { continue; } + hsDefault[name] = hsAdmin[name] = hsUser[name] = value; + } + } + µb.noDashboard = disableDashboard === true; + if ( Array.isArray(disabledPopupPanelParts) ) { + const partNameToBit = new Map([ + [ 'globalStats', 0b00010 ], + [ 'basicTools', 0b00100 ], + [ 'extraTools', 0b01000 ], + [ 'overviewPane', 0b10000 ], + ]); + let bits = hsDefault.popupPanelDisabledSections; + for ( const part of disabledPopupPanelParts ) { + const bit = partNameToBit.get(part); + if ( bit === undefined ) { continue; } + bits |= bit; + } + hsDefault.popupPanelDisabledSections = + hsAdmin.popupPanelDisabledSections = + hsUser.popupPanelDisabledSections = bits; + } + } + + const hs = results[1] instanceof Object && results[1].hiddenSettings || {}; + if ( Object.keys(hsAdmin).length === 0 && Object.keys(hs).length === 0 ) { + return; + } + + for ( const key in hsDefault ) { + if ( hsDefault.hasOwnProperty(key) === false ) { continue; } + if ( hsAdmin.hasOwnProperty(name) ) { continue; } + if ( typeof hs[key] !== typeof hsDefault[key] ) { continue; } + this.hiddenSettings[key] = hs[key]; + } + broadcast({ what: 'hiddenSettingsChanged' }); +}; + +// Note: Save only the settings which values differ from the default ones. +// This way the new default values in the future will properly apply for +// those which were not modified by the user. + +µb.saveHiddenSettings = function() { + vAPI.storage.set({ + hiddenSettings: this.getModifiedSettings( + this.hiddenSettings, + this.hiddenSettingsDefault + ) + }); +}; + +onBroadcast(msg => { + if ( msg.what !== 'hiddenSettingsChanged' ) { return; } + const µbhs = µb.hiddenSettings; + ubologSet(µbhs.consoleLogLevel === 'info'); + vAPI.net.setOptions({ + cnameIgnoreList: µbhs.cnameIgnoreList, + cnameIgnore1stParty: µbhs.cnameIgnore1stParty, + cnameIgnoreExceptions: µbhs.cnameIgnoreExceptions, + cnameIgnoreRootDocument: µbhs.cnameIgnoreRootDocument, + cnameMaxTTL: µbhs.cnameMaxTTL, + cnameReplayFullURL: µbhs.cnameReplayFullURL, + cnameUncloakProxied: µbhs.cnameUncloakProxied, + }); +}); + +/******************************************************************************/ + +µb.hiddenSettingsFromString = function(raw) { + const out = Object.assign({}, this.hiddenSettingsDefault); + const lineIter = new LineIterator(raw); + while ( lineIter.eot() === false ) { + const line = lineIter.next(); + const matches = /^\s*(\S+)\s+(.+)$/.exec(line); + if ( matches === null || matches.length !== 3 ) { continue; } + const name = matches[1]; + if ( out.hasOwnProperty(name) === false ) { continue; } + if ( this.hiddenSettingsAdmin.hasOwnProperty(name) ) { continue; } + const value = this.hiddenSettingValueFromString(name, matches[2]); + if ( value !== undefined ) { + out[name] = value; + } + } + return out; +}; + +µb.hiddenSettingValueFromString = function(name, value) { + if ( typeof name !== 'string' || typeof value !== 'string' ) { return; } + const hsDefault = this.hiddenSettingsDefault; + if ( hsDefault.hasOwnProperty(name) === false ) { return; } + let r; + switch ( typeof hsDefault[name] ) { + case 'boolean': + if ( value === 'true' ) { + r = true; + } else if ( value === 'false' ) { + r = false; + } + break; + case 'string': + r = value.trim(); + break; + case 'number': + if ( value.startsWith('0b') ) { + r = parseInt(value.slice(2), 2); + } else if ( value.startsWith('0x') ) { + r = parseInt(value.slice(2), 16); + } else { + r = parseInt(value, 10); + } + if ( isNaN(r) ) { r = undefined; } + break; + default: + break; + } + return r; +}; + +µb.stringFromHiddenSettings = function() { + const out = []; + for ( const key of Object.keys(this.hiddenSettings).sort() ) { + out.push(key + ' ' + this.hiddenSettings[key]); + } + return out.join('\n'); +}; + +/******************************************************************************/ + +µb.savePermanentFirewallRules = function() { + vAPI.storage.set({ + dynamicFilteringString: permanentFirewall.toString() + }); +}; + +/******************************************************************************/ + +µb.savePermanentURLFilteringRules = function() { + vAPI.storage.set({ + urlFilteringString: permanentURLFiltering.toString() + }); +}; + +/******************************************************************************/ + +µb.saveHostnameSwitches = function() { + vAPI.storage.set({ + hostnameSwitchesString: permanentSwitches.toString() + }); +}; + +/******************************************************************************/ + +µb.saveWhitelist = function() { + vAPI.storage.set({ + netWhitelist: this.arrayFromWhitelist(this.netWhitelist) + }); + this.netWhitelistModifyTime = Date.now(); +}; + +/******************************************************************************/ + +µb.isTrustedList = function(assetKey) { + if ( this.parsedTrustedListPrefixes.length === 0 ) { + this.parsedTrustedListPrefixes = + µb.hiddenSettings.trustedListPrefixes.split(/ +/).map(prefix => { + if ( prefix === '' ) { return; } + if ( prefix.startsWith('http://') ) { return; } + if ( prefix.startsWith('file:///') ) { return prefix; } + if ( prefix.startsWith('https://') === false ) { + return prefix.includes('://') ? undefined : prefix; + } + try { + const url = new URL(prefix); + if ( url.hostname.length > 0 ) { return url.href; } + } catch(_) { + } + }).filter(prefix => prefix !== undefined); + } + for ( const prefix of this.parsedTrustedListPrefixes ) { + if ( assetKey.startsWith(prefix) ) { return true; } + } + return false; +}; + +onBroadcast(msg => { + if ( msg.what !== 'hiddenSettingsChanged' ) { return; } + µb.parsedTrustedListPrefixes = []; +}); + +/******************************************************************************/ + +µb.loadSelectedFilterLists = async function() { + const bin = await vAPI.storage.get('selectedFilterLists'); + if ( bin instanceof Object && Array.isArray(bin.selectedFilterLists) ) { + this.selectedFilterLists = bin.selectedFilterLists; + return; + } + + // https://github.com/gorhill/uBlock/issues/747 + // Select default filter lists if first-time launch. + const lists = await io.metadata(); + this.saveSelectedFilterLists(this.autoSelectRegionalFilterLists(lists)); +}; + +µb.saveSelectedFilterLists = function(newKeys, append = false) { + const oldKeys = this.selectedFilterLists.slice(); + if ( append ) { + newKeys = newKeys.concat(oldKeys); + } + const newSet = new Set(newKeys); + // Purge unused filter lists from cache. + for ( const oldKey of oldKeys ) { + if ( newSet.has(oldKey) === false ) { + this.removeFilterList(oldKey); + } + } + newKeys = Array.from(newSet); + this.selectedFilterLists = newKeys; + return vAPI.storage.set({ selectedFilterLists: newKeys }); +}; + +/******************************************************************************/ + +µb.applyFilterListSelection = function(details) { + let selectedListKeySet = new Set(this.selectedFilterLists); + let importedLists = this.userSettings.importedLists.slice(); + + // Filter lists to select + if ( Array.isArray(details.toSelect) ) { + if ( details.merge ) { + for ( let i = 0, n = details.toSelect.length; i < n; i++ ) { + selectedListKeySet.add(details.toSelect[i]); + } + } else { + selectedListKeySet = new Set(details.toSelect); + } + } + + // Imported filter lists to remove + if ( Array.isArray(details.toRemove) ) { + for ( let i = 0, n = details.toRemove.length; i < n; i++ ) { + const assetKey = details.toRemove[i]; + selectedListKeySet.delete(assetKey); + const pos = importedLists.indexOf(assetKey); + if ( pos !== -1 ) { + importedLists.splice(pos, 1); + } + this.removeFilterList(assetKey); + } + } + + // Filter lists to import + if ( typeof details.toImport === 'string' ) { + // https://github.com/gorhill/uBlock/issues/1181 + // Try mapping the URL of an imported filter list to the assetKey + // of an existing stock list. + const assetKeyFromURL = url => { + const needle = url.replace(/^https?:/, ''); + const assets = this.availableFilterLists; + for ( const assetKey in assets ) { + const asset = assets[assetKey]; + if ( asset.content !== 'filters' ) { continue; } + if ( typeof asset.contentURL === 'string' ) { + if ( asset.contentURL.endsWith(needle) ) { return assetKey; } + continue; + } + if ( Array.isArray(asset.contentURL) === false ) { continue; } + for ( let i = 0, n = asset.contentURL.length; i < n; i++ ) { + if ( asset.contentURL[i].endsWith(needle) ) { + return assetKey; + } + } + } + return url; + }; + const importedSet = new Set(this.listKeysFromCustomFilterLists(importedLists)); + const toImportSet = new Set(this.listKeysFromCustomFilterLists(details.toImport)); + for ( const urlKey of toImportSet ) { + if ( importedSet.has(urlKey) ) { + selectedListKeySet.add(urlKey); + continue; + } + const assetKey = assetKeyFromURL(urlKey); + if ( assetKey === urlKey ) { + importedSet.add(urlKey); + } + selectedListKeySet.add(assetKey); + } + importedLists = Array.from(importedSet).sort(); + } + + const result = Array.from(selectedListKeySet); + if ( importedLists.join() !== this.userSettings.importedLists.join() ) { + this.userSettings.importedLists = importedLists; + this.saveUserSettings(); + } + this.saveSelectedFilterLists(result); +}; + +/******************************************************************************/ + +µb.listKeysFromCustomFilterLists = function(raw) { + const urls = typeof raw === 'string' + ? raw.trim().split(/[\n\r]+/) + : raw; + const out = new Set(); + const reIgnore = /^[!#]/; + const reValid = /^[a-z-]+:\/\/\S+/; + for ( const url of urls ) { + if ( reIgnore.test(url) || !reValid.test(url) ) { continue; } + // Ignore really bad lists. + if ( this.badLists.get(url) === true ) { continue; } + out.add(url); + } + return Array.from(out); +}; + +/******************************************************************************/ + +µb.saveUserFilters = function(content) { + // https://github.com/gorhill/uBlock/issues/1022 + // Be sure to end with an empty line. + content = content.trim(); + if ( content !== '' ) { content += '\n'; } + this.removeCompiledFilterList(this.userFiltersPath); + return io.put(this.userFiltersPath, content); +}; + +µb.loadUserFilters = function() { + return io.get(this.userFiltersPath); +}; + +µb.appendUserFilters = async function(filters, options) { + filters = filters.trim(); + if ( filters.length === 0 ) { return; } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/372 + // Auto comment using user-defined template. + let comment = ''; + if ( + options instanceof Object && + options.autoComment === true && + this.hiddenSettings.autoCommentFilterTemplate.indexOf('{{') !== -1 + ) { + const d = new Date(); + // Date in YYYY-MM-DD format - https://stackoverflow.com/a/50130338 + const ISO8601Date = new Date(d.getTime() + + (d.getTimezoneOffset()*60000)).toISOString().split('T')[0]; + const url = new URL(options.docURL); + comment = + '! ' + + this.hiddenSettings.autoCommentFilterTemplate + .replace('{{date}}', ISO8601Date) + .replace('{{time}}', d.toLocaleTimeString()) + .replace('{{hostname}}', url.hostname) + .replace('{{origin}}', url.origin) + .replace('{{url}}', url.href); + } + + const details = await this.loadUserFilters(); + if ( details.error ) { return; } + + // The comment, if any, will be applied if and only if it is different + // from the last comment found in the user filter list. + if ( comment !== '' ) { + const beg = details.content.lastIndexOf(comment); + const end = beg === -1 ? -1 : beg + comment.length; + if ( + end === -1 || + details.content.startsWith('\n', end) === false || + details.content.includes('\n!', end) + ) { + filters = '\n' + comment + '\n' + filters; + } + } + + // https://github.com/chrisaljoudi/uBlock/issues/976 + // If we reached this point, the filter quite probably needs to be + // added for sure: do not try to be too smart, trying to avoid + // duplicates at this point may lead to more issues. + await this.saveUserFilters(details.content.trim() + '\n' + filters); + + const compiledFilters = this.compileFilters(filters, { + assetKey: this.userFiltersPath, + trustedSource: true, + }); + const snfe = staticNetFilteringEngine; + const cfe = cosmeticFilteringEngine; + const acceptedCount = snfe.acceptedCount + cfe.acceptedCount; + const discardedCount = snfe.discardedCount + cfe.discardedCount; + this.applyCompiledFilters(compiledFilters, true); + const entry = this.availableFilterLists[this.userFiltersPath]; + const deltaEntryCount = + snfe.acceptedCount + + cfe.acceptedCount - acceptedCount; + const deltaEntryUsedCount = + deltaEntryCount - + (snfe.discardedCount + cfe.discardedCount - discardedCount); + entry.entryCount += deltaEntryCount; + entry.entryUsedCount += deltaEntryUsedCount; + vAPI.storage.set({ 'availableFilterLists': this.availableFilterLists }); + staticNetFilteringEngine.freeze(); + redirectEngine.freeze(); + staticExtFilteringEngine.freeze(); + this.selfieManager.destroy(); + + // https://www.reddit.com/r/uBlockOrigin/comments/cj7g7m/ + // https://www.reddit.com/r/uBlockOrigin/comments/cnq0bi/ + filteringBehaviorChanged(); + broadcast({ what: 'userFiltersUpdated' }); +}; + +µb.createUserFilters = function(details) { + this.appendUserFilters(details.filters, details); + // https://github.com/gorhill/uBlock/issues/1786 + if ( details.docURL === undefined ) { return; } + cosmeticFilteringEngine.removeFromSelectorCache( + hostnameFromURI(details.docURL) + ); +}; + +/******************************************************************************/ + +µb.autoSelectRegionalFilterLists = function(lists) { + const selectedListKeys = [ this.userFiltersPath ]; + for ( const key in lists ) { + if ( lists.hasOwnProperty(key) === false ) { continue; } + const list = lists[key]; + if ( list.content !== 'filters' ) { continue; } + if ( list.off !== true ) { + selectedListKeys.push(key); + continue; + } + if ( this.listMatchesEnvironment(list) ) { + selectedListKeys.push(key); + list.off = false; + } + } + return selectedListKeys; +}; + +/******************************************************************************/ + +µb.hasInMemoryFilter = function(raw) { + return this.inMemoryFilters.includes(raw); +}; + +µb.addInMemoryFilter = async function(raw) { + if ( this.inMemoryFilters.includes(raw) ){ return true; } + this.inMemoryFilters.push(raw); + this.inMemoryFiltersCompiled = ''; + await this.loadFilterLists(); + return true; +}; + +µb.removeInMemoryFilter = async function(raw) { + const pos = this.inMemoryFilters.indexOf(raw); + if ( pos === -1 ) { return false; } + this.inMemoryFilters.splice(pos, 1); + this.inMemoryFiltersCompiled = ''; + await this.loadFilterLists(); + return false; +}; + +µb.clearInMemoryFilters = async function() { + if ( this.inMemoryFilters.length === 0 ) { return; } + this.inMemoryFilters = []; + this.inMemoryFiltersCompiled = ''; + await this.loadFilterLists(); +}; + +/******************************************************************************/ + +µb.getAvailableLists = async function() { + const newAvailableLists = {}; + + // User filter list + newAvailableLists[this.userFiltersPath] = { + content: 'filters', + group: 'user', + title: i18n$('1pPageName'), + }; + + // Custom filter lists + const importedListKeys = new Set( + this.listKeysFromCustomFilterLists(this.userSettings.importedLists) + ); + for ( const listKey of importedListKeys ) { + const asset = { + content: 'filters', + contentURL: listKey, + external: true, + group: 'custom', + submitter: 'user', + title: '', + }; + newAvailableLists[listKey] = asset; + io.registerAssetSource(listKey, asset); + } + + // Load previously saved available lists -- these contains data + // computed at run-time, we will reuse this data if possible + const [ bin, registeredAssets, badlists ] = await Promise.all([ + Object.keys(this.availableFilterLists).length !== 0 + ? { availableFilterLists: this.availableFilterLists } + : vAPI.storage.get('availableFilterLists'), + io.metadata(), + this.badLists.size === 0 ? io.get('ublock-badlists') : false, + ]); + + if ( badlists instanceof Object ) { + for ( const line of badlists.content.split(/\s*[\n\r]+\s*/) ) { + if ( line === '' || line.startsWith('#') ) { continue; } + const fields = line.split(/\s+/); + const remove = fields.length === 2; + this.badLists.set(fields[0], remove); + } + } + + const oldAvailableLists = bin && bin.availableFilterLists || {}; + + for ( const [ assetKey, asset ] of Object.entries(registeredAssets) ) { + if ( asset.content !== 'filters' ) { continue; } + newAvailableLists[assetKey] = Object.assign({}, asset); + } + + // Load set of currently selected filter lists + const selectedListset = new Set(this.selectedFilterLists); + + // Remove imported filter lists which are already present in stock lists + for ( const [ stockAssetKey, stockEntry ] of Object.entries(newAvailableLists) ) { + if ( stockEntry.content !== 'filters' ) { continue; } + if ( stockEntry.group === 'user' ) { continue; } + if ( stockEntry.submitter === 'user' ) { continue; } + if ( stockAssetKey.includes('://') ) { continue; } + const contentURLs = Array.isArray(stockEntry.contentURL) + ? stockEntry.contentURL + : [ stockEntry.contentURL ]; + for ( const importedAssetKey of contentURLs ) { + const importedEntry = newAvailableLists[importedAssetKey]; + if ( importedEntry === undefined ) { continue; } + delete newAvailableLists[importedAssetKey]; + io.unregisterAssetSource(importedAssetKey); + this.removeFilterList(importedAssetKey); + if ( selectedListset.has(importedAssetKey) ) { + selectedListset.add(stockAssetKey); + selectedListset.delete(importedAssetKey); + } + importedListKeys.delete(importedAssetKey); + break; + } + } + + // Unregister lists in old listset not present in new listset. + // Convert a no longer existing stock list into an imported list, except + // when the removed stock list is deemed a "bad list". + for ( const [ assetKey, oldEntry ] of Object.entries(oldAvailableLists) ) { + if ( newAvailableLists[assetKey] !== undefined ) { continue; } + const on = selectedListset.delete(assetKey); + this.removeFilterList(assetKey); + io.unregisterAssetSource(assetKey); + if ( assetKey.includes('://') ) { continue; } + if ( on === false ) { continue; } + const listURL = Array.isArray(oldEntry.contentURL) + ? oldEntry.contentURL[0] + : oldEntry.contentURL; + if ( this.badLists.has(listURL) ) { continue; } + const newEntry = { + content: 'filters', + contentURL: listURL, + external: true, + group: 'custom', + submitter: 'user', + title: oldEntry.title || '' + }; + newAvailableLists[listURL] = newEntry; + io.registerAssetSource(listURL, newEntry); + importedListKeys.add(listURL); + selectedListset.add(listURL); + } + + // Remove unreferenced imported filter lists + for ( const [ assetKey, asset ] of Object.entries(newAvailableLists) ) { + if ( asset.submitter !== 'user' ) { continue; } + if ( importedListKeys.has(assetKey) ) { continue; } + selectedListset.delete(assetKey); + delete newAvailableLists[assetKey]; + this.removeFilterList(assetKey); + io.unregisterAssetSource(assetKey); + } + + // Mark lists as disabled/enabled according to selected listset + for ( const [ assetKey, asset ] of Object.entries(newAvailableLists) ) { + asset.off = selectedListset.has(assetKey) === false; + } + + // Reuse existing metadata + for ( const [ assetKey, oldEntry ] of Object.entries(oldAvailableLists) ) { + const newEntry = newAvailableLists[assetKey]; + if ( newEntry === undefined ) { continue; } + if ( oldEntry.entryCount !== undefined ) { + newEntry.entryCount = oldEntry.entryCount; + } + if ( oldEntry.entryUsedCount !== undefined ) { + newEntry.entryUsedCount = oldEntry.entryUsedCount; + } + // This may happen if the list name was pulled from the list content + // https://github.com/chrisaljoudi/uBlock/issues/982 + // There is no guarantee the title was successfully extracted from + // the list content + if ( + newEntry.title === '' && + typeof oldEntry.title === 'string' && + oldEntry.title !== '' + ) { + newEntry.title = oldEntry.title; + } + } + + if ( Array.from(importedListKeys).join('\n') !== this.userSettings.importedLists.join('\n') ) { + this.userSettings.importedLists = Array.from(importedListKeys); + this.saveUserSettings(); + } + + if ( Array.from(selectedListset).join() !== this.selectedFilterLists.join() ) { + this.saveSelectedFilterLists(Array.from(selectedListset)); + } + + return newAvailableLists; +}; + +/******************************************************************************/ + +{ + const loadedListKeys = []; + let loadingPromise; + let t0 = 0; + + const onDone = ( ) => { + ubolog(`loadFilterLists() took ${Date.now()-t0} ms`); + + staticNetFilteringEngine.freeze(); + staticExtFilteringEngine.freeze(); + redirectEngine.freeze(); + vAPI.net.unsuspend(); + filteringBehaviorChanged(); + + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); + + logger.writeOne({ + realm: 'message', + type: 'info', + text: 'Reloading all filter lists: done' + }); + + broadcast({ + what: 'staticFilteringDataChanged', + parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters, + ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, + listKeys: loadedListKeys + }); + + µb.selfieManager.destroy(); + lz4Codec.relinquish(); + µb.compiledFormatChanged = false; + + loadingPromise = undefined; + }; + + const applyCompiledFilters = (assetKey, compiled) => { + const snfe = staticNetFilteringEngine; + const sxfe = staticExtFilteringEngine; + let acceptedCount = snfe.acceptedCount + sxfe.acceptedCount; + let discardedCount = snfe.discardedCount + sxfe.discardedCount; + µb.applyCompiledFilters(compiled, assetKey === µb.userFiltersPath); + if ( µb.availableFilterLists.hasOwnProperty(assetKey) ) { + const entry = µb.availableFilterLists[assetKey]; + entry.entryCount = snfe.acceptedCount + sxfe.acceptedCount - + acceptedCount; + entry.entryUsedCount = entry.entryCount - + (snfe.discardedCount + sxfe.discardedCount - discardedCount); + } + loadedListKeys.push(assetKey); + }; + + const onFilterListsReady = lists => { + logger.writeOne({ + realm: 'message', + type: 'info', + text: 'Reloading all filter lists: start' + }); + + µb.availableFilterLists = lists; + + if ( vAPI.Net.canSuspend() ) { + vAPI.net.suspend(); + } + redirectEngine.reset(); + staticExtFilteringEngine.reset(); + staticNetFilteringEngine.reset(); + µb.selfieManager.destroy(); + staticFilteringReverseLookup.resetLists(); + + // We need to build a complete list of assets to pull first: this is + // because it *may* happens that some load operations are synchronous: + // This happens for assets which do not exist, or assets with no + // content. + const toLoad = []; + for ( const assetKey in lists ) { + if ( lists.hasOwnProperty(assetKey) === false ) { continue; } + if ( lists[assetKey].off ) { continue; } + toLoad.push( + µb.getCompiledFilterList(assetKey).then(details => { + applyCompiledFilters(details.assetKey, details.content); + }) + ); + } + + if ( µb.inMemoryFilters.length !== 0 ) { + if ( µb.inMemoryFiltersCompiled === '' ) { + µb.inMemoryFiltersCompiled = + µb.compileFilters(µb.inMemoryFilters.join('\n'), { + assetKey: 'in-memory', + trustedSource: true, + }); + } + if ( µb.inMemoryFiltersCompiled !== '' ) { + toLoad.push( + µb.applyCompiledFilters(µb.inMemoryFiltersCompiled, true) + ); + } + } + + return Promise.all(toLoad); + }; + + µb.loadFilterLists = function() { + if ( loadingPromise instanceof Promise ) { return loadingPromise; } + t0 = Date.now(); + loadedListKeys.length = 0; + loadingPromise = Promise.all([ + this.getAvailableLists().then(lists => onFilterListsReady(lists)), + this.loadRedirectResources(), + ]).then(( ) => { + onDone(); + }); + return loadingPromise; + }; +} + +/******************************************************************************/ + +µb.getCompiledFilterList = async function(assetKey) { + const compiledPath = 'compiled/' + assetKey; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 + // Verify that the list version matches that of the current compiled + // format. + if ( + this.compiledFormatChanged === false && + this.badLists.has(assetKey) === false + ) { + const compiledDetails = await io.get(compiledPath); + const compilerVersion = `${this.systemSettings.compiledMagic}\n`; + if ( compiledDetails.content.startsWith(compilerVersion) ) { + compiledDetails.assetKey = assetKey; + return compiledDetails; + } + } + + // Skip downloading really bad lists. + if ( this.badLists.get(assetKey) ) { + return { assetKey, content: '' }; + } + + const rawDetails = await io.get(assetKey, { + favorLocal: this.readyToFilter !== true, + silent: true, + }); + // Compiling an empty string results in an empty string. + if ( rawDetails.content === '' ) { + rawDetails.assetKey = assetKey; + return rawDetails; + } + + this.extractFilterListMetadata(assetKey, rawDetails.content); + + // Skip compiling bad lists. + if ( this.badLists.has(assetKey) ) { + return { assetKey, content: '' }; + } + + const compiledContent = this.compileFilters(rawDetails.content, { + assetKey, + trustedSource: this.isTrustedList(assetKey), + }); + io.put(compiledPath, compiledContent); + + return { assetKey, content: compiledContent }; +}; + +/******************************************************************************/ + +µb.extractFilterListMetadata = function(assetKey, raw) { + const listEntry = this.availableFilterLists[assetKey]; + if ( listEntry === undefined ) { return; } + // https://github.com/gorhill/uBlock/issues/313 + // Always try to fetch the name if this is an external filter list. + if ( listEntry.group !== 'custom' ) { return; } + const data = io.extractMetadataFromList(raw, [ 'Title', 'Homepage' ]); + const props = {}; + if ( data.title && data.title !== listEntry.title ) { + props.title = listEntry.title = orphanizeString(data.title); + } + if ( data.homepage && /^https?:\/\/\S+/.test(data.homepage) ) { + if ( data.homepage !== listEntry.supportURL ) { + props.supportURL = listEntry.supportURL = orphanizeString(data.homepage); + } + } + io.registerAssetSource(assetKey, props); +}; + +/******************************************************************************/ + +µb.removeCompiledFilterList = function(assetKey) { + io.remove('compiled/' + assetKey); +}; + +µb.removeFilterList = function(assetKey) { + this.removeCompiledFilterList(assetKey); + io.remove(assetKey); +}; + +/******************************************************************************/ + +µb.compileFilters = function(rawText, details = {}) { + const writer = new CompiledListWriter(); + + // Populate the writer with information potentially useful to the + // client compilers. + const trustedSource = details.trustedSource === true; + if ( details.assetKey ) { + writer.properties.set('name', details.assetKey); + writer.properties.set('trustedSource', trustedSource); + } + const assetName = details.assetKey ? details.assetKey : '?'; + const parser = new sfp.AstFilterParser({ + trustedSource, + maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH, + nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), + }); + const compiler = staticNetFilteringEngine.createCompiler(parser); + const lineIter = new LineIterator( + sfp.utils.preparser.prune(rawText, vAPI.webextFlavor.env) + ); + + compiler.start(writer); + + while ( lineIter.eot() === false ) { + let line = lineIter.next(); + + while ( line.endsWith(' \\') ) { + if ( lineIter.peek(4) !== ' ' ) { break; } + line = line.slice(0, -2).trim() + lineIter.next().trim(); + } + + parser.parse(line); + + if ( parser.isFilter() === false ) { continue; } + if ( parser.hasError() ) { + logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid filter (${assetName}): ${parser.raw}` + }); + continue; + } + + if ( parser.isExtendedFilter() ) { + staticExtFilteringEngine.compile(parser, writer); + continue; + } + + if ( parser.isNetworkFilter() === false ) { continue; } + + if ( compiler.compile(parser, writer) ) { continue; } + if ( compiler.error !== undefined ) { + logger.writeOne({ + realm: 'message', + type: 'error', + text: compiler.error + }); + } + } + + compiler.finish(writer); + parser.finish(); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 + // Embed version into compiled list itself: it is encoded in as the + // first digits followed by a whitespace. + const compiledContent + = `${this.systemSettings.compiledMagic}\n` + writer.toString(); + + return compiledContent; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1395 +// Added `firstparty` argument: to avoid discarding cosmetic filters when +// applying 1st-party filters. + +µb.applyCompiledFilters = function(rawText, firstparty) { + if ( rawText === '' ) { return; } + const reader = new CompiledListReader(rawText); + staticNetFilteringEngine.fromCompiled(reader); + staticExtFilteringEngine.fromCompiledContent(reader, { + skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters, + skipCosmetic: !firstparty && !this.userSettings.parseAllABPHideFilters + }); +}; + +/******************************************************************************/ + +µb.loadRedirectResources = async function() { + try { + const success = await redirectEngine.resourcesFromSelfie(io); + if ( success === true ) { return true; } + + const fetcher = (path, options = undefined) => { + if ( path.startsWith('/web_accessible_resources/') ) { + path += `?secret=${vAPI.warSecret.short()}`; + return io.fetch(path, options); + } + return io.fetchText(path); + }; + + const fetchPromises = [ + redirectEngine.loadBuiltinResources(fetcher) + ]; + + const userResourcesLocation = this.hiddenSettings.userResourcesLocation; + if ( userResourcesLocation !== 'unset' ) { + for ( const url of userResourcesLocation.split(/\s+/) ) { + fetchPromises.push(io.fetchText(url)); + } + } + + const results = await Promise.all(fetchPromises); + if ( Array.isArray(results) === false ) { return results; } + + let content = ''; + for ( let i = 1; i < results.length; i++ ) { + const result = results[i]; + if ( + result instanceof Object === false || + typeof result.content !== 'string' || + result.content === '' + ) { + continue; + } + content += '\n\n' + result.content; + } + + redirectEngine.resourcesFromString(content); + redirectEngine.selfieFromResources(io); + } catch(ex) { + ubolog(ex); + return false; + } + return true; +}; + +/******************************************************************************/ + +µb.loadPublicSuffixList = async function() { + const psl = publicSuffixList; + + // WASM is nice but not critical + if ( vAPI.canWASM && this.hiddenSettings.disableWebAssembly !== true ) { + const wasmModuleFetcher = function(path) { + return fetch( `${path}.wasm`, { + mode: 'same-origin' + }).then( + WebAssembly.compileStreaming + ).catch(reason => { + ubolog(reason); + }); + }; + let result = false; + try { + result = await psl.enableWASM(wasmModuleFetcher, + './lib/publicsuffixlist/wasm/' + ); + } catch(reason) { + ubolog(reason); + } + if ( result ) { + ubolog(`WASM PSL ready ${Date.now()-vAPI.T0} ms after launch`); + } + } + + try { + const result = await io.get(`compiled/${this.pslAssetKey}`); + if ( psl.fromSelfie(result.content, sparseBase64) ) { return; } + } catch (reason) { + ubolog(reason); + } + + const result = await io.get(this.pslAssetKey); + if ( result.content !== '' ) { + this.compilePublicSuffixList(result.content); + } +}; + +µb.compilePublicSuffixList = function(content) { + const psl = publicSuffixList; + psl.parse(content, punycode.toASCII); + io.put(`compiled/${this.pslAssetKey}`, psl.toSelfie(sparseBase64)); +}; + +/******************************************************************************/ + +// This is to be sure the selfie is generated in a sane manner: the selfie will +// be generated if the user doesn't change his filter lists selection for +// some set time. + +{ + // As of 2018-05-31: + // JSON.stringify-ing ourselves results in a better baseline + // memory usage at selfie-load time. For some reasons. + + const create = async function() { + vAPI.alarms.clear('createSelfie'); + createTimer.off(); + if ( µb.inMemoryFilters.length !== 0 ) { return; } + if ( Object.keys(µb.availableFilterLists).length === 0 ) { return; } + await Promise.all([ + io.put( + 'selfie/main', + JSON.stringify({ + magic: µb.systemSettings.selfieMagic, + availableFilterLists: µb.availableFilterLists, + }) + ), + redirectEngine.toSelfie('selfie/redirectEngine'), + staticExtFilteringEngine.toSelfie( + 'selfie/staticExtFilteringEngine' + ), + staticNetFilteringEngine.toSelfie(io, + 'selfie/staticNetFilteringEngine' + ), + ]); + lz4Codec.relinquish(); + µb.selfieIsInvalid = false; + }; + + const loadMain = async function() { + const details = await io.get('selfie/main'); + if ( + details instanceof Object === false || + typeof details.content !== 'string' || + details.content === '' + ) { + return false; + } + let selfie; + try { + selfie = JSON.parse(details.content); + } catch(ex) { + } + if ( selfie instanceof Object === false ) { return false; } + if ( selfie.magic !== µb.systemSettings.selfieMagic ) { return false; } + if ( selfie.availableFilterLists instanceof Object === false ) { return false; } + if ( Object.keys(selfie.availableFilterLists).length === 0 ) { return false; } + µb.availableFilterLists = selfie.availableFilterLists; + return true; + }; + + const load = async function() { + if ( µb.selfieIsInvalid ) { return false; } + try { + const results = await Promise.all([ + loadMain(), + redirectEngine.fromSelfie('selfie/redirectEngine'), + staticExtFilteringEngine.fromSelfie( + 'selfie/staticExtFilteringEngine' + ), + staticNetFilteringEngine.fromSelfie(io, + 'selfie/staticNetFilteringEngine' + ), + ]); + if ( results.every(v => v) ) { + return µb.loadRedirectResources(); + } + } + catch (reason) { + ubolog(reason); + } + destroy(); + return false; + }; + + const destroy = function() { + if ( µb.selfieIsInvalid === false ) { + io.remove(/^selfie\//); + µb.selfieIsInvalid = true; + } + if ( µb.wakeupReason === 'createSelfie' ) { + µb.wakeupReason = ''; + return createTimer.offon({ sec: 27 }); + } + vAPI.alarms.create('createSelfie', { + delayInMinutes: µb.hiddenSettings.selfieAfter + }); + createTimer.offon({ min: µb.hiddenSettings.selfieAfter }); + }; + + const createTimer = vAPI.defer.create(create); + + vAPI.alarms.onAlarm.addListener(alarm => { + if ( alarm.name !== 'createSelfie') { return; } + µb.wakeupReason = 'createSelfie'; + }); + + µb.selfieManager = { load, destroy }; +} + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/531 +// Overwrite user settings with admin settings if present. +// +// Admin settings match layout of a uBlock backup. Not all data is +// necessarily present, i.e. administrators may removed entries which +// values are left to the user's choice. + +µb.restoreAdminSettings = async function() { + let toOverwrite = {}; + let data; + try { + const store = await vAPI.adminStorage.get([ + 'adminSettings', + 'toOverwrite', + ]) || {}; + if ( store.toOverwrite instanceof Object ) { + toOverwrite = store.toOverwrite; + } + const json = store.adminSettings; + if ( typeof json === 'string' && json !== '' ) { + data = JSON.parse(json); + } else if ( json instanceof Object ) { + data = json; + } + } catch (ex) { + console.error(ex); + } + + if ( data instanceof Object === false ) { data = {}; } + + const bin = {}; + let binNotEmpty = false; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/666 + // Allows an admin to set their own 'assets.json' file, with their + // own set of stock assets. + if ( + typeof data.assetsBootstrapLocation === 'string' && + data.assetsBootstrapLocation !== '' + ) { + µb.assetsBootstrapLocation = data.assetsBootstrapLocation; + } + + if ( typeof data.userSettings === 'object' ) { + const µbus = this.userSettings; + const adminus = data.userSettings; + for ( const name in µbus ) { + if ( µbus.hasOwnProperty(name) === false ) { continue; } + if ( adminus.hasOwnProperty(name) === false ) { continue; } + bin[name] = adminus[name]; + binNotEmpty = true; + } + } + + // 'selectedFilterLists' is an array of filter list tokens. Each token + // is a reference to an asset in 'assets.json', or a URL for lists not + // present in 'assets.json'. + if ( + Array.isArray(toOverwrite.filterLists) && + toOverwrite.filterLists.length !== 0 + ) { + const importedLists = []; + for ( const list of toOverwrite.filterLists ) { + if ( /^[a-z-]+:\/\//.test(list) === false ) { continue; } + importedLists.push(list); + } + if ( importedLists.length !== 0 ) { + bin.importedLists = importedLists; + bin.externalLists = importedLists.join('\n'); + } + bin.selectedFilterLists = toOverwrite.filterLists; + binNotEmpty = true; + } else if ( Array.isArray(data.selectedFilterLists) ) { + bin.selectedFilterLists = data.selectedFilterLists; + binNotEmpty = true; + } + + if ( + Array.isArray(toOverwrite.trustedSiteDirectives) && + toOverwrite.trustedSiteDirectives.length !== 0 + ) { + µb.netWhitelistDefault = toOverwrite.trustedSiteDirectives.slice(); + bin.netWhitelist = toOverwrite.trustedSiteDirectives.slice(); + binNotEmpty = true; + } else if ( Array.isArray(data.whitelist) ) { + bin.netWhitelist = data.whitelist; + binNotEmpty = true; + } else if ( typeof data.netWhitelist === 'string' ) { + bin.netWhitelist = data.netWhitelist.split('\n'); + binNotEmpty = true; + } + + if ( typeof data.dynamicFilteringString === 'string' ) { + bin.dynamicFilteringString = data.dynamicFilteringString; + binNotEmpty = true; + } + + if ( typeof data.urlFilteringString === 'string' ) { + bin.urlFilteringString = data.urlFilteringString; + binNotEmpty = true; + } + + if ( typeof data.hostnameSwitchesString === 'string' ) { + bin.hostnameSwitchesString = data.hostnameSwitchesString; + binNotEmpty = true; + } + + if ( binNotEmpty ) { + vAPI.storage.set(bin); + } + + if ( + Array.isArray(toOverwrite.filters) && + toOverwrite.filters.length !== 0 + ) { + this.saveUserFilters(toOverwrite.filters.join('\n')); + } else if ( typeof data.userFilters === 'string' ) { + this.saveUserFilters(data.userFilters); + } +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/2344 +// Support multiple locales per filter list. +// https://github.com/gorhill/uBlock/issues/3210 +// Support ability to auto-enable a filter list based on user agent. +// https://github.com/gorhill/uBlock/pull/3860 +// Get current language using extensions API (instead of `navigator.language`) + +µb.listMatchesEnvironment = function(details) { + // Matches language? + if ( typeof details.lang === 'string' ) { + let re = this.listMatchesEnvironment.reLang; + if ( re === undefined ) { + const match = /^[a-z]+/.exec(i18n.getUILanguage()); + if ( match !== null ) { + re = new RegExp('\\b' + match[0] + '\\b'); + this.listMatchesEnvironment.reLang = re; + } + } + if ( re !== undefined && re.test(details.lang) ) { return true; } + } + // Matches user agent? + if ( typeof details.ua === 'string' ) { + let re = new RegExp('\\b' + this.escapeRegex(details.ua) + '\\b', 'i'); + if ( re.test(self.navigator.userAgent) ) { return true; } + } + return false; +}; + +/******************************************************************************/ + +{ + let next = 0; + let lastEmergencyUpdate = 0; + + const launchTimer = vAPI.defer.create(fetchDelay => { + next = 0; + io.updateStart({ fetchDelay, auto: true }); + }); + + µb.scheduleAssetUpdater = async function(details = {}) { + launchTimer.off(); + + if ( details.now ) { + next = 0; + io.updateStart(details); + return; + } + + if ( µb.userSettings.autoUpdate === false ) { + if ( Boolean(details.updateDelay) === false ) { + next = 0; + return; + } + } + + let updateDelay = details.updateDelay || + this.hiddenSettings.autoUpdatePeriod * 3600000; + + const now = Date.now(); + let needEmergencyUpdate = false; + + // Respect cooldown period before launching an emergency update. + const timeSinceLastEmergencyUpdate = (now - lastEmergencyUpdate) / 3600000; + if ( timeSinceLastEmergencyUpdate > 1 ) { + const entries = await io.getUpdateAges({ + filters: µb.selectedFilterLists, + internal: [ '*' ], + }); + for ( const entry of entries ) { + if ( entry.ageNormalized < 2 ) { continue; } + needEmergencyUpdate = true; + lastEmergencyUpdate = now; + break; + } + } + + // Use the new schedule if and only if it is earlier than the previous + // one. + if ( next !== 0 ) { + updateDelay = Math.min(updateDelay, Math.max(next - now, 0)); + } + + if ( needEmergencyUpdate ) { + updateDelay = Math.min(updateDelay, 15000); + } + + next = now + updateDelay; + + const fetchDelay = needEmergencyUpdate + ? 2000 + : this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 60000; + + launchTimer.on(updateDelay, fetchDelay); + }; +} + +/******************************************************************************/ + +µb.assetObserver = function(topic, details) { + // Do not update filter list if not in use. + // Also, ignore really bad lists, i.e. those which should not even be + // fetched from a remote server. + if ( topic === 'before-asset-updated' ) { + if ( details.type === 'filters' ) { + if ( + this.availableFilterLists.hasOwnProperty(details.assetKey) === false || + this.selectedFilterLists.indexOf(details.assetKey) === -1 || + this.badLists.get(details.assetKey) + ) { + return; + } + } + return true; + } + + // Compile the list while we have the raw version in memory + if ( topic === 'after-asset-updated' ) { + // Skip selfie-related content. + if ( details.assetKey.startsWith('selfie/') ) { return; } + const cached = typeof details.content === 'string' && + details.content !== ''; + if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { + if ( cached ) { + if ( this.selectedFilterLists.indexOf(details.assetKey) !== -1 ) { + this.extractFilterListMetadata( + details.assetKey, + details.content + ); + if ( this.badLists.has(details.assetKey) === false ) { + io.put( + 'compiled/' + details.assetKey, + this.compileFilters(details.content, { + assetKey: details.assetKey, + trustedSource: this.isTrustedList(details.assetKey), + }) + ); + } + } + } else { + this.removeCompiledFilterList(details.assetKey); + } + } else if ( details.assetKey === this.pslAssetKey ) { + if ( cached ) { + this.compilePublicSuffixList(details.content); + } + } else if ( details.assetKey === 'ublock-badlists' ) { + this.badLists = new Map(); + } + broadcast({ + what: 'assetUpdated', + key: details.assetKey, + cached, + }); + // https://github.com/gorhill/uBlock/issues/2585 + // Whenever an asset is overwritten, the current selfie is quite + // likely no longer valid. + this.selfieManager.destroy(); + return; + } + + // Update failed. + if ( topic === 'asset-update-failed' ) { + broadcast({ + what: 'assetUpdated', + key: details.assetKey, + failed: true, + }); + return; + } + + // Reload all filter lists if needed. + if ( topic === 'after-assets-updated' ) { + if ( details.assetKeys.length !== 0 ) { + // https://github.com/gorhill/uBlock/pull/2314#issuecomment-278716960 + if ( + this.hiddenSettings.userResourcesLocation !== 'unset' || + vAPI.webextFlavor.soup.has('devbuild') + ) { + redirectEngine.invalidateResourcesSelfie(io); + } + this.loadFilterLists(); + } + this.scheduleAssetUpdater(); + broadcast({ + what: 'assetsUpdated', + assetKeys: details.assetKeys + }); + return; + } + + // New asset source became available, if it's a filter list, should we + // auto-select it? + if ( topic === 'builtin-asset-source-added' ) { + if ( details.entry.content === 'filters' ) { + if ( + details.entry.off === true && + this.listMatchesEnvironment(details.entry) + ) { + this.saveSelectedFilterLists([ details.assetKey ], true); + } + } + return; + } + + if ( topic === 'assets.json-updated' ) { + const { newDict, oldDict } = details; + if ( newDict['assets.json'] === undefined ) { return; } + if ( oldDict['assets.json'] === undefined ) { return; } + const newDefaultListset = new Set(newDict['assets.json'].defaultListset || []); + const oldDefaultListset = new Set(oldDict['assets.json'].defaultListset || []); + if ( newDefaultListset.size === 0 ) { return; } + if ( oldDefaultListset.size === 0 ) { + Array.from(Object.entries(oldDict)) + .filter(a => + a[1].content === 'filters' && + a[1].off === undefined && + /^https?:\/\//.test(a[0]) === false + ) + .map(a => a[0]) + .forEach(a => oldDefaultListset.add(a)); + if ( oldDefaultListset.size === 0 ) { return; } + } + const selectedListset = new Set(this.selectedFilterLists); + let selectedListModified = false; + for ( const assetKey of oldDefaultListset ) { + if ( newDefaultListset.has(assetKey) ) { continue; } + selectedListset.delete(assetKey); + selectedListModified = true; + } + for ( const assetKey of newDefaultListset ) { + if ( oldDefaultListset.has(assetKey) ) { continue; } + selectedListset.add(assetKey); + selectedListModified = true; + } + if ( selectedListModified ) { + this.saveSelectedFilterLists(Array.from(selectedListset)); + } + return; + } +}; diff --git a/src/js/support.js b/src/js/support.js new file mode 100644 index 0000000..9bfd7cb --- /dev/null +++ b/src/js/support.js @@ -0,0 +1,335 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uBlockDashboard */ + +'use strict'; + +import { onBroadcast } from './broadcast.js'; +import { dom, qs$ } from './dom.js'; + +/******************************************************************************/ + +const uselessKeys = [ + 'hiddenSettings.benchmarkDatasetURL', + 'hiddenSettings.blockingProfiles', + 'hiddenSettings.consoleLogLevel', + 'hiddenSettings.uiPopupConfig', + 'userSettings.alwaysDetachLogger', + 'userSettings.firewallPaneMinimized', + 'userSettings.externalLists', + 'userSettings.importedLists', + 'userSettings.popupPanelSections', + 'userSettings.uiAccentCustom', + 'userSettings.uiAccentCustom0', + 'userSettings.uiTheme', +]; + +const sensitiveValues = [ + 'filterset (user)', + 'userSettings.popupPanelSections', + 'hiddenSettings.userResourcesLocation', + 'trustedset.added', + 'hostRuleset.added', + 'switchRuleset.added', + 'urlRuleset.added', +]; + +const sensitiveKeys = [ + 'listset.added', +]; + +/******************************************************************************/ + +function removeKey(data, prop) { + if ( data instanceof Object === false ) { return; } + const pos = prop.indexOf('.'); + if ( pos !== -1 ) { + const key = prop.slice(0, pos); + return removeKey(data[key], prop.slice(pos + 1)); + } + delete data[prop]; +} + +function redactValue(data, prop) { + if ( data instanceof Object === false ) { return; } + const pos = prop.indexOf('.'); + if ( pos !== -1 ) { + return redactValue(data[prop.slice(0, pos)], prop.slice(pos + 1)); + } + let value = data[prop]; + if ( value === undefined ) { return; } + if ( Array.isArray(value) ) { + if ( value.length !== 0 ) { + value = `[array of ${value.length} redacted]`; + } else { + value = '[empty]'; + } + } else { + value = '[redacted]'; + } + data[prop] = value; +} + +function redactKeys(data, prop) { + if ( data instanceof Object === false ) { return; } + const pos = prop.indexOf('.'); + if ( pos !== -1 ) { + return redactKeys(data[prop.slice(0, pos)], prop.slice(pos + 1)); + } + const obj = data[prop]; + if ( obj instanceof Object === false ) { return; } + let count = 1; + for ( const key in obj ) { + if ( key.startsWith('file://') === false ) { continue; } + const newkey = `[list name ${count} redacted]`; + obj[newkey] = obj[key]; + obj[key] = undefined; + count += 1; + } +} + +function patchEmptiness(data, prop) { + const entry = data[prop]; + if ( Array.isArray(entry) && entry.length === 0 ) { + data[prop] = '[empty]'; + return; + } + if ( entry instanceof Object === false ) { return; } + if ( Object.keys(entry).length === 0 ) { + data[prop] = '[none]'; + return; + } + for ( const key in entry ) { + patchEmptiness(entry, key); + } +} + +function configToMarkdown(collapse = false) { + const text = cmEditor.getValue().trim(); + return collapse + ? '<details>\n\n```yaml\n' + text + '\n```\n</details>' + : '```yaml\n' + text + '\n```\n'; +} + +function addDetailsToReportURL(id, collapse = false) { + const elem = qs$(`#${id}`); + const url = new URL(dom.attr(elem, 'data-url')); + url.searchParams.set('configuration', configToMarkdown(collapse)); + dom.attr(elem, 'data-url', url); +} + +function renderData(data, depth = 0) { + const indent = ' '.repeat(depth); + if ( Array.isArray(data) ) { + const out = []; + for ( const value of data ) { + out.push(renderData(value, depth)); + } + return out.join('\n'); + } + if ( typeof data !== 'object' || data === null ) { + return `${indent}${data}`; + } + const out = []; + for ( const [ name, value ] of Object.entries(data) ) { + if ( typeof value === 'object' && value !== null ) { + out.push(`${indent}${name}:`); + out.push(renderData(value, depth + 1)); + continue; + } + out.push(`${indent}${name}: ${value}`); + } + return out.join('\n'); +} + +async function showSupportData() { + const supportData = await vAPI.messaging.send('dashboard', { + what: 'getSupportData', + }); + const shownData = JSON.parse(JSON.stringify(supportData)); + uselessKeys.forEach(prop => { removeKey(shownData, prop); }); + const redacted = true; + if ( redacted ) { + sensitiveValues.forEach(prop => { redactValue(shownData, prop); }); + sensitiveKeys.forEach(prop => { redactKeys(shownData, prop); }); + } + for ( const prop in shownData ) { + patchEmptiness(shownData, prop); + } + if ( reportedPage !== null ) { + shownData.popupPanel = reportedPage.popupPanel; + } + const text = renderData(shownData); + cmEditor.setValue(text); + cmEditor.clearHistory(); + + addDetailsToReportURL('filterReport', true); + addDetailsToReportURL('bugReport', true); +} + +/******************************************************************************/ + +const reportedPage = (( ) => { + const url = new URL(window.location.href); + try { + const pageURL = url.searchParams.get('pageURL'); + if ( pageURL === null ) { return null; } + const parsedURL = new URL(pageURL); + parsedURL.username = ''; + parsedURL.password = ''; + parsedURL.hash = ''; + const select = qs$('select[name="url"]'); + dom.text(select.options[0], parsedURL.href); + if ( parsedURL.search !== '' ) { + const option = dom.create('option'); + parsedURL.search = ''; + dom.text(option, parsedURL.href); + select.append(option); + } + if ( parsedURL.pathname !== '/' ) { + const option = dom.create('option'); + parsedURL.pathname = ''; + dom.text(option, parsedURL.href); + select.append(option); + } + const shouldUpdateLists = url.searchParams.get('shouldUpdateLists'); + if ( shouldUpdateLists !== null ) { + dom.body.dataset.shouldUpdateLists = shouldUpdateLists; + } + dom.cl.add(dom.body, 'filterIssue'); + return { + hostname: parsedURL.hostname.replace(/^(m|mobile|www)\./, ''), + popupPanel: JSON.parse(url.searchParams.get('popupPanel')), + }; + } catch(ex) { + } + return null; +})(); + +function reportSpecificFilterType() { + return qs$('select[name="type"]').value; +} + +function reportSpecificFilterIssue() { + const githubURL = new URL( + 'https://github.com/uBlockOrigin/uAssets/issues/new?template=specific_report_from_ubo.yml' + ); + const issueType = reportSpecificFilterType(); + let title = `${reportedPage.hostname}: ${issueType}`; + if ( qs$('#isNSFW').checked ) { + title = `[nsfw] ${title}`; + } + githubURL.searchParams.set('title', title); + githubURL.searchParams.set( + 'url_address_of_the_web_page', + '`' + qs$('select[name="url"]').value + '`' + ); + githubURL.searchParams.set('category', issueType); + githubURL.searchParams.set('configuration', configToMarkdown(true)); + vAPI.messaging.send('default', { + what: 'gotoURL', + details: { url: githubURL.href, select: true, index: -1 }, + }); +} + +async function updateFilterLists() { + if ( dom.body.dataset.shouldUpdateLists === undefined ) { return false; } + dom.cl.add(dom.body, 'updating'); + const assetKeys = JSON.parse(dom.body.dataset.shouldUpdateLists); + vAPI.messaging.send('dashboard', { what: 'supportUpdateNow', assetKeys }); + return true; +} + +/******************************************************************************/ + +const cmEditor = new CodeMirror(qs$('#supportData'), { + autofocus: true, + readOnly: true, + styleActiveLine: true, +}); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +/******************************************************************************/ + +(async ( ) => { + await showSupportData(); + + dom.on('[data-url]', 'click', ev => { + const elem = ev.target.closest('[data-url]'); + const url = dom.attr(elem, 'data-url'); + if ( typeof url !== 'string' || url === '' ) { return; } + vAPI.messaging.send('default', { + what: 'gotoURL', + details: { url, select: true, index: -1, shiftKey: ev.shiftKey }, + }); + ev.preventDefault(); + }); + + if ( reportedPage !== null ) { + if ( dom.body.dataset.shouldUpdateLists ) { + dom.on('.supportEntry.shouldUpdate button', 'click', ev => { + if ( updateFilterLists() === false ) { return; } + ev.preventDefault(); + }); + } + + dom.on('[data-i18n="supportReportSpecificButton"]', 'click', ev => { + reportSpecificFilterIssue(); + ev.preventDefault(); + }); + + dom.on('[data-i18n="supportFindSpecificButton"]', 'click', ev => { + const url = new URL('https://github.com/uBlockOrigin/uAssets/issues'); + url.searchParams.set('q', `is:issue sort:updated-desc "${reportedPage.hostname}" in:title`); + vAPI.messaging.send('default', { + what: 'gotoURL', + details: { url: url.href, select: true, index: -1 }, + }); + ev.preventDefault(); + }); + + dom.on('#showSupportInfo', 'click', ev => { + const button = ev.target.closest('#showSupportInfo'); + dom.cl.add(button, 'hidden'); + dom.cl.add('.a.b.c.d', 'e'); + cmEditor.refresh(); + }); + } + + onBroadcast(msg => { + if ( msg.what === 'assetsUpdated' ) { + dom.cl.remove(dom.body, 'updating'); + dom.cl.add(dom.body, 'updated'); + return; + } + if ( msg.what === 'staticFilteringDataChanged' ) { + showSupportData(); + return; + } + }); + + dom.on('#selectAllButton', 'click', ( ) => { + cmEditor.focus(); + cmEditor.execCommand('selectAll'); + }); +})(); diff --git a/src/js/tab.js b/src/js/tab.js new file mode 100644 index 0000000..c505e5a --- /dev/null +++ b/src/js/tab.js @@ -0,0 +1,1178 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import contextMenu from './contextmenu.js'; +import logger from './logger.js'; +import scriptletFilteringEngine from './scriptlet-filtering.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import µb from './background.js'; +import webext from './webext.js'; +import { PageStore } from './pagestore.js'; +import { i18n$ } from './i18n.js'; + +import { + sessionFirewall, + sessionSwitches, + sessionURLFiltering, +} from './filtering-engines.js'; + +import { + domainFromHostname, + hostnameFromURI, + originFromURI, +} from './uri-utils.js'; + +/******************************************************************************/ +/******************************************************************************/ + +// https://github.com/gorhill/httpswitchboard/issues/303 +// Any scheme other than 'http' and 'https' is remapped into a fake +// URL which trick the rest of µBlock into being able to process an +// otherwise unmanageable scheme. µBlock needs web page to have a proper +// hostname to work properly, so just like the 'chromium-behind-the-scene' +// fake domain name, we map unknown schemes into a fake '{scheme}-scheme' +// hostname. This way, for a specific scheme you can create scope with +// rules which will apply only to that scheme. + +µb.normalizeTabURL = (( ) => { + const tabURLNormalizer = new URL('about:blank'); + + return (tabId, tabURL) => { + if ( tabId < 0 ) { + return 'http://behind-the-scene/'; + } + try { + tabURLNormalizer.href = tabURL; + } catch(ex) { + return tabURL; + } + const protocol = tabURLNormalizer.protocol.slice(0, -1); + if ( protocol === 'https' || protocol === 'http' ) { + return tabURLNormalizer.href; + } + + let fakeHostname = protocol + '-scheme'; + + if ( tabURLNormalizer.hostname !== '' ) { + fakeHostname = tabURLNormalizer.hostname + '.' + fakeHostname; + } else if ( protocol === 'about' && protocol.pathname !== '' ) { + fakeHostname = tabURLNormalizer.pathname + '.' + fakeHostname; + } + + return `http://${fakeHostname}/`; + }; +})(); + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/99 +// https://github.com/gorhill/uBlock/issues/991 +// +// popup: +// Test/close target URL +// popunder: +// Test/close opener URL +// +// popup filter match: +// 0 = false +// 1 = true +// +// opener: 0 0 1 1 +// target: 0 1 0 1 +// ---- ---- ---- ---- +// result: a b c d +// +// a: do nothing +// b: close target +// c: close opener +// d: close target + +const onPopupUpdated = (( ) => { + // https://github.com/gorhill/uBlock/commit/1d448b85b2931412508aa01bf899e0b6f0033626#commitcomment-14944764 + // See if two URLs are different, disregarding scheme -- because the + // scheme can be unilaterally changed by the browser. + // https://github.com/gorhill/uBlock/issues/1378 + // Maybe no link element was clicked. + // https://github.com/gorhill/uBlock/issues/3287 + // Do not bail out if the target URL has no hostname. + const areDifferentURLs = function(a, b) { + if ( b === '' ) { return true; } + if ( b.startsWith('about:') ) { return false; } + let pos = a.indexOf('://'); + if ( pos === -1 ) { return false; } + a = a.slice(pos); + pos = b.indexOf('://'); + if ( pos !== -1 ) { + b = b.slice(pos); + } + return b !== a; + }; + + const popupMatch = function( + fctxt, + rootOpenerURL, + localOpenerURL, + targetURL, + popupType = 'popup' + ) { + // https://github.com/chrisaljoudi/uBlock/issues/323 + // https://github.com/chrisaljoudi/uBlock/issues/1142 + // https://github.com/uBlockOrigin/uBlock-issues/issues/1616 + // Don't block if uBO is turned off in popup's context + if ( + µb.getNetFilteringSwitch(targetURL) === false || + µb.getNetFilteringSwitch(µb.normalizeTabURL(0, targetURL)) === false + ) { + return 0; + } + + fctxt.setTabOriginFromURL(rootOpenerURL) + .setDocOriginFromURL(localOpenerURL || rootOpenerURL) + .setURL(targetURL) + .setType('popup'); + + // https://github.com/gorhill/uBlock/issues/1735 + // Do not bail out on `data:` URI, they are commonly used for popups. + // https://github.com/uBlockOrigin/uAssets/issues/255 + // Do not bail out on `about:blank`: an `about:blank` popup can be + // opened, with the sole purpose to serve as an intermediary in + // a sequence of chained popups. + // https://github.com/uBlockOrigin/uAssets/issues/263#issuecomment-272615772 + // Do not bail out, period: the static filtering engine must be + // able to examine all sorts of URLs for popup filtering purpose. + + // Dynamic filtering makes sense only when we have a valid opener + // hostname. + // https://github.com/gorhill/uBlock/commit/1d448b85b2931412508aa01bf899e0b6f0033626#commitcomment-14944764 + // Ignore bad target URL. On Firefox, an `about:blank` tab may be + // opened for a new tab before it is filled in with the real target + // URL. + if ( fctxt.getTabHostname() !== '' && targetURL !== 'about:blank' ) { + // Check per-site switch first + // https://github.com/gorhill/uBlock/issues/3060 + // - The no-popups switch must apply only to popups, not to + // popunders. + if ( + popupType === 'popup' && + sessionSwitches.evaluateZ( + 'no-popups', + fctxt.getTabHostname() + ) + ) { + fctxt.filter = { + raw: 'no-popups: ' + sessionSwitches.z + ' true', + result: 1, + source: 'switch' + }; + return 1; + } + + // https://github.com/gorhill/uBlock/issues/581 + // Take into account popup-specific rules in dynamic URL + // filtering, OR generic allow rules. + let result = sessionURLFiltering.evaluateZ( + fctxt.getTabHostname(), + targetURL, + popupType + ); + if ( + result === 1 && sessionURLFiltering.type === popupType || + result === 2 + ) { + fctxt.filter = sessionURLFiltering.toLogData(); + return result; + } + + // https://github.com/gorhill/uBlock/issues/581 + // Take into account `allow` rules in dynamic filtering: `block` + // rules are ignored, as block rules are not meant to block + // specific types like `popup` (just like with static filters). + result = sessionFirewall.evaluateCellZY( + fctxt.getTabHostname(), + fctxt.getHostname(), + popupType + ); + if ( result === 2 ) { + fctxt.filter = sessionFirewall.toLogData(); + return 2; + } + } + + fctxt.type = popupType; + const result = staticNetFilteringEngine.matchRequest(fctxt, 0b0001); + if ( result !== 0 ) { + fctxt.filter = staticNetFilteringEngine.toLogData(); + return result; + } + + return 0; + }; + + const mapPopunderResult = function( + fctxt, + popunderURL, + popunderHostname, + result + ) { + if ( fctxt.filter === undefined || fctxt.filter !== 'static' ) { + return 0; + } + if ( fctxt.filter.isUntokenized() ) { + return 0; + } + if ( fctxt.filter.isPureHostname() ) { + return result; + } + const re = new RegExp(fctxt.filter.regex, 'i'); + const matches = re.exec(popunderURL); + if ( matches === null ) { return 0; } + const beg = matches.index; + const end = beg + matches[0].length; + const pos = popunderURL.indexOf(popunderHostname); + if ( pos === -1 ) { return 0; } + // https://github.com/gorhill/uBlock/issues/1471 + // We test whether the opener hostname as at least one character + // within matched portion of URL. + // https://github.com/gorhill/uBlock/issues/1903 + // Ignore filters which cause a match before the start of the + // hostname in the URL. + return beg >= pos && beg < pos + popunderHostname.length && end > pos + ? result + : 0; + }; + + const popunderMatch = function( + fctxt, + rootOpenerURL, + localOpenerURL, + targetURL + ) { + let result = popupMatch( + fctxt, + targetURL, + undefined, + rootOpenerURL, + 'popunder' + ); + if ( result === 1 ) { return result; } + + // https://github.com/gorhill/uBlock/issues/1010#issuecomment-186824878 + // Check the opener tab as if it were the newly opened tab: if there + // is a hit against a popup filter, and if the matching filter is not + // a broad one, we will consider the opener tab to be a popunder tab. + // For now, a "broad" filter is one which does not touch any part of + // the hostname part of the opener URL. + let popunderURL = rootOpenerURL, + popunderHostname = hostnameFromURI(popunderURL); + if ( popunderHostname === '' ) { return 0; } + + result = mapPopunderResult( + fctxt, + popunderURL, + popunderHostname, + popupMatch(fctxt, targetURL, undefined, popunderURL) + ); + if ( result !== 0 ) { return result; } + + // https://github.com/gorhill/uBlock/issues/1598 + // Try to find a match against origin part of the opener URL. + popunderURL = originFromURI(popunderURL); + if ( popunderURL === '' ) { return 0; } + + return mapPopunderResult( + fctxt, + popunderURL, + popunderHostname, + popupMatch(fctxt, targetURL, undefined, popunderURL) + ); + }; + + return function(targetTabId, openerDetails) { + // Opener details. + const openerTabId = openerDetails.tabId; + let tabContext = µb.tabContextManager.lookup(openerTabId); + if ( tabContext === null ) { return; } + const rootOpenerURL = tabContext.rawURL; + if ( rootOpenerURL === '' ) { return; } + const pageStore = µb.pageStoreFromTabId(openerTabId); + + // https://github.com/uBlockOrigin/uBlock-issues/discussions/2534#discussioncomment-5264792 + // An `about:blank` frame's context is that of the parent context + let localOpenerURL = openerDetails.frameId !== 0 + ? openerDetails.frameURL + : undefined; + if ( localOpenerURL === 'about:blank' && pageStore !== null ) { + let openerFrameId = openerDetails.frameId; + do { + const frame = pageStore.getFrameStore(openerFrameId); + if ( frame === null ) { break; } + openerFrameId = frame.parentId; + const parentFrame = pageStore.getFrameStore(openerFrameId); + if ( parentFrame === null ) { break; } + localOpenerURL = parentFrame.frameURL; + } while ( localOpenerURL === 'about:blank' && openerFrameId !== 0 ); + } + + // Popup details. + tabContext = µb.tabContextManager.lookup(targetTabId); + if ( tabContext === null ) { return; } + let targetURL = tabContext.rawURL; + if ( targetURL === '' ) { return; } + + // https://github.com/gorhill/uBlock/issues/341 + // Allow popups if uBlock is turned off in opener's context. + if ( µb.getNetFilteringSwitch(rootOpenerURL) === false ) { return; } + + // https://github.com/gorhill/uBlock/issues/1538 + if ( + µb.getNetFilteringSwitch( + µb.normalizeTabURL(openerTabId, rootOpenerURL) + ) === false + ) { + return; + } + + // If the page URL is that of our document-blocked URL, extract the URL + // of the page which was blocked. + targetURL = µb.pageURLFromMaybeDocumentBlockedURL(targetURL); + + // MUST be reset before code below is called. + const fctxt = µb.filteringContext.duplicate(); + + // Popup test. + let popupType = 'popup', + result = 0; + // https://github.com/gorhill/uBlock/issues/2919 + // If the target tab matches a clicked link, assume it's legit. + // https://github.com/uBlockOrigin/uBlock-issues/issues/1912 + // If the target also matches the last clicked link, assume it's + // legit. + if ( + areDifferentURLs(targetURL, openerDetails.trustedURL) && + areDifferentURLs(targetURL, µb.maybeGoodPopup.url) + ) { + result = popupMatch(fctxt, rootOpenerURL, localOpenerURL, targetURL); + } + + // Popunder test. + if ( result === 0 && openerDetails.popunder ) { + result = popunderMatch(fctxt, rootOpenerURL, localOpenerURL, targetURL); + if ( result === 1 ) { + popupType = 'popunder'; + } + } + + // Log only for when there was a hit against an actual filter (allow or block). + // https://github.com/gorhill/uBlock/issues/2776 + if ( logger.enabled ) { + fctxt.setRealm('network').setType(popupType); + if ( popupType === 'popup' ) { + fctxt.setURL(targetURL) + .setTabId(openerTabId) + .setTabOriginFromURL(rootOpenerURL) + .setDocOriginFromURL(localOpenerURL || rootOpenerURL); + } else { + fctxt.setURL(rootOpenerURL) + .setTabId(targetTabId) + .setTabOriginFromURL(targetURL) + .setDocOriginFromURL(targetURL); + } + fctxt.toLogger(); + } + + // Not blocked + if ( result !== 1 ) { return; } + + // Only if a popup was blocked do we report it in the dynamic + // filtering pane. + if ( pageStore ) { + pageStore.journalAddRequest(fctxt, result); + pageStore.popupBlockedCount += 1; + } + + // Blocked + if ( µb.userSettings.showIconBadge ) { + µb.updateToolbarIcon(openerTabId, 0b010); + } + + // It is a popup, block and remove the tab. + if ( popupType === 'popup' ) { + µb.unbindTabFromPageStore(targetTabId); + vAPI.tabs.remove(targetTabId, false); + } else { + µb.unbindTabFromPageStore(openerTabId); + vAPI.tabs.remove(openerTabId, true); + } + + return true; + }; +})(); + +/******************************************************************************/ +/****************************************************************************** + +To keep track from which context *exactly* network requests are made. This is +often tricky for various reasons, and the challenge is not specific to one +browser. + +The time at which a URL is assigned to a tab and the time when a network +request for a root document is made must be assumed to be unrelated: it's all +asynchronous. There is no guaranteed order in which the two events are fired. + +Also, other "anomalies" can occur: + +- a network request for a root document is fired without the corresponding +tab being really assigned a new URL +<https://github.com/chrisaljoudi/uBlock/issues/516> + +- a network request for a secondary resource is labeled with a tab id for +which no root document was pulled for that tab. +<https://github.com/chrisaljoudi/uBlock/issues/1001> + +- a network request for a secondary resource is made without the root +document to which it belongs being formally bound yet to the proper tab id, +causing a bad scope to be used for filtering purpose. +<https://github.com/chrisaljoudi/uBlock/issues/1205> +<https://github.com/chrisaljoudi/uBlock/issues/1140> + +So the solution here is to keep a lightweight data structure which only +purpose is to keep track as accurately as possible of which root document +belongs to which tab. That's the only purpose, and because of this, there are +no restrictions for when the URL of a root document can be associated to a tab. + +Before, the PageStore object was trying to deal with this, but it had to +enforce some restrictions so as to not descend into one of the above issues, or +other issues. The PageStore object can only be associated with a tab for which +a definitive navigation event occurred, because it collects information about +what occurred in the tab (for example, the number of requests blocked for a +page). + +The TabContext objects do not suffer this restriction, and as a result they +offer the most reliable picture of which root document URL is really associated +to which tab. Moreover, the TabObject can undo an association from a root +document, and automatically re-associate with the next most recent. This takes +care of <https://github.com/chrisaljoudi/uBlock/issues/516>. + +The PageStore object no longer cache the various information about which +root document it is currently bound. When it needs to find out, it will always +defer to the TabContext object, which will provide the real answer. This takes +case of <https://github.com/chrisaljoudi/uBlock/issues/1205>. In effect, the +master switch and dynamic filtering rules can be evaluated now properly even +in the absence of a PageStore object, this was not the case before. + +Also, the TabContext object will try its best to find a good candidate root +document URL for when none exists. This takes care of +<https://github.com/chrisaljoudi/uBlock/issues/1001>. + +The TabContext manager is self-contained, and it takes care to properly +housekeep itself. + +*/ + +µb.tabContextManager = (( ) => { + const tabContexts = new Map(); + + // https://github.com/chrisaljoudi/uBlock/issues/1001 + // This is to be used as last-resort fallback in case a tab is found to not + // be bound while network requests are fired for the tab. + let mostRecentRootDocURL = ''; + let mostRecentRootDocURLTimestamp = 0; + + const popupCandidates = new Map(); + + const PopupCandidate = class { + constructor(createDetails, openerDetails) { + this.targetTabId = createDetails.tabId; + this.opener = { + tabId: createDetails.sourceTabId, + tabURL: openerDetails[0].url, + frameId: createDetails.sourceFrameId, + frameURL: openerDetails[1].url, + popunder: false, + trustedURL: createDetails.sourceTabId === µb.maybeGoodPopup.tabId + ? µb.maybeGoodPopup.url + : '' + }; + this.selfDestructionTimer = vAPI.defer.create(( ) => { + this.destroy(); + }); + this.launchSelfDestruction(); + } + + destroy() { + this.selfDestructionTimer.off(); + popupCandidates.delete(this.targetTabId); + } + + launchSelfDestruction() { + this.selfDestructionTimer.offon(10000); + } + }; + + const popupCandidateTest = async function(targetTabId) { + for ( const [ tabId, candidate ] of popupCandidates ) { + if ( + targetTabId !== tabId && + targetTabId !== candidate.opener.tabId + ) { + continue; + } + // https://github.com/gorhill/uBlock/issues/3129 + // If the trigger is a change in the opener's URL, mark the entry + // as candidate for popunder filtering. + if ( targetTabId === candidate.opener.tabId ) { + candidate.opener.popunder = true; + } + const result = onPopupUpdated(tabId, candidate.opener); + if ( result === true ) { + candidate.destroy(); + } else { + candidate.launchSelfDestruction(); + } + } + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1184 + // Do not consider a tab opened from `about:newtab` to be a popup + // candidate. + + const onTabCreated = async function(createDetails) { + const { sourceTabId, sourceFrameId, tabId } = createDetails; + const popup = popupCandidates.get(tabId); + if ( popup === undefined ) { + let openerDetails; + try { + openerDetails = await Promise.all([ + webext.webNavigation.getFrame({ + tabId: createDetails.sourceTabId, + frameId: 0, + }), + webext.webNavigation.getFrame({ + tabId: sourceTabId, + frameId: sourceFrameId, + }), + ]); + } + catch (reason) { + return; + } + if ( + Array.isArray(openerDetails) === false || + openerDetails.length !== 2 || + openerDetails[1] === null || + openerDetails[1].url === 'about:newtab' + ) { + return; + } + popupCandidates.set( + tabId, + new PopupCandidate(createDetails, openerDetails) + ); + } + popupCandidateTest(tabId); + }; + + const gcPeriod = 10 * 60 * 1000; + + // A pushed entry is removed from the stack unless it is committed with + // a set time. + const StackEntry = function(url, commit) { + this.url = url; + this.committed = commit; + this.tstamp = Date.now(); + }; + + const TabContext = function(tabId) { + this.tabId = tabId; + this.stack = []; + this.rawURL = + this.normalURL = + this.origin = + this.rootHostname = + this.rootDomain = ''; + this.commitTimer = vAPI.defer.create(( ) => { + this.onCommit(); + }); + this.gcTimer = vAPI.defer.create(( ) => { + this.onGC(); + }); + this.onGCBarrier = false; + this.netFiltering = true; + this.netFilteringReadTime = 0; + + tabContexts.set(tabId, this); + }; + + TabContext.prototype.destroy = function() { + if ( vAPI.isBehindTheSceneTabId(this.tabId) ) { return; } + this.gcTimer.off(); + tabContexts.delete(this.tabId); + }; + + TabContext.prototype.onGC = async function() { + if ( vAPI.isBehindTheSceneTabId(this.tabId) ) { return; } + if ( this.onGCBarrier ) { return; } + this.onGCBarrier = true; + this.gcTimer.off(); + const tab = await vAPI.tabs.get(this.tabId); + if ( tab instanceof Object === false || tab.discarded === true ) { + this.destroy(); + } else { + this.gcTimer.on(gcPeriod); + } + this.onGCBarrier = false; + }; + + // https://github.com/gorhill/uBlock/issues/248 + // Stack entries have to be committed to stick. Non-committed stack + // entries are removed after a set delay. + TabContext.prototype.onCommit = function() { + if ( vAPI.isBehindTheSceneTabId(this.tabId) ) { return; } + this.commitTimer.off(); + // Remove uncommitted entries at the top of the stack. + let i = this.stack.length; + while ( i-- ) { + if ( this.stack[i].committed ) { break; } + } + // https://github.com/gorhill/uBlock/issues/300 + // If no committed entry was found, fall back on the bottom-most one + // as being the committed one by default. + if ( i === -1 && this.stack.length !== 0 ) { + this.stack[0].committed = true; + i = 0; + } + i += 1; + if ( i < this.stack.length ) { + this.stack.length = i; + this.update(); + } + }; + + // This takes care of orphanized tab contexts. Can't be started for all + // contexts, as the behind-the-scene context is permanent -- so we do not + // want to flush it. + TabContext.prototype.autodestroy = function() { + if ( vAPI.isBehindTheSceneTabId(this.tabId) ) { return; } + this.gcTimer.on(gcPeriod); + }; + + // Update just force all properties to be updated to match the most recent + // root URL. + // https://github.com/uBlockOrigin/uBlock-issues/issues/1954 + // In case of document-blocked page, use the blocked page URL as the + // context. + TabContext.prototype.update = function() { + this.netFilteringReadTime = 0; + if ( this.stack.length === 0 ) { + this.rawURL = + this.normalURL = + this.origin = + this.rootHostname = + this.rootDomain = ''; + return; + } + const stackEntry = this.stack[this.stack.length - 1]; + this.rawURL = µb.pageURLFromMaybeDocumentBlockedURL(stackEntry.url); + this.normalURL = µb.normalizeTabURL(this.tabId, this.rawURL); + this.origin = originFromURI(this.normalURL); + this.rootHostname = hostnameFromURI(this.origin); + this.rootDomain = + domainFromHostname(this.rootHostname) || + this.rootHostname; + }; + + // Called whenever a candidate root URL is spotted for the tab. + TabContext.prototype.push = function(url) { + if ( vAPI.isBehindTheSceneTabId(this.tabId) ) { + return; + } + const count = this.stack.length; + if ( count !== 0 && this.stack[count - 1].url === url ) { + return; + } + this.stack.push(new StackEntry(url)); + this.update(); + popupCandidateTest(this.tabId); + this.commitTimer.offon(500); + }; + + // This tells that the url is definitely the one to be associated with the + // tab, there is no longer any ambiguity about which root URL is really + // sitting in which tab. + TabContext.prototype.commit = function(url) { + if ( vAPI.isBehindTheSceneTabId(this.tabId) ) { return; } + if ( this.stack.length !== 0 ) { + const top = this.stack[this.stack.length - 1]; + if ( top.url === url && top.committed ) { return false; } + } + this.stack = [new StackEntry(url, true)]; + this.update(); + return true; + }; + + TabContext.prototype.getNetFilteringSwitch = function() { + if ( this.netFilteringReadTime > µb.netWhitelistModifyTime ) { + return this.netFiltering; + } + // https://github.com/chrisaljoudi/uBlock/issues/1078 + // Use both the raw and normalized URLs. + this.netFiltering = µb.getNetFilteringSwitch(this.normalURL); + if ( + this.netFiltering && + this.rawURL !== this.normalURL && + this.rawURL !== '' + ) { + this.netFiltering = µb.getNetFilteringSwitch(this.rawURL); + } + this.netFilteringReadTime = Date.now(); + return this.netFiltering; + }; + + // These are to be used for the API of the tab context manager. + + const push = function(tabId, url) { + let entry = tabContexts.get(tabId); + if ( entry === undefined ) { + entry = new TabContext(tabId); + entry.autodestroy(); + } + entry.push(url); + mostRecentRootDocURL = url; + mostRecentRootDocURLTimestamp = Date.now(); + return entry; + }; + + // Find a tab context for a specific tab. + const lookup = function(tabId) { + return tabContexts.get(tabId) || null; + }; + + // Find a tab context for a specific tab. If none is found, attempt to + // fix this. When all fail, the behind-the-scene context is returned. + const mustLookup = function(tabId) { + const entry = tabContexts.get(tabId); + if ( entry !== undefined ) { + return entry; + } + // https://github.com/chrisaljoudi/uBlock/issues/1025 + // Google Hangout popup opens without a root frame. So for now we will + // just discard that best-guess root frame if it is too far in the + // future, at which point it ceases to be a "best guess". + if ( + mostRecentRootDocURL !== '' && + mostRecentRootDocURLTimestamp + 500 < Date.now() + ) { + mostRecentRootDocURL = ''; + } + // https://github.com/chrisaljoudi/uBlock/issues/1001 + // Not a behind-the-scene request, yet no page store found for the + // tab id: we will thus bind the last-seen root document to the + // unbound tab. It's a guess, but better than ending up filtering + // nothing at all. + if ( mostRecentRootDocURL !== '' ) { + return push(tabId, mostRecentRootDocURL); + } + // If all else fail at finding a page store, re-categorize the + // request as behind-the-scene. At least this ensures that ultimately + // the user can still inspect/filter those net requests which were + // about to fall through the cracks. + // Example: Chromium + case #12 at + // http://raymondhill.net/ublock/popup.html + return tabContexts.get(vAPI.noTabId); + }; + + // https://github.com/gorhill/uBlock/issues/1735 + // Filter for popups if actually committing. + const commit = function(tabId, url) { + let entry = tabContexts.get(tabId); + if ( entry === undefined ) { + entry = push(tabId, url); + } else if ( entry.commit(url) ) { + popupCandidateTest(tabId); + } + return entry; + }; + + const exists = function(tabId) { + return tabContexts.get(tabId) !== undefined; + }; + + // Behind-the-scene tab context + { + const entry = new TabContext(vAPI.noTabId); + entry.stack.push(new StackEntry('', true)); + entry.rawURL = ''; + entry.normalURL = µb.normalizeTabURL(entry.tabId); + entry.origin = originFromURI(entry.normalURL); + entry.rootHostname = hostnameFromURI(entry.origin); + entry.rootDomain = domainFromHostname(entry.rootHostname); + } + + // Context object, typically to be used to feed filtering engines. + const contextJunkyard = []; + const Context = class { + constructor(tabId) { + this.init(tabId); + } + init(tabId) { + const tabContext = lookup(tabId); + this.rootHostname = tabContext.rootHostname; + this.rootDomain = tabContext.rootDomain; + this.pageHostname = + this.pageDomain = + this.requestURL = + this.origin = + this.requestHostname = + this.requestDomain = ''; + return this; + } + dispose() { + contextJunkyard.push(this); + } + }; + + const createContext = function(tabId) { + if ( contextJunkyard.length ) { + return contextJunkyard.pop().init(tabId); + } + return new Context(tabId); + }; + + return { + push, + commit, + lookup, + mustLookup, + exists, + createContext, + onTabCreated, + }; +})(); + +/******************************************************************************/ +/******************************************************************************/ + +vAPI.Tabs = class extends vAPI.Tabs { + onActivated(details) { + const { tabId } = details; + if ( vAPI.isBehindTheSceneTabId(tabId) ) { return; } + // https://github.com/uBlockOrigin/uBlock-issues/issues/757 + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore === null ) { + this.onNewTab(tabId); + return; + } + super.onActivated(details); + // https://github.com/uBlockOrigin/uBlock-issues/issues/680 + µb.updateToolbarIcon(tabId); + contextMenu.update(tabId); + } + + onClosed(tabId) { + super.onClosed(tabId); + if ( vAPI.isBehindTheSceneTabId(tabId) ) { return; } + µb.unbindTabFromPageStore(tabId); + contextMenu.update(); + } + + onCreated(details) { + super.onCreated(details); + µb.tabContextManager.onTabCreated(details); + } + + // When the DOM content of root frame is loaded, this means the tab + // content has changed. + // + // The webRequest.onBeforeRequest() won't be called for everything + // else than http/https. Thus, in such case, we will bind the tab as + // early as possible in order to increase the likelihood of a context + // properly setup if network requests are fired from within the tab. + // Example: Chromium + case #6 at + // http://raymondhill.net/ublock/popup.html + // https://github.com/uBlockOrigin/uBlock-issues/issues/688#issuecomment-748179731 + // For non-network URIs, defer scriptlet injection to content script. The + // reason for this is that we need the effective URL and this information + // is not available at this point. + // + // https://github.com/uBlockOrigin/uBlock-issues/issues/2343 + // uBO's isolated world in Firefox just does not work as expected at + // point, so we have to wait before injecting scriptlets. + onNavigation(details) { + super.onNavigation(details); + const { frameId, tabId, url } = details; + if ( frameId === 0 ) { + µb.tabContextManager.commit(tabId, url); + const pageStore = µb.bindTabToPageStore(tabId, 'tabCommitted', details); + if ( pageStore !== null ) { + pageStore.journalAddRootFrame('committed', url); + } + } + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore === null ) { return; } + pageStore.setFrameURL(details); + if ( pageStore.getNetFilteringSwitch() ) { + scriptletFilteringEngine.injectNow(details); + } + } + + async onNewTab(tabId) { + const tab = await vAPI.tabs.get(tabId); + if ( tab === null ) { return; } + const { id, url = '' } = tab; + if ( url === '' ) { return; } + µb.tabContextManager.commit(id, url); + µb.bindTabToPageStore(id, 'tabUpdated', tab); + contextMenu.update(id); + } + + // It may happen the URL in the tab changes, while the page's document + // stays the same (for instance, Google Maps). Without this listener, + // the extension icon won't be properly refreshed. + onUpdated(tabId, changeInfo, tab) { + super.onUpdated(tabId, changeInfo, tab); + if ( !tab.url || tab.url === '' ) { return; } + if ( !changeInfo.url ) { return; } + µb.tabContextManager.commit(tabId, changeInfo.url); + µb.bindTabToPageStore(tabId, 'tabUpdated', tab); + } +}; + +vAPI.tabs = new vAPI.Tabs(); + +/******************************************************************************/ +/******************************************************************************/ + +// Create an entry for the tab if it doesn't exist. + +µb.bindTabToPageStore = function(tabId, context, details = undefined) { + this.updateToolbarIcon(tabId, 0b111); + + // Do not create a page store for URLs which are of no interests + if ( this.tabContextManager.exists(tabId) === false ) { + this.unbindTabFromPageStore(tabId); + return null; + } + + // Reuse page store if one exists: this allows to guess if a tab is a popup + let pageStore = this.pageStores.get(tabId); + + // Tab is not bound + if ( pageStore === undefined ) { + pageStore = PageStore.factory(tabId, details); + this.pageStores.set(tabId, pageStore); + this.pageStoresToken = Date.now(); + return pageStore; + } + + // https://github.com/chrisaljoudi/uBlock/issues/516 + // Never rebind behind-the-scene scope. + if ( vAPI.isBehindTheSceneTabId(tabId) ) { + return pageStore; + } + + // https://github.com/chrisaljoudi/uBlock/issues/516 + // If context is 'beforeRequest', do not rebind, wait for confirmation. + if ( context === 'beforeRequest' ) { + pageStore.netFilteringCache.empty(); + return pageStore; + } + + // Rebind according to context. We rebind even if the URL did not change, + // as maybe the tab was force-reloaded, in which case the page stats must + // be all reset. + pageStore.reuse(context, details); + + this.pageStoresToken = Date.now(); + + return pageStore; +}; + +/******************************************************************************/ + +µb.unbindTabFromPageStore = function(tabId) { + const pageStore = this.pageStores.get(tabId); + if ( pageStore === undefined ) { return; } + pageStore.dispose(); + this.pageStores.delete(tabId); + this.pageStoresToken = Date.now(); +}; + +/******************************************************************************/ + +µb.pageStoreFromTabId = function(tabId) { + return this.pageStores.get(tabId) || null; +}; + +µb.mustPageStoreFromTabId = function(tabId) { + return this.pageStores.get(tabId) || this.pageStores.get(vAPI.noTabId); +}; + +/******************************************************************************/ + +// Permanent page store for behind-the-scene requests. Must never be removed. +// +// https://github.com/uBlockOrigin/uBlock-issues/issues/651 +// The whitelist status of the tabless page store will be determined by +// the document context (if present) of the network request. + +{ + const NoPageStore = class extends PageStore { + getNetFilteringSwitch(fctxt) { + if ( fctxt ) { + const docOrigin = fctxt.getDocOrigin(); + if ( docOrigin ) { + return µb.getNetFilteringSwitch(docOrigin); + } + } + return super.getNetFilteringSwitch(); + } + }; + const pageStore = new NoPageStore(vAPI.noTabId); + µb.pageStores.set(pageStore.tabId, pageStore); + pageStore.title = i18n$('logBehindTheScene'); +} + +/******************************************************************************/ + +// Update visual of extension icon. + +{ + const tabIdToDetails = new Map(); + + const computeBadgeColor = (bits) => { + let color = µb.blockingProfileColorCache.get(bits); + if ( color !== undefined ) { return color; } + let max = 0; + for ( const profile of µb.liveBlockingProfiles ) { + const v = bits & (profile.bits & ~1); + if ( v < max ) { break; } + color = profile.color; + max = v; + } + if ( color === undefined ) { + color = '#666'; + } + µb.blockingProfileColorCache.set(bits, color); + return color; + }; + + const updateBadge = (tabId) => { + let parts = tabIdToDetails.get(tabId); + tabIdToDetails.delete(tabId); + + let state = 0; + let badge = ''; + let color = '#666'; + + const pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore !== null ) { + state = pageStore.getNetFilteringSwitch() ? 1 : 0; + if ( state === 1 ) { + if ( (parts & 0b0010) !== 0 ) { + const blockCount = pageStore.counts.blocked.any; + if ( blockCount !== 0 ) { + badge = µb.formatCount(blockCount); + } + } + if ( (parts & 0b0100) !== 0 ) { + color = computeBadgeColor( + µb.blockingModeFromHostname(pageStore.tabHostname) + ); + } + } + } + + // https://www.reddit.com/r/uBlockOrigin/comments/d33d37/ + if ( µb.userSettings.showIconBadge === false ) { + parts |= 0b1000; + } + + vAPI.setIcon(tabId, { parts, state, badge, color }); + }; + + // parts: bit 0 = icon + // bit 1 = badge text + // bit 2 = badge color + // bit 3 = hide badge + + µb.updateToolbarIcon = function(tabId, newParts = 0b0111) { + if ( this.readyToFilter === false ) { return; } + if ( typeof tabId !== 'number' ) { return; } + if ( vAPI.isBehindTheSceneTabId(tabId) ) { return; } + const currentParts = tabIdToDetails.get(tabId); + if ( currentParts === newParts ) { return; } + if ( currentParts === undefined ) { + self.requestIdleCallback( + ( ) => updateBadge(tabId), + { timeout: 701 } + ); + } else { + newParts |= currentParts; + } + tabIdToDetails.set(tabId, newParts); + }; +} + +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/455 +// Stale page store entries janitor + +{ + let pageStoreJanitorSampleAt = 0; + let pageStoreJanitorSampleSize = 10; + + const checkTab = async tabId => { + const tab = await vAPI.tabs.get(tabId); + if ( tab instanceof Object && tab.discarded !== true ) { return; } + µb.unbindTabFromPageStore(tabId); + }; + + const pageStoreJanitor = function() { + const tabIds = Array.from(µb.pageStores.keys()).sort(); + if ( pageStoreJanitorSampleAt >= tabIds.length ) { + pageStoreJanitorSampleAt = 0; + } + const n = Math.min( + pageStoreJanitorSampleAt + pageStoreJanitorSampleSize, + tabIds.length + ); + for ( let i = pageStoreJanitorSampleAt; i < n; i++ ) { + const tabId = tabIds[i]; + if ( vAPI.isBehindTheSceneTabId(tabId) ) { continue; } + checkTab(tabId); + } + pageStoreJanitorSampleAt = n; + + pageStoreJanitorTimer.on(pageStoreJanitorPeriod); + }; + + const pageStoreJanitorTimer = vAPI.defer.create(pageStoreJanitor); + const pageStoreJanitorPeriod = { min: 15 }; + + pageStoreJanitorTimer.on(pageStoreJanitorPeriod); +} + +/******************************************************************************/ diff --git a/src/js/tasks.js b/src/js/tasks.js new file mode 100644 index 0000000..8358fd8 --- /dev/null +++ b/src/js/tasks.js @@ -0,0 +1,42 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals requestIdleCallback, cancelIdleCallback */ + +'use strict'; + +/******************************************************************************/ + +export function queueTask(func, timeout = 5000) { + if ( typeof requestIdleCallback === 'undefined' ) { + return setTimeout(func, 1); + } + + return requestIdleCallback(func, { timeout }); +} + +export function dropTask(id) { + if ( typeof cancelIdleCallback === 'undefined' ) { + return clearTimeout(id); + } + + return cancelIdleCallback(id); +} diff --git a/src/js/text-encode.js b/src/js/text-encode.js new file mode 100644 index 0000000..06c7b2c --- /dev/null +++ b/src/js/text-encode.js @@ -0,0 +1,275 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; + +/******************************************************************************/ + +const textEncode = (( ) => { + + if ( µb.canFilterResponseData !== true ) { return; } + + // charset aliases extracted from: + // https://github.com/inexorabletash/text-encoding/blob/b4e5bc26e26e51f56e3daa9f13138c79f49d3c34/lib/encoding.js#L342 + const normalizedCharset = new Map([ + [ 'utf8', 'utf-8' ], + [ 'unicode-1-1-utf-8', 'utf-8' ], + [ 'utf-8', 'utf-8' ], + + [ 'windows-1250', 'windows-1250' ], + [ 'cp1250', 'windows-1250' ], + [ 'x-cp1250', 'windows-1250' ], + + [ 'windows-1251', 'windows-1251' ], + [ 'cp1251', 'windows-1251' ], + [ 'x-cp1251', 'windows-1251' ], + + [ 'windows-1252', 'windows-1252' ], + [ 'ansi_x3.4-1968', 'windows-1252' ], + [ 'ascii', 'windows-1252' ], + [ 'cp1252', 'windows-1252' ], + [ 'cp819', 'windows-1252' ], + [ 'csisolatin1', 'windows-1252' ], + [ 'ibm819', 'windows-1252' ], + [ 'iso-8859-1', 'windows-1252' ], + [ 'iso-ir-100', 'windows-1252' ], + [ 'iso8859-1', 'windows-1252' ], + [ 'iso88591', 'windows-1252' ], + [ 'iso_8859-1', 'windows-1252' ], + [ 'iso_8859-1:1987', 'windows-1252' ], + [ 'l1', 'windows-1252' ], + [ 'latin1', 'windows-1252' ], + [ 'us-ascii', 'windows-1252' ], + [ 'x-cp1252', 'windows-1252' ], + ]); + + // http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1250.TXT + const cp1250_range0 = new Uint8Array([ + /* 0x0100 */ 0x00, 0x00, 0xC3, 0xE3, 0xA5, 0xB9, 0xC6, 0xE6, + /* 0x0108 */ 0x00, 0x00, 0x00, 0x00, 0xC8, 0xE8, 0xCF, 0xEF, + /* 0x0110 */ 0xD0, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0118 */ 0xCA, 0xEA, 0xCC, 0xEC, 0x00, 0x00, 0x00, 0x00, + /* 0x0120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0130 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0138 */ 0x00, 0xC5, 0xE5, 0x00, 0x00, 0xBC, 0xBE, 0x00, + /* 0x0140 */ 0x00, 0xA3, 0xB3, 0xD1, 0xF1, 0x00, 0x00, 0xD2, + /* 0x0148 */ 0xF2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0150 */ 0xD5, 0xF5, 0x00, 0x00, 0xC0, 0xE0, 0x00, 0x00, + /* 0x0158 */ 0xD8, 0xF8, 0x8C, 0x9C, 0x00, 0x00, 0xAA, 0xBA, + /* 0x0160 */ 0x8A, 0x9A, 0xDE, 0xFE, 0x8D, 0x9D, 0x00, 0x00, + /* 0x0168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xF9, + /* 0x0170 */ 0xDB, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0178 */ 0x00, 0x8F, 0x9F, 0xAF, 0xBF, 0x8E, 0x9E, 0x00 + ]); + + // http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1251.TXT + const cp1251_range0 = new Uint8Array([ + /* 0x0400 */ 0x00, 0xA8, 0x80, 0x81, 0xAA, 0xBD, 0xB2, 0xAF, + /* 0x0408 */ 0xA3, 0x8A, 0x8C, 0x8E, 0x8D, 0x00, 0xA1, 0x8F, + /* 0x0410 */ 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + /* 0x0418 */ 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + /* 0x0420 */ 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + /* 0x0428 */ 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + /* 0x0430 */ 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + /* 0x0438 */ 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + /* 0x0440 */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + /* 0x0448 */ 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, + /* 0x0450 */ 0x00, 0xB8, 0x90, 0x83, 0xBA, 0xBE, 0xB3, 0xBF, + /* 0x0458 */ 0xBC, 0x9A, 0x9C, 0x9E, 0x9D, 0x00, 0xA2, 0x9F, + /* 0x0460 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0468 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0470 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0478 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0480 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0488 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0490 */ 0xA5, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]); + + // https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT + const cp1252_range0 = new Uint8Array([ + /* 0x0150 */ 0x00, 0x00, 0x8C, 0x9C, 0x00, 0x00, 0x00, 0x00, + /* 0x0158 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0160 */ 0x8A, 0x9A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0170 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x0178 */ 0x9F, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x9E, 0x00 + ]); + + const cp125x_range0 = new Uint8Array([ + /* 0x2010 */ 0x00, 0x00, 0x00, 0x96, 0x97, 0x00, 0x00, 0x00, + /* 0x2018 */ 0x91, 0x92, 0x82, 0x00, 0x93, 0x94, 0x84, 0x00, + /* 0x2020 */ 0x86, 0x87, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, + /* 0x2028 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x2030 */ 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 0x2038 */ 0x00, 0x8B, 0x9B, 0x00, 0x00, 0x00, 0x00, 0x00 + ]); + + const encoders = { + 'windows-1250': function(buf) { + let i = 0, n = buf.byteLength, o = 0, c; + while ( i < n ) { + c = buf[i++]; + if ( c < 0x80 ) { + buf[o++] = c; + } else { + if ( (c & 0xE0) === 0xC0 ) { + c = (c & 0x1F) << 6; + c |= (buf[i++] & 0x3F); + } else if ( (c & 0xF0) === 0xE0 ) { + c = (c & 0x0F) << 12; + c |= (buf[i++] & 0x3F) << 6; + c |= (buf[i++] & 0x3F); + } else if ( (c & 0xF8) === 0xF0 ) { + c = (c & 0x07) << 18; + c |= (buf[i++] & 0x3F) << 12; + c |= (buf[i++] & 0x3F) << 6; + c |= (buf[i++] & 0x3F); + } + if ( c < 0x100 ) { + buf[o++] = c; + } else if ( c < 0x180 ) { + buf[o++] = cp1250_range0[c - 0x100]; + } else if ( c >= 0x2010 && c < 0x2040 ) { + buf[o++] = cp125x_range0[c - 0x2010]; + } else if ( c === 0x02C7 ) { + buf[o++] = 0xA1; + } else if ( c === 0x02D8 ) { + buf[o++] = 0xA2; + } else if ( c === 0x02D9 ) { + buf[o++] = 0xFF; + } else if ( c === 0x02DB ) { + buf[o++] = 0xB2; + } else if ( c === 0x02DD ) { + buf[o++] = 0xBD; + } else if ( c === 0x20AC ) { + buf[o++] = 0x88; + } else if ( c === 0x2122 ) { + buf[o++] = 0x99; + } + } + } + return buf.slice(0, o); + }, + 'windows-1251': function(buf) { + let i = 0, n = buf.byteLength, o = 0, c; + while ( i < n ) { + c = buf[i++]; + if ( c < 0x80 ) { + buf[o++] = c; + } else { + if ( (c & 0xE0) === 0xC0 ) { + c = (c & 0x1F) << 6; + c |= (buf[i++] & 0x3F); + } else if ( (c & 0xF0) === 0xE0 ) { + c = (c & 0x0F) << 12; + c |= (buf[i++] & 0x3F) << 6; + c |= (buf[i++] & 0x3F); + } else if ( (c & 0xF8) === 0xF0 ) { + c = (c & 0x07) << 18; + c |= (buf[i++] & 0x3F) << 12; + c |= (buf[i++] & 0x3F) << 6; + c |= (buf[i++] & 0x3F); + } + if ( c < 0x100 ) { + buf[o++] = c; + } else if ( c >= 0x400 && c < 0x4A0 ) { + buf[o++] = cp1251_range0[c - 0x400]; + } else if ( c >= 0x2010 && c < 0x2040 ) { + buf[o++] = cp125x_range0[c - 0x2010]; + } else if ( c === 0x20AC ) { + buf[o++] = 0x88; + } else if ( c === 0x2116 ) { + buf[o++] = 0xB9; + } else if ( c === 0x2122 ) { + buf[o++] = 0x99; + } + } + } + return buf.slice(0, o); + }, + 'windows-1252': function(buf) { + let i = 0, n = buf.byteLength, o = 0, c; + while ( i < n ) { + c = buf[i++]; + if ( c < 0x80 ) { + buf[o++] = c; + } else { + if ( (c & 0xE0) === 0xC0 ) { + c = (c & 0x1F) << 6; + c |= (buf[i++] & 0x3F); + } else if ( (c & 0xF0) === 0xE0 ) { + c = (c & 0x0F) << 12; + c |= (buf[i++] & 0x3F) << 6; + c |= (buf[i++] & 0x3F); + } else if ( (c & 0xF8) === 0xF0 ) { + c = (c & 0x07) << 18; + c |= (buf[i++] & 0x3F) << 12; + c |= (buf[i++] & 0x3F) << 6; + c |= (buf[i++] & 0x3F); + } + if ( c < 0x100 ) { + buf[o++] = c; + } else if ( c >= 0x150 && c < 0x180 ) { + buf[o++] = cp1252_range0[c - 0x150]; + } else if ( c >= 0x2010 && c < 0x2040 ) { + buf[o++] = cp125x_range0[c - 0x2010]; + } else if ( c === 0x192 ) { + buf[o++] = 0x83; + } else if ( c === 0x2C6 ) { + buf[o++] = 0x88; + } else if ( c === 0x2DC ) { + buf[o++] = 0x98; + } else if ( c === 0x20AC ) { + buf[o++] = 0x80; + } else if ( c === 0x2122 ) { + buf[o++] = 0x99; + } + } + } + return buf.slice(0, o); + } + }; + + return { + encode: function(charset, buf) { + return encoders.hasOwnProperty(charset) ? + encoders[charset](buf) : + buf; + }, + normalizeCharset: function(charset) { + if ( charset === undefined ) { + return 'utf-8'; + } + return normalizedCharset.get(charset.toLowerCase()); + } + }; +})(); + +/******************************************************************************/ + +export default textEncode; + +/******************************************************************************/ diff --git a/src/js/text-utils.js b/src/js/text-utils.js new file mode 100644 index 0000000..198a433 --- /dev/null +++ b/src/js/text-utils.js @@ -0,0 +1,107 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://bugs.chromium.org/p/v8/issues/detail?id=2869 +// orphanizeString is to work around String.slice() potentially causing +// the whole raw filter list to be held in memory just because we cut out +// the title as a substring. + +function orphanizeString(s) { + return JSON.parse(JSON.stringify(s)); +} + +/******************************************************************************/ + +class LineIterator { + constructor(text, offset) { + this.text = text; + this.textLen = this.text.length; + this.offset = offset || 0; + } + next(offset) { + if ( offset !== undefined ) { + this.offset += offset; + } + let lineEnd = this.text.indexOf('\n', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.text.indexOf('\r', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.textLen; + } + } + const line = this.text.slice(this.offset, lineEnd); + this.offset = lineEnd + 1; + return line; + } + peek(n) { + const offset = this.offset; + return this.text.slice(offset, offset + n); + } + charCodeAt(offset) { + return this.text.charCodeAt(this.offset + offset); + } + eot() { + return this.offset >= this.textLen; + } +} + +/******************************************************************************/ + +// The field iterator is less CPU-intensive than when using native +// String.split(). + +class FieldIterator { + constructor(sep) { + this.text = ''; + this.sep = sep; + this.sepLen = sep.length; + this.offset = 0; + } + first(text) { + this.text = text; + this.offset = 0; + return this.next(); + } + next() { + let end = this.text.indexOf(this.sep, this.offset); + if ( end === -1 ) { + end = this.text.length; + } + const field = this.text.slice(this.offset, end); + this.offset = end + this.sepLen; + return field; + } + remainder() { + return this.text.slice(this.offset); + } +} + +/******************************************************************************/ + +export { + FieldIterator, + LineIterator, + orphanizeString, +}; diff --git a/src/js/theme.js b/src/js/theme.js new file mode 100644 index 0000000..d3f9b00 --- /dev/null +++ b/src/js/theme.js @@ -0,0 +1,151 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +function getActualTheme(nominalTheme) { + let theme = nominalTheme || 'light'; + if ( nominalTheme === 'auto' ) { + if ( typeof self.matchMedia === 'function' ) { + const mql = self.matchMedia('(prefers-color-scheme: dark)'); + theme = mql instanceof Object && mql.matches === true + ? 'dark' + : 'light'; + } else { + theme = 'light'; + } + } + return theme; +} + +function setTheme(theme, propagate = false) { + theme = getActualTheme(theme); + let w = self; + for (;;) { + const rootcl = w.document.documentElement.classList; + if ( theme === 'dark' ) { + rootcl.add('dark'); + rootcl.remove('light'); + } else /* if ( theme === 'light' ) */ { + rootcl.add('light'); + rootcl.remove('dark'); + } + if ( propagate === false ) { break; } + if ( w === w.parent ) { break; } + w = w.parent; + try { void w.document; } catch(ex) { return; } + } +} + +function setAccentColor( + accentEnabled, + accentColor, + propagate, + stylesheet = '' +) { + if ( accentEnabled && stylesheet === '' && self.hsluv !== undefined ) { + const toRGB = hsl => self.hsluv.hsluvToRgb(hsl).map(a => Math.round(a * 255)).join(' '); + // Normalize first + const hsl = self.hsluv.hexToHsluv(accentColor); + hsl[0] = Math.round(hsl[0] * 10) / 10; + hsl[1] = Math.round(Math.min(100, Math.max(0, hsl[1]))); + // Use normalized result to derive all shades + const shades = [ 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95 ]; + const text = []; + text.push(':root.accented {'); + for ( const shade of shades ) { + hsl[2] = shade; + text.push(` --primary-${shade}: ${toRGB(hsl)};`); + } + text.push('}'); + hsl[1] = Math.min(25, hsl[1]); + hsl[2] = 80; + text.push( + ':root.light.accented {', + ` --button-surface-rgb: ${toRGB(hsl)};`, + '}', + ); + hsl[2] = 30; + text.push( + ':root.dark.accented {', + ` --button-surface-rgb: ${toRGB(hsl)};`, + '}', + ); + text.push(''); + stylesheet = text.join('\n'); + vAPI.messaging.send('dom', { what: 'uiAccentStylesheet', stylesheet }); + } + let w = self; + for (;;) { + const wdoc = w.document; + let style = wdoc.querySelector('style#accentColors'); + if ( style !== null ) { style.remove(); } + if ( accentEnabled ) { + style = wdoc.createElement('style'); + style.id = 'accentColors'; + style.textContent = stylesheet; + wdoc.head.append(style); + wdoc.documentElement.classList.add('accented'); + } else { + wdoc.documentElement.classList.remove('accented'); + } + if ( propagate === false ) { break; } + if ( w === w.parent ) { break; } + w = w.parent; + try { void w.document; } catch(ex) { break; } + } +} + +{ + // https://github.com/uBlockOrigin/uBlock-issues/issues/1044 + // Offer the possibility to bypass uBO's default styling + vAPI.messaging.send('dom', { what: 'uiStyles' }).then(response => { + if ( typeof response !== 'object' || response === null ) { return; } + setTheme(response.uiTheme); + if ( response.uiAccentCustom ) { + setAccentColor( + true, + response.uiAccentCustom0, + false, + response.uiAccentStylesheet + ); + } + if ( response.uiStyles !== 'unset' ) { + document.body.style.cssText = response.uiStyles; + } + }); + + const rootcl = document.documentElement.classList; + if ( vAPI.webextFlavor.soup.has('mobile') ) { + rootcl.add('mobile'); + } else { + rootcl.add('desktop'); + } + if ( window.matchMedia('(min-resolution: 150dpi)').matches ) { + rootcl.add('hidpi'); + } +} + +export { + getActualTheme, + setTheme, + setAccentColor, +}; diff --git a/src/js/traffic.js b/src/js/traffic.js new file mode 100644 index 0000000..bf34fd4 --- /dev/null +++ b/src/js/traffic.js @@ -0,0 +1,1261 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals browser */ + +'use strict'; + +/******************************************************************************/ + +import htmlFilteringEngine from './html-filtering.js'; +import httpheaderFilteringEngine from './httpheader-filtering.js'; +import logger from './logger.js'; +import scriptletFilteringEngine from './scriptlet-filtering.js'; +import staticNetFilteringEngine from './static-net-filtering.js'; +import textEncode from './text-encode.js'; +import µb from './background.js'; +import * as sfp from './static-filtering-parser.js'; +import * as fc from './filtering-context.js'; +import { isNetworkURI } from './uri-utils.js'; + +import { + sessionFirewall, + sessionSwitches, + sessionURLFiltering, +} from './filtering-engines.js'; + + +/******************************************************************************/ + +// Platform-specific behavior. + +// https://github.com/uBlockOrigin/uBlock-issues/issues/42 +// https://bugzilla.mozilla.org/show_bug.cgi?id=1376932 +// Add proper version number detection once issue is fixed in Firefox. +let dontCacheResponseHeaders = + vAPI.webextFlavor.soup.has('firefox'); + +// The real actual webextFlavor value may not be set in stone, so listen +// for possible future changes. +window.addEventListener('webextFlavor', function() { + dontCacheResponseHeaders = + vAPI.webextFlavor.soup.has('firefox'); +}, { once: true }); + +/******************************************************************************/ + +const patchLocalRedirectURL = url => url.charCodeAt(0) === 0x2F /* '/' */ + ? vAPI.getURL(url) + : url; + +/******************************************************************************/ + +// Intercept and filter web requests. + +const onBeforeRequest = function(details) { + const fctxt = µb.filteringContext.fromWebrequestDetails(details); + + // Special handling for root document. + // https://github.com/chrisaljoudi/uBlock/issues/1001 + // This must be executed regardless of whether the request is + // behind-the-scene + if ( fctxt.itype === fctxt.MAIN_FRAME ) { + return onBeforeRootFrameRequest(fctxt); + } + + // Special treatment: behind-the-scene requests + const tabId = details.tabId; + if ( tabId < 0 ) { + return onBeforeBehindTheSceneRequest(fctxt); + } + + // Lookup the page store associated with this tab id. + let pageStore = µb.pageStoreFromTabId(tabId); + if ( pageStore === null ) { + const tabContext = µb.tabContextManager.mustLookup(tabId); + if ( tabContext.tabId < 0 ) { + return onBeforeBehindTheSceneRequest(fctxt); + } + vAPI.tabs.onNavigation({ tabId, frameId: 0, url: tabContext.rawURL }); + pageStore = µb.pageStoreFromTabId(tabId); + } + + const result = pageStore.filterRequest(fctxt); + + pageStore.journalAddRequest(fctxt, result); + + if ( logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + + // Redirected + + if ( fctxt.redirectURL !== undefined ) { + return { redirectUrl: patchLocalRedirectURL(fctxt.redirectURL) }; + } + + // Not redirected + + // Blocked + if ( result === 1 ) { + return { cancel: true }; + } + + // Not blocked + if ( + fctxt.itype === fctxt.SUB_FRAME && + details.parentFrameId !== -1 && + details.aliasURL === undefined + ) { + pageStore.setFrameURL(details); + } + + if ( result === 2 ) { + return { cancel: false }; + } +}; + +/******************************************************************************/ + +const onBeforeRootFrameRequest = function(fctxt) { + const requestURL = fctxt.url; + + // Special handling for root document. + // https://github.com/chrisaljoudi/uBlock/issues/1001 + // This must be executed regardless of whether the request is + // behind-the-scene + const requestHostname = fctxt.getHostname(); + let result = 0; + let logData; + + // If the site is whitelisted, disregard strict blocking + const trusted = µb.getNetFilteringSwitch(requestURL) === false; + if ( trusted ) { + result = 2; + if ( logger.enabled ) { + logData = { engine: 'u', result: 2, raw: 'whitelisted' }; + } + } + + // Permanently unrestricted? + if ( + result === 0 && + sessionSwitches.evaluateZ('no-strict-blocking', requestHostname) + ) { + result = 2; + if ( logger.enabled ) { + logData = { + engine: 'u', + result: 2, + raw: `no-strict-blocking: ${sessionSwitches.z} true` + }; + } + } + + // Temporarily whitelisted? + if ( result === 0 && strictBlockBypasser.isBypassed(requestHostname) ) { + result = 2; + if ( logger.enabled ) { + logData = { + engine: 'u', + result: 2, + raw: 'no-strict-blocking: true (temporary)' + }; + } + } + + // Static filtering + if ( result === 0 ) { + ({ result, logData } = shouldStrictBlock(fctxt, logger.enabled)); + } + + const pageStore = µb.bindTabToPageStore(fctxt.tabId, 'beforeRequest'); + if ( pageStore !== null ) { + pageStore.journalAddRootFrame('uncommitted', requestURL); + pageStore.journalAddRequest(fctxt, result); + } + + if ( logger.enabled ) { + fctxt.setFilter(logData); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/760 + // Redirect non-blocked request? + if ( result !== 1 && trusted === false && pageStore !== null ) { + pageStore.redirectNonBlockedRequest(fctxt); + } + + if ( logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + + // Redirected + + if ( fctxt.redirectURL !== undefined ) { + return { redirectUrl: patchLocalRedirectURL(fctxt.redirectURL) }; + } + + // Not blocked + + if ( result !== 1 ) { return; } + + // No log data means no strict blocking (because we need to report why + // the blocking occurs. + if ( logData === undefined ) { return; } + + // Blocked + + const query = encodeURIComponent(JSON.stringify({ + url: requestURL, + hn: requestHostname, + dn: fctxt.getDomain() || requestHostname, + fs: logData.raw + })); + + vAPI.tabs.replace( + fctxt.tabId, + vAPI.getURL('document-blocked.html?details=') + query + ); + + return { cancel: true }; +}; + +/******************************************************************************/ + +// Strict blocking through static filtering +// +// https://github.com/chrisaljoudi/uBlock/issues/1128 +// Do not block if the match begins after the hostname, +// except when the filter is specifically of type `other`. +// https://github.com/gorhill/uBlock/issues/490 +// Removing this for the time being, will need a new, dedicated type. +// https://github.com/uBlockOrigin/uBlock-issues/issues/1501 +// Support explicit exception filters. +// +// Let result of match for specific `document` type be `rs` +// Let result of match for no specific type be `rg` *after* going through +// confirmation necessary for implicit matches +// Let `important` be `i` +// Let final result be logical combination of `rs` and `rg` as follow: +// +// | rs | +// +--------+--------+--------+--------| +// | 0 | 1 | 1i | 2 | +// --------+--------+--------+--------+--------+--------| +// | 0 | rg | rs | rs | rs | +// rg | 1 | rg | rs | rs | rs | +// | 1i | rg | rg | rs | rg | +// | 2 | rg | rg | rs | rs | +// --------+--------+--------+--------+--------+--------+ + +const shouldStrictBlock = function(fctxt, loggerEnabled) { + const snfe = staticNetFilteringEngine; + + // Explicit filtering: `document` option + const rs = snfe.matchRequest(fctxt, 0b0011); + const is = rs === 1 && snfe.isBlockImportant(); + let lds; + if ( rs !== 0 || loggerEnabled ) { + lds = snfe.toLogData(); + } + + // | rs | + // +--------+--------+--------+--------| + // | 0 | 1 | 1i | 2 | + // --------+--------+--------+--------+--------+--------| + // | 0 | rg | rs | x | rs | + // rg | 1 | rg | rs | x | rs | + // | 1i | rg | rg | x | rg | + // | 2 | rg | rg | x | rs | + // --------+--------+--------+--------+--------+--------+ + if ( rs === 1 && is ) { + return { result: rs, logData: lds }; + } + + // Implicit filtering: no `document` option + fctxt.type = 'no_type'; + let rg = snfe.matchRequest(fctxt, 0b0011); + fctxt.type = 'main_frame'; + const ig = rg === 1 && snfe.isBlockImportant(); + let ldg; + if ( rg !== 0 || loggerEnabled ) { + ldg = snfe.toLogData(); + if ( rg === 1 && validateStrictBlock(fctxt, ldg) === false ) { + rg = 0; ldg = undefined; + } + } + + // | rs | + // +--------+--------+--------+--------| + // | 0 | 1 | 1i | 2 | + // --------+--------+--------+--------+--------+--------| + // | 0 | x | rs | - | rs | + // rg | 1 | x | rs | - | rs | + // | 1i | x | x | - | x | + // | 2 | x | x | - | rs | + // --------+--------+--------+--------+--------+--------+ + if ( rs === 0 || rg === 1 && ig || rg === 2 && rs !== 2 ) { + return { result: rg, logData: ldg }; + } + + // | rs | + // +--------+--------+--------+--------| + // | 0 | 1 | 1i | 2 | + // --------+--------+--------+--------+--------+--------| + // | 0 | - | x | - | x | + // rg | 1 | - | x | - | x | + // | 1i | - | - | - | - | + // | 2 | - | - | - | x | + // --------+--------+--------+--------+--------+--------+ + return { result: rs, logData: lds }; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/3208 +// Mind case insensitivity. +// https://github.com/uBlockOrigin/uBlock-issues/issues/1147 +// Do not strict-block if the filter pattern does not contain at least one +// token character. + +const validateStrictBlock = function(fctxt, logData) { + if ( typeof logData.regex !== 'string' ) { return false; } + if ( typeof logData.raw === 'string' && /\w/.test(logData.raw) === false ) { + return false; + } + const url = fctxt.url; + const re = new RegExp(logData.regex, 'i'); + const match = re.exec(url.toLowerCase()); + if ( match === null ) { return false; } + + // https://github.com/chrisaljoudi/uBlock/issues/1128 + // https://github.com/chrisaljoudi/uBlock/issues/1212 + // Verify that the end of the match is anchored to the end of the + // hostname. + // https://github.com/uBlockOrigin/uAssets/issues/7619#issuecomment-653010310 + // Also match FQDN. + const hostname = fctxt.getHostname(); + const hnpos = url.indexOf(hostname); + const hnlen = hostname.length; + const end = match.index + match[0].length - hnpos - hnlen; + return end === 0 || end === 1 || + end === 2 && url.charCodeAt(hnpos + hnlen) === 0x2E /* '.' */; +}; + +/******************************************************************************/ + +// Intercept and filter behind-the-scene requests. + +const onBeforeBehindTheSceneRequest = function(fctxt) { + const pageStore = µb.pageStoreFromTabId(fctxt.tabId); + if ( pageStore === null ) { return; } + + // https://github.com/gorhill/uBlock/issues/3150 + // Ability to globally block CSP reports MUST also apply to + // behind-the-scene network requests. + + let result = 0; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/339 + // Need to also test against `-scheme` since tabOrigin is normalized. + // Not especially elegant but for now this accomplishes the purpose of + // not dealing with network requests fired from a synthetic scope, + // that is unless advanced user mode is enabled. + + if ( + fctxt.tabOrigin.endsWith('-scheme') === false && + isNetworkURI(fctxt.tabOrigin) || + µb.userSettings.advancedUserEnabled || + fctxt.itype === fctxt.CSP_REPORT + ) { + result = pageStore.filterRequest(fctxt); + + // The "any-tab" scope is not whitelist-able, and in such case we must + // use the origin URL as the scope. Most such requests aren't going to + // be blocked, so we test for whitelisting and modify the result only + // when the request is being blocked. + // + // https://github.com/uBlockOrigin/uBlock-issues/issues/1478 + // Also remove potential redirection when request is to be + // whitelisted. + if ( + result === 1 && + µb.getNetFilteringSwitch(fctxt.tabOrigin) === false + ) { + result = 2; + fctxt.redirectURL = undefined; + fctxt.filter = { engine: 'u', result: 2, raw: 'whitelisted' }; + } + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/1204 + onBeforeBehindTheSceneRequest.journalAddRequest(fctxt, result); + + if ( logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + + // Redirected + + if ( fctxt.redirectURL !== undefined ) { + return { redirectUrl: patchLocalRedirectURL(fctxt.redirectURL) }; + } + + // Blocked? + + if ( result === 1 ) { + return { cancel: true }; + } +}; + +// https://github.com/uBlockOrigin/uBlock-issues/issues/1204 +// Report the tabless network requests to all page stores matching the +// document origin. This is an approximation, there is unfortunately no +// way to know for sure which exact page triggered a tabless network +// request. + +{ + const pageStores = new Set(); + let hostname = ''; + let pageStoresToken = 0; + + const reset = function() { + hostname = ''; + pageStores.clear(); + pageStoresToken = 0; + }; + + const gc = ( ) => { + if ( pageStoresToken !== µb.pageStoresToken ) { return reset(); } + gcTimer.on(30011); + }; + + const gcTimer = vAPI.defer.create(gc); + + onBeforeBehindTheSceneRequest.journalAddRequest = (fctxt, result) => { + const docHostname = fctxt.getDocHostname(); + if ( + docHostname !== hostname || + pageStoresToken !== µb.pageStoresToken + ) { + hostname = docHostname; + pageStores.clear(); + for ( const pageStore of µb.pageStores.values() ) { + if ( pageStore.tabHostname !== docHostname ) { continue; } + pageStores.add(pageStore); + } + pageStoresToken = µb.pageStoresToken; + gcTimer.offon(30011); + } + for ( const pageStore of pageStores ) { + pageStore.journalAddRequest(fctxt, result); + } + }; +} + +/******************************************************************************/ + +// To handle: +// - Media elements larger than n kB +// - Scriptlet injection (requires ability to modify response body) +// - HTML filtering (requires ability to modify response body) +// - CSP injection + +const onHeadersReceived = function(details) { + + const fctxt = µb.filteringContext.fromWebrequestDetails(details); + const isRootDoc = fctxt.itype === fctxt.MAIN_FRAME; + + let pageStore = µb.pageStoreFromTabId(fctxt.tabId); + if ( pageStore === null ) { + if ( isRootDoc === false ) { return; } + pageStore = µb.bindTabToPageStore(fctxt.tabId, 'beforeRequest'); + } + if ( pageStore.getNetFilteringSwitch(fctxt) === false ) { return; } + + if ( fctxt.itype === fctxt.IMAGE || fctxt.itype === fctxt.MEDIA ) { + const result = foilLargeMediaElement(details, fctxt, pageStore); + if ( result !== undefined ) { return result; } + } + + // Keep in mind response headers will be modified in-place if needed, so + // `details.responseHeaders` will always point to the modified response + // headers. + const { responseHeaders } = details; + if ( Array.isArray(responseHeaders) === false ) { return; } + + if ( isRootDoc === false ) { + const result = pageStore.filterOnHeaders(fctxt, responseHeaders); + if ( result !== 0 ) { + if ( logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + if ( result === 1 ) { + pageStore.journalAddRequest(fctxt, 1); + return { cancel: true }; + } + } + } + + const mime = mimeFromHeaders(responseHeaders); + + // https://github.com/gorhill/uBlock/issues/2813 + // Disable the blocking of large media elements if the document is itself + // a media element: the resource was not prevented from loading so no + // point to further block large media elements for the current document. + if ( isRootDoc ) { + if ( reMediaContentTypes.test(mime) ) { + pageStore.allowLargeMediaElementsUntil = 0; + // Fall-through: this could be an SVG document, which supports + // script tags. + } + } + + if ( bodyFilterer.canFilter(fctxt, details) ) { + const jobs = []; + // `replace=` filter option + const replaceDirectives = + staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'replace'); + if ( replaceDirectives ) { + jobs.push({ + fn: textResponseFilterer, + args: [ replaceDirectives ], + }); + } + // html filtering + if ( mime === 'text/html' || mime === 'application/xhtml+xml' ) { + const selectors = htmlFilteringEngine.retrieve(fctxt); + if ( selectors ) { + jobs.push({ + fn: htmlResponseFilterer, + args: [ selectors ], + }); + } + } + if ( jobs.length !== 0 ) { + bodyFilterer.doFilter(fctxt, jobs); + } + } + + let modifiedHeaders = false; + if ( httpheaderFilteringEngine.apply(fctxt, responseHeaders) === true ) { + modifiedHeaders = true; + } + if ( injectCSP(fctxt, pageStore, responseHeaders) === true ) { + modifiedHeaders = true; + } + if ( injectPP(fctxt, pageStore, responseHeaders) === true ) { + modifiedHeaders = true; + } + + // https://bugzilla.mozilla.org/show_bug.cgi?id=1376932 + // Prevent document from being cached by the browser if we modified it, + // either through HTML filtering and/or modified response headers. + // https://github.com/uBlockOrigin/uBlock-issues/issues/229 + // Use `no-cache` instead of `no-cache, no-store, must-revalidate`, this + // allows Firefox's offline mode to work as expected. + if ( modifiedHeaders && dontCacheResponseHeaders ) { + const cacheControl = µb.hiddenSettings.cacheControlForFirefox1376932; + if ( cacheControl !== 'unset' ) { + let i = headerIndexFromName('cache-control', responseHeaders); + if ( i !== -1 ) { + responseHeaders[i].value = cacheControl; + } else { + responseHeaders.push({ name: 'Cache-Control', value: cacheControl }); + } + modifiedHeaders = true; + } + } + + if ( modifiedHeaders ) { + return { responseHeaders }; + } +}; + +const reMediaContentTypes = /^(?:audio|image|video)\//; + +/******************************************************************************/ + +const mimeFromHeaders = headers => { + if ( Array.isArray(headers) === false ) { return ''; } + return mimeFromContentType(headerValueFromName('content-type', headers)); +}; + +const mimeFromContentType = contentType => { + const match = reContentTypeMime.exec(contentType); + if ( match === null ) { return ''; } + return match[0].toLowerCase(); +}; + +const reContentTypeMime = /^[^;]+/i; + +/******************************************************************************/ + +function textResponseFilterer(session, directives) { + const applied = []; + for ( const directive of directives ) { + if ( directive.refs instanceof Object === false ) { continue; } + if ( directive.result !== 1 ) { + applied.push(directive); + continue; + } + const { refs } = directive; + if ( refs.$cache === null ) { + refs.$cache = sfp.parseReplaceValue(refs.value); + } + const cache = refs.$cache; + if ( cache === undefined ) { continue; } + cache.re.lastIndex = 0; + if ( cache.re.test(session.getString()) !== true ) { continue; } + cache.re.lastIndex = 0; + session.setString(session.getString().replace( + cache.re, + cache.replacement + )); + applied.push(directive); + } + if ( applied.length === 0 ) { return; } + if ( logger.enabled !== true ) { return; } + session.setRealm('network') + .pushFilters(applied.map(a => a.logData())) + .toLogger(); +} + +/******************************************************************************/ + +function htmlResponseFilterer(session, selectors) { + if ( htmlResponseFilterer.domParser === null ) { + htmlResponseFilterer.domParser = new DOMParser(); + htmlResponseFilterer.xmlSerializer = new XMLSerializer(); + } + + const doc = htmlResponseFilterer.domParser.parseFromString( + session.getString(), + session.mime + ); + + if ( selectors === undefined ) { return; } + if ( htmlFilteringEngine.apply(doc, session, selectors) !== true ) { return; } + + // https://stackoverflow.com/questions/6088972/get-doctype-of-an-html-as-string-with-javascript/10162353#10162353 + const doctypeStr = [ + doc.doctype instanceof Object ? + htmlResponseFilterer.xmlSerializer.serializeToString(doc.doctype) + '\n' : + '', + doc.documentElement.outerHTML, + ].join('\n'); + session.setString(doctypeStr); +} +htmlResponseFilterer.domParser = null; +htmlResponseFilterer.xmlSerializer = null; + + +/******************************************************************************* + + The response body filterer is responsible for: + + - Realize static network filter option `replace=` + - HTML filtering + +**/ + +const bodyFilterer = (( ) => { + const sessions = new Map(); + const reContentTypeCharset = /charset=['"]?([^'" ]+)/i; + const otherValidMimes = new Set([ + 'application/javascript', + 'application/json', + 'application/mpegurl', + 'application/vnd.api+json', + 'application/vnd.apple.mpegurl', + 'application/vnd.apple.mpegurl.audio', + 'application/x-javascript', + 'application/x-mpegurl', + 'application/xhtml+xml', + 'application/xml', + 'audio/mpegurl', + 'audio/x-mpegurl', + ]); + const BINARY_TYPES = fc.FONT | fc.IMAGE | fc.MEDIA | fc.WEBSOCKET; + const MAX_BUFFER_LENGTH = 3 * 1024 * 1024; + + let textDecoder, textEncoder; + let mime = ''; + let charset = ''; + + const contentTypeFromDetails = details => { + switch ( details.type ) { + case 'script': + return 'text/javascript; charset=utf-8'; + case 'stylesheet': + return 'text/css'; + default: + break; + } + return ''; + }; + + const charsetFromContentType = contentType => { + const match = reContentTypeCharset.exec(contentType); + if ( match === null ) { return; } + return match[1].toLowerCase(); + }; + + const charsetFromMime = mime => { + switch ( mime ) { + case 'application/xml': + case 'application/xhtml+xml': + case 'text/html': + case 'text/css': + return; + default: + break; + } + return 'utf-8'; + }; + + const charsetFromStream = bytes => { + if ( bytes.length < 3 ) { return; } + if ( bytes[0] === 0xEF && bytes[1] === 0xBB && bytes[2] === 0xBF ) { + return 'utf-8'; + } + let i = -1; + while ( i < 65536 ) { + i += 1; + /* c */ if ( bytes[i+0] !== 0x63 ) { continue; } + /* h */ if ( bytes[i+1] !== 0x68 ) { continue; } + /* a */ if ( bytes[i+2] !== 0x61 ) { continue; } + /* r */ if ( bytes[i+3] !== 0x72 ) { continue; } + /* s */ if ( bytes[i+4] !== 0x73 ) { continue; } + /* e */ if ( bytes[i+5] !== 0x65 ) { continue; } + /* t */ if ( bytes[i+6] !== 0x74 ) { continue; } + break; + } + if ( (i - 40) >= 65536 ) { return; } + i += 8; + // find first alpha character + let j = -1; + while ( j < 8 ) { + j += 1; + const c = bytes[i+j]; + if ( c >= 0x41 && c <= 0x5A ) { break; } + if ( c >= 0x61 && c <= 0x7A ) { break; } + } + if ( j === 8 ) { return; } + i += j; + // Collect characters until first non charset-name-character + const chars = []; + j = 0; + while ( j < 24 ) { + const c = bytes[i+j]; + if ( c < 0x2D ) { break; } + if ( c > 0x2D && c < 0x30 ) { break; } + if ( c > 0x39 && c < 0x41 ) { break; } + if ( c > 0x5A && c < 0x61 ) { break; } + if ( c > 0x7A ) { break; } + chars.push(c); + j += 1; + } + if ( j === 20 ) { return; } + return String.fromCharCode(...chars).toLowerCase(); + }; + + const streamClose = (session, buffer) => { + if ( buffer !== undefined ) { + session.stream.write(buffer); + } else if ( session.buffer !== undefined ) { + session.stream.write(session.buffer); + } + session.stream.close(); + }; + + const onStreamData = function(ev) { + const session = sessions.get(this); + if ( session === undefined ) { + this.write(ev.data); + this.disconnect(); + return; + } + if ( this.status !== 'transferringdata' ) { + if ( this.status !== 'finishedtransferringdata' ) { + sessions.delete(this); + this.disconnect(); + return; + } + } + if ( session.buffer === null ) { + session.buffer = new Uint8Array(ev.data); + return; + } + const buffer = new Uint8Array( + session.buffer.byteLength + ev.data.byteLength + ); + buffer.set(session.buffer); + buffer.set(new Uint8Array(ev.data), session.buffer.byteLength); + session.buffer = buffer; + if ( session.buffer.length >= MAX_BUFFER_LENGTH ) { + sessions.delete(this); + this.write(session.buffer); + this.disconnect(); + } + }; + + const onStreamStop = function() { + const session = sessions.get(this); + sessions.delete(this); + if ( session === undefined || session.buffer === null ) { + this.close(); + return; + } + if ( this.status !== 'finishedtransferringdata' ) { return; } + + // If encoding is still unknown, try to extract from stream data + if ( session.charset === undefined ) { + const charsetFound = charsetFromStream(session.buffer); + if ( charsetFound === undefined ) { return streamClose(session); } + const charsetUsed = textEncode.normalizeCharset(charsetFound); + if ( charsetUsed === undefined ) { return streamClose(session); } + session.charset = charsetUsed; + } + + while ( session.jobs.length !== 0 ) { + const job = session.jobs.shift(); + job.fn(session, ...job.args); + } + if ( session.modified !== true ) { return streamClose(session); } + + if ( textEncoder === undefined ) { + textEncoder = new TextEncoder(); + } + let encodedStream = textEncoder.encode(session.str); + + if ( session.charset !== 'utf-8' ) { + encodedStream = textEncode.encode(session.charset, encodedStream); + } + + streamClose(session, encodedStream); + }; + + const onStreamError = function() { + sessions.delete(this); + }; + + return class Session extends µb.FilteringContext { + constructor(fctxt, mime, charset, jobs) { + super(fctxt); + this.stream = null; + this.buffer = null; + this.mime = mime; + this.charset = charset; + this.str = null; + this.modified = false; + this.jobs = jobs; + } + getString() { + if ( this.str !== null ) { return this.str; } + if ( textDecoder !== undefined ) { + if ( textDecoder.encoding !== this.charset ) { + textDecoder = undefined; + } + } + if ( textDecoder === undefined ) { + textDecoder = new TextDecoder(this.charset); + } + this.str = textDecoder.decode(this.buffer); + return this.str; + } + setString(s) { + this.str = s; + this.modified = true; + } + static doFilter(fctxt, jobs) { + if ( jobs.length === 0 ) { return; } + const session = new Session(fctxt, mime, charset, jobs); + session.stream = browser.webRequest.filterResponseData(session.id); + session.stream.ondata = onStreamData; + session.stream.onstop = onStreamStop; + session.stream.onerror = onStreamError; + sessions.set(session.stream, session); + } + static canFilter(fctxt, details) { + if ( µb.canFilterResponseData !== true ) { return; } + + if ( (fctxt.itype & BINARY_TYPES) !== 0 ) { return; } + + if ( fctxt.method !== fc.METHOD_GET ) { + if ( fctxt.method !== fc.METHOD_POST ) { + return; + } + } + + // https://github.com/gorhill/uBlock/issues/3478 + const statusCode = details.statusCode || 0; + if ( statusCode === 0 ) { return; } + + const hostname = fctxt.getHostname(); + if ( hostname === '' ) { return; } + + // https://bugzilla.mozilla.org/show_bug.cgi?id=1426789 + const headers = details.responseHeaders; + const disposition = headerValueFromName('content-disposition', headers); + if ( disposition !== '' ) { + if ( disposition.startsWith('inline') === false ) { return; } + } + + mime = 'text/plain'; + charset = 'utf-8'; + const contentType = headerValueFromName('content-type', headers) || + contentTypeFromDetails(details); + if ( contentType !== '' ) { + mime = mimeFromContentType(contentType); + if ( mime === undefined ) { return; } + if ( mime.startsWith('text/') === false ) { + if ( otherValidMimes.has(mime) === false ) { return; } + } + charset = charsetFromContentType(contentType); + if ( charset !== undefined ) { + charset = textEncode.normalizeCharset(charset); + if ( charset === undefined ) { return; } + } else { + charset = charsetFromMime(mime); + } + } + + return true; + } + }; +})(); + +/******************************************************************************/ + +const injectCSP = function(fctxt, pageStore, responseHeaders) { + const cspSubsets = []; + const requestType = fctxt.type; + + // Start collecting policies >>>>>>>> + + // ======== built-in policies + + const builtinDirectives = []; + + if ( pageStore.filterScripting(fctxt, true) === 1 ) { + builtinDirectives.push(µb.cspNoScripting); + if ( logger.enabled ) { + fctxt.setRealm('network').setType('scripting').toLogger(); + } + } + // https://github.com/uBlockOrigin/uBlock-issues/issues/422 + // We need to derive a special context for filtering `inline-script`, + // as the embedding document for this "resource" will always be the + // frame itself, not that of the parent of the frame. + else { + const fctxt2 = fctxt.duplicate(); + fctxt2.type = 'inline-script'; + fctxt2.setDocOriginFromURL(fctxt.url); + const result = pageStore.filterRequest(fctxt2); + if ( result === 1 ) { + builtinDirectives.push(µb.cspNoInlineScript); + } + if ( result === 2 && logger.enabled ) { + fctxt2.setRealm('network').toLogger(); + } + } + + // https://github.com/gorhill/uBlock/issues/1539 + // - Use a CSP to also forbid inline fonts if remote fonts are blocked. + fctxt.type = 'inline-font'; + if ( pageStore.filterRequest(fctxt) === 1 ) { + builtinDirectives.push(µb.cspNoInlineFont); + if ( logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + } + + if ( builtinDirectives.length !== 0 ) { + cspSubsets[0] = builtinDirectives.join(', '); + } + + // ======== filter-based policies + + // Static filtering. + + fctxt.type = requestType; + const staticDirectives = + staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'csp'); + if ( staticDirectives !== undefined ) { + for ( const directive of staticDirectives ) { + if ( directive.result !== 1 ) { continue; } + cspSubsets.push(directive.value); + } + } + + // URL filtering `allow` rules override static filtering. + if ( + cspSubsets.length !== 0 && + sessionURLFiltering.evaluateZ( + fctxt.getTabHostname(), + fctxt.url, + 'csp' + ) === 2 + ) { + if ( logger.enabled ) { + fctxt.setRealm('network') + .setType('csp') + .setFilter(sessionURLFiltering.toLogData()) + .toLogger(); + } + return; + } + + // Dynamic filtering `allow` rules override static filtering. + if ( + cspSubsets.length !== 0 && + µb.userSettings.advancedUserEnabled && + sessionFirewall.evaluateCellZY( + fctxt.getTabHostname(), + fctxt.getTabHostname(), + '*' + ) === 2 + ) { + if ( logger.enabled ) { + fctxt.setRealm('network') + .setType('csp') + .setFilter(sessionFirewall.toLogData()) + .toLogger(); + } + return; + } + + // <<<<<<<< All policies have been collected + + // Static CSP policies will be applied. + + if ( logger.enabled && staticDirectives !== undefined ) { + fctxt.setRealm('network') + .pushFilters(staticDirectives.map(a => a.logData())) + .toLogger(); + } + + if ( cspSubsets.length === 0 ) { return; } + + µb.updateToolbarIcon(fctxt.tabId, 0b0010); + + // Use comma to merge CSP directives. + // Ref.: https://www.w3.org/TR/CSP2/#implementation-considerations + // + // https://github.com/gorhill/uMatrix/issues/967 + // Inject a new CSP header rather than modify an existing one, except + // if the current environment does not support merging headers: + // Firefox 58/webext and less can't merge CSP headers, so we will merge + // them here. + + responseHeaders.push({ + name: 'Content-Security-Policy', + value: cspSubsets.join(', ') + }); + + return true; +}; + +/******************************************************************************/ + +const injectPP = function(fctxt, pageStore, responseHeaders) { + const permissions = []; + const directives = staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'permissions'); + if ( directives !== undefined ) { + for ( const directive of directives ) { + if ( directive.result !== 1 ) { continue; } + permissions.push(directive.value.replace('|', ', ')); + } + } + + if ( logger.enabled && directives !== undefined ) { + fctxt.setRealm('network') + .pushFilters(directives.map(a => a.logData())) + .toLogger(); + } + + if ( permissions.length === 0 ) { return; } + + µb.updateToolbarIcon(fctxt.tabId, 0x02); + + responseHeaders.push({ + name: 'permissions-policy', + value: permissions.join(', ') + }); + + return true; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1163 +// "Block elements by size". +// https://github.com/gorhill/uBlock/issues/1390#issuecomment-187310719 +// Do not foil when the media element is fetched from the browser +// cache. This works only when the webext API supports the `fromCache` +// property (Firefox). + +const foilLargeMediaElement = function(details, fctxt, pageStore) { + if ( details.fromCache === true ) { return; } + + let size = 0; + if ( µb.userSettings.largeMediaSize !== 0 ) { + const headers = details.responseHeaders; + const i = headerIndexFromName('content-length', headers); + if ( i === -1 ) { return; } + size = parseInt(headers[i].value, 10) || 0; + } + + const result = pageStore.filterLargeMediaElement(fctxt, size); + if ( result === 0 ) { return; } + + if ( logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + + return { cancel: true }; +}; + +/******************************************************************************/ + +// Caller must ensure headerName is normalized to lower case. + +const headerIndexFromName = function(headerName, headers) { + let i = headers.length; + while ( i-- ) { + if ( headers[i].name.toLowerCase() === headerName ) { + return i; + } + } + return -1; +}; + +const headerValueFromName = function(headerName, headers) { + const i = headerIndexFromName(headerName, headers); + return i !== -1 ? headers[i].value : ''; +}; + +/******************************************************************************/ + +const strictBlockBypasser = { + hostnameToDeadlineMap: new Map(), + cleanupTimer: vAPI.defer.create(( ) => { + strictBlockBypasser.cleanup(); + }), + + cleanup: function() { + for ( const [ hostname, deadline ] of this.hostnameToDeadlineMap ) { + if ( deadline <= Date.now() ) { + this.hostnameToDeadlineMap.delete(hostname); + } + } + }, + + revokeTime: function() { + return Date.now() + µb.hiddenSettings.strictBlockingBypassDuration * 1000; + }, + + bypass: function(hostname) { + if ( typeof hostname !== 'string' || hostname === '' ) { return; } + this.hostnameToDeadlineMap.set(hostname, this.revokeTime()); + }, + + isBypassed: function(hostname) { + if ( this.hostnameToDeadlineMap.size === 0 ) { return false; } + this.cleanupTimer.on({ sec: µb.hiddenSettings.strictBlockingBypassDuration + 10 }); + for (;;) { + const deadline = this.hostnameToDeadlineMap.get(hostname); + if ( deadline !== undefined ) { + if ( deadline > Date.now() ) { + this.hostnameToDeadlineMap.set(hostname, this.revokeTime()); + return true; + } + this.hostnameToDeadlineMap.delete(hostname); + } + const pos = hostname.indexOf('.'); + if ( pos === -1 ) { break; } + hostname = hostname.slice(pos + 1); + } + return false; + } +}; + +/******************************************************************************/ + +// https://github.com/uBlockOrigin/uBlock-issues/issues/2350 +// Added scriptlet injection attempt at onResponseStarted time as per +// https://github.com/AdguardTeam/AdguardBrowserExtension/issues/1029 and +// https://github.com/AdguardTeam/AdguardBrowserExtension/blob/9ab85be5/Extension/src/background/webrequest.js#L620 + +const webRequest = { + onBeforeRequest, + + start: (( ) => { + vAPI.net = new vAPI.Net(); + if ( vAPI.Net.canSuspend() ) { + vAPI.net.suspend(); + } + + return ( ) => { + vAPI.net.setSuspendableListener(onBeforeRequest); + vAPI.net.addListener( + 'onHeadersReceived', + onHeadersReceived, + { urls: [ 'http://*/*', 'https://*/*' ] }, + [ 'blocking', 'responseHeaders' ] + ); + vAPI.net.addListener( + 'onResponseStarted', + details => { + if ( details.tabId === -1 ) { return; } + const pageStore = µb.pageStoreFromTabId(details.tabId); + if ( pageStore === null ) { return; } + if ( pageStore.getNetFilteringSwitch() === false ) { return; } + scriptletFilteringEngine.injectNow(details); + }, + { + types: [ 'main_frame', 'sub_frame' ], + urls: [ 'http://*/*', 'https://*/*' ] + } + ); + vAPI.defer.once({ sec: µb.hiddenSettings.toolbarWarningTimeout }).then(( ) => { + if ( vAPI.net.hasUnprocessedRequest() === false ) { return; } + vAPI.net.removeUnprocessedRequest(); + return vAPI.tabs.getCurrent(); + }).then(tab => { + if ( tab instanceof Object === false ) { return; } + µb.updateToolbarIcon(tab.id, 0b0110); + }); + vAPI.net.unsuspend({ all: true }); + }; + })(), + + strictBlockBypass: hostname => { + strictBlockBypasser.bypass(hostname); + }, +}; + +/******************************************************************************/ + +export default webRequest; + +/******************************************************************************/ diff --git a/src/js/ublock.js b/src/js/ublock.js new file mode 100644 index 0000000..e963377 --- /dev/null +++ b/src/js/ublock.js @@ -0,0 +1,700 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import io from './assets.js'; +import µb from './background.js'; +import { broadcast, filteringBehaviorChanged, onBroadcast } from './broadcast.js'; +import contextMenu from './contextmenu.js'; +import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import { redirectEngine } from './redirect-engine.js'; +import { hostnameFromURI } from './uri-utils.js'; + +import { + permanentFirewall, + sessionFirewall, + permanentSwitches, + sessionSwitches, + permanentURLFiltering, + sessionURLFiltering, +} from './filtering-engines.js'; + +/******************************************************************************/ +/******************************************************************************/ + +// https://github.com/chrisaljoudi/uBlock/issues/405 +// Be more flexible with whitelist syntax + +// Any special regexp char will be escaped +const whitelistDirectiveEscape = /[-\/\\^$+?.()|[\]{}]/g; + +// All `*` will be expanded into `.*` +const whitelistDirectiveEscapeAsterisk = /\*/g; + +// Remember encountered regexps for reuse. +const directiveToRegexpMap = new Map(); + +// Probably manually entered whitelist directive +const isHandcraftedWhitelistDirective = function(directive) { + return directive.startsWith('/') && directive.endsWith('/') || + directive.indexOf('/') !== -1 && directive.indexOf('*') !== -1; +}; + +const matchDirective = function(url, hostname, directive) { + // Directive is a plain hostname. + if ( directive.indexOf('/') === -1 ) { + return hostname.endsWith(directive) && + (hostname.length === directive.length || + hostname.charAt(hostname.length - directive.length - 1) === '.'); + } + // Match URL exactly. + if ( + directive.startsWith('/') === false && + directive.indexOf('*') === -1 + ) { + return url === directive; + } + // Transpose into a regular expression. + let re = directiveToRegexpMap.get(directive); + if ( re === undefined ) { + let reStr; + if ( directive.startsWith('/') && directive.endsWith('/') ) { + reStr = directive.slice(1, -1); + } else { + reStr = directive.replace(whitelistDirectiveEscape, '\\$&') + .replace(whitelistDirectiveEscapeAsterisk, '.*'); + } + re = new RegExp(reStr); + directiveToRegexpMap.set(directive, re); + } + return re.test(url); +}; + +const matchBucket = function(url, hostname, bucket, start) { + if ( bucket ) { + for ( let i = start || 0, n = bucket.length; i < n; i++ ) { + if ( matchDirective(url, hostname, bucket[i]) ) { + return i; + } + } + } + return -1; +}; + +/******************************************************************************/ + +µb.getNetFilteringSwitch = function(url) { + const hostname = hostnameFromURI(url); + let key = hostname; + for (;;) { + if ( matchBucket(url, hostname, this.netWhitelist.get(key)) !== -1 ) { + return false; + } + const pos = key.indexOf('.'); + if ( pos === -1 ) { break; } + key = key.slice(pos + 1); + } + if ( matchBucket(url, hostname, this.netWhitelist.get('//')) !== -1 ) { + return false; + } + return true; +}; + +/******************************************************************************/ + +µb.toggleNetFilteringSwitch = function(url, scope, newState) { + const currentState = this.getNetFilteringSwitch(url); + if ( newState === undefined ) { + newState = !currentState; + } + if ( newState === currentState ) { + return currentState; + } + + const netWhitelist = this.netWhitelist; + const pos = url.indexOf('#'); + let targetURL = pos !== -1 ? url.slice(0, pos) : url; + const targetHostname = hostnameFromURI(targetURL); + let key = targetHostname; + let directive = scope === 'page' ? targetURL : targetHostname; + + // Add to directive list + if ( newState === false ) { + let bucket = netWhitelist.get(key); + if ( bucket === undefined ) { + bucket = []; + netWhitelist.set(key, bucket); + } + bucket.push(directive); + this.saveWhitelist(); + filteringBehaviorChanged({ hostname: targetHostname }); + return true; + } + + // Remove all directives which cause current URL to be whitelisted + for (;;) { + const bucket = netWhitelist.get(key); + if ( bucket !== undefined ) { + let i; + for (;;) { + i = matchBucket(targetURL, targetHostname, bucket, i); + if ( i === -1 ) { break; } + directive = bucket.splice(i, 1)[0]; + if ( isHandcraftedWhitelistDirective(directive) ) { + netWhitelist.get('#').push(`# ${directive}`); + } + } + if ( bucket.length === 0 ) { + netWhitelist.delete(key); + } + } + const pos = key.indexOf('.'); + if ( pos === -1 ) { break; } + key = key.slice(pos + 1); + } + const bucket = netWhitelist.get('//'); + if ( bucket !== undefined ) { + let i; + for (;;) { + i = matchBucket(targetURL, targetHostname, bucket, i); + if ( i === -1 ) { break; } + directive = bucket.splice(i, 1)[0]; + if ( isHandcraftedWhitelistDirective(directive) ) { + netWhitelist.get('#').push(`# ${directive}`); + } + } + if ( bucket.length === 0 ) { + netWhitelist.delete('//'); + } + } + this.saveWhitelist(); + filteringBehaviorChanged({ direction: 1 }); + return true; +}; + +/******************************************************************************/ + +µb.arrayFromWhitelist = function(whitelist) { + const out = new Set(); + for ( const bucket of whitelist.values() ) { + for ( const directive of bucket ) { + out.add(directive); + } + } + return Array.from(out).sort((a, b) => a.localeCompare(b)); +}; + +µb.stringFromWhitelist = function(whitelist) { + return this.arrayFromWhitelist(whitelist).join('\n'); +}; + +/******************************************************************************/ + +µb.whitelistFromArray = function(lines) { + const whitelist = new Map(); + + // Comment bucket must always be ready to be used. + whitelist.set('#', []); + + // New set of directives, scrap cached data. + directiveToRegexpMap.clear(); + + for ( let line of lines ) { + line = line.trim(); + + // https://github.com/gorhill/uBlock/issues/171 + // Skip empty lines + if ( line === '' ) { continue; } + + let key, directive; + + // Don't throw out commented out lines: user might want to fix them + if ( line.startsWith('#') ) { + key = '#'; + directive = line; + } + // Plain hostname + else if ( line.indexOf('/') === -1 ) { + if ( this.reWhitelistBadHostname.test(line) ) { + key = '#'; + directive = '# ' + line; + } else { + key = directive = line; + } + } + // Regex-based (ensure it is valid) + else if ( + line.length > 2 && + line.startsWith('/') && + line.endsWith('/') + ) { + key = '//'; + directive = line; + try { + const re = new RegExp(directive.slice(1, -1)); + directiveToRegexpMap.set(directive, re); + } catch(ex) { + key = '#'; + directive = '# ' + line; + } + } + // URL, possibly wildcarded: there MUST be at least one hostname + // label (or else it would be just impossible to make an efficient + // dict. + else { + const matches = this.reWhitelistHostnameExtractor.exec(line); + if ( !matches || matches.length !== 2 ) { + key = '#'; + directive = '# ' + line; + } else { + key = matches[1]; + directive = line; + } + } + + // https://github.com/gorhill/uBlock/issues/171 + // Skip empty keys + if ( key === '' ) { continue; } + + // Be sure this stays fixed: + // https://github.com/chrisaljoudi/uBlock/issues/185 + let bucket = whitelist.get(key); + if ( bucket === undefined ) { + bucket = []; + whitelist.set(key, bucket); + } + bucket.push(directive); + } + return whitelist; +}; + +µb.whitelistFromString = function(s) { + return this.whitelistFromArray(s.split('\n')); +}; + +// https://github.com/gorhill/uBlock/issues/3717 +µb.reWhitelistBadHostname = /[^a-z0-9.\-_\[\]:]/; +µb.reWhitelistHostnameExtractor = /([a-z0-9.\-_\[\]]+)(?::[\d*]+)?\/(?:[^\x00-\x20\/]|$)[^\x00-\x20]*$/; + +/******************************************************************************/ + +µb.changeUserSettings = function(name, value) { + let us = this.userSettings; + + // Return all settings if none specified. + if ( name === undefined ) { + us = JSON.parse(JSON.stringify(us)); + us.noCosmeticFiltering = sessionSwitches.evaluate('no-cosmetic-filtering', '*') === 1; + us.noLargeMedia = sessionSwitches.evaluate('no-large-media', '*') === 1; + us.noRemoteFonts = sessionSwitches.evaluate('no-remote-fonts', '*') === 1; + us.noScripting = sessionSwitches.evaluate('no-scripting', '*') === 1; + us.noCSPReports = sessionSwitches.evaluate('no-csp-reports', '*') === 1; + return us; + } + + if ( typeof name !== 'string' || name === '' ) { return; } + + if ( value === undefined ) { + return us[name]; + } + + // Pre-change + switch ( name ) { + case 'largeMediaSize': + if ( typeof value !== 'number' ) { + value = parseInt(value, 10) || 0; + } + value = Math.ceil(Math.max(value, 0)); + break; + default: + break; + } + + // Change -- but only if the user setting actually exists. + const mustSave = us.hasOwnProperty(name) && value !== us[name]; + if ( mustSave ) { + us[name] = value; + } + + // Post-change + switch ( name ) { + case 'advancedUserEnabled': + if ( value === true ) { + us.popupPanelSections |= 0b11111; + } + break; + case 'autoUpdate': + this.scheduleAssetUpdater({ updateDelay: value ? 2000 : 0 }); + break; + case 'cnameUncloakEnabled': + if ( vAPI.net.canUncloakCnames === true ) { + vAPI.net.setOptions({ cnameUncloakEnabled: value === true }); + } + break; + case 'collapseBlocked': + if ( value === false ) { + cosmeticFilteringEngine.removeFromSelectorCache('*', 'net'); + } + break; + case 'contextMenuEnabled': + contextMenu.update(null); + break; + case 'hyperlinkAuditingDisabled': + if ( this.privacySettingsSupported ) { + vAPI.browserSettings.set({ 'hyperlinkAuditing': !value }); + } + break; + case 'noCosmeticFiltering': + case 'noLargeMedia': + case 'noRemoteFonts': + case 'noScripting': + case 'noCSPReports': + let switchName; + switch ( name ) { + case 'noCosmeticFiltering': + switchName = 'no-cosmetic-filtering'; break; + case 'noLargeMedia': + switchName = 'no-large-media'; break; + case 'noRemoteFonts': + switchName = 'no-remote-fonts'; break; + case 'noScripting': + switchName = 'no-scripting'; break; + case 'noCSPReports': + switchName = 'no-csp-reports'; break; + default: + break; + } + if ( switchName === undefined ) { break; } + let switchState = value ? 1 : 0; + sessionSwitches.toggle(switchName, '*', switchState); + if ( permanentSwitches.toggle(switchName, '*', switchState) ) { + this.saveHostnameSwitches(); + } + break; + case 'prefetchingDisabled': + if ( this.privacySettingsSupported ) { + vAPI.browserSettings.set({ 'prefetching': !value }); + } + break; + case 'webrtcIPAddressHidden': + if ( this.privacySettingsSupported ) { + vAPI.browserSettings.set({ 'webrtcIPAddress': !value }); + } + break; + default: + break; + } + + if ( mustSave ) { + this.saveUserSettings(); + } +}; + +/******************************************************************************/ + +// https://www.reddit.com/r/uBlockOrigin/comments/8524cf/my_custom_scriptlets_doesnt_work_what_am_i_doing/ + +µb.changeHiddenSettings = function(hs) { + const mustReloadResources = + hs.userResourcesLocation !== this.hiddenSettings.userResourcesLocation; + this.hiddenSettings = hs; + this.saveHiddenSettings(); + if ( mustReloadResources ) { + redirectEngine.invalidateResourcesSelfie(io); + this.loadRedirectResources(); + } + broadcast({ what: 'hiddenSettingsChanged' }); +}; + +/******************************************************************************/ + +µb.elementPickerExec = async function( + tabId, + frameId, + targetElement, + zap = false, +) { + if ( vAPI.isBehindTheSceneTabId(tabId) ) { return; } + + this.epickerArgs.target = targetElement || ''; + this.epickerArgs.zap = zap; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/40 + // The element picker needs this library + if ( zap !== true ) { + vAPI.tabs.executeScript(tabId, { + file: '/lib/diff/swatinem_diff.js', + runAt: 'document_end', + }); + } + + await vAPI.tabs.executeScript(tabId, { + file: '/js/scriptlets/epicker.js', + frameId, + runAt: 'document_end', + }); + + // https://github.com/uBlockOrigin/uBlock-issues/issues/168 + // Force activate the target tab once the element picker has been + // injected. + vAPI.tabs.select(tabId); +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/2033 +// Always set own rules, trying to be fancy to avoid setting seemingly +// (but not really) redundant rules led to this issue. + +µb.toggleFirewallRule = function(details) { + const { desHostname, requestType, action } = details; + let { srcHostname } = details; + + if ( action !== 0 ) { + sessionFirewall.setCell( + srcHostname, + desHostname, + requestType, + action + ); + } else { + sessionFirewall.unsetCell( + srcHostname, + desHostname, + requestType + ); + } + + // https://github.com/chrisaljoudi/uBlock/issues/731#issuecomment-73937469 + if ( details.persist ) { + if ( action !== 0 ) { + permanentFirewall.setCell( + srcHostname, + desHostname, + requestType, + action + ); + } else { + permanentFirewall.unsetCell( + srcHostname, + desHostname, + requestType + ); + } + this.savePermanentFirewallRules(); + } + + // https://github.com/gorhill/uBlock/issues/1662 + // Flush all cached `net` cosmetic filters if we are dealing with a + // collapsible type: any of the cached entries could be a resource on the + // target page. + if ( + (srcHostname !== '*') && + ( + requestType === '*' || + requestType === 'image' || + requestType === '3p' || + requestType === '3p-frame' + ) + ) { + srcHostname = '*'; + } + + // https://github.com/chrisaljoudi/uBlock/issues/420 + cosmeticFilteringEngine.removeFromSelectorCache(srcHostname, 'net'); + + // Flush caches + filteringBehaviorChanged({ + direction: action === 1 ? 1 : 0, + hostname: srcHostname, + }); + + if ( details.tabId === undefined ) { return; } + + if ( requestType.startsWith('3p') ) { + this.updateToolbarIcon(details.tabId, 0b100); + } + + if ( requestType === '3p' && action === 3 ) { + vAPI.tabs.executeScript(details.tabId, { + file: '/js/scriptlets/load-3p-css.js', + allFrames: true, + runAt: 'document_idle', + }); + } +}; + +/******************************************************************************/ + +µb.toggleURLFilteringRule = function(details) { + let changed = sessionURLFiltering.setRule( + details.context, + details.url, + details.type, + details.action + ); + if ( changed === false ) { return; } + + cosmeticFilteringEngine.removeFromSelectorCache(details.context, 'net'); + + if ( details.persist !== true ) { return; } + + changed = permanentURLFiltering.setRule( + details.context, + details.url, + details.type, + details.action + ); + + if ( changed ) { + this.savePermanentFirewallRules(); + } +}; + +/******************************************************************************/ + +µb.toggleHostnameSwitch = function(details) { + const newState = typeof details.state === 'boolean' + ? details.state + : sessionSwitches.evaluateZ(details.name, details.hostname) === false; + let changed = sessionSwitches.toggleZ( + details.name, + details.hostname, + !!details.deep, + newState + ); + if ( changed === false ) { return; } + + // Take per-switch action if needed + switch ( details.name ) { + case 'no-scripting': + this.updateToolbarIcon(details.tabId, 0b100); + break; + case 'no-cosmetic-filtering': { + const scriptlet = newState ? 'cosmetic-off' : 'cosmetic-on'; + vAPI.tabs.executeScript(details.tabId, { + file: `/js/scriptlets/${scriptlet}.js`, + allFrames: true, + }); + break; + } + case 'no-large-media': + const pageStore = this.pageStoreFromTabId(details.tabId); + if ( pageStore !== null ) { + pageStore.temporarilyAllowLargeMediaElements(!newState); + } + break; + default: + break; + } + + // Flush caches if needed + if ( newState ) { + switch ( details.name ) { + case 'no-scripting': + case 'no-remote-fonts': + filteringBehaviorChanged({ + direction: details.state ? 1 : 0, + hostname: details.hostname, + }); + break; + default: + break; + } + } + + if ( details.persist !== true ) { return; } + + changed = permanentSwitches.toggleZ( + details.name, + details.hostname, + !!details.deep, + newState + ); + if ( changed ) { + this.saveHostnameSwitches(); + } +}; + +/******************************************************************************/ + +µb.blockingModeFromHostname = function(hn) { + let bits = 0; + if ( sessionSwitches.evaluateZ('no-scripting', hn) ) { + bits |= 0b00000010; + } + if ( this.userSettings.advancedUserEnabled ) { + if ( sessionFirewall.evaluateCellZY(hn, '*', '3p') === 1 ) { + bits |= 0b00000100; + } + if ( sessionFirewall.evaluateCellZY(hn, '*', '3p-script') === 1 ) { + bits |= 0b00001000; + } + if ( sessionFirewall.evaluateCellZY(hn, '*', '3p-frame') === 1 ) { + bits |= 0b00010000; + } + } + return bits; +}; + +{ + const parse = function() { + const s = µb.hiddenSettings.blockingProfiles; + const profiles = []; + s.split(/\s+/).forEach(s => { + let pos = s.indexOf('/'); + if ( pos === -1 ) { + pos = s.length; + } + const bits = parseInt(s.slice(0, pos), 2); + if ( isNaN(bits) ) { return; } + const color = s.slice(pos + 1); + profiles.push({ bits, color: color !== '' ? color : '#666' }); + }); + µb.liveBlockingProfiles = profiles; + µb.blockingProfileColorCache.clear(); + }; + + parse(); + + onBroadcast(msg => { + if ( msg.what !== 'hiddenSettingsChanged' ) { return; } + parse(); + }); +} + +/******************************************************************************/ + +µb.pageURLFromMaybeDocumentBlockedURL = function(pageURL) { + if ( pageURL.startsWith(vAPI.getURL('/document-blocked.html?')) ) { + try { + const url = new URL(pageURL); + return JSON.parse(url.searchParams.get('details')).url; + } catch(ex) { + } + } + return pageURL; +}; + +/******************************************************************************/ diff --git a/src/js/uri-utils.js b/src/js/uri-utils.js new file mode 100644 index 0000000..273b151 --- /dev/null +++ b/src/js/uri-utils.js @@ -0,0 +1,175 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; +import punycode from '../lib/punycode.js'; + +/******************************************************************************/ + +// Originally: +// https://github.com/gorhill/uBlock/blob/8b5733a58d3acf9fb62815e14699c986bd1c2fdc/src/js/uritools.js + +const reHostnameFromCommonURL = + /^https:\/\/[0-9a-z._-]+[0-9a-z]\//; +const reAuthorityFromURI = + /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; +const reHostFromNakedAuthority = + /^[0-9a-z._-]+[0-9a-z]$/i; +const reHostFromAuthority = + /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; +const reIPv6FromAuthority = + /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; +const reMustNormalizeHostname = + /[^0-9a-z._-]/; +const reOriginFromURI = + /^[^:\/?#]+:\/\/[^\/?#]+/; +const reHostnameFromNetworkURL = + /^(?:http|ws|ftp)s?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])(?::\d+)?\//; +const reIPAddressNaive = + /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; +const reNetworkURI = + /^(?:ftps?|https?|wss?):\/\//; + +// For performance purpose, as simple tests as possible +const reIPv4VeryCoarse = /\.\d+$/; +const reHostnameVeryCoarse = /[g-z_\-]/; + +/******************************************************************************/ + +function domainFromHostname(hostname) { + return reIPAddressNaive.test(hostname) + ? hostname + : publicSuffixList.getDomain(hostname); +} + +function domainFromURI(uri) { + if ( !uri ) { return ''; } + return domainFromHostname(hostnameFromURI(uri)); +} + +function entityFromDomain(domain) { + const pos = domain.indexOf('.'); + return pos !== -1 ? domain.slice(0, pos) + '.*' : ''; +} + +function hostnameFromURI(uri) { + let match = reHostnameFromCommonURL.exec(uri); + if ( match !== null ) { return match[0].slice(8, -1); } + match = reAuthorityFromURI.exec(uri); + if ( match === null ) { return ''; } + const authority = match[1].slice(2); + if ( reHostFromNakedAuthority.test(authority) ) { + return authority.toLowerCase(); + } + match = reHostFromAuthority.exec(authority); + if ( match === null ) { + match = reIPv6FromAuthority.exec(authority); + if ( match === null ) { return ''; } + } + let hostname = match[1]; + while ( hostname.endsWith('.') ) { + hostname = hostname.slice(0, -1); + } + if ( reMustNormalizeHostname.test(hostname) ) { + hostname = punycode.toASCII(hostname.toLowerCase()); + } + return hostname; +} + +function hostnameFromNetworkURL(url) { + const matches = reHostnameFromNetworkURL.exec(url); + return matches !== null ? matches[1] : ''; +} + +function originFromURI(uri) { + let match = reHostnameFromCommonURL.exec(uri); + if ( match !== null ) { return match[0].slice(0, -1); } + match = reOriginFromURI.exec(uri); + return match !== null ? match[0].toLowerCase() : ''; +} + +function isNetworkURI(uri) { + return reNetworkURI.test(uri); +} + +/******************************************************************************/ + +function toBroaderHostname(hostname) { + const pos = hostname.indexOf('.'); + if ( pos !== -1 ) { + return hostname.slice(pos + 1); + } + return hostname !== '*' && hostname !== '' ? '*' : ''; +} + +function toBroaderIPv4Address(ipaddress) { + if ( ipaddress === '*' || ipaddress === '' ) { return ''; } + const pos = ipaddress.lastIndexOf('.'); + if ( pos === -1 ) { return '*'; } + return ipaddress.slice(0, pos); +} + +function toBroaderIPv6Address(ipaddress) { + return ipaddress !== '*' && ipaddress !== '' ? '*' : ''; +} + +function decomposeHostname(hostname, out) { + if ( out.length !== 0 && out[0] === hostname ) { + return out; + } + let broadenFn; + if ( reHostnameVeryCoarse.test(hostname) === false ) { + if ( reIPv4VeryCoarse.test(hostname) ) { + broadenFn = toBroaderIPv4Address; + } else if ( hostname.startsWith('[') ) { + broadenFn = toBroaderIPv6Address; + } + } + if ( broadenFn === undefined ) { + broadenFn = toBroaderHostname; + } + out[0] = hostname; + let i = 1; + for (;;) { + hostname = broadenFn(hostname); + if ( hostname === '' ) { break; } + out[i++] = hostname; + } + out.length = i; + return out; +} + +/******************************************************************************/ + +export { + decomposeHostname, + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromNetworkURL, + hostnameFromURI, + isNetworkURI, + originFromURI, +}; diff --git a/src/js/url-net-filtering.js b/src/js/url-net-filtering.js new file mode 100644 index 0000000..39befc7 --- /dev/null +++ b/src/js/url-net-filtering.js @@ -0,0 +1,336 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import { LineIterator } from './text-utils.js'; +import { decomposeHostname } from './uri-utils.js'; + +/******************************************************************************* + + The purpose of log filtering is to create ad hoc filtering rules, to + diagnose and assist in the creation of custom filters. + + buckets: map of [hostname + type] + bucket: array of rule entries, sorted from shorter to longer url + rule entry: { url, action } + +*******************************************************************************/ + +const actionToNameMap = { + 1: 'block', + 2: 'allow', + 3: 'noop' +}; + +const nameToActionMap = Object.create(null); +Object.assign(nameToActionMap, { + 'block': 1, + 'allow': 2, + 'noop': 3 +}); + +const knownInvalidTypes = new Set([ + 'doc', + 'main_frame', +]); + +const intToActionMap = new Map([ + [ 1, ' block' ], + [ 2, ' allow' ], + [ 3, ' noop' ] +]); + +const decomposedSource = []; + +/******************************************************************************/ + +class RuleEntry { + constructor(url, action) { + this.url = url; + this.action = action; + } +} + +/******************************************************************************/ + +function indexOfURL(entries, url) { + // TODO: binary search -- maybe, depends on common use cases + const urlLen = url.length; + // URLs must be ordered by increasing length. + for ( let i = 0; i < entries.length; i++ ) { + const entry = entries[i]; + if ( entry.url.length > urlLen ) { break; } + if ( entry.url === url ) { return i; } + } + return -1; +} + +/******************************************************************************/ + +function indexOfMatch(entries, url) { + const urlLen = url.length; + let i = entries.length; + while ( i-- ) { + if ( entries[i].url.length <= urlLen ) { + break; + } + } + if ( i !== -1 ) { + do { + if ( url.startsWith(entries[i].url) ) { + return i; + } + } while ( i-- ); + } + return -1; +} + +/******************************************************************************/ + +function indexFromLength(entries, len) { + // TODO: binary search -- maybe, depends on common use cases + // URLs must be ordered by increasing length. + for ( let i = 0; i < entries.length; i++ ) { + if ( entries[i].url.length > len ) { return i; } + } + return -1; +} + +/******************************************************************************/ + +function addRuleEntry(entries, url, action) { + const entry = new RuleEntry(url, action); + const i = indexFromLength(entries, url.length); + if ( i === -1 ) { + entries.push(entry); + } else { + entries.splice(i, 0, entry); + } +} + +/******************************************************************************/ + +class DynamicURLRuleFiltering { + constructor() { + this.reset(); + } + + reset() { + this.rules = new Map(); + // registers, filled with result of last evaluation + this.context = ''; + this.url = ''; + this.type = ''; + this.r = 0; + this.changed = false; + } + + assign(other) { + // Remove rules not in other + for ( const key of this.rules.keys() ) { + if ( other.rules.has(key) === false ) { + this.rules.delete(key); + } + } + // Add/change rules in other + for ( const entry of other.rules ) { + this.rules.set(entry[0], entry[1].slice()); + } + this.changed = true; + } + + setRule(srcHostname, url, type, action) { + if ( action === 0 ) { + return this.removeRule(srcHostname, url, type); + } + const bucketKey = srcHostname + ' ' + type; + let entries = this.rules.get(bucketKey); + if ( entries === undefined ) { + entries = []; + this.rules.set(bucketKey, entries); + } + const i = indexOfURL(entries, url); + if ( i !== -1 ) { + const entry = entries[i]; + if ( entry.action === action ) { return false; } + entry.action = action; + } else { + addRuleEntry(entries, url, action); + } + this.changed = true; + return true; + } + + removeRule(srcHostname, url, type) { + const bucketKey = srcHostname + ' ' + type; + const entries = this.rules.get(bucketKey); + if ( entries === undefined ) { return false; } + const i = indexOfURL(entries, url); + if ( i === -1 ) { return false; } + entries.splice(i, 1); + if ( entries.length === 0 ) { + this.rules.delete(bucketKey); + } + this.changed = true; + return true; + } + + evaluateZ(context, target, type) { + this.r = 0; + if ( this.rules.size === 0 ) { return 0; } + decomposeHostname(context, decomposedSource); + for ( const srchn of decomposedSource ) { + this.context = srchn; + let entries = this.rules.get(`${srchn} ${type}`); + if ( entries !== undefined ) { + const i = indexOfMatch(entries, target); + if ( i !== -1 ) { + const entry = entries[i]; + this.url = entry.url; + this.type = type; + this.r = entry.action; + return this.r; + } + } + entries = this.rules.get(`${srchn} *`); + if ( entries !== undefined ) { + const i = indexOfMatch(entries, target); + if ( i !== -1 ) { + const entry = entries[i]; + this.url = entry.url; + this.type = '*'; + this.r = entry.action; + return this.r; + } + } + } + return 0; + } + + mustAllowCellZ(context, target, type) { + return this.evaluateZ(context, target, type).r === 2; + } + + mustBlockOrAllow() { + return this.r === 1 || this.r === 2; + } + + toLogData() { + if ( this.r === 0 ) { return; } + const { context, url, type } = this; + return { + source: 'dynamicUrl', + result: this.r, + rule: [ context, url, type, intToActionMap.get(this.r) ], + raw: `${context} ${url} ${type} ${intToActionMap.get(this.r)}`, + }; + } + + copyRules(other, context, urls, type) { + let i = urls.length; + while ( i-- ) { + const url = urls[i]; + other.evaluateZ(context, url, type); + const otherOwn = other.r !== 0 && + other.context === context && + other.url === url && + other.type === type; + this.evaluateZ(context, url, type); + const thisOwn = this.r !== 0 && + this.context === context && + this.url === url && + this.type === type; + if ( otherOwn && !thisOwn || other.r !== this.r ) { + this.setRule(context, url, type, other.r); + this.changed = true; + } + if ( !otherOwn && thisOwn ) { + this.removeRule(context, url, type); + this.changed = true; + } + } + return this.changed; + } + + toArray() { + const out = []; + for ( const [ key, entries ] of this.rules ) { + let pos = key.indexOf(' '); + const hn = key.slice(0, pos); + pos = key.lastIndexOf(' '); + const type = key.slice(pos + 1); + for ( const { url, action } of entries ) { + out.push(`${hn} ${url} ${type} ${actionToNameMap[action]}`); + } + } + return out; + } + + toString() { + return this.toArray().sort().join('\n'); + } + + fromString(text) { + this.reset(); + const lineIter = new LineIterator(text); + while ( lineIter.eot() === false ) { + this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); + } + } + + validateRuleParts(parts) { + if ( parts.length !== 4 ) { return; } + if ( parts[1].indexOf('://') <= 0 ) { return; } + if ( + /[^a-z_-]/.test(parts[2]) && parts[2] !== '*' || + knownInvalidTypes.has(parts[2]) + ) { + return; + } + if ( nameToActionMap[parts[3]] === undefined ) { return; } + return parts; + } + + addFromRuleParts(parts) { + if ( this.validateRuleParts(parts) !== undefined ) { + this.setRule(parts[0], parts[1], parts[2], nameToActionMap[parts[3]]); + return true; + } + return false; + } + + removeFromRuleParts(parts) { + if ( this.validateRuleParts(parts) !== undefined ) { + this.removeRule(parts[0], parts[1], parts[2]); + return true; + } + return false; + } +} + +/******************************************************************************/ + +export default DynamicURLRuleFiltering; + +/******************************************************************************/ diff --git a/src/js/utils.js b/src/js/utils.js new file mode 100644 index 0000000..e48e963 --- /dev/null +++ b/src/js/utils.js @@ -0,0 +1,136 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import µb from './background.js'; + +/******************************************************************************/ + +µb.formatCount = function(count) { + if ( typeof count !== 'number' ) { return ''; } + const s = `${count}`; + if ( count < 1000 ) { return s; } + if ( count < 10000 ) { + return '>' + s.slice(0,1) + 'k'; + } + if ( count < 100000 ) { + return s.slice(0,2) + 'k'; + } + if ( count < 1000000 ) { + return s.slice(0,3) + 'k'; + } + return s.slice(0,-6) + 'M'; +}; + +/******************************************************************************/ + +µb.dateNowToSensibleString = function() { + const now = new Date(Date.now() - (new Date()).getTimezoneOffset() * 60000); + return now.toISOString().replace(/\.\d+Z$/, '') + .replace(/:/g, '.') + .replace('T', '_'); +}; + +/******************************************************************************/ + +µb.openNewTab = function(details) { + if ( details.url.startsWith('logger-ui.html') ) { + if ( details.shiftKey ) { + this.changeUserSettings( + 'alwaysDetachLogger', + !this.userSettings.alwaysDetachLogger + ); + } + if ( this.userSettings.alwaysDetachLogger ) { + details.popup = this.hiddenSettings.loggerPopupType; + const url = new URL(vAPI.getURL(details.url)); + url.searchParams.set('popup', '1'); + details.url = url.href; + let popupLoggerBox; + try { + popupLoggerBox = JSON.parse( + vAPI.localStorage.getItem('popupLoggerBox') + ); + } catch(ex) { + } + if ( popupLoggerBox !== undefined ) { + details.box = popupLoggerBox; + } + } + } + vAPI.tabs.open(details); +}; + +/******************************************************************************/ + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions + +µb.escapeRegex = function(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +}; + +/******************************************************************************/ + +// TODO: properly compare arrays + +µb.getModifiedSettings = function(edit, orig = {}) { + const out = {}; + for ( const prop in edit ) { + if ( orig.hasOwnProperty(prop) && edit[prop] !== orig[prop] ) { + out[prop] = edit[prop]; + } + } + return out; +}; + +µb.settingValueFromString = function(orig, name, s) { + if ( typeof name !== 'string' || typeof s !== 'string' ) { return; } + if ( orig.hasOwnProperty(name) === false ) { return; } + let r; + switch ( typeof orig[name] ) { + case 'boolean': + if ( s === 'true' ) { + r = true; + } else if ( s === 'false' ) { + r = false; + } + break; + case 'string': + r = s.trim(); + break; + case 'number': + if ( s.startsWith('0b') ) { + r = parseInt(s.slice(2), 2); + } else if ( s.startsWith('0x') ) { + r = parseInt(s.slice(2), 16); + } else { + r = parseInt(s, 10); + } + if ( isNaN(r) ) { r = undefined; } + break; + default: + break; + } + return r; +}; diff --git a/src/js/wasm/README.md b/src/js/wasm/README.md new file mode 100644 index 0000000..32aef07 --- /dev/null +++ b/src/js/wasm/README.md @@ -0,0 +1,24 @@ +### For code reviewers + +All `wasm` files in that directory where created by compiling the +corresponding `wat` file using the command (using `hntrie.wat`/`hntrie.wasm` +as example): + + wat2wasm hntrie.wat -o hntrie.wasm + +Assuming: + +- The command is executed from within the present directory. + +### `wat2wasm` tool + +The `wat2wasm` tool can be downloaded from an official WebAssembly project: +<https://github.com/WebAssembly/wabt/releases>. + +### `wat2wasm` tool online + +You can also use the following online `wat2wasm` tool: +<https://webassembly.github.io/wabt/demo/wat2wasm/>. + +Just paste the whole content of the `wat` file to compile into the WAT pane. +Click "Download" button to retrieve the resulting `wasm` file.
\ No newline at end of file diff --git a/src/js/wasm/biditrie.wasm b/src/js/wasm/biditrie.wasm Binary files differnew file mode 100644 index 0000000..5bfc6b7 --- /dev/null +++ b/src/js/wasm/biditrie.wasm diff --git a/src/js/wasm/biditrie.wat b/src/js/wasm/biditrie.wat new file mode 100644 index 0000000..a6c80ba --- /dev/null +++ b/src/js/wasm/biditrie.wat @@ -0,0 +1,728 @@ +;; +;; uBlock Origin - a comprehensive, efficient content blocker +;; Copyright (C) 2019-present Raymond Hill +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see {http://www.gnu.org/licenses/}. +;; +;; Home: https://github.com/gorhill/uBlock +;; File: biditrie.wat +;; Description: WebAssembly code used by src/js/biditrie.js +;; How to compile: See README.md in this directory. + +(module +;; +;; module start +;; + +(memory (import "imports" "memory") 1) +(func $extraHandler (import "imports" "extraHandler") (param i32 i32 i32) (result i32)) + +;; Trie container +;; +;; Memory layout, byte offset: +;; const HAYSTACK_START = 0; +;; const HAYSTACK_SIZE = 2048; // i32 / i8 +;; const HAYSTACK_SIZE_SLOT = HAYSTACK_SIZE >>> 2; // 512 / 2048 +;; const TRIE0_SLOT = HAYSTACK_SIZE_SLOT + 1; // 513 / 2052 +;; const TRIE1_SLOT = HAYSTACK_SIZE_SLOT + 2; // 514 / 2056 +;; const CHAR0_SLOT = HAYSTACK_SIZE_SLOT + 3; // 515 / 2060 +;; const CHAR1_SLOT = HAYSTACK_SIZE_SLOT + 4; // 516 / 2064 +;; const RESULT_L_SLOT = HAYSTACK_SIZE_SLOT + 5; // 517 / 2068 +;; const RESULT_R_SLOT = HAYSTACK_SIZE_SLOT + 6; // 518 / 2072 +;; const RESULT_IU_SLOT = HAYSTACK_SIZE_SLOT + 7; // 519 / 2076 +;; const TRIE0_START = HAYSTACK_SIZE_SLOT + 8 << 2; // 2080 +;; + +;; +;; Public functions +;; + +;; +;; unsigned int matches(icell, ai) +;; +;; Test whether the trie at icell matches the haystack content at position ai. +;; +(func (export "matches") + (param $icell i32) ;; start offset in haystack + (param $ai i32) ;; offset in haystack + (result i32) ;; result: 0 = no match, 1 = match + (local $char0 i32) + (local $aR i32) + (local $al i32) + (local $bl i32) + (local $x i32) + (local $y i32) + ;; trie index is a uint32 offset, need to convert to uint8 offset + local.get $icell + i32.const 2 + i32.shl + local.set $icell + ;; const buf32 = this.buf32; + ;; const buf8 = this.buf8; + ;; const char0 = buf32[CHAR0_SLOT]; + i32.const 2060 + i32.load align=4 + local.set $char0 + ;; const aR = buf32[HAYSTACK_SIZE_SLOT]; + i32.const 2048 + i32.load align=4 + local.set $aR + ;; let al = ai; + local.get $ai + local.set $al + block $matchFound + block $matchNotFound + ;; for (;;) { + loop $mainLoop + ;; x = buf8[al]; + local.get $al + i32.load8_u + local.set $x + ;; al += 1; + local.get $al + i32.const 1 + i32.add + local.set $al + ;; // find matching segment + ;; for (;;) { + block $nextSegment loop $findSegment + ;; y = buf32[icell+SEGMENT_INFO]; + local.get $icell + i32.load offset=8 align=4 + local.tee $y + ;; bl = char0 + (y & 0x00FFFFFF); + i32.const 0x00FFFFFF + i32.and + local.get $char0 + i32.add + local.tee $bl + ;; if ( buf8[bl] === x ) { + i32.load8_u + local.get $x + i32.eq + if + ;; y = (y >>> 24) - 1; + local.get $y + i32.const 24 + i32.shr_u + i32.const 1 + i32.sub + local.tee $y + ;; if ( n !== 0 ) { + if + ;; x = al + y; + local.get $y + local.get $al + i32.add + local.tee $x + ;; if ( x > aR ) { return 0; } + local.get $aR + i32.gt_u + br_if $matchNotFound + ;; for (;;) { + loop + ;; bl += 1; + local.get $bl + i32.const 1 + i32.add + local.tee $bl + ;; if ( buf8[bl] !== buf8[al] ) { return 0; } + i32.load8_u + local.get $al + i32.load8_u + i32.ne + br_if $matchNotFound + ;; al += 1; + local.get $al + i32.const 1 + i32.add + local.tee $al + ;; if ( al === x ) { break; } + local.get $x + i32.ne + br_if 0 + end + ;; } + end + br $nextSegment + end + ;; icell = buf32[icell+CELL_OR]; + local.get $icell + i32.load offset=4 align=4 + i32.const 2 + i32.shl + local.tee $icell + ;; if ( icell === 0 ) { return 0; } + i32.eqz + br_if $matchNotFound + br $findSegment + ;; } + end end + ;; // next segment + ;; icell = buf32[icell+CELL_AND]; + local.get $icell + i32.load align=4 + i32.const 2 + i32.shl + local.tee $icell + ;; const x = buf32[icell+BCELL_EXTRA]; + i32.load offset=8 align=4 + local.tee $x + ;; if ( x <= BCELL_EXTRA_MAX ) { + i32.const 0x00FFFFFF + i32.le_u + if + ;; if ( x !== 0 && this.matchesExtra(ai, al, x) !== 0 ) { + ;; return 1; + ;; } + local.get $x + if + local.get $ai + local.get $al + local.get $x + call $matchesExtra + br_if $matchFound + end + ;; x = buf32[icell+BCELL_ALT_AND]; + local.get $icell + i32.load offset=4 align=4 + i32.const 2 + i32.shl + local.tee $x + ;; if ( x !== 0 && this.matchesLeft(x, ai, al) !== 0 ) { + if + local.get $x + local.get $ai + local.get $al + call $matchesLeft + br_if $matchFound + ;; } + end + ;; icell = buf32[icell+BCELL_NEXT_AND]; + local.get $icell + i32.load align=4 + i32.const 2 + i32.shl + local.tee $icell + ;; if ( icell === 0 ) { return 0; } + i32.eqz + br_if $matchNotFound + ;; } + end + ;; if ( al === aR ) { return 0; } + local.get $al + local.get $aR + i32.ne + br_if $mainLoop + ;; } + end ;; $mainLoop + end ;; $matchNotFound + i32.const 0 + return + end ;; $matchFound + i32.const 1 + return +) + +;; +;; unsigned int matchesLeft(icell, ar, r) +;; +;; Test whether the trie at icell matches the haystack content at position ai. +;; +(func $matchesLeft + (param $icell i32) ;; start offset in haystack + (param $ar i32) ;; offset of where to start in haystack + (param $r i32) ;; right bound of match so far + (result i32) ;; result: 0 = no match, 1 = match + (local $char0 i32) + (local $bl i32) + (local $br i32) + (local $x i32) + (local $y i32) + ;; const buf32 = this.buf32; + ;; const buf8 = this.buf8; + ;; const char0 = buf32[CHAR0_SLOT]; + i32.const 2060 + i32.load align=4 + local.set $char0 + block $matchFound + block $matchNotFound + ;; for (;;) { + loop $mainLoop + ;; if ( ar === 0 ) { return 0; } + local.get $ar + i32.eqz + br_if $matchNotFound + ;; ar -= 1; + local.get $ar + i32.const 1 + i32.sub + local.tee $ar + ;; x = buf8[ar]; + i32.load8_u + local.set $x + ;; // find matching segment + ;; for (;;) { + block $nextSegment loop $findSegment + ;; y = buf32[icell+SEGMENT_INFO]; + local.get $icell + i32.load offset=8 align=4 + local.tee $y + ;; br = char0 + (y & 0x00FFFFFF); + i32.const 0x00FFFFFF + i32.and + local.get $char0 + i32.add + local.tee $br + ;; y = (y >>> 24) - 1; + local.get $y + i32.const 24 + i32.shr_u + i32.const 1 + i32.sub + local.tee $y + ;; br += y; + i32.add + local.tee $br + ;; if ( buf8[br] === x ) { + i32.load8_u + local.get $x + i32.eq + if + ;; // all characters in segment must match + ;; if ( y !== 0 ) { + local.get $y + if + ;; x = ar - y; + local.get $ar + local.get $y + i32.sub + local.tee $x + ;; if ( x < 0 ) { return 0; } + i32.const 0 + i32.lt_s + br_if $matchNotFound + ;; for (;;) { + loop + ;; ar -= 1; br -= 1; + ;; if ( buf8[ar] !== buf8[br] ) { return 0; } + local.get $ar + i32.const 1 + i32.sub + local.tee $ar + i32.load8_u + local.get $br + i32.const 1 + i32.sub + local.tee $br + i32.load8_u + i32.ne + br_if $matchNotFound + ;; if ( ar === x ) { break; } + local.get $ar + local.get $x + i32.ne + br_if 0 + end + ;; } + end + br $nextSegment + end + ;; icell = buf32[icell+CELL_OR]; + local.get $icell + i32.load offset=4 align=4 + i32.const 2 + i32.shl + local.tee $icell + ;; if ( icell === 0 ) { return 0; } + i32.eqz + br_if $matchNotFound + br $findSegment + ;; } + end end + ;; // next segment + ;; icell = buf32[icell+CELL_AND]; + local.get $icell + i32.load align=4 + i32.const 2 + i32.shl + local.tee $icell + ;; const x = buf32[icell+BCELL_EXTRA]; + i32.load offset=8 align=4 + local.tee $x + ;; if ( x <= BCELL_EXTRA_MAX ) { + i32.const 0x00FFFFFF + i32.le_u + if + ;; if ( x !== 0 && this.matchesExtra(ar, r, x) !== 0 ) { + ;; return 1; + ;; } + local.get $x + if + local.get $ar + local.get $r + local.get $x + call $matchesExtra + br_if $matchFound + end + ;; icell = buf32[icell+BCELL_NEXT_AND]; + local.get $icell + i32.load align=4 + i32.const 2 + i32.shl + local.tee $icell + ;; if ( icell === 0 ) { return 0; } + i32.eqz + br_if $matchNotFound + ;; } + end + br $mainLoop + ;; } + end ;; $mainLoop + end ;; $matchNotFound + i32.const 0 + return + end ;; $matchFound + i32.const 1 + return +) + +;; +;; int matchExtra(l, r, ix) +;; +;; Test whether extra handler returns a match. +;; +(func $matchesExtra + (param $l i32) ;; left bound of match so far + (param $r i32) ;; right bound of match so far + (param $ix i32) ;; extra token + (result i32) ;; result: 0 = no match, 1 = match + (local $iu i32) ;; filter unit + block $fail + block $succeed + ;; if ( ix !== 1 ) { + ;; const iu = this.extraHandler(l, r, ix); + ;; if ( iu === 0 ) { return 0; } + local.get $ix + i32.const 1 + i32.ne + if + local.get $l + local.get $r + local.get $ix + call $extraHandler + local.tee $iu + i32.eqz + br_if $fail + ;; } else { + ;; iu = -1; + else + i32.const -1 + local.set $iu + ;; } + end + ;; this.buf32[RESULT_IU_SLOT] = iu; + i32.const 2076 + local.get $iu + i32.store align=4 + ;; this.buf32[RESULT_L_SLOT] = l; + i32.const 2068 + local.get $l + i32.store align=4 + ;; this.buf32[RESULT_R_SLOT] = r; + i32.const 2072 + local.get $r + i32.store align=4 + end ;; $succeed + i32.const 1 + return + end ;; $fail + i32.const 0 +) + +;; +;; unsigned int startsWith(haystackLeft, haystackRight, needleLeft, needleLen) +;; +;; Test whether the string at needleOffset and of length needleLen matches +;; the haystack at offset haystackOffset. +;; +(func (export "startsWith") + (param $haystackLeft i32) ;; start offset in haystack + (param $haystackRight i32) ;; end offset in haystack + (param $needleLeft i32) ;; start of needle in character buffer + (param $needleLen i32) ;; number of characters to match + (result i32) ;; result: 0 = no match, 1 = match + (local $needleRight i32) + block $fail + block $succeed + ;; + ;; if ( haystackLeft < 0 || (haystackLeft + needleLen) > haystackRight ) { + ;; return 0; + ;; } + local.get $haystackLeft + i32.const 0 + i32.lt_s + br_if $fail + local.get $haystackLeft + local.get $needleLen + i32.add + local.get $haystackRight + i32.gt_u + br_if $fail + ;; const charCodes = this.buf8; + ;; needleLeft += this.buf32[CHAR0_SLOT]; + local.get $needleLeft + i32.const 2060 ;; CHAR0_SLOT memory address + i32.load align=4 ;; CHAR0 memory address + i32.add ;; needle memory address + local.tee $needleLeft + ;; const needleRight = needleLeft + needleLen; + local.get $needleLen + i32.add + local.set $needleRight + ;; while ( charCodes[haystackLeft] === charCodes[needleLeft] ) { + loop $compare + local.get $haystackLeft + i32.load8_u + local.get $needleLeft + i32.load8_u + i32.ne + br_if $fail + ;; needleLeft += 1; + local.get $needleLeft + i32.const 1 + i32.add + local.tee $needleLeft + ;; if ( needleLeft === needleRight ) { return 1; } + local.get $needleRight + i32.eq + br_if $succeed + ;; haystackLeft += 1; + i32.const 1 + local.get $haystackLeft + i32.add + local.set $haystackLeft + br $compare + end + ;; } + ;; return 1; + end ;; $succeed + i32.const 1 + return + ;; return 0; + end ;; $fail + i32.const 0 +) + +;; +;; int indexOf(haystackLeft, haystackEnd, needleLeft, needleLen) +;; +;; Test whether the string at needleOffset and of length needleLen is found in +;; the haystack at or to the left of haystackLeft, but not farther than +;; haystackEnd. +;; +(func (export "indexOf") + (param $haystackLeft i32) ;; start offset in haystack + (param $haystackEnd i32) ;; end offset in haystack + (param $needleLeft i32) ;; start of needle in character buffer + (param $needleLen i32) ;; number of characters to match + (result i32) ;; result: index of match, -1 = no match + (local $needleRight i32) + (local $i i32) + (local $j i32) + (local $c0 i32) + block $fail + block $succeed + ;; if ( needleLen === 0 ) { return haystackLeft; } + local.get $needleLen + i32.eqz + br_if $succeed + ;; haystackEnd -= needleLen; + local.get $haystackEnd + local.get $needleLen + i32.sub + local.tee $haystackEnd + ;; if ( haystackEnd < haystackLeft ) { return -1; } + local.get $haystackLeft + i32.lt_s + br_if $fail + ;; needleLeft += this.buf32[CHAR0_SLOT]; + local.get $needleLeft + i32.const 2060 ;; CHAR0_SLOT memory address + i32.load align=4 ;; CHAR0 memory address + i32.add ;; needle memory address + local.tee $needleLeft + ;; const needleRight = needleLeft + needleLen; + local.get $needleLen + i32.add + local.set $needleRight + ;; const charCodes = this.buf8; + ;; for (;;) { + loop $mainLoop + ;; let i = haystackLeft; + ;; let j = needleLeft; + local.get $haystackLeft + local.set $i + local.get $needleLeft + local.set $j + ;; while ( charCodes[i] === charCodes[j] ) { + block $breakMatchChars loop $matchChars + local.get $i + i32.load8_u + local.get $j + i32.load8_u + i32.ne + br_if $breakMatchChars + ;; j += 1; + local.get $j + i32.const 1 + i32.add + local.tee $j + ;; if ( j === needleRight ) { return haystackLeft; } + local.get $needleRight + i32.eq + br_if $succeed + ;; i += 1; + local.get $i + i32.const 1 + i32.add + local.set $i + br $matchChars + ;; } + end end + ;; haystackLeft += 1; + local.get $haystackLeft + i32.const 1 + i32.add + local.tee $haystackLeft + ;; if ( haystackLeft > haystackEnd ) { break; } + local.get $haystackEnd + i32.gt_u + br_if $fail + br $mainLoop + ;; } + end + end ;; $succeed + local.get $haystackLeft + return + end ;; $fail + ;; return -1; + i32.const -1 +) + +;; +;; int lastIndexOf(haystackBeg, haystackEnd, needleLeft, needleLen) +;; +;; Test whether the string at needleOffset and of length needleLen is found in +;; the haystack at or to the right of haystackBeg, but not farther than +;; haystackEnd. +;; +(func (export "lastIndexOf") + (param $haystackBeg i32) ;; start offset in haystack + (param $haystackEnd i32) ;; end offset in haystack + (param $needleLeft i32) ;; start of needle in character buffer + (param $needleLen i32) ;; number of characters to match + (result i32) ;; result: index of match, -1 = no match + (local $haystackLeft i32) + (local $needleRight i32) + (local $i i32) + (local $j i32) + (local $c0 i32) + ;; if ( needleLen === 0 ) { return haystackBeg; } + local.get $needleLen + i32.eqz + if + local.get $haystackBeg + return + end + block $fail + block $succeed + ;; let haystackLeft = haystackEnd - needleLen; + local.get $haystackEnd + local.get $needleLen + i32.sub + local.tee $haystackLeft + ;; if ( haystackLeft < haystackBeg ) { return -1; } + local.get $haystackBeg + i32.lt_s + br_if $fail + ;; needleLeft += this.buf32[CHAR0_SLOT]; + local.get $needleLeft + i32.const 2060 ;; CHAR0_SLOT memory address + i32.load align=4 ;; CHAR0 memory address + i32.add ;; needle memory address + local.tee $needleLeft + ;; const needleRight = needleLeft + needleLen; + local.get $needleLen + i32.add + local.set $needleRight + ;; const charCodes = this.buf8; + ;; for (;;) { + loop $mainLoop + ;; let i = haystackLeft; + ;; let j = needleLeft; + local.get $haystackLeft + local.set $i + local.get $needleLeft + local.set $j + ;; while ( charCodes[i] === charCodes[j] ) { + block $breakMatchChars loop $matchChars + local.get $i + i32.load8_u + local.get $j + i32.load8_u + i32.ne + br_if $breakMatchChars + ;; j += 1; + local.get $j + i32.const 1 + i32.add + local.tee $j + ;; if ( j === needleRight ) { return haystackLeft; } + local.get $needleRight + i32.eq + br_if $succeed + ;; i += 1; + local.get $i + i32.const 1 + i32.add + local.set $i + br $matchChars + ;; } + end end + ;; if ( haystackLeft === haystackBeg ) { break; } + ;; haystackLeft -= 1; + local.get $haystackLeft + local.get $haystackBeg + i32.eq + br_if $fail + local.get $haystackLeft + i32.const 1 + i32.sub + local.set $haystackLeft + br $mainLoop + ;; } + end + end ;; $succeed + local.get $haystackLeft + return + end ;; $fail + ;; return -1; + i32.const -1 +) + +;; +;; module end +;; +) diff --git a/src/js/wasm/hntrie.wasm b/src/js/wasm/hntrie.wasm Binary files differnew file mode 100644 index 0000000..9067f42 --- /dev/null +++ b/src/js/wasm/hntrie.wasm diff --git a/src/js/wasm/hntrie.wat b/src/js/wasm/hntrie.wat new file mode 100644 index 0000000..666c44e --- /dev/null +++ b/src/js/wasm/hntrie.wat @@ -0,0 +1,724 @@ +;; +;; uBlock Origin - a comprehensive, efficient content blocker +;; Copyright (C) 2018-present Raymond Hill +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see {http://www.gnu.org/licenses/}. +;; +;; Home: https://github.com/gorhill/uBlock +;; File: hntrie.wat +;; Description: WebAssembly code used by src/js/hntrie.js +;; How to compile: See README.md in this directory. + +(module +;; +;; module start +;; + +(func $growBuf (import "imports" "growBuf")) +(memory (import "imports" "memory") 1) + +;; Trie container +;; +;; Memory layout, byte offset: +;; 0-254: needle being processed +;; 255: length of needle +;; 256-259: offset to start of trie data section (=> trie0) +;; 260-263: offset to end of trie data section (=> trie1) +;; 264-267: offset to start of character data section (=> char0) +;; 268-271: offset to end of character data section (=> char1) +;; 272: start of trie data section +;; + +;; +;; Public functions +;; + +;; +;; unsigned int matches(icell) +;; +;; Test whether the currently set needle matches the trie at specified trie +;; offset. +;; +(func (export "matches") + (param $iroot i32) ;; offset to root cell of the trie + (result i32) ;; result = match index, -1 = miss + (local $icell i32) ;; offset to the current cell + (local $char0 i32) ;; offset to first character data + (local $ineedle i32) ;; current needle offset + (local $c i32) + (local $v i32) + (local $n i32) + (local $i0 i32) + (local $i1 i32) + ;; + i32.const 264 ;; start of char section is stored at addr 264 + i32.load + local.set $char0 + ;; let ineedle = this.buf[255]; + i32.const 255 ;; addr of needle is stored at addr 255 + i32.load8_u + local.set $ineedle + ;; let icell = this.buf32[iroot+0]; + local.get $iroot + i32.const 2 + i32.shl + i32.load + i32.const 2 + i32.shl + local.tee $icell + ;; if ( icell === 0 ) { return -1; } + i32.eqz + if + i32.const -1 + return + end + ;; for (;;) { + block $noSegment loop $nextSegment + ;; if ( ineedle === 0 ) { return -1; } + local.get $ineedle + i32.eqz + if + i32.const -1 + return + end + ;; ineedle -= 1; + local.get $ineedle + i32.const -1 + i32.add + local.tee $ineedle + ;; let c = this.buf[ineedle]; + i32.load8_u + local.set $c + ;; for (;;) { + block $foundSegment loop $findSegment + ;; v = this.buf32[icell+2]; + local.get $icell + i32.load offset=8 + local.tee $v + ;; i0 = char0 + (v >>> 8); + i32.const 8 + i32.shr_u + local.get $char0 + i32.add + local.tee $i0 + ;; if ( this.buf[i0] === c ) { break; } + i32.load8_u + local.get $c + i32.eq + br_if $foundSegment + ;; icell = this.buf32[icell+0]; + local.get $icell + i32.load + i32.const 2 + i32.shl + local.tee $icell + i32.eqz + if + i32.const -1 + return + end + br 0 + end end + ;; let n = v & 0x7F; + local.get $v + i32.const 0x7F + i32.and + local.tee $n + ;; if ( n > 1 ) { + i32.const 1 + i32.gt_u + if + ;; n -= 1; + local.get $n + i32.const -1 + i32.add + local.tee $n + ;; if ( n > ineedle ) { return -1; } + local.get $ineedle + i32.gt_u + if + i32.const -1 + return + end + local.get $i0 + i32.const 1 + i32.add + local.tee $i0 + ;; const i1 = i0 + n; + local.get $n + i32.add + local.set $i1 + ;; do { + loop + ;; ineedle -= 1; + local.get $ineedle + i32.const -1 + i32.add + local.tee $ineedle + ;; if ( this.buf[i0] !== this.buf[ineedle] ) { return -1; } + i32.load8_u + local.get $i0 + i32.load8_u + i32.ne + if + i32.const -1 + return + end + ;; i0 += 1; + local.get $i0 + i32.const 1 + i32.add + local.tee $i0 + ;; } while ( i0 < i1 ); + local.get $i1 + i32.lt_u + br_if 0 + end + end + ;; if ( (v & 0x80) !== 0 ) { + local.get $v + i32.const 0x80 + i32.and + if + ;; if ( ineedle === 0 || buf8[ineedle-1] === 0x2E /* '.' */ ) { + ;; return ineedle; + ;; } + local.get $ineedle + i32.eqz + if + i32.const 0 + return + end + local.get $ineedle + i32.const -1 + i32.add + i32.load8_u + i32.const 0x2E + i32.eq + if + local.get $ineedle + return + end + end + ;; icell = this.buf32[icell+1]; + local.get $icell + i32.load offset=4 + i32.const 2 + i32.shl + local.tee $icell + ;; if ( icell === 0 ) { break; } + br_if 0 + end end + ;; return -1; + i32.const -1 +) + +;; +;; unsigned int add(icell) +;; +;; Add a new hostname to a trie which root cell is passed as argument. +;; +(func (export "add") + (param $iroot i32) ;; index of root cell of the trie + (result i32) ;; result: 0 not added, 1 = added + (local $icell i32) ;; index of current cell in the trie + (local $lhnchar i32) ;; number of characters left to process in hostname + (local $char0 i32) ;; offset to start of character data section + (local $v i32) ;; integer value describing a segment + (local $isegchar0 i32) ;; offset to start of current segment's character data + (local $isegchar i32) + (local $lsegchar i32) ;; number of character in current segment + (local $inext i32) ;; index of next cell to process + (local $boundaryBit i32) ;; the boundary bit state of the current cell + ;; + ;; let lhnchar = this.buf[255]; + i32.const 255 + i32.load8_u + local.tee $lhnchar + ;; if ( lhnchar === 0 ) { return 0; } + i32.eqz + if + i32.const 0 + return + end + ;; if ( + ;; (this.buf32[HNBIGTRIE_CHAR0_SLOT] - this.buf32[HNBIGTRIE_TRIE1_SLOT]) < 24 || + ;; (this.buf.length - this.buf32[HNBIGTRIE_CHAR1_SLOT]) < 256 + ;; ) { + ;; this.growBuf(); + ;; } + i32.const 264 + i32.load + i32.const 260 + i32.load + i32.sub + i32.const 24 + i32.lt_u + if + call $growBuf + else + memory.size + i32.const 16 + i32.shl + i32.const 268 + i32.load + i32.sub + i32.const 256 + i32.lt_u + if + call $growBuf + end + end + ;; let icell = this.buf32[iroot+0]; + local.get $iroot + i32.const 2 + i32.shl + local.tee $iroot + i32.load + i32.const 2 + i32.shl + local.tee $icell + ;; if ( this.buf32[icell+2] === 0 ) { + i32.eqz + if + ;; this.buf32[iroot+0] = this.addLeafCell(lhnchar); + ;; return 1; + local.get $iroot + local.get $lhnchar + call $addLeafCell + i32.store + i32.const 1 + return + end + ;; const char0 = this.buf32[HNBIGTRIE_CHAR0_SLOT]; + i32.const 264 + i32.load + local.set $char0 + ;; for (;;) { + loop $nextSegment + ;; const v = this.buf32[icell+2]; + local.get $icell + i32.load offset=8 + local.tee $v + ;; let isegchar0 = char0 + (v >>> 8); + i32.const 8 + i32.shr_u + local.get $char0 + i32.add + local.tee $isegchar0 + ;; if ( this.buf[isegchar0] !== this.buf[lhnchar-1] ) { + i32.load8_u + local.get $lhnchar + i32.const -1 + i32.add + i32.load8_u + i32.ne + if + ;; inext = this.buf32[icell+0]; + local.get $icell + i32.load + local.tee $inext + ;; if ( inext === 0 ) { + i32.eqz + if + ;; this.buf32[icell+0] = this.addLeafCell(lhnchar); + local.get $icell + local.get $lhnchar + call $addLeafCell + i32.store + ;; return 1; + i32.const 1 + return + end + ;; icell = inext; + local.get $inext + i32.const 2 + i32.shl + local.set $icell + br $nextSegment + end + ;; let isegchar = 1; + i32.const 1 + local.set $isegchar + ;; lhnchar -= 1; + local.get $lhnchar + i32.const -1 + i32.add + local.set $lhnchar + ;; const lsegchar = v & 0x7F; + local.get $v + i32.const 0x7F + i32.and + local.tee $lsegchar + ;; if ( lsegchar !== 1 ) { + i32.const 1 + i32.ne + if + ;; for (;;) { + block $mismatch loop + ;; if ( isegchar === lsegchar ) { break; } + local.get $isegchar + local.get $lsegchar + i32.eq + br_if $mismatch + local.get $lhnchar + i32.eqz + br_if $mismatch + ;; if ( this.buf[isegchar0+isegchar] !== this.buf[lhnchar-1] ) { break; } + local.get $isegchar0 + local.get $isegchar + i32.add + i32.load8_u + local.get $lhnchar + i32.const -1 + i32.add + i32.load8_u + i32.ne + br_if $mismatch + ;; isegchar += 1; + local.get $isegchar + i32.const 1 + i32.add + local.set $isegchar + ;; lhnchar -= 1; + local.get $lhnchar + i32.const -1 + i32.add + local.set $lhnchar + br 0 + end end + end + ;; const boundaryBit = v & 0x80; + local.get $v + i32.const 0x80 + i32.and + local.set $boundaryBit + ;; if ( isegchar === lsegchar ) { + local.get $isegchar + local.get $lsegchar + i32.eq + if + ;; if ( lhnchar === 0 ) { + local.get $lhnchar + i32.eqz + if + ;; if ( boundaryBit !== 0 ) { return 0; } + local.get $boundaryBit + if + i32.const 0 + return + end + ;; this.buf32[icell+2] = v | 0x80; + local.get $icell + local.get $v + i32.const 0x80 + i32.or + i32.store offset=8 + else + ;; if ( boundaryBit !== 0 ) { + local.get $boundaryBit + if + ;; if ( this.buf[lhnchar-1] === 0x2E /* '.' */ ) { return -1; } + local.get $lhnchar + i32.const -1 + i32.add + i32.load8_u + i32.const 0x2E + i32.eq + if + i32.const -1 + return + end + end + ;; inext = this.buf32[icell+1]; + local.get $icell + i32.load offset=4 + local.tee $inext + ;; if ( inext !== 0 ) { + if + ;; icell = inext; + local.get $inext + i32.const 2 + i32.shl + local.set $icell + ;; continue; + br $nextSegment + end + ;; this.buf32[icell+1] = this.addLeafCell(lhnchar); + local.get $icell + local.get $lhnchar + call $addLeafCell + i32.store offset=4 + end + else + ;; isegchar0 -= char0; + local.get $icell + local.get $isegchar0 + local.get $char0 + i32.sub + local.tee $isegchar0 + ;; this.buf32[icell+2] = isegchar0 << 8 | isegchar; + i32.const 8 + i32.shl + local.get $isegchar + i32.or + i32.store offset=8 + ;; inext = this.addCell( + ;; 0, + ;; this.buf32[icell+1], + ;; isegchar0 + isegchar << 8 | boundaryBit | lsegchar - isegchar + ;; ); + local.get $icell + i32.const 0 + local.get $icell + i32.load offset=4 + local.get $isegchar0 + local.get $isegchar + i32.add + i32.const 8 + i32.shl + local.get $boundaryBit + i32.or + local.get $lsegchar + local.get $isegchar + i32.sub + i32.or + call $addCell + local.tee $inext + ;; this.buf32[icell+1] = inext; + i32.store offset=4 + ;; if ( lhnchar !== 0 ) { + local.get $lhnchar + if + ;; this.buf32[inext+0] = this.addLeafCell(lhnchar); + local.get $inext + i32.const 2 + i32.shl + local.get $lhnchar + call $addLeafCell + i32.store + else + ;; this.buf32[icell+2] |= 0x80; + local.get $icell + local.get $icell + i32.load offset=8 + i32.const 0x80 + i32.or + i32.store offset=8 + end + end + ;; return 1; + i32.const 1 + return + end + ;; + i32.const 1 +) + +;; +;; Private functions +;; + +;; +;; unsigned int addCell(idown, iright, v) +;; +;; Add a new cell, return cell index. +;; +(func $addCell + (param $idown i32) + (param $iright i32) + (param $v i32) + (result i32) ;; result: index of added cell + (local $icell i32) + ;; + ;; let icell = this.buf32[HNBIGTRIE_TRIE1_SLOT]; + ;; this.buf32[HNBIGTRIE_TRIE1_SLOT] = icell + 12; + i32.const 260 + i32.const 260 + i32.load + local.tee $icell + i32.const 12 + i32.add + i32.store + ;; this.buf32[icell+0] = idown; + local.get $icell + local.get $idown + i32.store + ;; this.buf32[icell+1] = iright; + local.get $icell + local.get $iright + i32.store offset=4 + ;; this.buf32[icell+2] = v; + local.get $icell + local.get $v + i32.store offset=8 + ;; return icell; + local.get $icell + i32.const 2 + i32.shr_u +) + +;; +;; unsigned int addLeafCell(lsegchar) +;; +;; Add a new cell, return cell index. +;; +(func $addLeafCell + (param $lsegchar i32) + (result i32) ;; result: index of added cell + (local $r i32) + (local $i i32) + ;; const r = this.buf32[TRIE1_SLOT] >>> 2; + i32.const 260 + i32.load + local.tee $r + ;; let i = r; + local.set $i + ;; while ( lsegchar > 127 ) { + block $lastSegment loop + local.get $lsegchar + i32.const 127 + i32.le_u + br_if $lastSegment + ;; this.buf32[i+0] = 0; + local.get $i + i32.const 0 + i32.store + ;; this.buf32[i+1] = i + 3; + local.get $i + local.get $i + i32.const 12 + i32.add + i32.const 2 + i32.shr_u + i32.store offset=4 + ;; this.buf32[i+2] = this.addSegment(lsegchar, lsegchar - 127); + local.get $i + local.get $lsegchar + local.get $lsegchar + i32.const 127 + i32.sub + call $addSegment + i32.store offset=8 + ;; lsegchar -= 127; + local.get $lsegchar + i32.const 127 + i32.sub + local.set $lsegchar + ;; i += 3; + local.get $i + i32.const 12 + i32.add + local.set $i + br 0 + end end + ;; this.buf32[i+0] = 0; + local.get $i + i32.const 0 + i32.store + ;; this.buf32[i+1] = 0; + local.get $i + i32.const 0 + i32.store offset=4 + ;; this.buf32[i+2] = this.addSegment(lsegchar, 0) | 0x80; + local.get $i + local.get $lsegchar + i32.const 0 + call $addSegment + i32.const 0x80 + i32.or + i32.store offset=8 + ;; this.buf32[TRIE1_SLOT] = i + 3 << 2; + i32.const 260 + local.get $i + i32.const 12 + i32.add + i32.store + ;; return r; + local.get $r + i32.const 2 + i32.shr_u +) + +;; +;; unsigned int addSegment(lsegchar, lsegend) +;; +;; Store a segment of characters and return a segment descriptor. The segment +;; is created from the character data in the needle buffer. +;; +(func $addSegment + (param $lsegchar i32) + (param $lsegend i32) + (result i32) ;; result: segment descriptor + (local $char1 i32) ;; offset to end of character data section + (local $isegchar i32) ;; relative offset to first character of segment + (local $i i32) ;; iterator + ;; + ;; if ( lsegchar === 0 ) { return 0; } + local.get $lsegchar + i32.eqz + if + i32.const 0 + return + end + ;; let char1 = this.buf32[HNBIGTRIE_CHAR1_SLOT]; + i32.const 268 + i32.load + local.tee $char1 + ;; const isegchar = char1 - this.buf32[HNBIGTRIE_CHAR0_SLOT]; + i32.const 264 + i32.load + i32.sub + local.set $isegchar + ;; let i = lsegchar; + local.get $lsegchar + local.set $i + ;; do { + loop + ;; this.buf[char1++] = this.buf[--i]; + local.get $char1 + local.get $i + i32.const -1 + i32.add + local.tee $i + i32.load8_u + i32.store8 + local.get $char1 + i32.const 1 + i32.add + local.set $char1 + ;; } while ( i !== lsegend ); + local.get $i + local.get $lsegend + i32.ne + br_if 0 + end + ;; this.buf32[HNBIGTRIE_CHAR1_SLOT] = char1; + i32.const 268 + local.get $char1 + i32.store + ;; return isegchar << 8 | lsegchar - lsegend; + local.get $isegchar + i32.const 8 + i32.shl + local.get $lsegchar + local.get $lsegend + i32.sub + i32.or +) + +;; +;; module end +;; +) diff --git a/src/js/whitelist.js b/src/js/whitelist.js new file mode 100644 index 0000000..e7905ee --- /dev/null +++ b/src/js/whitelist.js @@ -0,0 +1,258 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2014-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uBlockDashboard */ + +'use strict'; + +import { i18n$ } from './i18n.js'; +import { dom, qs$ } from './dom.js'; + +/******************************************************************************/ + +const reComment = /^\s*#\s*/; + +const directiveFromLine = function(line) { + const match = reComment.exec(line); + return match === null + ? line.trim() + : line.slice(match.index + match[0].length).trim(); +}; + +/******************************************************************************/ + +CodeMirror.defineMode("ubo-whitelist-directives", function() { + const reRegex = /^\/.+\/$/; + + return { + token: function(stream) { + const line = stream.string.trim(); + stream.skipToEnd(); + if ( reBadHostname === undefined ) { + return null; + } + if ( reComment.test(line) ) { + return 'comment'; + } + if ( line.indexOf('/') === -1 ) { + if ( reBadHostname.test(line) ) { return 'error'; } + if ( whitelistDefaultSet.has(line.trim()) ) { + return 'keyword'; + } + return null; + } + if ( reRegex.test(line) ) { + try { + new RegExp(line.slice(1, -1)); + } catch(ex) { + return 'error'; + } + return null; + } + if ( reHostnameExtractor.test(line) === false ) { + return 'error'; + } + if ( whitelistDefaultSet.has(line.trim()) ) { + return 'keyword'; + } + return null; + } + }; +}); + +let reBadHostname; +let reHostnameExtractor; +let whitelistDefaultSet = new Set(); + +/******************************************************************************/ + +const messaging = vAPI.messaging; +const noopFunc = function(){}; + +let cachedWhitelist = ''; + +const cmEditor = new CodeMirror(qs$('#whitelist'), { + autofocus: true, + lineNumbers: true, + lineWrapping: true, + styleActiveLine: true, +}); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +/******************************************************************************/ + +const getEditorText = function() { + let text = cmEditor.getValue().replace(/\s+$/, ''); + return text === '' ? text : text + '\n'; +}; + +const setEditorText = function(text) { + cmEditor.setValue(text.replace(/\s+$/, '') + '\n'); +}; + +/******************************************************************************/ + +const whitelistChanged = function() { + const whitelistElem = qs$('#whitelist'); + const bad = qs$(whitelistElem, '.cm-error') !== null; + const changedWhitelist = getEditorText().trim(); + const changed = changedWhitelist !== cachedWhitelist; + qs$('#whitelistApply').disabled = !changed || bad; + qs$('#whitelistRevert').disabled = !changed; + CodeMirror.commands.save = changed && !bad ? applyChanges : noopFunc; +}; + +cmEditor.on('changes', whitelistChanged); + +/******************************************************************************/ + +const renderWhitelist = async function() { + const details = await messaging.send('dashboard', { + what: 'getWhitelist', + }); + + const first = reBadHostname === undefined; + if ( first ) { + reBadHostname = new RegExp(details.reBadHostname); + reHostnameExtractor = new RegExp(details.reHostnameExtractor); + whitelistDefaultSet = new Set(details.whitelistDefault); + } + const toAdd = new Set(whitelistDefaultSet); + for ( const line of details.whitelist ) { + const directive = directiveFromLine(line); + if ( whitelistDefaultSet.has(directive) === false ) { continue; } + toAdd.delete(directive); + if ( toAdd.size === 0 ) { break; } + } + if ( toAdd.size !== 0 ) { + details.whitelist.push(...Array.from(toAdd).map(a => `# ${a}`)); + } + details.whitelist.sort((a, b) => { + const ad = directiveFromLine(a); + const bd = directiveFromLine(b); + const abuiltin = whitelistDefaultSet.has(ad); + if ( abuiltin !== whitelistDefaultSet.has(bd) ) { + return abuiltin ? -1 : 1; + } + return ad.localeCompare(bd); + }); + const whitelistStr = details.whitelist.join('\n').trim(); + cachedWhitelist = whitelistStr; + setEditorText(whitelistStr); + if ( first ) { + cmEditor.clearHistory(); + } +}; + +/******************************************************************************/ + +const handleImportFilePicker = function() { + const file = this.files[0]; + if ( file === undefined || file.name === '' ) { return; } + if ( file.type.indexOf('text') !== 0 ) { return; } + const fr = new FileReader(); + fr.onload = ev => { + if ( ev.type !== 'load' ) { return; } + const content = uBlockDashboard.mergeNewLines( + getEditorText().trim(), + fr.result.trim() + ); + setEditorText(content); + }; + fr.readAsText(file); +}; + +/******************************************************************************/ + +const startImportFilePicker = function() { + const input = qs$('#importFilePicker'); + // Reset to empty string, this will ensure an change event is properly + // triggered if the user pick a file, even if it is the same as the last + // one picked. + input.value = ''; + input.click(); +}; + +/******************************************************************************/ + +const exportWhitelistToFile = function() { + const val = getEditorText(); + if ( val === '' ) { return; } + const filename = + i18n$('whitelistExportFilename') + .replace('{{datetime}}', uBlockDashboard.dateNowToSensibleString()) + .replace(/ +/g, '_'); + vAPI.download({ + 'url': `data:text/plain;charset=utf-8,${encodeURIComponent(val + '\n')}`, + 'filename': filename + }); +}; + +/******************************************************************************/ + +const applyChanges = async function() { + cachedWhitelist = getEditorText().trim(); + await messaging.send('dashboard', { + what: 'setWhitelist', + whitelist: cachedWhitelist, + }); + renderWhitelist(); +}; + +const revertChanges = function() { + setEditorText(cachedWhitelist); +}; + +/******************************************************************************/ + +const getCloudData = function() { + return getEditorText(); +}; + +const setCloudData = function(data, append) { + if ( typeof data !== 'string' ) { return; } + if ( append ) { + data = uBlockDashboard.mergeNewLines(getEditorText().trim(), data); + } + setEditorText(data.trim()); +}; + +self.cloud.onPush = getCloudData; +self.cloud.onPull = setCloudData; + +/******************************************************************************/ + +self.hasUnsavedData = function() { + return getEditorText().trim() !== cachedWhitelist; +}; + +/******************************************************************************/ + +dom.on('#importWhitelistFromFile', 'click', startImportFilePicker); +dom.on('#importFilePicker', 'change', handleImportFilePicker); +dom.on('#exportWhitelistToFile', 'click', exportWhitelistToFile); +dom.on('#whitelistApply', 'click', ( ) => { applyChanges(); }); +dom.on('#whitelistRevert', 'click', revertChanges); + +renderWhitelist(); + +/******************************************************************************/ |