Adding upstream version 2020.10.7.upstream/2020.10.7upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

11 files changed, 374 insertions, 0 deletions

diff --git a/scripts/chromedriver.sh b/scripts/chromedriver.sh
new file mode 100755
index 0000000..3bd55d6
--- /dev/null
+++ b/scripts/chromedriver.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+TEMPFILE=$(mktemp)
+CHROME="$1"
+
+trap 'rm $TEMPFILE' EXIT
+
+# install the appropriate version of ChromeDriver
+chrome_version=$("$CHROME" --product-version | cut -d . -f 1-3)
+chromedriver_version_url=https://chromedriver.storage.googleapis.com/LATEST_RELEASE_"$chrome_version"
+chromedriver_version=$(wget "$chromedriver_version_url" -q -O -)
+echo "Setting up ChromeDriver version $chromedriver_version ..."
+chromedriver_url=https://chromedriver.storage.googleapis.com/"$chromedriver_version"/chromedriver_linux64.zip
+wget -q -O "$TEMPFILE" "$chromedriver_url"
+sudo unzip -q -o "$TEMPFILE" chromedriver -d /usr/local/bin/
+sudo chmod a+x /usr/local/bin/chromedriver
+
+# check that chromedriver is now present
+type chromedriver >/dev/null 2>&1 || {
+  echo "Failed to install ChromeDriver!"
+  exit 1
+}
diff --git a/scripts/convertpsl.py b/scripts/convertpsl.py
new file mode 100755
index 0000000..3cd78be
--- /dev/null
+++ b/scripts/convertpsl.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+# script based on
+# https://github.com/adblockplus/buildtools/blob/d090e00610a58cebc78478ae33e896e6b949fc12/publicSuffixListUpdater.py
+
+from __future__ import print_function
+
+import json
+import sys
+
+def convert(psl_text):
+    suffixes = {}
+
+    for line in psl_text:
+        line = line.rstrip()
+        if line.startswith('//') or '.' not in line:
+            continue
+        if line.startswith('*.'):
+            suffixes[line[2:]] = 2
+        elif line.startswith('!'):
+            suffixes[line[1:]] = 0
+        else:
+            suffixes[line] = 1
+
+    return suffixes
+
+
+if __name__ == '__main__':
+    with open(sys.argv[1], 'r+') as f:
+        psl = convert(f)
+        f.seek(0)
+        text = 'window.publicSuffixes = %s;' % (
+            json.dumps(psl, sort_keys=True, indent=2, separators=(',', ': '))
+        )
+        print(text, file=f)
+        f.truncate()
diff --git a/scripts/fix_placeholders.py b/scripts/fix_placeholders.py
new file mode 100755
index 0000000..e940dcc
--- /dev/null
+++ b/scripts/fix_placeholders.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import json
+
+from collections import OrderedDict
+from glob import glob
+
+SOURCE_LOCALE = 'src/_locales/en_US/messages.json'
+
+
+def fix_locale(locale, placeholder_keys):
+    # read in locale, preserving existing ordering
+    with open(locale, 'r') as f:
+        data = json.load(f, object_pairs_hook=OrderedDict)
+
+    # restore missing placeholders
+    for key in placeholder_keys:
+        if key in data and "placeholders" not in data[key]:
+            data[key]["placeholders"] = source_data[key]["placeholders"]
+
+    with open(locale, 'w') as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)
+
+
+if __name__ == '__main__':
+    with open(SOURCE_LOCALE, 'r') as f:
+        source_data = json.load(f, object_pairs_hook=OrderedDict)
+
+    # get keys of locale messages with placeholders
+    placeholder_keys = []
+    for key in source_data:
+        if "placeholders" in source_data[key]:
+            placeholder_keys.append(key)
+
+    # fix all locales
+    for locale in glob('src/_locales/*/*.json'):
+        if locale == SOURCE_LOCALE:
+            continue
+
+        fix_locale(locale, placeholder_keys)
diff --git a/scripts/generate-legacy-yellowlist.sh b/scripts/generate-legacy-yellowlist.sh
new file mode 100755
index 0000000..b74712e
--- /dev/null
+++ b/scripts/generate-legacy-yellowlist.sh
@@ -0,0 +1 @@
+sed -e "s/^\([^\!].*\)$/@@||\1^\$third-party/g" src/data/yellowlist.txt
diff --git a/scripts/run_travis.sh b/scripts/run_travis.sh
new file mode 100755
index 0000000..fa20fdb
--- /dev/null
+++ b/scripts/run_travis.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+toplevel=$(git rev-parse --show-toplevel)
+testdir=${toplevel}/tests/selenium
+
+function run_lint {
+  if ! make -C "$toplevel" lint; then
+    echo "Linting errors"
+    exit 1
+  fi
+}
+
+function run_selenium {
+  # autodiscover and run the tests
+  pytest --capture=no --verbose --durations=10 "$testdir"
+}
+
+if [ "$INFO" == "lint" ]; then
+    echo "running lint tests"
+    run_lint
+else
+    case $BROWSER in
+      *chrome*)
+        echo "running tests on chrome"
+        run_selenium
+        ;;
+      *firefox*)
+        echo "running tests on firefox"
+        run_selenium
+        ;;
+      *)
+        echo "bad INFO variable, got $INFO"
+        exit 1
+        ;;
+    esac
+fi
diff --git a/scripts/setup_travis.sh b/scripts/setup_travis.sh
new file mode 100755
index 0000000..5e4bbf0
--- /dev/null
+++ b/scripts/setup_travis.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+toplevel=$(git rev-parse --show-toplevel)
+
+function setup_firefox {
+    # Install the latest version of geckodriver
+    version=$(curl -sI https://github.com/mozilla/geckodriver/releases/latest | grep -i "^Location: " | sed 's/.*\///' | tr -d '\r')
+
+    # check that we got something
+    if [ -z "$version" ]; then
+      echo "Failed to determine the latest geckodriver version!"
+      exit 1
+    fi
+
+    # Geckodriver distribution is MacOS or Linux specific
+    os="$(uname -s)"
+    if [[ $os == "Darwin" ]]; then
+      os_dist="macos.tar.gz"
+    else
+      os_dist="linux64.tar.gz"
+    fi
+
+    echo "Setting up geckodriver version $version ..."
+    url="https://github.com/mozilla/geckodriver/releases/download/${version}/geckodriver-${version}-${os_dist}"
+    wget -q -O /tmp/geckodriver.tar.gz "$url"
+    sudo tar -xvf /tmp/geckodriver.tar.gz -C /usr/local/bin/
+    sudo chmod a+x /usr/local/bin/geckodriver
+
+    # check that geckodriver is now present
+    type geckodriver >/dev/null 2>&1 || {
+      echo "Failed to install geckodriver!"
+      exit 1
+    }
+}
+
+function browser_setup {
+  # install python stuff
+  pip install -r "$toplevel"/tests/requirements.txt
+}
+
+function setup_lint {
+  # "--production" to skip installing devDependencies modules
+  npm install --production || exit 1
+}
+
+# check that the desired browser is present as it might fail to install
+# for example: https://travis-ci.org/EFForg/privacybadger/jobs/362381214
+function check_browser {
+  type "$BROWSER" >/dev/null 2>&1 || {
+    echo "$BROWSER seems to be missing!"
+    exit 1
+  }
+
+  # print the version
+  echo "Found $("$BROWSER" --version)"
+}
+
+case $INFO in
+  *chrome*)
+    check_browser
+    "$toplevel"/scripts/chromedriver.sh "$BROWSER"
+    browser_setup
+    ;;
+  *firefox*) # Install the latest version of geckodriver
+    check_browser
+    setup_firefox
+    browser_setup
+    ;;
+  *lint*)
+    setup_lint
+    ;;
+  *)
+    echo "bad INFO variable, got $INFO"
+    exit 1
+    ;;
+esac
diff --git a/scripts/updategoogle.py b/scripts/updategoogle.py
new file mode 100755
index 0000000..a3046d4
--- /dev/null
+++ b/scripts/updategoogle.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+from collections import OrderedDict
+
+def convert(text):
+    patterns = []
+    for domain in text.split():
+        patterns.append("https://www" + domain + "/*")
+        patterns.append("http://www" + domain + "/*")
+    return patterns
+
+def update_manifest(tempfile_path, manifest_path):
+    with open(manifest_path, 'r') as f:
+        manifest = json.load(f, object_pairs_hook=OrderedDict)
+
+    with open(tempfile_path, 'r+') as f:
+        # tempfile_path contains Google's supported domains
+        match_patterns = convert(f.read())
+
+        scripts_idx = -1
+        for idx, entry in enumerate(manifest['content_scripts']):
+            if "js/firstparties/google-search.js" in entry['js']:
+                scripts_idx = idx
+                break
+        if scripts_idx == -1:
+            print("Failed to locate the Google Search content script in the manifest!")
+            sys.exit(1)
+
+        manifest['content_scripts'][scripts_idx]['matches'] = match_patterns
+
+        # overwrite tempfile_path with the updated manifest
+        f.seek(0)
+        # print() auto-adds a trailing newline
+        print(
+            json.dumps(
+                manifest,
+                sort_keys=False,
+                indent=2,
+                separators=(',', ': ')
+            ),
+            file=f
+        )
+        f.truncate()
+
+if __name__ == '__main__':
+    # argv[1]: the path to a copy of https://www.google.com/supported_domains
+    # argv[2]: the path to the extension manifest
+    update_manifest(sys.argv[1], sys.argv[2])
diff --git a/scripts/updategoogle.sh b/scripts/updategoogle.sh
new file mode 100755
index 0000000..d446861
--- /dev/null
+++ b/scripts/updategoogle.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+GOOGLE_DOMAINS_URL=https://www.google.com/supported_domains
+MANIFEST_PATH=src/manifest.json
+TEMPFILE=$(mktemp)
+
+trap 'rm $TEMPFILE' EXIT
+
+echo "fetching Google Search domains ..."
+if wget -q -T 30 -O "$TEMPFILE" -- $GOOGLE_DOMAINS_URL && [ -s "$TEMPFILE" ]; then
+	./scripts/updategoogle.py "$TEMPFILE" "$MANIFEST_PATH"
+	if cmp -s "$TEMPFILE" $MANIFEST_PATH; then
+		echo "    no Google Search domain updates"
+	else
+		cp "$TEMPFILE" $MANIFEST_PATH
+		echo "    updated Google Search domains in $MANIFEST_PATH"
+		echo "    please verify, update Google's MDFP list, and commit both!"
+		exit 1
+	fi
+else
+	echo "    failed to fetch $GOOGLE_DOMAINS_URL"
+	echo "    aborting build!"
+	exit 1
+fi
diff --git a/scripts/updatepsl.sh b/scripts/updatepsl.sh
new file mode 100755
index 0000000..fa91741
--- /dev/null
+++ b/scripts/updatepsl.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Update the Public Suffix List (psl)
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+PSL_PATH=src/lib/publicSuffixList.js
+PSL_URL=https://publicsuffix.org/list/public_suffix_list.dat
+TEMPFILE=$(mktemp)
+
+trap 'rm $TEMPFILE' EXIT
+
+echo "fetching Public Suffix List ..."
+if wget -q -T 30 -O "$TEMPFILE" -- $PSL_URL && [ -s "$TEMPFILE" ]; then
+	python scripts/convertpsl.py "$TEMPFILE"
+	if cmp -s "$TEMPFILE" $PSL_PATH; then
+		echo "    no PSL updates"
+	else
+		cp "$TEMPFILE" $PSL_PATH
+		echo "    updated PSL at $PSL_PATH"
+		echo "    please verify and commit!"
+		exit 1
+	fi
+else
+	echo "    failed to fetch PSL from $PSL_URL"
+	echo "    aborting build!"
+	exit 1
+fi
diff --git a/scripts/updateseeddata.sh b/scripts/updateseeddata.sh
new file mode 100755
index 0000000..bb1d221
--- /dev/null
+++ b/scripts/updateseeddata.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Update the pre-trained "seed" tracker list
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+SEED_PATH=src/data/seed.json
+SEED_URL=https://raw.githubusercontent.com/EFForg/badger-sett/master/results.json
+TEMPFILE=$(mktemp)
+
+trap 'rm $TEMPFILE' EXIT
+
+echo "fetching seed tracker lists..."
+if wget -q -T 30 -O "$TEMPFILE" -- $SEED_URL && [ -s "$TEMPFILE" ]; then
+  if ! python scripts/verify_json.py "$TEMPFILE"; then
+    echo "    new seed data is not formatted correctly"
+    echo "    aborting build!"
+    exit 1
+  fi
+
+  if cmp -s "$TEMPFILE" $SEED_PATH; then
+    echo "    no seed data updates"
+  else
+    cp "$TEMPFILE" $SEED_PATH
+    echo "    updated seed data at $SEED_PATH"
+    echo "    please verify and commit!"
+    exit 1
+  fi
+else
+  echo "    failed to fetch seed data from $SEED_URL"
+  echo "    aborting build!"
+  exit 1
+fi
diff --git a/scripts/verify_json.py b/scripts/verify_json.py
new file mode 100644
index 0000000..e5a17ac
--- /dev/null
+++ b/scripts/verify_json.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+KEYS = set(['snitch_map', 'action_map', 'version'])
+
+with open(sys.argv[1]) as f:
+    try:
+        js = json.load(f)
+        if set(js.keys()) == KEYS:
+            sys.exit(0)
+        else:
+            print("json keys %s are not correct" % js.keys())
+            sys.exit(1)
+    except Exception as e:
+        print("error parsing json:", e)
+        sys.exit(1)