summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/chromedriver.sh25
-rwxr-xr-xscripts/convertpsl.py36
-rwxr-xr-xscripts/fix_placeholders.py40
-rwxr-xr-xscripts/generate-legacy-yellowlist.sh1
-rwxr-xr-xscripts/run_travis.sh36
-rwxr-xr-xscripts/setup_travis.sh79
-rwxr-xr-xscripts/updategoogle.py51
-rwxr-xr-xscripts/updategoogle.sh27
-rwxr-xr-xscripts/updatepsl.sh28
-rwxr-xr-xscripts/updateseeddata.sh33
-rw-r--r--scripts/verify_json.py18
11 files changed, 374 insertions, 0 deletions
diff --git a/scripts/chromedriver.sh b/scripts/chromedriver.sh
new file mode 100755
index 0000000..3bd55d6
--- /dev/null
+++ b/scripts/chromedriver.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+TEMPFILE=$(mktemp)
+CHROME="$1"
+
+trap 'rm $TEMPFILE' EXIT
+
+# install the appropriate version of ChromeDriver
+chrome_version=$("$CHROME" --product-version | cut -d . -f 1-3)
+chromedriver_version_url=https://chromedriver.storage.googleapis.com/LATEST_RELEASE_"$chrome_version"
+chromedriver_version=$(wget "$chromedriver_version_url" -q -O -)
+echo "Setting up ChromeDriver version $chromedriver_version ..."
+chromedriver_url=https://chromedriver.storage.googleapis.com/"$chromedriver_version"/chromedriver_linux64.zip
+wget -q -O "$TEMPFILE" "$chromedriver_url"
+sudo unzip -q -o "$TEMPFILE" chromedriver -d /usr/local/bin/
+sudo chmod a+x /usr/local/bin/chromedriver
+
+# check that chromedriver is now present
+type chromedriver >/dev/null 2>&1 || {
+ echo "Failed to install ChromeDriver!"
+ exit 1
+}
diff --git a/scripts/convertpsl.py b/scripts/convertpsl.py
new file mode 100755
index 0000000..3cd78be
--- /dev/null
+++ b/scripts/convertpsl.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+# script based on
+# https://github.com/adblockplus/buildtools/blob/d090e00610a58cebc78478ae33e896e6b949fc12/publicSuffixListUpdater.py
+
+from __future__ import print_function
+
+import json
+import sys
+
+def convert(psl_text):
+ suffixes = {}
+
+ for line in psl_text:
+ line = line.rstrip()
+ if line.startswith('//') or '.' not in line:
+ continue
+ if line.startswith('*.'):
+ suffixes[line[2:]] = 2
+ elif line.startswith('!'):
+ suffixes[line[1:]] = 0
+ else:
+ suffixes[line] = 1
+
+ return suffixes
+
+
+if __name__ == '__main__':
+ with open(sys.argv[1], 'r+') as f:
+ psl = convert(f)
+ f.seek(0)
+ text = 'window.publicSuffixes = %s;' % (
+ json.dumps(psl, sort_keys=True, indent=2, separators=(',', ': '))
+ )
+ print(text, file=f)
+ f.truncate()
diff --git a/scripts/fix_placeholders.py b/scripts/fix_placeholders.py
new file mode 100755
index 0000000..e940dcc
--- /dev/null
+++ b/scripts/fix_placeholders.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import json
+
+from collections import OrderedDict
+from glob import glob
+
+SOURCE_LOCALE = 'src/_locales/en_US/messages.json'
+
+
+def fix_locale(locale, placeholder_keys):
+ # read in locale, preserving existing ordering
+ with open(locale, 'r') as f:
+ data = json.load(f, object_pairs_hook=OrderedDict)
+
+ # restore missing placeholders
+ for key in placeholder_keys:
+ if key in data and "placeholders" not in data[key]:
+ data[key]["placeholders"] = source_data[key]["placeholders"]
+
+ with open(locale, 'w') as f:
+ json.dump(data, f, ensure_ascii=False, indent=4)
+
+
+if __name__ == '__main__':
+ with open(SOURCE_LOCALE, 'r') as f:
+ source_data = json.load(f, object_pairs_hook=OrderedDict)
+
+ # get keys of locale messages with placeholders
+ placeholder_keys = []
+ for key in source_data:
+ if "placeholders" in source_data[key]:
+ placeholder_keys.append(key)
+
+ # fix all locales
+ for locale in glob('src/_locales/*/*.json'):
+ if locale == SOURCE_LOCALE:
+ continue
+
+ fix_locale(locale, placeholder_keys)
diff --git a/scripts/generate-legacy-yellowlist.sh b/scripts/generate-legacy-yellowlist.sh
new file mode 100755
index 0000000..b74712e
--- /dev/null
+++ b/scripts/generate-legacy-yellowlist.sh
@@ -0,0 +1 @@
+sed -e "s/^\([^\!].*\)$/@@||\1^\$third-party/g" src/data/yellowlist.txt
diff --git a/scripts/run_travis.sh b/scripts/run_travis.sh
new file mode 100755
index 0000000..fa20fdb
--- /dev/null
+++ b/scripts/run_travis.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+toplevel=$(git rev-parse --show-toplevel)
+testdir=${toplevel}/tests/selenium
+
+function run_lint {
+ if ! make -C "$toplevel" lint; then
+ echo "Linting errors"
+ exit 1
+ fi
+}
+
+function run_selenium {
+ # autodiscover and run the tests
+ pytest --capture=no --verbose --durations=10 "$testdir"
+}
+
+if [ "$INFO" == "lint" ]; then
+ echo "running lint tests"
+ run_lint
+else
+ case $BROWSER in
+ *chrome*)
+ echo "running tests on chrome"
+ run_selenium
+ ;;
+ *firefox*)
+ echo "running tests on firefox"
+ run_selenium
+ ;;
+ *)
+ echo "bad INFO variable, got $INFO"
+ exit 1
+ ;;
+ esac
+fi
diff --git a/scripts/setup_travis.sh b/scripts/setup_travis.sh
new file mode 100755
index 0000000..5e4bbf0
--- /dev/null
+++ b/scripts/setup_travis.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+toplevel=$(git rev-parse --show-toplevel)
+
+function setup_firefox {
+ # Install the latest version of geckodriver
+ version=$(curl -sI https://github.com/mozilla/geckodriver/releases/latest | grep -i "^Location: " | sed 's/.*\///' | tr -d '\r')
+
+ # check that we got something
+ if [ -z "$version" ]; then
+ echo "Failed to determine the latest geckodriver version!"
+ exit 1
+ fi
+
+ # Geckodriver distribution is MacOS or Linux specific
+ os="$(uname -s)"
+ if [[ $os == "Darwin" ]]; then
+ os_dist="macos.tar.gz"
+ else
+ os_dist="linux64.tar.gz"
+ fi
+
+ echo "Setting up geckodriver version $version ..."
+ url="https://github.com/mozilla/geckodriver/releases/download/${version}/geckodriver-${version}-${os_dist}"
+ wget -q -O /tmp/geckodriver.tar.gz "$url"
+ sudo tar -xvf /tmp/geckodriver.tar.gz -C /usr/local/bin/
+ sudo chmod a+x /usr/local/bin/geckodriver
+
+ # check that geckodriver is now present
+ type geckodriver >/dev/null 2>&1 || {
+ echo "Failed to install geckodriver!"
+ exit 1
+ }
+}
+
+function browser_setup {
+ # install python stuff
+ pip install -r "$toplevel"/tests/requirements.txt
+}
+
+function setup_lint {
+ # "--production" to skip installing devDependencies modules
+ npm install --production || exit 1
+}
+
+# check that the desired browser is present as it might fail to install
+# for example: https://travis-ci.org/EFForg/privacybadger/jobs/362381214
+function check_browser {
+ type "$BROWSER" >/dev/null 2>&1 || {
+ echo "$BROWSER seems to be missing!"
+ exit 1
+ }
+
+ # print the version
+ echo "Found $("$BROWSER" --version)"
+}
+
+case $INFO in
+ *chrome*)
+ check_browser
+ "$toplevel"/scripts/chromedriver.sh "$BROWSER"
+ browser_setup
+ ;;
+ *firefox*) # Install the latest version of geckodriver
+ check_browser
+ setup_firefox
+ browser_setup
+ ;;
+ *lint*)
+ setup_lint
+ ;;
+ *)
+ echo "bad INFO variable, got $INFO"
+ exit 1
+ ;;
+esac
diff --git a/scripts/updategoogle.py b/scripts/updategoogle.py
new file mode 100755
index 0000000..a3046d4
--- /dev/null
+++ b/scripts/updategoogle.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+from collections import OrderedDict
+
+def convert(text):
+ patterns = []
+ for domain in text.split():
+ patterns.append("https://www" + domain + "/*")
+ patterns.append("http://www" + domain + "/*")
+ return patterns
+
+def update_manifest(tempfile_path, manifest_path):
+ with open(manifest_path, 'r') as f:
+ manifest = json.load(f, object_pairs_hook=OrderedDict)
+
+ with open(tempfile_path, 'r+') as f:
+ # tempfile_path contains Google's supported domains
+ match_patterns = convert(f.read())
+
+ scripts_idx = -1
+ for idx, entry in enumerate(manifest['content_scripts']):
+ if "js/firstparties/google-search.js" in entry['js']:
+ scripts_idx = idx
+ break
+ if scripts_idx == -1:
+ print("Failed to locate the Google Search content script in the manifest!")
+ sys.exit(1)
+
+ manifest['content_scripts'][scripts_idx]['matches'] = match_patterns
+
+ # overwrite tempfile_path with the updated manifest
+ f.seek(0)
+ # print() auto-adds a trailing newline
+ print(
+ json.dumps(
+ manifest,
+ sort_keys=False,
+ indent=2,
+ separators=(',', ': ')
+ ),
+ file=f
+ )
+ f.truncate()
+
+if __name__ == '__main__':
+ # argv[1]: the path to a copy of https://www.google.com/supported_domains
+ # argv[2]: the path to the extension manifest
+ update_manifest(sys.argv[1], sys.argv[2])
diff --git a/scripts/updategoogle.sh b/scripts/updategoogle.sh
new file mode 100755
index 0000000..d446861
--- /dev/null
+++ b/scripts/updategoogle.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+GOOGLE_DOMAINS_URL=https://www.google.com/supported_domains
+MANIFEST_PATH=src/manifest.json
+TEMPFILE=$(mktemp)
+
+trap 'rm $TEMPFILE' EXIT
+
+echo "fetching Google Search domains ..."
+if wget -q -T 30 -O "$TEMPFILE" -- $GOOGLE_DOMAINS_URL && [ -s "$TEMPFILE" ]; then
+ ./scripts/updategoogle.py "$TEMPFILE" "$MANIFEST_PATH"
+ if cmp -s "$TEMPFILE" $MANIFEST_PATH; then
+ echo " no Google Search domain updates"
+ else
+ cp "$TEMPFILE" $MANIFEST_PATH
+ echo " updated Google Search domains in $MANIFEST_PATH"
+ echo " please verify, update Google's MDFP list, and commit both!"
+ exit 1
+ fi
+else
+ echo " failed to fetch $GOOGLE_DOMAINS_URL"
+ echo " aborting build!"
+ exit 1
+fi
diff --git a/scripts/updatepsl.sh b/scripts/updatepsl.sh
new file mode 100755
index 0000000..fa91741
--- /dev/null
+++ b/scripts/updatepsl.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Update the Public Suffix List (psl)
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+PSL_PATH=src/lib/publicSuffixList.js
+PSL_URL=https://publicsuffix.org/list/public_suffix_list.dat
+TEMPFILE=$(mktemp)
+
+trap 'rm $TEMPFILE' EXIT
+
+echo "fetching Public Suffix List ..."
+if wget -q -T 30 -O "$TEMPFILE" -- $PSL_URL && [ -s "$TEMPFILE" ]; then
+ python scripts/convertpsl.py "$TEMPFILE"
+ if cmp -s "$TEMPFILE" $PSL_PATH; then
+ echo " no PSL updates"
+ else
+ cp "$TEMPFILE" $PSL_PATH
+ echo " updated PSL at $PSL_PATH"
+ echo " please verify and commit!"
+ exit 1
+ fi
+else
+ echo " failed to fetch PSL from $PSL_URL"
+ echo " aborting build!"
+ exit 1
+fi
diff --git a/scripts/updateseeddata.sh b/scripts/updateseeddata.sh
new file mode 100755
index 0000000..bb1d221
--- /dev/null
+++ b/scripts/updateseeddata.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Update the pre-trained "seed" tracker list
+
+# stop on errors (nonzero exit codes), uninitialized vars
+set -eu
+
+SEED_PATH=src/data/seed.json
+SEED_URL=https://raw.githubusercontent.com/EFForg/badger-sett/master/results.json
+TEMPFILE=$(mktemp)
+
+trap 'rm $TEMPFILE' EXIT
+
+echo "fetching seed tracker lists..."
+if wget -q -T 30 -O "$TEMPFILE" -- $SEED_URL && [ -s "$TEMPFILE" ]; then
+ if ! python scripts/verify_json.py "$TEMPFILE"; then
+ echo " new seed data is not formatted correctly"
+ echo " aborting build!"
+ exit 1
+ fi
+
+ if cmp -s "$TEMPFILE" $SEED_PATH; then
+ echo " no seed data updates"
+ else
+ cp "$TEMPFILE" $SEED_PATH
+ echo " updated seed data at $SEED_PATH"
+ echo " please verify and commit!"
+ exit 1
+ fi
+else
+ echo " failed to fetch seed data from $SEED_URL"
+ echo " aborting build!"
+ exit 1
+fi
diff --git a/scripts/verify_json.py b/scripts/verify_json.py
new file mode 100644
index 0000000..e5a17ac
--- /dev/null
+++ b/scripts/verify_json.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+KEYS = set(['snitch_map', 'action_map', 'version'])
+
+with open(sys.argv[1]) as f:
+ try:
+ js = json.load(f)
+ if set(js.keys()) == KEYS:
+ sys.exit(0)
+ else:
+ print("json keys %s are not correct" % js.keys())
+ sys.exit(1)
+ except Exception as e:
+ print("error parsing json:", e)
+ sys.exit(1)