diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/chromedriver.sh | 25 | ||||
| -rwxr-xr-x | scripts/convertpsl.py | 36 | ||||
| -rwxr-xr-x | scripts/fix_placeholders.py | 40 | ||||
| -rwxr-xr-x | scripts/generate-legacy-yellowlist.sh | 1 | ||||
| -rwxr-xr-x | scripts/run_travis.sh | 36 | ||||
| -rwxr-xr-x | scripts/setup_travis.sh | 79 | ||||
| -rwxr-xr-x | scripts/updategoogle.py | 51 | ||||
| -rwxr-xr-x | scripts/updategoogle.sh | 27 | ||||
| -rwxr-xr-x | scripts/updatepsl.sh | 28 | ||||
| -rwxr-xr-x | scripts/updateseeddata.sh | 33 | ||||
| -rw-r--r-- | scripts/verify_json.py | 18 |
11 files changed, 374 insertions, 0 deletions
diff --git a/scripts/chromedriver.sh b/scripts/chromedriver.sh new file mode 100755 index 0000000..3bd55d6 --- /dev/null +++ b/scripts/chromedriver.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# stop on errors (nonzero exit codes), uninitialized vars +set -eu + +TEMPFILE=$(mktemp) +CHROME="$1" + +trap 'rm $TEMPFILE' EXIT + +# install the appropriate version of ChromeDriver +chrome_version=$("$CHROME" --product-version | cut -d . -f 1-3) +chromedriver_version_url=https://chromedriver.storage.googleapis.com/LATEST_RELEASE_"$chrome_version" +chromedriver_version=$(wget "$chromedriver_version_url" -q -O -) +echo "Setting up ChromeDriver version $chromedriver_version ..." +chromedriver_url=https://chromedriver.storage.googleapis.com/"$chromedriver_version"/chromedriver_linux64.zip +wget -q -O "$TEMPFILE" "$chromedriver_url" +sudo unzip -q -o "$TEMPFILE" chromedriver -d /usr/local/bin/ +sudo chmod a+x /usr/local/bin/chromedriver + +# check that chromedriver is now present +type chromedriver >/dev/null 2>&1 || { + echo "Failed to install ChromeDriver!" + exit 1 +} diff --git a/scripts/convertpsl.py b/scripts/convertpsl.py new file mode 100755 index 0000000..3cd78be --- /dev/null +++ b/scripts/convertpsl.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# script based on +# https://github.com/adblockplus/buildtools/blob/d090e00610a58cebc78478ae33e896e6b949fc12/publicSuffixListUpdater.py + +from __future__ import print_function + +import json +import sys + +def convert(psl_text): + suffixes = {} + + for line in psl_text: + line = line.rstrip() + if line.startswith('//') or '.' not in line: + continue + if line.startswith('*.'): + suffixes[line[2:]] = 2 + elif line.startswith('!'): + suffixes[line[1:]] = 0 + else: + suffixes[line] = 1 + + return suffixes + + +if __name__ == '__main__': + with open(sys.argv[1], 'r+') as f: + psl = convert(f) + f.seek(0) + text = 'window.publicSuffixes = %s;' % ( + json.dumps(psl, sort_keys=True, indent=2, separators=(',', ': ')) + ) + print(text, file=f) + f.truncate() diff --git a/scripts/fix_placeholders.py b/scripts/fix_placeholders.py new file mode 100755 index 0000000..e940dcc --- /dev/null +++ b/scripts/fix_placeholders.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import json + +from collections import OrderedDict +from glob import glob + +SOURCE_LOCALE = 'src/_locales/en_US/messages.json' + + +def fix_locale(locale, placeholder_keys): + # read in locale, preserving existing ordering + with open(locale, 'r') as f: + data = json.load(f, object_pairs_hook=OrderedDict) + + # restore missing placeholders + for key in placeholder_keys: + if key in data and "placeholders" not in data[key]: + data[key]["placeholders"] = source_data[key]["placeholders"] + + with open(locale, 'w') as f: + json.dump(data, f, ensure_ascii=False, indent=4) + + +if __name__ == '__main__': + with open(SOURCE_LOCALE, 'r') as f: + source_data = json.load(f, object_pairs_hook=OrderedDict) + + # get keys of locale messages with placeholders + placeholder_keys = [] + for key in source_data: + if "placeholders" in source_data[key]: + placeholder_keys.append(key) + + # fix all locales + for locale in glob('src/_locales/*/*.json'): + if locale == SOURCE_LOCALE: + continue + + fix_locale(locale, placeholder_keys) diff --git a/scripts/generate-legacy-yellowlist.sh b/scripts/generate-legacy-yellowlist.sh new file mode 100755 index 0000000..b74712e --- /dev/null +++ b/scripts/generate-legacy-yellowlist.sh @@ -0,0 +1 @@ +sed -e "s/^\([^\!].*\)$/@@||\1^\$third-party/g" src/data/yellowlist.txt diff --git a/scripts/run_travis.sh b/scripts/run_travis.sh new file mode 100755 index 0000000..fa20fdb --- /dev/null +++ b/scripts/run_travis.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +toplevel=$(git rev-parse --show-toplevel) +testdir=${toplevel}/tests/selenium + +function run_lint { + if ! make -C "$toplevel" lint; then + echo "Linting errors" + exit 1 + fi +} + +function run_selenium { + # autodiscover and run the tests + pytest --capture=no --verbose --durations=10 "$testdir" +} + +if [ "$INFO" == "lint" ]; then + echo "running lint tests" + run_lint +else + case $BROWSER in + *chrome*) + echo "running tests on chrome" + run_selenium + ;; + *firefox*) + echo "running tests on firefox" + run_selenium + ;; + *) + echo "bad INFO variable, got $INFO" + exit 1 + ;; + esac +fi diff --git a/scripts/setup_travis.sh b/scripts/setup_travis.sh new file mode 100755 index 0000000..5e4bbf0 --- /dev/null +++ b/scripts/setup_travis.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +# stop on errors (nonzero exit codes), uninitialized vars +set -eu + +toplevel=$(git rev-parse --show-toplevel) + +function setup_firefox { + # Install the latest version of geckodriver + version=$(curl -sI https://github.com/mozilla/geckodriver/releases/latest | grep -i "^Location: " | sed 's/.*\///' | tr -d '\r') + + # check that we got something + if [ -z "$version" ]; then + echo "Failed to determine the latest geckodriver version!" + exit 1 + fi + + # Geckodriver distribution is MacOS or Linux specific + os="$(uname -s)" + if [[ $os == "Darwin" ]]; then + os_dist="macos.tar.gz" + else + os_dist="linux64.tar.gz" + fi + + echo "Setting up geckodriver version $version ..." + url="https://github.com/mozilla/geckodriver/releases/download/${version}/geckodriver-${version}-${os_dist}" + wget -q -O /tmp/geckodriver.tar.gz "$url" + sudo tar -xvf /tmp/geckodriver.tar.gz -C /usr/local/bin/ + sudo chmod a+x /usr/local/bin/geckodriver + + # check that geckodriver is now present + type geckodriver >/dev/null 2>&1 || { + echo "Failed to install geckodriver!" + exit 1 + } +} + +function browser_setup { + # install python stuff + pip install -r "$toplevel"/tests/requirements.txt +} + +function setup_lint { + # "--production" to skip installing devDependencies modules + npm install --production || exit 1 +} + +# check that the desired browser is present as it might fail to install +# for example: https://travis-ci.org/EFForg/privacybadger/jobs/362381214 +function check_browser { + type "$BROWSER" >/dev/null 2>&1 || { + echo "$BROWSER seems to be missing!" + exit 1 + } + + # print the version + echo "Found $("$BROWSER" --version)" +} + +case $INFO in + *chrome*) + check_browser + "$toplevel"/scripts/chromedriver.sh "$BROWSER" + browser_setup + ;; + *firefox*) # Install the latest version of geckodriver + check_browser + setup_firefox + browser_setup + ;; + *lint*) + setup_lint + ;; + *) + echo "bad INFO variable, got $INFO" + exit 1 + ;; +esac diff --git a/scripts/updategoogle.py b/scripts/updategoogle.py new file mode 100755 index 0000000..a3046d4 --- /dev/null +++ b/scripts/updategoogle.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import json +import sys + +from collections import OrderedDict + +def convert(text): + patterns = [] + for domain in text.split(): + patterns.append("https://www" + domain + "/*") + patterns.append("http://www" + domain + "/*") + return patterns + +def update_manifest(tempfile_path, manifest_path): + with open(manifest_path, 'r') as f: + manifest = json.load(f, object_pairs_hook=OrderedDict) + + with open(tempfile_path, 'r+') as f: + # tempfile_path contains Google's supported domains + match_patterns = convert(f.read()) + + scripts_idx = -1 + for idx, entry in enumerate(manifest['content_scripts']): + if "js/firstparties/google-search.js" in entry['js']: + scripts_idx = idx + break + if scripts_idx == -1: + print("Failed to locate the Google Search content script in the manifest!") + sys.exit(1) + + manifest['content_scripts'][scripts_idx]['matches'] = match_patterns + + # overwrite tempfile_path with the updated manifest + f.seek(0) + # print() auto-adds a trailing newline + print( + json.dumps( + manifest, + sort_keys=False, + indent=2, + separators=(',', ': ') + ), + file=f + ) + f.truncate() + +if __name__ == '__main__': + # argv[1]: the path to a copy of https://www.google.com/supported_domains + # argv[2]: the path to the extension manifest + update_manifest(sys.argv[1], sys.argv[2]) diff --git a/scripts/updategoogle.sh b/scripts/updategoogle.sh new file mode 100755 index 0000000..d446861 --- /dev/null +++ b/scripts/updategoogle.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# stop on errors (nonzero exit codes), uninitialized vars +set -eu + +GOOGLE_DOMAINS_URL=https://www.google.com/supported_domains +MANIFEST_PATH=src/manifest.json +TEMPFILE=$(mktemp) + +trap 'rm $TEMPFILE' EXIT + +echo "fetching Google Search domains ..." +if wget -q -T 30 -O "$TEMPFILE" -- $GOOGLE_DOMAINS_URL && [ -s "$TEMPFILE" ]; then + ./scripts/updategoogle.py "$TEMPFILE" "$MANIFEST_PATH" + if cmp -s "$TEMPFILE" $MANIFEST_PATH; then + echo " no Google Search domain updates" + else + cp "$TEMPFILE" $MANIFEST_PATH + echo " updated Google Search domains in $MANIFEST_PATH" + echo " please verify, update Google's MDFP list, and commit both!" + exit 1 + fi +else + echo " failed to fetch $GOOGLE_DOMAINS_URL" + echo " aborting build!" + exit 1 +fi diff --git a/scripts/updatepsl.sh b/scripts/updatepsl.sh new file mode 100755 index 0000000..fa91741 --- /dev/null +++ b/scripts/updatepsl.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Update the Public Suffix List (psl) + +# stop on errors (nonzero exit codes), uninitialized vars +set -eu + +PSL_PATH=src/lib/publicSuffixList.js +PSL_URL=https://publicsuffix.org/list/public_suffix_list.dat +TEMPFILE=$(mktemp) + +trap 'rm $TEMPFILE' EXIT + +echo "fetching Public Suffix List ..." +if wget -q -T 30 -O "$TEMPFILE" -- $PSL_URL && [ -s "$TEMPFILE" ]; then + python scripts/convertpsl.py "$TEMPFILE" + if cmp -s "$TEMPFILE" $PSL_PATH; then + echo " no PSL updates" + else + cp "$TEMPFILE" $PSL_PATH + echo " updated PSL at $PSL_PATH" + echo " please verify and commit!" + exit 1 + fi +else + echo " failed to fetch PSL from $PSL_URL" + echo " aborting build!" + exit 1 +fi diff --git a/scripts/updateseeddata.sh b/scripts/updateseeddata.sh new file mode 100755 index 0000000..bb1d221 --- /dev/null +++ b/scripts/updateseeddata.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Update the pre-trained "seed" tracker list + +# stop on errors (nonzero exit codes), uninitialized vars +set -eu + +SEED_PATH=src/data/seed.json +SEED_URL=https://raw.githubusercontent.com/EFForg/badger-sett/master/results.json +TEMPFILE=$(mktemp) + +trap 'rm $TEMPFILE' EXIT + +echo "fetching seed tracker lists..." +if wget -q -T 30 -O "$TEMPFILE" -- $SEED_URL && [ -s "$TEMPFILE" ]; then + if ! python scripts/verify_json.py "$TEMPFILE"; then + echo " new seed data is not formatted correctly" + echo " aborting build!" + exit 1 + fi + + if cmp -s "$TEMPFILE" $SEED_PATH; then + echo " no seed data updates" + else + cp "$TEMPFILE" $SEED_PATH + echo " updated seed data at $SEED_PATH" + echo " please verify and commit!" + exit 1 + fi +else + echo " failed to fetch seed data from $SEED_URL" + echo " aborting build!" + exit 1 +fi diff --git a/scripts/verify_json.py b/scripts/verify_json.py new file mode 100644 index 0000000..e5a17ac --- /dev/null +++ b/scripts/verify_json.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import json +import sys + +KEYS = set(['snitch_map', 'action_map', 'version']) + +with open(sys.argv[1]) as f: + try: + js = json.load(f) + if set(js.keys()) == KEYS: + sys.exit(0) + else: + print("json keys %s are not correct" % js.keys()) + sys.exit(1) + except Exception as e: + print("error parsing json:", e) + sys.exit(1) |