diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/requirements.txt | 3 | ||||
| -rw-r--r-- | tests/selenium/.flake8 | 2 | ||||
| -rw-r--r-- | tests/selenium/breakage_test.py | 30 | ||||
| -rw-r--r-- | tests/selenium/clobbering_test.py | 102 | ||||
| -rw-r--r-- | tests/selenium/cookie_test.py | 163 | ||||
| -rw-r--r-- | tests/selenium/dnt_test.py | 324 | ||||
| -rw-r--r-- | tests/selenium/fingerprinting_test.py | 105 | ||||
| -rw-r--r-- | tests/selenium/options_test.py | 327 | ||||
| -rw-r--r-- | tests/selenium/pbtest.py | 498 | ||||
| -rw-r--r-- | tests/selenium/pbtest_org_test.py | 71 | ||||
| -rw-r--r-- | tests/selenium/popup_test.py | 361 | ||||
| -rw-r--r-- | tests/selenium/qunit_test.py | 38 | ||||
| -rw-r--r-- | tests/selenium/service_workers_test.py | 56 | ||||
| -rw-r--r-- | tests/selenium/storage_test.py | 71 | ||||
| -rw-r--r-- | tests/selenium/super_cookie_test.py | 120 | ||||
| -rw-r--r-- | tests/selenium/surrogates_test.py | 104 | ||||
| -rw-r--r-- | tests/selenium/website_testbed/first-party.html | 13 | ||||
| -rw-r--r-- | tests/selenium/website_testbed/first-party.js | 25 | ||||
| -rw-r--r-- | tests/selenium/widgets_test.py | 344 |
19 files changed, 2757 insertions, 0 deletions
diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..8a2509d --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +selenium +xvfbwrapper +pytest diff --git a/tests/selenium/.flake8 b/tests/selenium/.flake8 new file mode 100644 index 0000000..2a1147a --- /dev/null +++ b/tests/selenium/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E501, E731 diff --git a/tests/selenium/breakage_test.py b/tests/selenium/breakage_test.py new file mode 100644 index 0000000..2970610 --- /dev/null +++ b/tests/selenium/breakage_test.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest +import pbtest +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + + +class BreakageTest(pbtest.PBSeleniumTest): + """Make sure the extension doesn't break common sites and use cases. + e.g. we should be able to load a website, search on Google. + TODO: Add tests to simulate most common web use cases: + e.g. play Youtube videos, login to popular services, tweet some text, + add Reddit comments etc.""" + + def test_should_load_eff_org(self): + self.load_url("https://www.eff.org") + WebDriverWait(self.driver, 10).until( + EC.title_contains("Electronic Frontier Foundation")) + + def test_should_search_google(self): + self.load_url("https://www.google.com/") + qry_el = self.driver.find_element_by_name("q") + qry_el.send_keys("EFF") # search term + qry_el.submit() + WebDriverWait(self.driver, 10).until(EC.title_contains("EFF")) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/clobbering_test.py b/tests/selenium/clobbering_test.py new file mode 100644 index 0000000..724b83b --- /dev/null +++ b/tests/selenium/clobbering_test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest + +import pbtest + + +class ClobberingTest(pbtest.PBSeleniumTest): + def test_localstorage_clobbering(self): + LOCALSTORAGE_TESTS = [ + # (test result element ID, expected stored, expected empty) + ('get-item', "qwerty", "null"), + ('get-property', "asdf", "undefined"), + ('get-item-proto', "qwerty", "null"), + ('get-item-srcdoc', "qwerty", "null"), + ('get-property-srcdoc', "asdf", "undefined"), + ('get-item-frames', "qwerty", "null"), + ('get-property-frames', "asdf", "undefined"), + ] + # page loads a frame that writes to and reads from localStorage + # TODO remove delays from fixture once race condition (https://crbug.com/478183) is fixed + FIXTURE_URL = "https://privacybadger-tests.eff.org/html/clobbering.html" + FRAME_DOMAIN = "efforg.github.io" + + # first allow localStorage to be set + self.load_url(FIXTURE_URL) + self.wait_for_and_switch_to_frame('iframe') + for selector, expected, _ in LOCALSTORAGE_TESTS: + # wait for each test to run + self.wait_for_script( + "return document.getElementById('%s')" + ".textContent != '...';" % selector, + timeout=2, + message=( + "Timed out waiting for localStorage (%s) to finish ... " + "This probably means the fixture " + "errored out somewhere." % selector + ) + ) + self.assertEqual( + self.txt_by_css("#" + selector), expected, + "localStorage (%s) was not read successfully" + "for some reason" % selector + ) + + # mark the frame domain for cookieblocking + self.cookieblock_domain(FRAME_DOMAIN) + + # now rerun and check results for various localStorage access tests + self.load_url(FIXTURE_URL) + self.wait_for_and_switch_to_frame('iframe') + for selector, _, expected in LOCALSTORAGE_TESTS: + # wait for each test to run + self.wait_for_script( + "return document.getElementById('%s')" + ".textContent != '...';" % selector, + timeout=2, + message=( + "Timed out waiting for localStorage (%s) to finish ... " + "This probably means the fixture " + "errored out somewhere." % selector + ) + ) + self.assertEqual( + self.txt_by_css("#" + selector), expected, + "localStorage (%s) was read despite cookieblocking" % selector + ) + + def test_referrer_header(self): + FIXTURE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "referrer.html" + ) + THIRD_PARTY_DOMAIN = "httpbin.org" + + def verify_referrer_header(expected, failure_message): + self.load_url(FIXTURE_URL) + self.wait_for_script( + "return document.getElementById('referrer').textContent != '';") + referrer = self.txt_by_css("#referrer") + self.assertEqual(referrer[0:8], "Referer=", "Unexpected page output") + self.assertEqual(referrer[8:], expected, failure_message) + + # verify base case + verify_referrer_header( + FIXTURE_URL, + "Unexpected default referrer header" + ) + + # cookieblock the domain fetched by the fixture + self.cookieblock_domain(THIRD_PARTY_DOMAIN) + + # recheck what the referrer header looks like now after cookieblocking + verify_referrer_header( + "https://efforg.github.io/", + "Referrer header does not appear to be origin-only" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/cookie_test.py b/tests/selenium/cookie_test.py new file mode 100644 index 0000000..93e70af --- /dev/null +++ b/tests/selenium/cookie_test.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +import unittest + +import pbtest + +from popup_test import get_domain_slider_state + + +class CookieTest(pbtest.PBSeleniumTest): + """Basic test to make sure the PB doesn't mess up with the cookies.""" + + def assert_pass_opera_cookie_test(self, url, test_name): + self.load_url(url) + self.assertEqual("PASS", self.txt_by_css("#result"), + "Cookie test failed: %s" % test_name) + + def test_should_pass_std_cookie_test(self): + self.assert_pass_opera_cookie_test(( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "first_party_cookie.html" + ), "Set 1st party cookie") + + def test_cookie_tracker_detection(self): + """Tests basic cookie tracking. The tracking site has no DNT file, + and gets blocked by PB. + + Visits three sites, all of which have an iframe that points to a fourth site + that reads and writes a cookie. The third party cookie will be picked up by + PB after each of the site loads, but no action will be taken. Then the first + site will be reloaded, and the UI will show the third party domain as blocked.""" + + SITE1_URL = "https://ddrybktjfxh4.cloudfront.net/" + SITE2_URL = "https://d3syxqe9po5ji0.cloudfront.net/" + SITE3_URL = "https://d3b37ucnz1m2l2.cloudfront.net/" + + THIRD_PARTY_DOMAIN = "efforg.github.io" + + # enable local learning + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#local-learning-checkbox').click() + + # remove pre-trained domains + self.js( + "chrome.extension.getBackgroundPage()." + "badger.storage.clearTrackerData();" + ) + + # load the first site with the third party code that reads and writes a cookie + self.load_url(SITE1_URL) + self.load_pb_ui(SITE1_URL) + # TODO it takes another visit (or a page reload) + # TODO to show the domain as not-yet-blocked-but-tracking? + #self.assertIn(THIRD_PARTY_DOMAIN, self.notYetBlocked) + self.close_window_with_url(SITE1_URL) + + # go to second site + self.load_url(SITE2_URL) + self.load_pb_ui(SITE2_URL) + self.assertIn(THIRD_PARTY_DOMAIN, self.notYetBlocked) + self.close_window_with_url(SITE2_URL) + + # go to third site + self.load_url(SITE3_URL) + self.load_pb_ui(SITE3_URL) + self.assertIn(THIRD_PARTY_DOMAIN, self.notYetBlocked) + self.close_window_with_url(SITE3_URL) + + # revisiting the first site should cause + # the third-party domain to be blocked + self.load_url(SITE1_URL) + self.load_pb_ui(SITE1_URL) + self.assertIn(THIRD_PARTY_DOMAIN, self.blocked) + + def load_pb_ui(self, target_url): + """Show the PB popup as a new tab. + + If Selenium would let us just programmatically launch an extension from its icon, + we wouldn't need this method. Alas it will not. + + But! We can open a new tab and set the url to the extension's popup html page and + test away. That's how most devs test extensions. But**2!! PB's popup code uses + the current tab's url to report the current tracker status. And since we changed + the current tab's url when we loaded the popup as a tab, the popup loses all the + blocker status information from the original tab. + + The workaround is to execute a new convenience function in the popup codebase that + looks for a given url in the tabs and, if it finds a match, refreshes the popup + with the associated tabid. Then the correct status information will be displayed + in the popup.""" + + self.open_window() + self.load_url(self.popup_url) + + # get the popup populated with status information for the correct url + self.switch_to_window_with_url(self.popup_url) + self.js(""" +/** + * if the query url pattern matches a tab, switch the module's tab object to that tab + */ +(function (url) { + chrome.tabs.query({url}, function (tabs) { + if (!tabs || !tabs.length) { + return; + } + chrome.runtime.sendMessage({ + type: "getPopupData", + tabId: tabs[0].id, + tabUrl: tabs[0].url + }, (response) => { + setPopupData(response); + refreshPopup(); + window.DONE_REFRESHING = true; + }); + }); +}(arguments[0]));""", target_url) + + # wait for popup to be ready + self.wait_for_script( + "return typeof window.DONE_REFRESHING != 'undefined' &&" + "window.POPUP_INITIALIZED &&" + "window.SLIDERS_DONE" + ) + + self.get_tracker_state() + + def get_tracker_state(self): + """Parse the UI to group all third party origins into their respective action states.""" + self.notYetBlocked = {} + self.cookieBlocked = {} + self.blocked = {} + + self.driver.switch_to.window(self.driver.current_window_handle) + + domain_divs = self.driver.find_elements_by_css_selector( + "#blockedResourcesInner > div.clicker[data-origin]") + for div in domain_divs: + origin = div.get_attribute('data-origin') + + # assert that this origin is never duplicated in the UI + self.assertNotIn(origin, self.notYetBlocked) + self.assertNotIn(origin, self.cookieBlocked) + self.assertNotIn(origin, self.blocked) + + # get slider state for given origin + action_type = get_domain_slider_state(self.driver, origin) + + # non-tracking domains are hidden by default + # so if we see a slider set to "allow", + # it must be in the tracking-but-not-yet-blocked section + if action_type == 'allow': + self.notYetBlocked[origin] = True + elif action_type == 'cookieblock': + self.cookieBlocked[origin] = True + elif action_type == 'block': + self.blocked[origin] = True + else: + self.fail("what is this?!? %s" % action_type) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/dnt_test.py b/tests/selenium/dnt_test.py new file mode 100644 index 0000000..fb3fdf8 --- /dev/null +++ b/tests/selenium/dnt_test.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import json +import unittest + +import pbtest + +from functools import partial + +from selenium.common.exceptions import NoSuchElementException + +from pbtest import retry_until + + +class DntTest(pbtest.PBSeleniumTest): + """Tests to make sure DNT policy checking works as expected.""" + + CHECK_FOR_DNT_POLICY_JS = ( + "chrome.extension.getBackgroundPage()." + "badger.checkForDNTPolicy(" + " arguments[0]," + " r => window.DNT_CHECK_RESULT = r" + ");" + ) + + # TODO switch to non-delayed version + # https://gist.github.com/ghostwords/9fc6900566a2f93edd8e4a1e48bbaa28 + # once race condition (https://crbug.com/478183) is fixed + NAVIGATOR_DNT_TEST_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "navigator_donottrack_delayed.html" + ) + + def get_first_party_headers(self, url): + self.load_url(url) + + text = self.driver.find_element_by_tag_name('body').text + + try: + headers = json.loads(text)['headers'] + except ValueError: + print("\nFailed to parse JSON from {}".format(repr(text))) + return None + + return headers + + def domain_was_recorded(self, domain): + return self.js( + "return (Object.keys(" + " chrome.extension.getBackgroundPage()." + " badger.storage.action_map.getItemClones()" + ").indexOf(arguments[0]) != -1);", + domain + ) + + def domain_was_detected(self, domain): + return self.js( + "return (Object.keys(chrome.extension.getBackgroundPage().badger.tabData).some(tab_id => {" + " let origins = chrome.extension.getBackgroundPage().badger.tabData[tab_id].origins;" + " return origins.hasOwnProperty(arguments[0]);" + "}));", + domain + ) + + def domain_was_blocked(self, domain): + return self.js( + "return (Object.keys(chrome.extension.getBackgroundPage().badger.tabData).some(tab_id => {" + " let origins = chrome.extension.getBackgroundPage().badger.tabData[tab_id].origins;" + " return (" + " origins.hasOwnProperty(arguments[0]) &&" + " chrome.extension.getBackgroundPage().constants.BLOCKED_ACTIONS.has(origins[arguments[0]])" + " );" + "}));", + domain + ) + + @pbtest.repeat_if_failed(3) + def test_dnt_policy_check_should_happen_for_blocked_domains(self): + PAGE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "dnt.html" + ) + DNT_DOMAIN = "www.eff.org" + + # mark a DNT-compliant domain for blocking + self.block_domain(DNT_DOMAIN) + + # visit a page that loads a resource from that DNT-compliant domain + self.open_window() + self.load_url(PAGE_URL) + + # switch back to Badger's options page + self.switch_to_window_with_url(self.options_url) + + # verify that the domain is blocked + self.assertTrue(self.domain_was_detected(DNT_DOMAIN), + msg="Domain should have been detected.") + self.assertTrue(self.domain_was_blocked(DNT_DOMAIN), + msg="DNT-compliant resource should have been blocked at first.") + + def reload_and_see_if_unblocked(): + # switch back to the page with the DNT-compliant resource + self.switch_to_window_with_url(PAGE_URL) + + # reload it + self.load_url(PAGE_URL) + + # switch back to Badger's options page + self.switch_to_window_with_url(self.options_url) + + return ( + self.domain_was_detected(DNT_DOMAIN) and + self.domain_was_blocked(DNT_DOMAIN) + ) + + # verify that the domain is allowed + was_blocked = retry_until( + reload_and_see_if_unblocked, + tester=lambda x: not x, + msg="Waiting a bit for DNT check to complete and retrying ...") + + self.assertFalse(was_blocked, + msg="DNT-compliant resource should have gotten unblocked.") + + def test_dnt_policy_check_should_not_set_cookies(self): + TEST_DOMAIN = "dnt-test.trackersimulator.org" + TEST_URL = "https://{}/".format(TEST_DOMAIN) + + # verify that the domain itself doesn't set cookies + self.load_url(TEST_URL) + self.assertEqual(len(self.driver.get_cookies()), 0, + "No cookies initially") + + # directly visit a DNT policy URL known to set cookies + self.load_url(TEST_URL + ".well-known/dnt-policy.txt") + self.assertEqual(len(self.driver.get_cookies()), 1, + "DNT policy URL set a cookie") + + # verify we got a cookie + self.load_url(TEST_URL) + self.assertEqual(len(self.driver.get_cookies()), 1, + "We still have just one cookie") + + # clear cookies and verify + self.driver.delete_all_cookies() + self.load_url(TEST_URL) + self.assertEqual(len(self.driver.get_cookies()), 0, + "No cookies again") + + self.load_url(self.options_url) + # perform a DNT policy check + self.js(DntTest.CHECK_FOR_DNT_POLICY_JS, TEST_DOMAIN) + # wait until checkForDNTPolicy completed + self.wait_for_script("return window.DNT_CHECK_RESULT === false") + + # check that we didn't get cookied by the DNT URL + self.load_url(TEST_URL) + self.assertEqual(len(self.driver.get_cookies()), 0, + "Shouldn't have any cookies after the DNT check") + + def test_dnt_policy_check_should_not_send_cookies(self): + TEST_DOMAIN = "dnt-request-cookies-test.trackersimulator.org" + TEST_URL = "https://{}/".format(TEST_DOMAIN) + + # directly visit a DNT policy URL known to set cookies + self.load_url(TEST_URL + ".well-known/dnt-policy.txt") + self.assertEqual(len(self.driver.get_cookies()), 1, + "DNT policy URL set a cookie") + + # how to check we didn't send a cookie along with request? + # the DNT policy URL used by this test returns "cookies=X" + # where X is the number of cookies it got + # MEGAHACK: make sha1 of "cookies=0" a valid DNT hash + self.load_url(self.options_url) + self.js( + "chrome.extension.getBackgroundPage()." + "badger.storage.updateDntHashes({" + " 'cookies=0 test policy': 'f63ee614ebd77f8634b92633c6bb809a64b9a3d7'" + "});" + ) + + # perform a DNT policy check + self.js(DntTest.CHECK_FOR_DNT_POLICY_JS, TEST_DOMAIN) + # wait until checkForDNTPolicy completed + self.wait_for_script("return typeof window.DNT_CHECK_RESULT != 'undefined';") + # get the result + result = self.js("return window.DNT_CHECK_RESULT;") + self.assertTrue(result, "No cookies were sent") + + def test_should_not_record_nontracking_domains(self): + FIXTURE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "recording_nontracking_domains.html" + ) + TRACKING_DOMAIN = "dnt-request-cookies-test.trackersimulator.org" + NON_TRACKING_DOMAIN = "www.eff.org" + + # clear pre-trained/seed tracker data + self.load_url(self.options_url) + self.js("chrome.extension.getBackgroundPage().badger.storage.clearTrackerData();") + + # enable local learning + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#local-learning-checkbox').click() + + # visit a page containing two third-party resources, + # one from a cookie-tracking domain + # and one from a non-tracking domain + self.load_url(FIXTURE_URL) + + # verify both domains are present on the page + try: + selector = "iframe[src*='%s']" % TRACKING_DOMAIN + self.driver.find_element_by_css_selector(selector) + except NoSuchElementException: + self.fail("Unable to find the tracking domain on the page") + try: + selector = "img[src*='%s']" % NON_TRACKING_DOMAIN + self.driver.find_element_by_css_selector(selector) + except NoSuchElementException: + self.fail("Unable to find the non-tracking domain on the page") + + self.load_url(self.options_url) + + # verify that the cookie-tracking domain was recorded + self.assertTrue( + self.domain_was_recorded(TRACKING_DOMAIN), + "Tracking domain should have gotten recorded" + ) + + # verify that the non-tracking domain was not recorded + self.assertFalse( + self.domain_was_recorded(NON_TRACKING_DOMAIN), + "Non-tracking domain should not have gotten recorded" + ) + + def test_first_party_dnt_header(self): + TEST_URL = "https://httpbin.org/get" + headers = retry_until(partial(self.get_first_party_headers, TEST_URL), + times=8) + self.assertTrue(headers is not None, "It seems we failed to get headers") + self.assertIn('Dnt', headers, "DNT header should have been present") + self.assertIn('Sec-Gpc', headers, "GPC header should have been present") + self.assertEqual(headers['Dnt'], "1", + 'DNT header should have been set to "1"') + self.assertEqual(headers['Sec-Gpc'], "1", + 'Sec-Gpc header should have been set to "1"') + + def test_no_dnt_header_when_disabled_on_site(self): + TEST_URL = "https://httpbin.org/get" + self.disable_badger_on_site(TEST_URL) + headers = retry_until(partial(self.get_first_party_headers, TEST_URL), + times=8) + self.assertTrue(headers is not None, "It seems we failed to get headers") + self.assertNotIn('Dnt', headers, "DNT header should have been missing") + self.assertNotIn('Sec-Gpc', headers, "GPC header should have been missing") + + def test_no_dnt_header_when_dnt_disabled(self): + TEST_URL = "https://httpbin.org/get" + + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#enable_dnt_checkbox').click() + + headers = retry_until(partial(self.get_first_party_headers, TEST_URL), + times=8) + self.assertTrue(headers is not None, "It seems we failed to get headers") + self.assertNotIn('Dnt', headers, "DNT header should have been missing") + self.assertNotIn('Sec-Gpc', headers, "GPC header should have been missing") + + def test_navigator_object(self): + self.load_url(DntTest.NAVIGATOR_DNT_TEST_URL, wait_for_body_text=True) + + self.assertEqual( + self.driver.find_element_by_tag_name('body').text, + 'no tracking (navigator.doNotTrack="1")', + "navigator.DoNotTrack should have been set to \"1\"" + ) + self.assertEqual( + self.js("return navigator.globalPrivacyControl"), + "1", + "navigator.globalPrivacyControl should have been set to \"1\"" + ) + + def test_navigator_unmodified_when_disabled_on_site(self): + self.disable_badger_on_site(DntTest.NAVIGATOR_DNT_TEST_URL) + + self.load_url(DntTest.NAVIGATOR_DNT_TEST_URL, wait_for_body_text=True) + + # navigator.doNotTrack defaults to null in Chrome, "unspecified" in Firefox + self.assertEqual( + self.driver.find_element_by_tag_name('body').text[0:5], + 'unset', + "navigator.DoNotTrack should have been left unset" + ) + self.assertEqual( + self.js("return navigator.globalPrivacyControl"), + None, + "navigator.globalPrivacyControl should have been left unset" + ) + + def test_navigator_unmodified_when_dnt_disabled(self): + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#enable_dnt_checkbox').click() + + self.load_url(DntTest.NAVIGATOR_DNT_TEST_URL, wait_for_body_text=True) + + # navigator.doNotTrack defaults to null in Chrome, "unspecified" in Firefox + self.assertEqual( + self.driver.find_element_by_tag_name('body').text[0:5], + 'unset', + "navigator.DoNotTrack should have been left unset" + ) + self.assertEqual( + self.js("return navigator.globalPrivacyControl"), + None, + "navigator.globalPrivacyControl should have been left unset" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/fingerprinting_test.py b/tests/selenium/fingerprinting_test.py new file mode 100644 index 0000000..0fadd2b --- /dev/null +++ b/tests/selenium/fingerprinting_test.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest + +import pbtest + +from functools import partial + +from pbtest import retry_until + + +class FingerprintingTest(pbtest.PBSeleniumTest): + """Tests to make sure fingerprinting detection works as expected.""" + + def detected_fingerprinting(self, domain): + return self.js("""let tracker_origin = window.getBaseDomain("{}"); +let tabData = chrome.extension.getBackgroundPage().badger.tabData; +return ( + Object.keys(tabData).some(tab_id => {{ + let fpData = tabData[tab_id].fpData; + return fpData && + fpData.hasOwnProperty(tracker_origin) && + fpData[tracker_origin].canvas && + fpData[tracker_origin].canvas.fingerprinting === true; + }}) +);""".format(domain)) + + def detected_tracking(self, domain, page_url): + return self.js("""let tracker_origin = window.getBaseDomain("{}"), + site_origin = window.getBaseDomain((new URI("{}")).host), + map = chrome.extension.getBackgroundPage().badger.storage.snitch_map.getItemClones(); + +return ( + map.hasOwnProperty(tracker_origin) && + map[tracker_origin].indexOf(site_origin) != -1 +);""".format(domain, page_url)) + + def get_fillText_source(self): + return self.js(""" + const canvas = document.getElementById("writetome"); + const ctx = canvas.getContext("2d"); + return ctx.fillText.toString(); + """) + + def setUp(self): + # enable local learning + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#local-learning-checkbox').click() + + # TODO can fail because our content script runs too late: https://crbug.com/478183 + @pbtest.repeat_if_failed(3) + def test_canvas_fingerprinting_detection(self): + FIXTURE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "fingerprinting.html" + ) + FINGERPRINTING_DOMAIN = "cdn.jsdelivr.net" + + # clear pre-trained/seed tracker data + self.load_url(self.options_url) + self.js("chrome.extension.getBackgroundPage().badger.storage.clearTrackerData();") + + # visit the page + self.load_url(FIXTURE_URL) + + # now open a new window (to avoid clearing badger.tabData) + # and verify results + self.open_window() + + # check that we detected the fingerprinting domain as a tracker + self.load_url(self.options_url) + # TODO unnecessary retrying? + self.assertTrue( + retry_until(partial(self.detected_tracking, FINGERPRINTING_DOMAIN, FIXTURE_URL)), + "Canvas fingerprinting resource was detected as a tracker.") + + # check that we detected canvas fingerprinting + self.assertTrue( + self.detected_fingerprinting(FINGERPRINTING_DOMAIN), + "Canvas fingerprinting resource was detected as a fingerprinter." + ) + + # Privacy Badger overrides a few functions on canvas contexts to check for fingerprinting. + # In previous versions, it would restore the native function after a single call. Unfortunately, + # this would wipe out polyfills that had also overridden the same functions, such as the very + # popular "hidpi-canvas-polyfill". + def test_canvas_polyfill_clobbering(self): + FIXTURE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "fingerprinting_canvas_hidpi.html" + ) + + # visit the page + self.load_url(FIXTURE_URL) + + # check that we did not restore the native function (should be hipdi polyfill) + self.assertNotIn("[native code]", self.get_fillText_source(), + "Canvas context fillText is not native version (polyfill has been retained)." + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/options_test.py b/tests/selenium/options_test.py new file mode 100644 index 0000000..ce95588 --- /dev/null +++ b/tests/selenium/options_test.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import time +import unittest + +import pbtest + +from selenium.common.exceptions import ( + ElementNotInteractableException, + ElementNotVisibleException, + NoSuchElementException, + TimeoutException, +) +from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + + +class OptionsTest(pbtest.PBSeleniumTest): + """Make sure the options page works correctly.""" + + def assert_slider_state(self, origin, action, failure_msg): + clicker = self.driver.find_element_by_css_selector( + 'div[data-origin="{}"]'.format(origin)) + self.assertEqual( + clicker.get_attribute("class"), + "clicker userset", + failure_msg + ) + + switches_div = clicker.find_element_by_css_selector(".switch-container") + self.assertEqual( + switches_div.get_attribute("class"), + "switch-container " + action, + failure_msg + ) + + def find_origin_by_xpath(self, origin): + origins = self.driver.find_element_by_id("blockedResourcesInner") + return origins.find_element_by_xpath(( + './/div[@data-origin="{origin}"]' + # test that "origin" is one of the classes on the element: + # https://stackoverflow.com/a/1390680 + '//div[contains(concat(" ", normalize-space(@class), " "), " origin ")]' + '//span[text()="{origin}"]' + ).format(origin=origin)) + + def select_domain_list_tab(self): + self.find_el_by_css('a[href="#tab-tracking-domains"]').click() + try: + self.driver.find_element_by_id('show-tracking-domains-checkbox').click() + except (ElementNotInteractableException, ElementNotVisibleException): + # The list will be loaded directly if we're opening the tab for the second time in this test + pass + + def select_manage_data_tab(self): + self.find_el_by_css('a[href="#tab-manage-data"]').click() + + def check_tracker_messages(self, error_message, many, none): + self.assertEqual(many, + self.driver.find_element_by_id("options_domain_list_trackers").is_displayed(), error_message) + self.assertEqual(none, + self.driver.find_element_by_id("options_domain_list_no_trackers").is_displayed(), error_message) + + def load_options_page(self): + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + + def clear_seed_data(self): + """Clear the seed dataset to make test checks easier""" + self.load_options_page() + self.js("chrome.extension.getBackgroundPage().badger.storage.clearTrackerData();") + + def add_test_origin(self, origin, action): + """Add given origin to backend storage.""" + self.load_options_page() + self.js(( + "chrome.extension.getBackgroundPage()" + ".badger.storage.setupHeuristicAction('{}', '{}');" + ).format(origin, action)) + + def user_overwrite(self, origin, action): + # Get the slider that corresponds to this radio button + origin_div = self.find_el_by_css('div[data-origin="' + origin + '"]') + slider = origin_div.find_element_by_css_selector('.switch-toggle') + + # Click on the correct place over the slider to block this origin + click_action = ActionChains(self.driver) + if action == 'block': + # Top left (+2px) + click_action.move_to_element_with_offset(slider, 2, 2) + if action == 'cookieblock': + # Top middle + click_action.move_to_element_with_offset(slider, slider.size['width']/2, 2) + if action == 'allow': + # Top right + click_action.move_to_element_with_offset(slider, slider.size['width']-2, 2) + click_action.click() + click_action.perform() + + def test_page_title(self): + self.load_options_page() + localized_title = self.js('return chrome.i18n.getMessage("options_title")') + try: + WebDriverWait(self.driver, 3).until( + EC.title_contains(localized_title)) + except TimeoutException: + self.fail("Unexpected title for the Options page. Got (%s)," + " expected (%s)" + % (self.driver.title, localized_title)) + + def test_added_origin_display(self): + """Ensure origin and tracker message is displayed when there is 1 origin.""" + self.clear_seed_data() + + self.add_test_origin("pbtest.org", "block") + + self.load_options_page() + self.select_domain_list_tab() + + error_message = "The 'multiple tracker' message should be displayed after adding an origin" + self.check_tracker_messages(error_message, many=True, none=False) + + try: + self.find_origin_by_xpath("pbtest.org") + except NoSuchElementException: + self.fail("Tracking origin is not displayed") + + def test_added_multiple_origins_display(self): + """Ensure origin and tracker count is displayed when there are multiple origins.""" + self.clear_seed_data() + + self.add_test_origin("pbtest.org", "block") + self.add_test_origin("pbtest1.org", "block") + + self.load_options_page() + self.select_domain_list_tab() + + error_message = "The 'multiple tracker' message should be displayed after adding 2 origins" + self.check_tracker_messages(error_message, many=True, none=False) + + # check tracker count + self.assertEqual( + self.driver.find_element_by_id("options_domain_list_trackers").text, + "Privacy Badger has decided to block 2 potential tracking domains so far", + "Origin tracker count should be 2 after adding origin" + ) + + # Check those origins are displayed. + try: + self.find_origin_by_xpath("pbtest.org") + self.find_origin_by_xpath("pbtest1.org") + except NoSuchElementException: + self.fail("Tracking origin is not displayed") + + def test_removed_origin_display(self): + """Ensure origin is removed properly.""" + self.clear_seed_data() + self.add_test_origin("pbtest.org", "block") + + self.load_options_page() + self.select_domain_list_tab() + + # Remove displayed origin. + remove_origin_element = self.find_el_by_xpath( + './/div[@data-origin="pbtest.org"]' + '//a[@class="removeOrigin"]') + remove_origin_element.click() + + # Make sure the alert is present. Otherwise we get intermittent errors. + WebDriverWait(self.driver, 3).until(EC.alert_is_present()) + self.driver.switch_to.alert.accept() + + # Check that only the 'no trackers' message is displayed. + try: + WebDriverWait(self.driver, 5).until( + EC.visibility_of_element_located((By.ID, "options_domain_list_no_trackers"))) + except TimeoutException: + self.fail("There should be a 'no trackers' message after deleting origin") + + error_message = "Only the 'no trackers' message should be displayed before adding an origin" + self.assertFalse( + self.driver.find_element_by_id( + "options_domain_list_trackers").is_displayed(), error_message) + + # Check that no origins are displayed. + try: + origins = self.driver.find_element_by_id("blockedResourcesInner") + except NoSuchElementException: + origins = None + error_message = "Origin should not be displayed after removal" + self.assertIsNone(origins, error_message) + + def test_reset_data(self): + self.load_options_page() + self.select_domain_list_tab() + + # make sure the default tracker list includes many trackers + error_message = "By default, the tracker list should contain many trackers" + self.check_tracker_messages(error_message, many=True, none=False) + + # get the number of trackers in the seed data + default_summary_text = self.driver.find_element_by_id("options_domain_list_trackers").text + + # Click on the "remove all data" button to empty the tracker lists, and + # click "OK" in the popup that ensues + self.select_manage_data_tab() + self.driver.find_element_by_id('removeAllData').click() + self.driver.switch_to.alert.accept() + time.sleep(1) # wait for page to reload + + # now make sure the tracker list is empty + self.select_domain_list_tab() + error_message = "No trackers should be displayed after removing all data" + self.check_tracker_messages(error_message, many=False, none=True) + + # add new blocked domains + self.add_test_origin("pbtest.org", "block") + self.add_test_origin("pbtest1.org", "block") + + # reload the options page + self.load_options_page() + self.select_domain_list_tab() + + # make sure only two trackers are displayed now + self.assertEqual( + self.driver.find_element_by_id("options_domain_list_trackers").text, + "Privacy Badger has decided to block 2 potential tracking domains so far", + "Origin tracker count should be 2 after clearing and adding origins" + ) + + # click the "reset data" button to restore seed data and get rid of the + # domains we learned + self.select_manage_data_tab() + self.driver.find_element_by_id('resetData').click() + self.driver.switch_to.alert.accept() + time.sleep(1) + + # make sure the same number of trackers are displayed as by default + self.select_domain_list_tab() + error_message = "After resetting data, tracker count should return to default" + self.assertEqual(self.driver.find_element_by_id("options_domain_list_trackers").text, + default_summary_text, error_message) + + def tracking_user_overwrite(self, original_action, overwrite_action): + """Ensure preferences are persisted when a user overwrites pb's default behaviour for an origin.""" + self.clear_seed_data() + self.add_test_origin("pbtest.org", original_action) + + self.load_options_page() + self.wait_for_script("return window.OPTIONS_INITIALIZED") + # enable learning to reveal the show-not-yet-blocked checkbox + self.find_el_by_css('#local-learning-checkbox').click() + self.select_domain_list_tab() + self.find_el_by_css('#tracking-domains-show-not-yet-blocked').click() + # wait for sliders to finish rendering + self.wait_for_script("return window.SLIDERS_DONE") + + # Change user preferences + self.user_overwrite("pbtest.org", overwrite_action) + + # Re-open the tab + self.load_options_page() + self.select_domain_list_tab() + self.find_el_by_css('#tracking-domains-show-not-yet-blocked').click() + # wait for sliders to finish rendering + self.wait_for_script("return window.SLIDERS_DONE") + + # Check the user preferences for the origins are still displayed + failure_msg = ( + "Origin should be displayed as {} after user overwrite of " + "PB's decision to {}".format(overwrite_action, original_action) + ) + self.assert_slider_state("pbtest.org", overwrite_action, failure_msg) + + def test_tracking_user_overwrite_allowed_block(self): + self.tracking_user_overwrite('allow', 'block') + + def test_tracking_user_overwrite_allowed_cookieblock(self): + self.tracking_user_overwrite('allow', 'cookieblock') + + def test_tracking_user_overwrite_cookieblocked_allow(self): + self.tracking_user_overwrite('cookieblock', 'allow') + + def test_tracking_user_overwrite_cookieblocked_block(self): + self.tracking_user_overwrite('cookieblock', 'block') + + def test_tracking_user_overwrite_blocked_allow(self): + self.tracking_user_overwrite('block', 'allow') + + def test_tracking_user_overwrite_blocked_cookieblock(self): + self.tracking_user_overwrite('block', 'cookieblock') + + # early-warning check for the open_in_tab attribute of options_ui + # https://github.com/EFForg/privacybadger/pull/1775#pullrequestreview-76940251 + def test_options_ui_open_in_tab(self): + # open options page manually, keeping the new user intro page + self.open_window() + self.load_options_page() + + # switch to new user intro page + self.switch_to_window_with_url(self.first_run_url) + + # save open windows + handles_before = set(self.driver.window_handles) + + # open options page using dedicated chrome API + self.js("chrome.runtime.openOptionsPage();") + + # if we switched to the previously manually opened options page, all good + # if we haven't, this must mean options_ui's open_in_tab no longer works + new_handles = set(self.driver.window_handles).difference(handles_before) + num_newly_opened_windows = len(new_handles) + + if num_newly_opened_windows: + self.driver.switch_to.window(new_handles.pop()) + + self.assertEqual(num_newly_opened_windows, 0, + "Expected to switch to existing options page, " + "opened a new page ({}) instead: {}".format( + self.driver.title, self.driver.current_url)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/pbtest.py b/tests/selenium/pbtest.py new file mode 100644 index 0000000..b502b4b --- /dev/null +++ b/tests/selenium/pbtest.py @@ -0,0 +1,498 @@ +# -*- coding: UTF-8 -*- + +import json +import os +import subprocess +import tempfile +import time +import unittest + +from contextlib import contextmanager +from functools import wraps +from shutil import copytree + +from selenium import webdriver +from selenium.common.exceptions import ( + NoSuchWindowException, + TimeoutException, + WebDriverException, +) +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.by import By + +try: + from xvfbwrapper import Xvfb +except ImportError: + print("\n\nxvfbwrapper Python package import failed") + print("headless mode (ENABLE_XVFB=1) is not supported") + + +SEL_DEFAULT_WAIT_TIMEOUT = 30 + +BROWSER_TYPES = ['chrome', 'firefox'] +BROWSER_NAMES = ['google-chrome', 'google-chrome-stable', 'google-chrome-beta', 'firefox'] + +parse_stdout = lambda res: res.strip().decode('utf-8') + +run_shell_command = lambda command: parse_stdout(subprocess.check_output(command)) + +GIT_ROOT = run_shell_command(['git', 'rev-parse', '--show-toplevel']) + + +class WindowNotFoundException(Exception): + pass + + +def unix_which(command, silent=False): + try: + return run_shell_command(['which', command]) + except subprocess.CalledProcessError as e: + if silent: + return None + raise e + + +def get_browser_type(string): + for t in BROWSER_TYPES: + if t in string.lower(): + return t + raise ValueError("couldn't get browser type from %s" % string) + + +def get_browser_name(string): + if ('/' in string) or ('\\' in string): # it's a path + return os.path.basename(string) + + # it's a browser type + for bn in BROWSER_NAMES: + if string in bn and unix_which(bn, silent=True): + return os.path.basename(unix_which(bn)) + raise ValueError('Could not get browser name from %s' % string) + + +class Shim: + _browser_msg = '''BROWSER should be one of: +* /path/to/a/browser +* a browser executable name so we can find the browser with "which $BROWSER" +* something from BROWSER_TYPES +''' + __doc__ = 'Chooses the correct driver and extension_url based on the BROWSER environment\nvariable. ' + _browser_msg + + def __init__(self): + print("\n\nConfiguring the test run ...") + + browser = os.environ.get('BROWSER') + + # get browser_path and browser_type first + if browser is None: + raise ValueError("The BROWSER environment variable is not set. " + self._browser_msg) + + if ("/" in browser) or ("\\" in browser): # path to a browser binary + self.browser_path = browser + self.browser_type = get_browser_type(self.browser_path) + elif unix_which(browser, silent=True): # executable browser name like 'google-chrome-stable' + self.browser_path = unix_which(browser) + self.browser_type = get_browser_type(browser) + elif get_browser_type(browser): # browser type like 'firefox' or 'chrome' + bname = get_browser_name(browser) + self.browser_path = unix_which(bname) + self.browser_type = browser + else: + raise ValueError("could not infer BROWSER from %s" % browser) + + self.extension_path = os.path.join(GIT_ROOT, 'src') + + if self.browser_type == 'chrome': + # this extension ID and the "key" property in manifest.json + # must both be derived from the same private key + self.info = { + 'extension_id': 'mcgekeccgjgcmhnhbabplanchdogjcnh' + } + self.manager = self.chrome_manager + self.base_url = 'chrome-extension://%s/' % self.info['extension_id'] + + # make extension ID constant across runs + self.fix_chrome_extension_id() + + elif self.browser_type == 'firefox': + self.info = { + 'extension_id': 'jid1-MnnxcxisBPnSXQ@jetpack', + 'uuid': 'd56a5b99-51b6-4e83-ab23-796216679614' + } + self.manager = self.firefox_manager + self.base_url = 'moz-extension://%s/' % self.info['uuid'] + + print('\nUsing browser path: %s\nwith browser type: %s\nand extension path: %s\n' % ( + self.browser_path, self.browser_type, self.extension_path)) + + def fix_chrome_extension_id(self): + # create temp directory + self.tmp_dir = tempfile.TemporaryDirectory() + new_extension_path = os.path.join(self.tmp_dir.name, "src") + + # copy extension sources there + copytree(self.extension_path, new_extension_path) + + # update manifest.json + manifest_path = os.path.join(new_extension_path, "manifest.json") + with open(manifest_path, "r") as f: + manifest = json.load(f) + # this key and the extension ID must both be derived from the same private key + manifest['key'] = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArMdgFkGsm7nOBr/9qkx8XEcmYSu1VkIXXK94oXLz1VKGB0o2MN+mXL/Dsllgkh61LZgK/gVuFFk89e/d6Vlsp9IpKLANuHgyS98FKx1+3sUoMujue+hyxulEGxXXJKXhk0kGxWdE0IDOamFYpF7Yk0K8Myd/JW1U2XOoOqJRZ7HR6is1W6iO/4IIL2/j3MUioVqu5ClT78+fE/Fn9b/DfzdX7RxMNza9UTiY+JCtkRTmm4ci4wtU1lxHuVmWiaS45xLbHphQr3fpemDlyTmaVoE59qG5SZZzvl6rwDah06dH01YGSzUF1ezM2IvY9ee1nMSHEadQRQ2sNduNZWC9gwIDAQAB" # noqa:E501 pylint:disable=line-too-long + with open(manifest_path, "w") as f: + json.dump(manifest, f) + + # update self.extension_path + self.extension_path = new_extension_path + + @property + def wants_xvfb(self): + if self.on_travis or bool(int(os.environ.get('ENABLE_XVFB', 0))): + try: + Xvfb + except NameError: + print("\nHeadless mode not supported: install xvfbwrapper first") + return False + return True + return False + + @property + def on_travis(self): + if "TRAVIS" in os.environ: + return True + return False + + @contextmanager + def chrome_manager(self): + opts = ChromeOptions() + if self.on_travis: # github.com/travis-ci/travis-ci/issues/938 + opts.add_argument("--no-sandbox") + opts.add_argument("--load-extension=" + self.extension_path) + opts.binary_location = self.browser_path + opts.add_experimental_option("prefs", {"profile.block_third_party_cookies": False}) + + # TODO not yet in Firefox (w/o hacks anyway): + # https://github.com/mozilla/geckodriver/issues/284#issuecomment-456073771 + opts.set_capability("loggingPrefs", {'browser': 'ALL'}) + + for i in range(5): + try: + driver = webdriver.Chrome(options=opts) + except WebDriverException as e: + if i == 0: print("") + print("Chrome WebDriver initialization failed:") + print(str(e) + "Retrying ...") + else: + break + + try: + yield driver + finally: + driver.quit() + + @contextmanager + def firefox_manager(self): + ffp = webdriver.FirefoxProfile() + # make extension ID constant across runs + ffp.set_preference('extensions.webextensions.uuids', '{"%s": "%s"}' % + (self.info['extension_id'], self.info['uuid'])) + + for i in range(5): + try: + opts = FirefoxOptions() + # to produce a trace-level geckodriver.log, + # remove the service_log_path argument to Firefox() + # and uncomment the line below + #opts.log.level = "trace" + driver = webdriver.Firefox( + firefox_profile=ffp, + firefox_binary=self.browser_path, + options=opts, + service_log_path=os.path.devnull) + except WebDriverException as e: + if i == 0: print("") + print("Firefox WebDriver initialization failed:") + print(str(e) + "Retrying ...") + else: + break + + driver.install_addon(self.extension_path, temporary=True) + + try: + yield driver + finally: + driver.quit() + + +shim = Shim() # create the browser shim + + +def if_firefox(wrapper): + ''' + A test decorator that applies the function `wrapper` to the test if the + browser is firefox. Ex: + + @if_firefox(unittest.skip("broken on ff")) + def test_stuff(self): + ... + ''' + def test_catcher(test): + if shim.browser_type == 'firefox': + return wraps(test)(wrapper)(test) + return test + + return test_catcher + + +def retry_until(fun, tester=None, times=5, msg="Waiting a bit and retrying ..."): + """ + Execute function `fun` until either its return is truthy + (or if `tester` is set, until the result of calling `tester` with `fun`'s return is truthy), + or it gets executed X times, where X = `times` + 1. + """ + for i in range(times): + result = fun() + + if tester is not None: + if tester(result): + break + elif result: + break + + if i == 0: + print("") + print(msg) + + time.sleep(2 ** i) + + return result + + +attempts = {} # used to count test retries +def repeat_if_failed(ntimes): # noqa + ''' + A decorator that retries the test if it fails `ntimes`. The TestCase must + be used on a subclass of unittest.TestCase. NB: this just registers function + to be retried. The try/except logic is in PBSeleniumTest.run. + ''' + def test_catcher(test): + attempts[test.__name__] = ntimes + + @wraps(test) + def caught(*args, **kwargs): + return test(*args, **kwargs) + return caught + return test_catcher + + +class PBSeleniumTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.manager = shim.manager + cls.base_url = shim.base_url + cls.wants_xvfb = shim.wants_xvfb + if cls.wants_xvfb: + cls.vdisplay = Xvfb(width=1280, height=720) + cls.vdisplay.start() + + # setting DBUS_SESSION_BUS_ADDRESS to nonsense prevents frequent + # hangs of chromedriver (possibly due to crbug.com/309093) + os.environ["DBUS_SESSION_BUS_ADDRESS"] = "/dev/null" + cls.proj_root = GIT_ROOT + + @classmethod + def tearDownClass(cls): + if cls.wants_xvfb: + cls.vdisplay.stop() + + def init(self, driver): + self.driver = driver + self.js = self.driver.execute_script + self.bg_url = self.base_url + "_generated_background_page.html" + self.options_url = self.base_url + "skin/options.html" + self.popup_url = self.base_url + "skin/popup.html" + self.first_run_url = self.base_url + "skin/firstRun.html" + self.test_url = self.base_url + "tests/index.html" + + def run(self, result=None): + nretries = attempts.get(result.name, 1) + for i in range(nretries): + try: + with self.manager() as driver: + self.init(driver) + + # wait for Badger's storage, listeners, ... + self.load_url(self.options_url) + self.wait_for_script( + "return chrome.extension.getBackgroundPage()." + "badger.INITIALIZED" + ) + + driver.close() + if driver.window_handles: + driver.switch_to.window(driver.window_handles[0]) + + super(PBSeleniumTest, self).run(result) + + # retry test magic + if result.name in attempts and result._excinfo: # pylint:disable=protected-access + raise Exception(result._excinfo.pop()) # pylint:disable=protected-access + + break + + except Exception: + if i == nretries - 1: + raise + + wait_secs = 2 ** i + print('\nRetrying {} after {} seconds ...'.format( + result, wait_secs)) + time.sleep(wait_secs) + continue + + def open_window(self): + if self.driver.current_url.startswith("moz-extension://"): + # work around https://bugzilla.mozilla.org/show_bug.cgi?id=1491443 + self.wait_for_script("return typeof chrome != 'undefined' && chrome && chrome.extension") + self.js( + "delete window.__new_window_created;" + "chrome.windows.create({}, function () {" + "window.__new_window_created = true;" + "});" + ) + self.wait_for_script("return window.__new_window_created") + else: + self.js('window.open()') + + self.driver.switch_to.window(self.driver.window_handles[-1]) + + def load_url(self, url, wait_for_body_text=False, retries=5): + """Load a URL and wait before returning.""" + for i in range(retries): + try: + self.driver.get(url) + break + except TimeoutException as e: + if i < retries - 1: + time.sleep(2 ** i) + continue + raise e + # work around geckodriver/marionette/Firefox timeout handling, + # for example: https://travis-ci.org/EFForg/privacybadger/jobs/389429089 + except WebDriverException as e: + if str(e).startswith("Reached error page") and i < retries - 1: + time.sleep(2 ** i) + continue + raise e + self.driver.switch_to.window(self.driver.current_window_handle) + + if wait_for_body_text: + retry_until( + lambda: self.driver.find_element_by_tag_name('body').text, + msg="Waiting for document.body.textContent to get populated ..." + ) + + def txt_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT): + """Find an element by CSS selector and return its text.""" + return self.find_el_by_css( + css_selector, visible_only=False, timeout=timeout).text + + def find_el_by_css(self, css_selector, visible_only=True, timeout=SEL_DEFAULT_WAIT_TIMEOUT): + condition = ( + EC.visibility_of_element_located if visible_only + else EC.presence_of_element_located + ) + return WebDriverWait(self.driver, timeout).until( + condition((By.CSS_SELECTOR, css_selector))) + + def find_el_by_xpath(self, xpath, timeout=SEL_DEFAULT_WAIT_TIMEOUT): + return WebDriverWait(self.driver, timeout).until( + EC.visibility_of_element_located((By.XPATH, xpath))) + + def wait_for_script( + self, + script, + *script_args, + timeout=SEL_DEFAULT_WAIT_TIMEOUT, + message="Timed out waiting for execute_script to eval to True" + ): + """Variant of self.js that executes script continuously until it + returns True.""" + return WebDriverWait(self.driver, timeout).until( + lambda driver: driver.execute_script(script, *script_args), + message + ) + + def wait_for_text(self, selector, text, timeout=SEL_DEFAULT_WAIT_TIMEOUT): + return WebDriverWait(self.driver, timeout).until( + EC.text_to_be_present_in_element( + (By.CSS_SELECTOR, selector), text)) + + def wait_for_and_switch_to_frame(self, selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT): + return WebDriverWait(self.driver, timeout).until( + EC.frame_to_be_available_and_switch_to_it( + (By.CSS_SELECTOR, selector))) + + def switch_to_window_with_url(self, url, max_tries=5): + """Point the driver to the first window that matches this url.""" + + for _ in range(max_tries): + for w in self.driver.window_handles: + try: + self.driver.switch_to.window(w) + if self.driver.current_url != url: + continue + except NoSuchWindowException: + pass + else: + return + + time.sleep(1) + + raise WindowNotFoundException("Failed to find window for " + url) + + + def close_window_with_url(self, url, max_tries=5): + self.switch_to_window_with_url(url, max_tries) + + if len(self.driver.window_handles) == 1: + # open another window to avoid implicit session deletion + self.open_window() + self.switch_to_window_with_url(url, max_tries) + + self.driver.close() + self.driver.switch_to.window(self.driver.window_handles[0]) + + def block_domain(self, domain): + self.load_url(self.options_url) + self.js(( + "(function (domain) {" + " let bg = chrome.extension.getBackgroundPage();" + " let base_domain = window.getBaseDomain(domain);" + " bg.badger.heuristicBlocking.blocklistOrigin(domain, base_domain);" + "}(arguments[0]));" + ), domain) + + def cookieblock_domain(self, domain): + self.load_url(self.options_url) + self.js(( + "(function (domain) {" + " let bg = chrome.extension.getBackgroundPage();" + " bg.badger.storage.setupHeuristicAction(domain, bg.constants.COOKIEBLOCK);" + "}(arguments[0]));" + ), domain) + + def disable_badger_on_site(self, url): + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('a[href="#tab-allowlist"]').click() + self.driver.find_element_by_id('new-disabled-site-input').send_keys(url) + self.driver.find_element_by_css_selector('#add-disabled-site').click() + + @property + def logs(self): + # TODO not yet in Firefox + return [log.get('message') for log in self.driver.get_log('browser')] diff --git a/tests/selenium/pbtest_org_test.py b/tests/selenium/pbtest_org_test.py new file mode 100644 index 0000000..df381d9 --- /dev/null +++ b/tests/selenium/pbtest_org_test.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import pbtest +import unittest + +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +# where to run the acceptance tests +PBTEST_ORG_URL = "https://pbtest.org/tracker" + +# the id of the element where test results are reported +PBTEST_ORG_TEST_RESULTS_TABLE_ID = "results" + +# unicode characters we look in the results to tell if a test passed or failed +PASS = u'Pass' +FAIL = u'Fail' + + +class PBTestDotOrgTest(pbtest.PBSeleniumTest): + """Run the pbtest.org website acceptance tests. Loads the pbtest.org test + suite and assert that none of the tests failed or are 'undefined'.""" + + @unittest.skip("Until we understand and fix the intermittent pbtest.org failures.") + #@pbtest.repeat_if_failed(5) # TODO doesn't work with unittest.skip above + def test_should_pass_pbtest_org_suite(self): + driver = self.driver + driver.delete_all_cookies() + results = {'passed': [], 'failed': [], 'undefined': []} + self.load_url(PBTEST_ORG_URL) + WebDriverWait(driver, 100).until( + EC.presence_of_element_located(( + By.XPATH, + "//*[@id='buttons'][contains(@style, 'display: block')]"))) + for el in driver.find_elements_by_class_name('complimentary_text'): + if not el.is_displayed(): + continue + + test_text = el.find_element_by_xpath('../..').text + if PASS in el.text: + results['passed'].append(test_text) + elif FAIL in el.text: + results['failed'].append(test_text) + elif u'undefined' in el.text: + results['undefined'].append(test_text) + else: + raise ValueError("Malformed test result") + + # now we have all the completed test results. + # print a summary + print("\npbtest_org test results: %d passed, %d failed, %d undefined" % + (len(results['passed']), len(results['failed']), + len(results['undefined']))) + failed_tests = ([t for t in results['failed']] + + [t for t in results['undefined']]) + + firefox_failures = [u'Does Privacy Badger Honor the Cookie Block List \u2717 Fail'] + # ignore this failure on firefox + if pbtest.shim.browser_type == 'firefox' and failed_tests == firefox_failures: + return + + fail_msg = "%d tests failed:\n * %s" % ( + len(failed_tests), + "\n * ".join(failed_tests).replace(u'\u2717', 'x'), + ) + self.assertTrue(len(failed_tests) == 0, msg=fail_msg) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/popup_test.py b/tests/selenium/popup_test.py new file mode 100644 index 0000000..e11701c --- /dev/null +++ b/tests/selenium/popup_test.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import time +import unittest + +import pbtest + +from selenium.common.exceptions import TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.support.ui import WebDriverWait + + +def get_domain_slider_state(driver, domain): + label = driver.find_element_by_css_selector( + 'input[name="{}"][checked]'.format(domain)) + return label.get_attribute('value') + + +class PopupTest(pbtest.PBSeleniumTest): + """Make sure the popup works correctly.""" + + def clear_seed_data(self): + self.load_url(self.options_url) + self.js("chrome.extension.getBackgroundPage().badger.storage.clearTrackerData();") + + def wait_for_page_to_start_loading(self, url, timeout=20): + """Wait until the title element is present. Use it to work around + Firefox not updating self.driver.current_url fast enough.""" + try: + WebDriverWait(self.driver, timeout).until( + expected_conditions.presence_of_element_located( + (By.CSS_SELECTOR, "title"))) + except TimeoutException: + # TODO debug info + print("\n") + print("driver.current_url=" + self.driver.current_url) + print() + print(self.driver.page_source[:5000]) + print("...\n") + + self.fail("Timed out waiting for %s to start loading" % url) + + def open_popup(self, show_nag=False, origins=None): + """Open popup and optionally close overlay.""" + + DUMMY_PAGE_URL = "https://efforg.github.io/privacybadger-test-fixtures/" + + # hack to get tabData populated for the popup's tab + # to get the popup shown for regular pages + # as opposed to special (no-tabData) browser pages + self.open_window() + self.load_url(DUMMY_PAGE_URL) + + self.open_window() + self.load_url(self.popup_url) + self.wait_for_script("return window.POPUP_INITIALIZED") + + # override tab ID (to get regular page popup instead of + # special browser page popup), + # optionally set the domains the popup should report, + # optionally ask for the new user welcome page reminder + popup_js = ( + "(function (origins, show_nag, DUMMY_PAGE_URL) {" + "chrome.tabs.query({ url: DUMMY_PAGE_URL }, (tabs) => {" + " chrome.runtime.sendMessage({" + " type: 'getPopupData'," + " tabId: tabs[0].id" + " }, (response) => {" + " response.seenComic = !show_nag;" + " response.origins = origins;" + " setPopupData(response);" + " refreshPopup();" + " showNagMaybe();" + " window.DONE_REFRESHING = true;" + " });" + "});" + "}(arguments[0], arguments[1], arguments[2]));" + ) + self.js(popup_js, origins if origins else {}, show_nag, DUMMY_PAGE_URL) + # wait until the async getTab function is done + self.wait_for_script( + "return typeof window.DONE_REFRESHING != 'undefined'", + timeout=5, + message="Timed out waiting for getTab() to complete." + ) + + # wait for any sliders to finish rendering + self.wait_for_script("return window.SLIDERS_DONE") + + def get_enable_button(self): + """Get enable button on popup.""" + return self.driver.find_element_by_id("activate_site_btn") + + def get_disable_button(self): + """Get disable button on popup.""" + return self.driver.find_element_by_id("deactivate_site_btn") + + def test_welcome_page_reminder_overlay(self): + """Ensure overlay links to new user welcome page.""" + + # first close the welcome page if already open + try: + self.close_window_with_url(self.first_run_url, max_tries=1) + except pbtest.WindowNotFoundException: + pass + + self.open_popup(show_nag=True) + self.driver.find_element_by_id("intro-reminder-btn").click() + + # Look for first run page and return if found. + self.switch_to_window_with_url(self.first_run_url) + + def test_help_button(self): + """Ensure FAQ website is opened when help button is clicked.""" + + FAQ_URL = "https://privacybadger.org/#faq" + + try: + self.switch_to_window_with_url(FAQ_URL, max_tries=1) + except pbtest.WindowNotFoundException: + pass + else: + self.fail("FAQ should not already be open") + + self.open_popup() + self.driver.find_element_by_id("help").click() + + self.switch_to_window_with_url(FAQ_URL) + + def test_options_button(self): + """Ensure options page is opened when button is clicked.""" + self.open_popup() + self.driver.find_element_by_id("options").click() + self.switch_to_window_with_url(self.options_url) + + @pbtest.repeat_if_failed(5) + def test_trackers_link(self): + """Ensure trackers link opens EFF website.""" + + EFF_URL = "https://privacybadger.org/#What-is-a-third-party-tracker" + + self.open_popup() + + # Get all possible tracker links (none, one, multiple) + trackers_links = self.driver.find_elements_by_css_selector("#pbInstructions a") + if not trackers_links: + self.fail("Unable to find trackers link on popup") + + # Get the one that's displayed on the page that this test is using + for link in trackers_links: + if link.is_displayed(): + trackers_link = link + + trackers_link.click() + + # Make sure EFF website not opened in same window. + if self.driver.current_url != self.popup_url: + self.fail("EFF website not opened in new window") + + # Look for EFF website and return if found. + self.switch_to_window_with_url(EFF_URL) + + self.wait_for_page_to_start_loading(EFF_URL) + + self.assertEqual(self.driver.current_url, EFF_URL, + "EFF website should open after clicking trackers link on popup") + + # Verify EFF website contains the linked anchor element. + faq_selector = 'a[href="{}"]'.format(EFF_URL[EFF_URL.index('#'):]) + try: + WebDriverWait(self.driver, pbtest.SEL_DEFAULT_WAIT_TIMEOUT).until( + expected_conditions.presence_of_element_located( + (By.CSS_SELECTOR, faq_selector))) + except TimeoutException: + self.fail("Unable to find expected element ({}) on EFF website".format(faq_selector)) + + def test_toggling_sliders(self): + """Ensure toggling sliders is persisted.""" + self.clear_seed_data() + + # enable learning to show not-yet-blocked domains in popup + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#local-learning-checkbox').click() + + DOMAIN = "example.com" + DOMAIN_ID = DOMAIN.replace(".", "-") + + self.open_popup(origins={DOMAIN:"allow"}) + + # click input with JavaScript to avoid "Element ... is not clickable" / + # "Other element would receive the click" Selenium limitation + self.js("$('#block-{}').click()".format(DOMAIN_ID)) + + # retrieve the new action + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('a[href="#tab-tracking-domains"]').click() + new_action = get_domain_slider_state(self.driver, DOMAIN) + + self.assertEqual(new_action, "block", + "The domain should be blocked on options page.") + + # test toggling some more + self.open_popup(origins={DOMAIN:"user_block"}) + + self.assertTrue( + self.driver.find_element_by_id("block-" + DOMAIN_ID).is_selected(), + "The domain should be shown as blocked in popup." + ) + + # change to "cookieblock" + self.js("$('#cookieblock-{}').click()".format(DOMAIN_ID)) + # change again to "block" + self.js("$('#block-{}').click()".format(DOMAIN_ID)) + + # retrieve the new action + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('a[href="#tab-tracking-domains"]').click() + new_action = get_domain_slider_state(self.driver, DOMAIN) + + self.assertEqual(new_action, "block", + "The domain should still be blocked on options page.") + + def test_reverting_control(self): + """Test restoring control of a domain to Privacy Badger.""" + self.clear_seed_data() + + DOMAIN = "example.com" + DOMAIN_ID = DOMAIN.replace(".", "-") + + # record the domain as cookieblocked by Badger + self.cookieblock_domain(DOMAIN) + + self.open_popup(origins={DOMAIN:"cookieblock"}) + + # set the domain to user control + # click input with JavaScript to avoid "Element ... is not clickable" / + # "Other element would receive the click" Selenium limitation + self.js("$('#block-{}').click()".format(DOMAIN_ID)) + + # restore control to Badger + self.driver.find_element_by_css_selector( + 'div[data-origin="{}"] a.honeybadgerPowered'.format(DOMAIN) + ).click() + + # get back to a valid window handle as the window just got closed + self.driver.switch_to.window(self.driver.window_handles[0]) + + # verify the domain is no longer user controlled + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('a[href="#tab-tracking-domains"]').click() + + # assert the action is not what we manually clicked + action = get_domain_slider_state(self.driver, DOMAIN) + self.assertEqual(action, "cookieblock", + "Domain's action should have been restored.") + + # assert the undo arrow is not displayed + self.driver.find_element_by_css_selector('a[href="#tab-tracking-domains"]').click() + self.driver.find_element_by_id('show-tracking-domains-checkbox').click() + self.assertFalse( + self.driver.find_element_by_css_selector( + 'div[data-origin="{}"] a.honeybadgerPowered'.format(DOMAIN) + ).is_displayed(), + "Undo arrow should not be displayed." + ) + + def test_disable_enable_buttons(self): + """Ensure disable/enable buttons change popup state.""" + + DISPLAYED_ERROR = " should not be displayed on popup" + NOT_DISPLAYED_ERROR = " should be displayed on popup" + + self.open_popup() + + self.get_disable_button().click() + + # get back to a valid window handle as the window just got closed + self.driver.switch_to.window(self.driver.window_handles[0]) + self.open_popup() + + # Check that popup state changed after disabling. + disable_button = self.get_disable_button() + self.assertFalse(disable_button.is_displayed(), + "Disable button" + DISPLAYED_ERROR) + enable_button = self.get_enable_button() + self.assertTrue(enable_button.is_displayed(), + "Enable button" + NOT_DISPLAYED_ERROR) + + enable_button.click() + + self.driver.switch_to.window(self.driver.window_handles[0]) + self.open_popup() + + # Check that popup state changed after re-enabling. + disable_button = self.get_disable_button() + self.assertTrue(disable_button.is_displayed(), + "Disable button" + NOT_DISPLAYED_ERROR) + enable_button = self.get_enable_button() + self.assertFalse(enable_button.is_displayed(), + "Enable button" + DISPLAYED_ERROR) + + def test_error_button(self): + """Ensure error button opens report error overlay.""" + self.open_popup() + + # TODO: selenium firefox has a bug where is_displayed() is always True + # for these elements. But we should use is_displayed when this is fixed. + #overlay_input = self.driver.find_element_by_id("error_input") + #self.assertTrue(overlay_input.is_displayed(), "User input" + error_message) + + # assert error reporting menu is not open + self.assertTrue(len(self.driver.find_elements_by_class_name('active')) == 0, + 'error reporting should not be open') + + # Click error button to open overlay for reporting sites. + error_button = self.driver.find_element_by_id("error") + error_button.click() + time.sleep(1) + + # check error is open + self.assertTrue(len(self.driver.find_elements_by_class_name('active')) == 1, + 'error reporting should be open') + + overlay_close = self.driver.find_element_by_id("report_close") + overlay_close.click() + time.sleep(1) + self.assertTrue(len(self.driver.find_elements_by_class_name('active')) == 0, + 'error reporting should be closed again') + + @pbtest.repeat_if_failed(5) + def test_donate_button(self): + """Ensure donate button opens EFF website.""" + + EFF_URL = "https://supporters.eff.org/donate/support-privacy-badger" + + self.open_popup() + + donate_button = self.driver.find_element_by_id("donate") + + donate_button.click() + + # Make sure EFF website not opened in same window. + if self.driver.current_url != self.popup_url: + self.fail("EFF website not opened in new window") + + # Look for EFF website and return if found. + self.switch_to_window_with_url(EFF_URL) + + self.wait_for_page_to_start_loading(EFF_URL) + + self.assertEqual(self.driver.current_url, EFF_URL, + "EFF website should open after clicking donate button on popup") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/qunit_test.py b/tests/selenium/qunit_test.py new file mode 100644 index 0000000..f6afa4a --- /dev/null +++ b/tests/selenium/qunit_test.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest + +import pbtest + +from selenium.common.exceptions import TimeoutException + + +class QUnitTest(pbtest.PBSeleniumTest): + + def test_run_qunit_tests(self): + self.load_url(self.test_url) + + try: + # this text appears when tests finish running + self.txt_by_css( + "#qunit-testresult-display > span.total", + timeout=120 + ) + except TimeoutException as exc: + self.fail("Cannot find the results of QUnit tests %s" % exc) + + print("\nQUnit summary:") + print(self.txt_by_css("#qunit-testresult-display")) + + failed_test_els = self.driver.find_elements_by_css_selector( + ".fail .test-name" + ) + fail_msg = "The following QUnit tests failed:\n * {}".format( + "\n * ".join([el.text for el in failed_test_els]) + ) + + self.assertTrue(len(failed_test_els) == 0, msg=fail_msg) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/service_workers_test.py b/tests/selenium/service_workers_test.py new file mode 100644 index 0000000..00da1f5 --- /dev/null +++ b/tests/selenium/service_workers_test.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest + +import pbtest + + +class ServiceWorkersTest(pbtest.PBSeleniumTest): + """Verifies interaction with sites that use Service Worker caches""" + + def get_tab_data_domains(self): + domains = self.js( + "let tabData = chrome.extension.getBackgroundPage().badger.tabData;" + "return (Object.keys(tabData).map(tab_id => {" + " return tabData[tab_id].frames[0].host;" + "}));" + ) + return domains + + def test_returning_to_sw_cached_page(self): + FIXTURE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "service_workers.html" + ) + + # visit the Service Worker page to activate the worker + self.load_url(FIXTURE_URL) + + # Service Workers are off by default in Firefox 60 ESR + if not self.js("return 'serviceWorker' in navigator;"): + self.skipTest("Service Workers are disabled") + + # wait for the worker to initialize its cache + self.wait_for_script("return window.WORKER_READY;") + + # visit a different site (doesn't matter what it is, + # just needs to be an http site with a different domain) + self.load_url("https://dnt-test.trackersimulator.org/") + + # return to the SW page + self.driver.back() + + # now open a new window (to avoid clearing badger.tabData) + # and verify results + self.open_window() + self.load_url(self.options_url) + domains = self.get_tab_data_domains() + self.assertIn("efforg.github.io", domains, + "SW page URL was not correctly attributed") + self.assertEqual(len(domains), 1, + "tabData contains an unexpected number of entries") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/storage_test.py b/tests/selenium/storage_test.py new file mode 100644 index 0000000..d8e6c64 --- /dev/null +++ b/tests/selenium/storage_test.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest +import pbtest +from time import sleep + +# time to wait for loading privacy policy from eff.org +POLICY_DOWNLOAD_TIMEOUT = 20 +PB_POLICY_HASH_LEN = 40 # https://www.eff.org/files/dnt-policies.json + + +class StorageTest(pbtest.PBSeleniumTest): + """Privacy Badger storage initialization tests.""" + + def check_policy_download(self): + timeout = POLICY_DOWNLOAD_TIMEOUT + dnt_hashes_not_empty = ( + "return (" + "chrome.extension.getBackgroundPage()." + "badger.storage.getStore('dnt_hashes') != {}" + ")" + ) + # give updatePrivacyPolicyHashes() some time to download the policy hash + while (timeout > 0 and not self.js(dnt_hashes_not_empty)): + sleep(1) + timeout -= 1 + + # make sure we didn't time out + self.assertGreater(timeout, 0, "Timed out waiting for DNT hashes") + # now check the downloaded policy hash + get_dnt_hashes = ( + "return (" + "chrome.extension.getBackgroundPage()." + "badger.storage.getStore('dnt_hashes')." + "getItemClones()" + ")" + ) + policy_hashes = self.js(get_dnt_hashes) + for policy_hash in policy_hashes.keys(): + self.assertEqual(PB_POLICY_HASH_LEN, len(policy_hash)) + + def test_should_init_storage_entries(self): + self.load_url(self.options_url) + + self.check_policy_download() + self.assertEqual( + self.js( + "return chrome.extension.getBackgroundPage()." + "constants.YELLOWLIST_URL" + ), + "https://www.eff.org/files/cookieblocklist_new.txt" + ) + + disabled_sites = self.js( + "return chrome.extension.getBackgroundPage()." + "badger.getSettings().getItem('disabledSites')" + ) + self.assertFalse( + len(disabled_sites), + "Shouldn't have any disabledSites after installation" + ) + + self.assertTrue(self.js( + "return chrome.extension.getBackgroundPage()." + "badger.getSettings().getItem('checkForDNTPolicy')" + ), "Should start with DNT policy enabled") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/super_cookie_test.py b/tests/selenium/super_cookie_test.py new file mode 100644 index 0000000..de6c5dd --- /dev/null +++ b/tests/selenium/super_cookie_test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest + +import pbtest + +from functools import partial + + +class SupercookieTest(pbtest.PBSeleniumTest): + """Make sure we detect potential supercookies. """ + + def get_snitch_map_for(self, origin): + self.open_window() # don't replace the test page to allow for retrying + self.load_url(self.options_url) + + CHECK_SNITCH_MAP_JS = ( + "return chrome.extension.getBackgroundPage()" + ".badger.storage.getStore('snitch_map')" + ".getItemClones()[arguments[0]];" + ) + + return self.js(CHECK_SNITCH_MAP_JS, origin) + + def setUp(self): + # enable local learning + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('#local-learning-checkbox').click() + + # test for https://github.com/EFForg/privacybadger/pull/1403 + # TODO remove retrying entire test after we revert 879a74f807999a2135e4d48bb5efbd8a1beff4f8 + @pbtest.repeat_if_failed(5) + def test_async_tracking_attribution_bug(self): + FIRST_PARTY_BASE = "eff.org" + THIRD_PARTY_BASE = "efforg.github.io" + + self.load_url(( + "https://privacybadger-tests.{}/html/" + "async_localstorage_attribution_bug.html" + ).format(FIRST_PARTY_BASE)) + + # the above HTML page reloads itself furiously to trigger our bug + # we need to wait for it to finish reloading + self.wait_for_script("return window.DONE_RELOADING === true") + + # the HTML page contains: + + # an iframe from THIRD_PARTY_BASE that writes to localStorage + self.assertEqual( + pbtest.retry_until(partial(self.get_snitch_map_for, THIRD_PARTY_BASE)), + [FIRST_PARTY_BASE], + msg="Frame sets localStorage but was not flagged as a tracker.") + + # and an image from raw.githubusercontent.com that doesn't do any tracking + self.assertFalse(self.get_snitch_map_for("raw.githubusercontent.com"), + msg="Image is not a tracker but was flagged as one.") + + + def test_should_detect_ls_of_third_party_frame(self): + FIRST_PARTY_BASE = "eff.org" + THIRD_PARTY_BASE = "efforg.github.io" + + self.assertFalse(self.get_snitch_map_for(THIRD_PARTY_BASE)) + + self.load_url(( + "https://privacybadger-tests.{}/html/" + "localstorage.html" + ).format(FIRST_PARTY_BASE)) + + # TODO We get some intermittent failures for this test. + # It seems we sometimes miss the setting of localStorage items + # because the script runs after we already checked what's in localStorage. + # We can work around this race condition by reloading the page. + self.driver.refresh() + + self.assertEqual( + pbtest.retry_until(partial(self.get_snitch_map_for, THIRD_PARTY_BASE), times=3), + [FIRST_PARTY_BASE] + ) + + def test_should_not_detect_low_entropy_ls_of_third_party_frame(self): + FIRST_PARTY_BASE = "eff.org" + THIRD_PARTY_BASE = "efforg.github.io" + self.assertFalse(self.get_snitch_map_for(THIRD_PARTY_BASE)) + self.load_url(( + "https://privacybadger-tests.{}/html/" + "localstorage_low_entropy.html" + ).format(FIRST_PARTY_BASE)) + self.driver.refresh() + self.assertFalse(self.get_snitch_map_for(THIRD_PARTY_BASE)) + + def test_should_not_detect_first_party_ls(self): + BASE_DOMAIN = "efforg.github.io" + self.load_url(( + "https://{}/privacybadger-test-fixtures/html/" + "localstorage/set_ls.html" + ).format(BASE_DOMAIN)) + self.driver.refresh() + self.assertFalse(self.get_snitch_map_for(BASE_DOMAIN)) + + def test_should_not_detect_ls_of_third_party_script(self): + FIRST_PARTY_BASE = "eff.org" + THIRD_PARTY_BASE = "efforg.github.io" + + # a third-party script included by the top page (not a 3rd party frame) + self.load_url(( + "https://privacybadger-tests.{}/html/" + "localstorage_from_third_party_script.html" + ).format(FIRST_PARTY_BASE)) + + self.driver.refresh() + + self.assertFalse(self.get_snitch_map_for(FIRST_PARTY_BASE)) + self.assertFalse(self.get_snitch_map_for(THIRD_PARTY_BASE)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/surrogates_test.py b/tests/selenium/surrogates_test.py new file mode 100644 index 0000000..eff654f --- /dev/null +++ b/tests/selenium/surrogates_test.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest +import pbtest + +from selenium.common.exceptions import TimeoutException + +from pbtest import retry_until + + +class SurrogatesTest(pbtest.PBSeleniumTest): + """Integration tests to verify surrogate script functionality.""" + + FIXTURE_URL = ( + "https://efforg.github.io/privacybadger-test-fixtures/html/" + "ga_surrogate.html" + ) + + def load_ga_js_test_page(self, timeout=12): + self.load_url(SurrogatesTest.FIXTURE_URL) + try: + self.wait_for_and_switch_to_frame('iframe', timeout=timeout) + self.wait_for_text('h1', "It worked!", timeout=timeout) + return True + except TimeoutException: + return False + + def test_ga_js_surrogate(self): + # clear pre-trained/seed tracker data + self.load_url(self.options_url) + self.js("chrome.extension.getBackgroundPage().badger.storage.clearTrackerData();") + + # verify the surrogate is present + self.load_url(self.options_url) + self.assertTrue(self.js( + "let bg = chrome.extension.getBackgroundPage();" + "const sdb = bg.require('surrogatedb');" + "return sdb.hostnames.hasOwnProperty('www.google-analytics.com');" + ), "Surrogate is missing but should be present.") + + # verify site loads + self.assertTrue( + self.load_ga_js_test_page(), + "Page failed to load even before we did anything." + ) + + # block ga.js (known to break the site) + self.block_domain("www.google-analytics.com") + # back up the surrogate definition before removing it + ga_backup = self.js( + "let bg = chrome.extension.getBackgroundPage();" + "const sdb = bg.require('surrogatedb');" + "return JSON.stringify(sdb.hostnames['www.google-analytics.com']);" + ) + # now remove the surrogate + self.js( + "let bg = chrome.extension.getBackgroundPage();" + "const sdb = bg.require('surrogatedb');" + "delete sdb.hostnames['www.google-analytics.com'];" + ) + + # wait until this happens + self.wait_for_script( + "let bg = chrome.extension.getBackgroundPage();" + "const sdb = bg.require('surrogatedb');" + "return !sdb.hostnames.hasOwnProperty('www.google-analytics.com');", + timeout=5, + message="Timed out waiting for surrogate to get removed." + ) + + # verify site breaks + self.assertFalse( + self.load_ga_js_test_page(), + "Page loaded successfully when it should have failed." + ) + + # re-enable surrogate + self.open_window() + self.load_url(self.options_url) + self.js( + "let bg = chrome.extension.getBackgroundPage();" + "const sdb = bg.require('surrogatedb');" + "sdb.hostnames['www.google-analytics.com'] = JSON.parse('%s');" % ga_backup + ) + + # wait until this happens + self.wait_for_script( + "let bg = chrome.extension.getBackgroundPage();" + "const sdb = bg.require('surrogatedb');" + "return sdb.hostnames.hasOwnProperty('www.google-analytics.com');", + timeout=5, + message="Timed out waiting for surrogate to get readded." + ) + + # verify site loads again + self.assertTrue( + retry_until(self.load_ga_js_test_page), + "Page failed to load after surrogation." + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selenium/website_testbed/first-party.html b/tests/selenium/website_testbed/first-party.html new file mode 100644 index 0000000..13713c4 --- /dev/null +++ b/tests/selenium/website_testbed/first-party.html @@ -0,0 +1,13 @@ +<!DOCTYPE html> +<html> + <head> + <script src=first-party.js></script> + </head> + <body> + <p>Welcome to the cookie tracker test site. I'm creating a localcookie for this domain. + </p> + <button id="newwindowbutton" class="button" onClick="window.open();">Click this button to open a new window</button> + <p>I feel like iframing in a third party website:</p> + <iframe src="https://efforg.github.io/privacybadger-test-fixtures/html/cookie_frame.html" width="200" height="200">where's my iframe?</iframe> + </body> +</html> diff --git a/tests/selenium/website_testbed/first-party.js b/tests/selenium/website_testbed/first-party.js new file mode 100644 index 0000000..7fdebe8 --- /dev/null +++ b/tests/selenium/website_testbed/first-party.js @@ -0,0 +1,25 @@ +function setExpire() { + var now = new Date(); + var time = now.getTime(); + var expireTime = time + 864000; + now.setTime(expireTime); + return ";expires=" + now.toGMTString(); +} + +function setPath() { + return ";path=/"; +} + +function setSameSite() { + return ";SameSite=None;Secure"; +} + +function updateCookie() { + var oldcookie = document.cookie; + var val = "1234567890"; + console.log("read cookie: " + oldcookie); + document.cookie = "localtest=" + encodeURIComponent(val) + setExpire() + setPath() + setSameSite(); + console.log("updating cookie to:" + document.cookie); +} + +updateCookie(); diff --git a/tests/selenium/widgets_test.py b/tests/selenium/widgets_test.py new file mode 100644 index 0000000..d8bc2b5 --- /dev/null +++ b/tests/selenium/widgets_test.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import unittest + +import pbtest + +from time import sleep + +from selenium.common.exceptions import ( + NoSuchElementException, + StaleElementReferenceException, + TimeoutException +) +from selenium.webdriver.common.keys import Keys + + +class WidgetsTest(pbtest.PBSeleniumTest): + + FIXTURES_URL = "https://efforg.github.io/privacybadger-test-fixtures/html/" + BASIC_FIXTURE_URL = FIXTURES_URL + "widget_basic.html" + DYNAMIC_FIXTURE_URL = FIXTURES_URL + "widget_dynamic.html" + THIRD_PARTY_DOMAIN = "privacybadger-tests.eff.org" + TYPE3_WIDGET_NAME = "Type 3 Widget" + TYPE4_WIDGET_NAME = "Type 4 Widget" + TYPE4_WIDGET_CLASS = "pb-type4-test-widget" + + def setUp(self): + self.set_up_widgets() + + def set_up_widgets(self): + """Reinitializes Privacy Badger's widget replacement definitions.""" + + widgetsJson = { + self.TYPE3_WIDGET_NAME: { + "domain": self.THIRD_PARTY_DOMAIN, + "buttonSelectors": [ + "iframe#pb-type3-test-widget" + ], + "replacementButton": { + "unblockDomains": [ + self.THIRD_PARTY_DOMAIN + ], + "type": 3 + } + }, + self.TYPE4_WIDGET_NAME: { + "domains": [ + self.THIRD_PARTY_DOMAIN + ], + "buttonSelectors": [ + "div." + self.TYPE4_WIDGET_CLASS + ], + "scriptSelectors": [ + "script." + self.TYPE4_WIDGET_CLASS + ], + "replacementButton": { + "unblockDomains": [ + self.THIRD_PARTY_DOMAIN + ], + "type": 4 + } + } + } + + # reinitialize widgets using above JSON + self.load_url(self.options_url) + self.js(( + "(function (widgetsJson) {" + " let bg = chrome.extension.getBackgroundPage();" + " bg.badger.widgetList = bg.widgetLoader.initializeWidgets(widgetsJson);" + "}(arguments[0]));" + ), widgetsJson) + + def switch_to_frame(self, selector): + self.wait_for_and_switch_to_frame(selector, timeout=1) + + def assert_widget(self, kind="type3"): + if kind == "type3": + self._assert_type3_widget() + elif kind == "type4": + self._assert_type4_widget() + else: + self.fail("Unknown widget type") + + def _assert_type3_widget(self): + try: + self.switch_to_frame('iframe[src]') + except (StaleElementReferenceException, TimeoutException): + self.fail("Unable to find widget frame") + + try: + self.wait_for_text('body', "Hello world!") + except TimeoutException: + self.fail("Unable to find expected widget text") + + self.driver.switch_to.default_content() + + def _assert_type4_widget(self): + try: + self.wait_for_text('div.' + self.TYPE4_WIDGET_CLASS, + "A third-party widget script was here") + except TimeoutException: + self.fail("Unable to find expected widget output") + + def assert_replacement(self, widget_name=None): + if not widget_name: + widget_name = self.TYPE3_WIDGET_NAME + + try: + self.switch_to_frame('iframe[srcdoc*="{}"]'.format(widget_name)) + except (StaleElementReferenceException, TimeoutException): + self.fail("Unable to find widget placeholder frame") + + try: + self.find_el_by_css("button[id^='btn-once-']") + self.find_el_by_css("button[id^='btn-site-']") + except TimeoutException: + self.fail("Unable to find expected widget placeholder buttons") + + self.driver.switch_to.default_content() + + def assert_widget_blocked(self): + try: + self.switch_to_frame('iframe[src]') + except TimeoutException: + self.fail("Widget frame should still be here") + + self.assertFalse( + self.txt_by_css('body'), "Widget frame should be empty") + + self.driver.switch_to.default_content() + + def assert_no_widget(self): + try: + self.switch_to_frame('iframe[src]') + self.fail("Widget frame should be missing") + except TimeoutException: + pass + self.driver.switch_to.default_content() + + def assert_no_replacement(self, widget_name=None): + if not widget_name: + widget_name = self.TYPE3_WIDGET_NAME + try: + self.switch_to_frame('iframe[srcdoc*="{}"]'.format(widget_name)) + self.fail("Widget placeholder frame should be missing") + except TimeoutException: + pass + self.driver.switch_to.default_content() + + def activate_widget(self, widget_name=None, once=True): + if not widget_name: + widget_name = self.TYPE3_WIDGET_NAME + id_prefix = 'btn-once' if once else 'btn-site' + self.switch_to_frame('iframe[srcdoc*="{}"]'.format(widget_name)) + self.find_el_by_css("button[id^='%s']" % id_prefix).click() + self.driver.switch_to.default_content() + + def test_replacement_basic(self): + # visit the basic widget fixture + self.load_url(self.BASIC_FIXTURE_URL) + # verify the widget is present + self.assert_widget() + + # block the test widget's domain + self.block_domain(self.THIRD_PARTY_DOMAIN) + + # revisit the fixture + self.load_url(self.BASIC_FIXTURE_URL) + # verify the widget got replaced + self.assert_replacement() + + def test_replacement_dynamic(self): + # visit the dynamic widget fixture + self.load_url(self.DYNAMIC_FIXTURE_URL) + # verify the widget is initially missing + self.assert_no_widget() + + # verify the widget shows up once you click on the trigger element + self.find_el_by_css('#widget-trigger').click() + self.assert_widget() + + # block the test widget's domain + self.block_domain(self.THIRD_PARTY_DOMAIN) + + # revisit the fixture + self.load_url(self.DYNAMIC_FIXTURE_URL) + # click on the trigger element + self.find_el_by_css('#widget-trigger').click() + # verify the widget got replaced + self.assert_replacement() + + def test_activation(self): + self.block_domain(self.THIRD_PARTY_DOMAIN) + self.load_url(self.BASIC_FIXTURE_URL) + self.assert_replacement() + + # click the "allow once" button + self.activate_widget() + + # verify the original widget is restored + self.assert_widget() + + # verify the type 4 widget is still replaced + try: + self.driver.find_element_by_css_selector( + 'div.' + self.TYPE4_WIDGET_CLASS) + self.fail("Widget output container div should be missing") + except NoSuchElementException: + pass + self.assert_replacement(self.TYPE4_WIDGET_NAME) + + self.activate_widget(self.TYPE4_WIDGET_NAME) + + # assert all script attributes were copied + script_el = self.driver.find_element_by_css_selector( + 'script.' + self.TYPE4_WIDGET_CLASS) + self.assertEqual(script_el.get_attribute('async'), "true") + self.assertEqual(script_el.get_attribute('data-foo'), "bar") + + self.assert_widget("type4") + + def test_activation_site(self): + self.block_domain(self.THIRD_PARTY_DOMAIN) + self.load_url(self.BASIC_FIXTURE_URL) + self.assert_replacement() + + # click the "allow once" button + self.activate_widget() + + # verify the original widget is restored + self.assert_widget() + + # open a new window (to get around widget activation caching) + self.open_window() + self.load_url(self.BASIC_FIXTURE_URL) + + # verify the widget got replaced + self.assert_replacement() + + # click the "allow on site" button + self.activate_widget(once=False) + + # verify the original widget is restored + self.assert_widget() + + # open a new window (to get around widget activation caching) + self.open_window() + self.load_url(self.BASIC_FIXTURE_URL) + + # verify basic widget is neither replaced nor blocked + self.assert_no_replacement() + self.assert_widget() + + def test_disabling_site(self): + self.block_domain(self.THIRD_PARTY_DOMAIN) + + self.disable_badger_on_site(self.BASIC_FIXTURE_URL) + + # verify basic widget is neither replaced nor blocked + self.load_url(self.BASIC_FIXTURE_URL) + self.assert_no_replacement() + self.assert_widget() + # type 4 replacement should also be missing + self.assert_no_replacement(self.TYPE4_WIDGET_NAME) + # while the type 4 widget script should have executed + self.assert_widget("type4") + + # verify dynamic widget is neither replaced nor blocked + self.load_url(self.DYNAMIC_FIXTURE_URL) + self.find_el_by_css('#widget-trigger').click() + self.assert_no_replacement() + self.assert_widget() + + def test_disabling_all_replacement(self): + self.block_domain(self.THIRD_PARTY_DOMAIN) + + # disable widget replacement + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('a[href="#tab-manage-widgets"]').click() + self.driver.find_element_by_id('replace-widgets-checkbox').click() + + # verify basic widget is no longer replaced + self.load_url(self.BASIC_FIXTURE_URL) + self.assert_no_replacement() + self.assert_widget_blocked() + # type 4 replacement should also be missing + self.assert_no_replacement(self.TYPE4_WIDGET_NAME) + # type 4 widget should also have gotten blocked + try: + widget_div = self.driver.find_element_by_css_selector( + 'div.pb-type4-test-widget') + except NoSuchElementException: + self.fail("Widget div should still be here") + # check the div's text a few times to make sure it stays empty + for _ in range(3): + self.assertFalse(widget_div.text, + "Widget output container should remain empty") + sleep(1) + + # verify dynamic widget is no longer replaced + self.load_url(self.DYNAMIC_FIXTURE_URL) + self.find_el_by_css('#widget-trigger').click() + self.assert_no_replacement() + self.assert_widget_blocked() + + def test_disabling_replacement_for_one_widget(self): + self.block_domain(self.THIRD_PARTY_DOMAIN) + + # add the widget to the list of exceptions + self.load_url(self.options_url) + self.wait_for_script("return window.OPTIONS_INITIALIZED") + self.find_el_by_css('a[href="#tab-manage-widgets"]').click() + self.find_el_by_css('input[type="search"]').send_keys( + self.TYPE3_WIDGET_NAME, Keys.ENTER) + + # verify basic widget is no longer replaced + self.load_url(self.BASIC_FIXTURE_URL) + self.assert_no_replacement() + self.assert_widget_blocked() + # verify the type 4 widget is still replaced + self.assert_replacement(self.TYPE4_WIDGET_NAME) + + # verify dynamic widget is no longer replaced + self.load_url(self.DYNAMIC_FIXTURE_URL) + self.find_el_by_css('#widget-trigger').click() + self.assert_no_replacement() + self.assert_widget_blocked() + + def test_no_replacement_when_cookieblocked(self): + self.cookieblock_domain(self.THIRD_PARTY_DOMAIN) + self.load_url(self.BASIC_FIXTURE_URL) + + self.assert_no_replacement() + self.assert_no_replacement(self.TYPE4_WIDGET_NAME) + + self.assert_widget() + self.assert_widget("type4") + + +if __name__ == "__main__": + unittest.main() |