# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import asyncio import os import random from arsenic.errors import UnknownArsenicError, UnknownError from condprof.helpers import TabSwitcher, execute_async_script, is_mobile from condprof.util import get_credentials, logger BOOKMARK_FREQUENCY = 5 MAX_URLS = 150 MAX_BOOKMARKS = 250 CallErrors = asyncio.TimeoutError, UnknownError, UnknownArsenicError class Builder: def __init__(self, options): self.options = options self.words = self._read_lines("words.txt") self.urls = self._build_url_list(self._read_lines("urls.txt")) self.sync_js = self._load_script("sync") self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS) self.bookmark_js = self._load_script("bookmark") self.platform = options.get("platform", "") self.mobile = is_mobile(self.platform) self.max_urls = options.get("max_urls", MAX_URLS) # see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar if self.mobile: self.max_urls = min(self.max_urls, 20) logger.info("platform: %s" % self.platform) logger.info("max_urls: %s" % self.max_urls) self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY) # we're syncing only on desktop for now self.syncing = not self.mobile if self.syncing: self.username, self.password = get_credentials() if self.username is None: raise ValueError("Sync operations need an FxA username and password") else: self.username, self.password = None, None def _load_script(self, name): return "\n".join(self._read_lines("%s.js" % name)) def _read_lines(self, filename): path = os.path.join(os.path.dirname(__file__), filename) with open(path) as f: return f.readlines() def _build_url_list(self, urls): url_list = [] for url in urls: url = url.strip() if url.startswith("#"): continue for word in self.words: word = word.strip() if word.startswith("#"): continue url_list.append((url.format(word), word)) random.shuffle(url_list) return url_list async def add_bookmark(self, session, url, title): logger.info("Adding bookmark to %s" % url) return await execute_async_script( session, self.bookmark_js, url, title, self.max_bookmarks ) async def sync(self, session, metadata): if not self.syncing: return # now that we've visited all pages, we want to upload to FXSync logger.info("Syncing profile to FxSync") logger.info("Username is %s, password is %s" % (self.username, self.password)) script_res = await execute_async_script( session, self.sync_js, self.username, self.password, "https://accounts.stage.mozaws.net", ) if script_res is None: script_res = {} metadata["logs"] = script_res.get("logs", {}) metadata["result"] = script_res.get("result", 0) metadata["result_message"] = script_res.get("result_message", "SUCCESS") return metadata async def _visit_url(self, current, session, url, word): await asyncio.wait_for(session.get(url), 5) if current % self.bookmark_frequency == 0 and not self.mobile: await asyncio.wait_for(self.add_bookmark(session, url, word), 5) async def __call__(self, session): metadata = {} tabs = TabSwitcher(session, self.options) await tabs.create_windows() visited = 0 for current, (url, word) in enumerate(self.urls): logger.info("%d/%d %s" % (current + 1, self.max_urls, url)) retries = 0 while retries < 3: try: await self._visit_url(current, session, url, word) visited += 1 break except CallErrors: await asyncio.sleep(1.0) retries += 1 if current == self.max_urls - 1: break # switch to the next tab try: await asyncio.wait_for(tabs.switch(), 5) except CallErrors: # if we can't switch, it's ok pass metadata["visited_url"] = visited await self.sync(session, metadata) return metadata async def full(session, options): builder = Builder(options) return await builder(session)