1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import random
import os
import asyncio
from arsenic.errors import UnknownError
from condprof.util import logger, get_credentials
from condprof.helpers import TabSwitcher, execute_async_script, is_mobile
BOOKMARK_FREQUENCY = 5
MAX_URLS = 150
MAX_BOOKMARKS = 250
CallErrors = asyncio.TimeoutError, UnknownError
class Builder:
def __init__(self, options):
self.options = options
self.words = self._read_lines("words.txt")
self.urls = self._build_url_list(self._read_lines("urls.txt"))
self.sync_js = self._load_script("sync")
self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS)
self.bookmark_js = self._load_script("bookmark")
self.platform = options.get("platform", "")
self.mobile = is_mobile(self.platform)
self.max_urls = options.get("max_urls", MAX_URLS)
# see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar
if self.mobile:
self.max_urls = min(self.max_urls, 20)
logger.info("platform: %s" % self.platform)
logger.info("max_urls: %s" % self.max_urls)
self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY)
# we're syncing only on desktop for now
self.syncing = not self.mobile
if self.syncing:
self.username, self.password = get_credentials()
if self.username is None:
raise ValueError("Sync operations need an FxA username and password")
else:
self.username, self.password = None, None
def _load_script(self, name):
return "\n".join(self._read_lines("%s.js" % name))
def _read_lines(self, filename):
path = os.path.join(os.path.dirname(__file__), filename)
with open(path) as f:
return f.readlines()
def _build_url_list(self, urls):
url_list = []
for url in urls:
url = url.strip()
if url.startswith("#"):
continue
for word in self.words:
word = word.strip()
if word.startswith("#"):
continue
url_list.append((url.format(word), word))
random.shuffle(url_list)
return url_list
async def add_bookmark(self, session, url, title):
logger.info("Adding bookmark to %s" % url)
return await execute_async_script(
session, self.bookmark_js, url, title, self.max_bookmarks
)
async def sync(self, session, metadata):
if not self.syncing:
return
# now that we've visited all pages, we want to upload to FXSync
logger.info("Syncing profile to FxSync")
logger.info("Username is %s, password is %s" % (self.username, self.password))
script_res = await execute_async_script(
session,
self.sync_js,
self.username,
self.password,
"https://accounts.stage.mozaws.net",
)
if script_res is None:
script_res = {}
metadata["logs"] = script_res.get("logs", {})
metadata["result"] = script_res.get("result", 0)
metadata["result_message"] = script_res.get("result_message", "SUCCESS")
return metadata
async def _visit_url(self, current, session, url, word):
await asyncio.wait_for(session.get(url), 5)
if current % self.bookmark_frequency == 0 and not self.mobile:
await asyncio.wait_for(self.add_bookmark(session, url, word), 5)
async def __call__(self, session):
metadata = {}
tabs = TabSwitcher(session, self.options)
await tabs.create_windows()
visited = 0
for current, (url, word) in enumerate(self.urls):
logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
retries = 0
while retries < 3:
try:
await self._visit_url(current, session, url, word)
visited += 1
break
except CallErrors:
await asyncio.sleep(1.0)
retries += 1
if current == self.max_urls - 1:
break
# switch to the next tab
try:
await asyncio.wait_for(tabs.switch(), 5)
except CallErrors:
# if we can't switch, it's ok
pass
metadata["visited_url"] = visited
await self.sync(session, metadata)
return metadata
async def full(session, options):
builder = Builder(options)
return await builder(session)
|
