1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import asyncio
import os
import random
from arsenic.errors import UnknownArsenicError, UnknownError
from condprof.helpers import TabSwitcher, execute_async_script, is_mobile
from condprof.util import get_credentials, logger
BOOKMARK_FREQUENCY = 5
MAX_URLS = 150
MAX_BOOKMARKS = 250
CallErrors = asyncio.TimeoutError, UnknownError, UnknownArsenicError
class Builder:
def __init__(self, options):
self.options = options
self.words = self._read_lines("words.txt")
self.urls = self._build_url_list(self._read_lines("urls.txt"))
self.sync_js = self._load_script("sync")
self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS)
self.bookmark_js = self._load_script("bookmark")
self.platform = options.get("platform", "")
self.mobile = is_mobile(self.platform)
self.max_urls = options.get("max_urls", MAX_URLS)
# see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar
if self.mobile:
self.max_urls = min(self.max_urls, 20)
logger.info("platform: %s" % self.platform)
logger.info("max_urls: %s" % self.max_urls)
self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY)
# we're syncing only on desktop for now
self.syncing = not self.mobile
if self.syncing:
self.username, self.password = get_credentials()
if self.username is None:
raise ValueError("Sync operations need an FxA username and password")
else:
self.username, self.password = None, None
def _load_script(self, name):
return "\n".join(self._read_lines("%s.js" % name))
def _read_lines(self, filename):
path = os.path.join(os.path.dirname(__file__), filename)
with open(path) as f:
return f.readlines()
def _build_url_list(self, urls):
url_list = []
for url in urls:
url = url.strip()
if url.startswith("#"):
continue
for word in self.words:
word = word.strip()
if word.startswith("#"):
continue
url_list.append((url.format(word), word))
random.shuffle(url_list)
return url_list
async def add_bookmark(self, session, url, title):
logger.info("Adding bookmark to %s" % url)
return await execute_async_script(
session, self.bookmark_js, url, title, self.max_bookmarks
)
async def sync(self, session, metadata):
if not self.syncing:
return
# now that we've visited all pages, we want to upload to FXSync
logger.info("Syncing profile to FxSync")
logger.info("Username is %s, password is %s" % (self.username, self.password))
script_res = await execute_async_script(
session,
self.sync_js,
self.username,
self.password,
"https://accounts.stage.mozaws.net",
)
if script_res is None:
script_res = {}
metadata["logs"] = script_res.get("logs", {})
metadata["result"] = script_res.get("result", 0)
metadata["result_message"] = script_res.get("result_message", "SUCCESS")
return metadata
async def _visit_url(self, current, session, url, word):
await asyncio.wait_for(session.get(url), 5)
if current % self.bookmark_frequency == 0 and not self.mobile:
await asyncio.wait_for(self.add_bookmark(session, url, word), 5)
async def __call__(self, session):
metadata = {}
tabs = TabSwitcher(session, self.options)
await tabs.create_windows()
visited = 0
for current, (url, word) in enumerate(self.urls):
logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
retries = 0
while retries < 3:
try:
await self._visit_url(current, session, url, word)
visited += 1
break
except CallErrors:
await asyncio.sleep(1.0)
retries += 1
if current == self.max_urls - 1:
break
# switch to the next tab
try:
await asyncio.wait_for(tabs.switch(), 5)
except CallErrors:
# if we can't switch, it's ok
pass
metadata["visited_url"] = visited
await self.sync(session, metadata)
return metadata
async def full(session, options):
builder = Builder(options)
return await builder(session)
|
