1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
#! /usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Script name: talos_from_code.py
# Purpose: Read from a talos.json file the different files to download for a talos job
# Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com>
# Target: Python 2.5
#
import json
import os
import re
import sys
from optparse import OptionParser
import six
def main():
"""
This script downloads a talos.json file which indicates which files to download
for a talos job.
See a talos.json file for a better understand:
https://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
"""
parser = OptionParser()
parser.add_option(
"--talos-json-url",
dest="talos_json_url",
type="string",
help="It indicates from where to download the talos.json file.",
)
(options, args) = parser.parse_args()
# 1) check that the url was passed
if options.talos_json_url is None:
print("You need to specify --talos-json-url.")
sys.exit(1)
# 2) try to download the talos.json file
try:
jsonFilename = download_file(options.talos_json_url)
except Exception as e:
print("ERROR: We tried to download the talos.json file but something failed.")
print("ERROR: %s" % str(e))
sys.exit(1)
# 3) download the necessary files
print("INFO: talos.json URL: %s" % options.talos_json_url)
try:
key = "talos.zip"
entity = get_value(jsonFilename, key)
if passesRestrictions(options.talos_json_url, entity["url"]):
# the key is at the same time the filename e.g. talos.zip
print(
"INFO: Downloading %s as %s"
% (entity["url"], os.path.join(entity["path"], key))
)
download_file(entity["url"], entity["path"], key)
else:
print(
"ERROR: You have tried to download a file "
+ "from: %s " % entity["url"]
+ "which is a location different than http://talos-bundles.pvt.build.mozilla.org/"
)
print("ERROR: This is only allowed for the certain branches.")
sys.exit(1)
except Exception as e:
print("ERROR: %s" % str(e))
sys.exit(1)
def passesRestrictions(talosJsonUrl, fileUrl):
"""
Only certain branches are exempted from having to host their downloadable files
in talos-bundles.pvt.build.mozilla.org
"""
if (
talosJsonUrl.startswith("http://hg.mozilla.org/try/")
or talosJsonUrl.startswith("https://hg.mozilla.org/try/")
or talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/")
or talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/")
or talosJsonUrl.startswith("http://hg.mozilla.org/projects/ash/")
or talosJsonUrl.startswith("https://hg.mozilla.org/projects/ash/")
):
return True
else:
p = re.compile("^http://talos-bundles.pvt.build.mozilla.org/")
m = p.match(fileUrl)
if m is None:
return False
return True
def get_filename_from_url(url):
"""
This returns the filename of the file we're trying to download
"""
parsed = six.moves.urllib.parse.urlsplit(url.rstrip("/"))
if parsed.path != "":
return parsed.path.rsplit("/", 1)[-1]
else:
print(
"ERROR: We were trying to download a file from %s "
+ "but the URL seems to be incorrect."
)
sys.exit(1)
def download_file(url, path="", saveAs=None):
"""
It downloads a file from URL to the indicated path
"""
req = six.moves.urllib.request.Request(url)
f = six.moves.urllib.request.urlopen(req)
if path != "" and not os.path.isdir(path):
try:
os.makedirs(path)
print("INFO: directory %s created" % path)
except Exception as e:
print("ERROR: %s" % str(e))
sys.exit(1)
filename = saveAs if saveAs else get_filename_from_url(url)
local_file = open(os.path.join(path, filename), "wb")
local_file.write(f.read())
local_file.close()
return filename
def get_value(json_filename, key):
"""
It loads up a JSON file and returns the value for the given string
"""
f = open(json_filename, "r")
return json.load(f)[key]
if __name__ == "__main__":
main()
|
