1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#! /usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Script name: talos_from_code.py
# Purpose: Read from a talos.json file the different files to download for a talos job
# Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com>
# Target: Python 2.5
#
from optparse import OptionParser
import json
import re
import urllib2
import urlparse
import sys
import os
def main():
'''
This script downloads a talos.json file which indicates which files to download
for a talos job.
See a talos.json file for a better understand:
http://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
'''
parser = OptionParser()
parser.add_option("--talos-json-url", dest="talos_json_url", type="string",
help="It indicates from where to download the talos.json file.")
(options, args) = parser.parse_args()
# 1) check that the url was passed
if options.talos_json_url is None:
print("You need to specify --talos-json-url.")
sys.exit(1)
# 2) try to download the talos.json file
try:
jsonFilename = download_file(options.talos_json_url)
except Exception as e:
print("ERROR: We tried to download the talos.json file but something failed.")
print("ERROR: %s" % str(e))
sys.exit(1)
# 3) download the necessary files
print("INFO: talos.json URL: %s" % options.talos_json_url)
try:
key = 'talos.zip'
entity = get_value(jsonFilename, key)
if passesRestrictions(options.talos_json_url, entity["url"]):
# the key is at the same time the filename e.g. talos.zip
print("INFO: Downloading %s as %s" %
(entity["url"], os.path.join(entity["path"], key)))
download_file(entity["url"], entity["path"], key)
else:
print("ERROR: You have tried to download a file " +
"from: %s " % entity["url"] +
"which is a location different than http://talos-bundles.pvt.build.mozilla.org/")
print("ERROR: This is only allowed for the certain branches.")
sys.exit(1)
except Exception as e:
print("ERROR: %s" % str(e))
sys.exit(1)
def passesRestrictions(talosJsonUrl, fileUrl):
'''
Only certain branches are exempted from having to host their downloadable files
in talos-bundles.pvt.build.mozilla.org
'''
if talosJsonUrl.startswith("http://hg.mozilla.org/try/") or \
talosJsonUrl.startswith("https://hg.mozilla.org/try/") or \
talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") or \
talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/") or \
talosJsonUrl.startswith("http://hg.mozilla.org/projects/ash/") or \
talosJsonUrl.startswith("https://hg.mozilla.org/projects/ash/"):
return True
else:
p = re.compile('^http://talos-bundles.pvt.build.mozilla.org/')
m = p.match(fileUrl)
if m is None:
return False
return True
def get_filename_from_url(url):
'''
This returns the filename of the file we're trying to download
'''
parsed = urlparse.urlsplit(url.rstrip('/'))
if parsed.path != '':
return parsed.path.rsplit('/', 1)[-1]
else:
print("ERROR: We were trying to download a file from %s " +
"but the URL seems to be incorrect.")
sys.exit(1)
def download_file(url, path="", saveAs=None):
'''
It downloads a file from URL to the indicated path
'''
req = urllib2.Request(url)
f = urllib2.urlopen(req)
if path != "" and not os.path.isdir(path):
try:
os.makedirs(path)
print("INFO: directory %s created" % path)
except Exception as e:
print("ERROR: %s" % str(e))
sys.exit(1)
filename = saveAs if saveAs else get_filename_from_url(url)
local_file = open(os.path.join(path, filename), 'wb')
local_file.write(f.read())
local_file.close()
return filename
def get_value(json_filename, key):
'''
It loads up a JSON file and returns the value for the given string
'''
f = open(json_filename, 'r')
return json.load(f)[key]
if __name__ == '__main__':
main()
|
