diff options
Diffstat (limited to 'testing/tps')
| -rw-r--r-- | testing/tps/.gitignore | 4 | ||||
| -rw-r--r-- | testing/tps/README | 42 | ||||
| -rw-r--r-- | testing/tps/config/config.json.in | 24 | ||||
| -rwxr-xr-x | testing/tps/create_venv.py | 194 | ||||
| -rw-r--r-- | testing/tps/pages/microsummary1.txt | 1 | ||||
| -rw-r--r-- | testing/tps/pages/microsummary2.txt | 1 | ||||
| -rw-r--r-- | testing/tps/pages/microsummary3.txt | 1 | ||||
| -rw-r--r-- | testing/tps/pages/page1.html | 15 | ||||
| -rw-r--r-- | testing/tps/pages/page2.html | 15 | ||||
| -rw-r--r-- | testing/tps/pages/page3.html | 15 | ||||
| -rw-r--r-- | testing/tps/pages/page4.html | 15 | ||||
| -rw-r--r-- | testing/tps/pages/page5.html | 15 | ||||
| -rw-r--r-- | testing/tps/setup.py | 48 | ||||
| -rw-r--r-- | testing/tps/tps/__init__.py | 6 | ||||
| -rw-r--r-- | testing/tps/tps/cli.py | 128 | ||||
| -rw-r--r-- | testing/tps/tps/firefoxrunner.py | 84 | ||||
| -rw-r--r-- | testing/tps/tps/phase.py | 69 | ||||
| -rw-r--r-- | testing/tps/tps/testrunner.py | 491 |
18 files changed, 1168 insertions, 0 deletions
diff --git a/testing/tps/.gitignore b/testing/tps/.gitignore new file mode 100644 index 000000000..65f8b6e05 --- /dev/null +++ b/testing/tps/.gitignore @@ -0,0 +1,4 @@ +# These files are added by running the TPS test suite. +build/ +dist/ +tps.egg-info/ diff --git a/testing/tps/README b/testing/tps/README new file mode 100644 index 000000000..50280682e --- /dev/null +++ b/testing/tps/README @@ -0,0 +1,42 @@ +TPS is a test automation framework for Firefox Sync. See +https://developer.mozilla.org/en/TPS for documentation. + +Installation +============ + +TPS requires several packages to operate properly. To install TPS and +required packages, use the INSTALL.sh script, provided: + + ./INSTALL.sh /path/to/create/virtualenv + +This script will create a virtalenv and install TPS into it. TPS can then +be run by activating the virtualenv and executing: + + runtps --binary=/path/to/firefox + + +Configuration +============= +To edit the TPS configuration, do not edit config/config.json.in in the tree. +Instead, edit config.json inside your virtualenv; it will be located at +something like: + + (linux): /path/to/virtualenv/lib/python2.6/site-packages/tps-0.2.40-py2.6.egg/tps/config.json + (win): /path/to/virtualenv/Lib/site-packages/tps-0.2.40-py2.6.egg/tps/config.json + + +Setting Up Test Accounts +======================== + +Firefox Accounts +---------------- +To create a test account for using the Firefox Account authentication perform the +following steps: + +1. Go to a URL like http://restmail.net/mail/%account_prefix%@restmail.net +2. Go to https://accounts.firefox.com/signup?service=sync&context=fx_desktop_v1 +3. Sign in with the previous chosen email address and a password +4. Go back to the Restmail URL, reload the page +5. Search for the verification link and open that page + +Now you will be able to use your setup Firefox Account for Sync. diff --git a/testing/tps/config/config.json.in b/testing/tps/config/config.json.in new file mode 100644 index 000000000..53ccb016c --- /dev/null +++ b/testing/tps/config/config.json.in @@ -0,0 +1,24 @@ +{ + "sync_account": { + "username": "__SYNC_ACCOUNT_USERNAME__", + "password": "__SYNC_ACCOUNT_PASSWORD__", + "passphrase": "__SYNC_ACCOUNT_PASSPHRASE__" + }, + "fx_account": { + "username": "__FX_ACCOUNT_USERNAME__", + "password": "__FX_ACCOUNT_PASSWORD__" + }, + "email": { + "username": "crossweave@mozilla.com", + "password": "", + "passednotificationlist": ["crossweave@mozilla.com"], + "notificationlist": ["crossweave@mozilla.com"] + }, + "auth_type": "fx_account", + "es": "localhost:9200", + "os": "Ubuntu", + "platform": "linux64", + "serverURL": null, + "extensiondir": "__EXTENSIONDIR__", + "testdir": "__TESTDIR__" +} diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py new file mode 100755 index 000000000..3d5417f8a --- /dev/null +++ b/testing/tps/create_venv.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This scripts sets up a virtualenv and installs TPS into it. +It's probably best to specify a path NOT inside the repo, otherwise +all the virtualenv files will show up in e.g. hg status. +""" + +import optparse +import os +import shutil +import subprocess +import sys +import urllib2 +import zipfile + + +here = os.path.dirname(os.path.abspath(__file__)) +usage_message = """ +*********************************************************************** + +To run TPS, activate the virtualenv using: + source {TARGET}/{BIN_NAME} + +To change your TPS config, please edit the file: + {TARGET}/config.json + +To execute tps use: + runtps --binary=/path/to/firefox + +See runtps --help for all options + +*********************************************************************** +""" + +# Link to the folder, which contains the zip archives of virtualenv +URL_VIRTUALENV = 'https://codeload.github.com/pypa/virtualenv/zip/' +VERSION_VIRTUALENV = '1.11.6' + + +if sys.platform == 'win32': + bin_name = os.path.join('Scripts', 'activate.bat') + activate_env = os.path.join('Scripts', 'activate_this.py') + python_env = os.path.join('Scripts', 'python.exe') +else: + bin_name = os.path.join('bin', 'activate') + activate_env = os.path.join('bin', 'activate_this.py') + python_env = os.path.join('bin', 'python') + + +def download(url, target): + """Downloads the specified url to the given target.""" + response = urllib2.urlopen(url) + with open(target, 'wb') as f: + f.write(response.read()) + + return target + + +def setup_virtualenv(target, python_bin=None): + script_path = os.path.join(here, 'virtualenv-%s' % VERSION_VIRTUALENV, + 'virtualenv.py') + + print 'Downloading virtualenv %s' % VERSION_VIRTUALENV + zip_path = download(URL_VIRTUALENV + VERSION_VIRTUALENV, + os.path.join(here, 'virtualenv.zip')) + + try: + with zipfile.ZipFile(zip_path, 'r') as f: + f.extractall(here) + + print 'Creating new virtual environment' + cmd_args = [sys.executable, script_path, target] + + if python_bin: + cmd_args.extend(['-p', python_bin]) + + subprocess.check_call(cmd_args) + finally: + try: + os.remove(zip_path) + except OSError: + pass + + shutil.rmtree(os.path.dirname(script_path), ignore_errors=True) + + +def update_configfile(source, target, replacements): + lines = [] + + with open(source) as config: + for line in config: + for source_string, target_string in replacements.iteritems(): + if target_string: + line = line.replace(source_string, target_string) + lines.append(line) + + with open(target, 'w') as config: + for line in lines: + config.write(line) + + +def main(): + parser = optparse.OptionParser('Usage: %prog [options] path_to_venv') + parser.add_option('--password', + type='string', + dest='password', + metavar='FX_ACCOUNT_PASSWORD', + default=None, + help='The Firefox Account password.') + parser.add_option('-p', '--python', + type='string', + dest='python', + metavar='PYTHON_BIN', + default=None, + help='The Python interpreter to use.') + parser.add_option('--sync-passphrase', + type='string', + dest='sync_passphrase', + metavar='SYNC_ACCOUNT_PASSPHRASE', + default=None, + help='The old Firefox Sync account passphrase.') + parser.add_option('--sync-password', + type='string', + dest='sync_password', + metavar='SYNC_ACCOUNT_PASSWORD', + default=None, + help='The old Firefox Sync account password.') + parser.add_option('--sync-username', + type='string', + dest='sync_username', + metavar='SYNC_ACCOUNT_USERNAME', + default=None, + help='The old Firefox Sync account username.') + parser.add_option('--username', + type='string', + dest='username', + metavar='FX_ACCOUNT_USERNAME', + default=None, + help='The Firefox Account username.') + + (options, args) = parser.parse_args(args=None, values=None) + + if len(args) != 1: + parser.error('Path to the environment has to be specified') + target = args[0] + assert(target) + + setup_virtualenv(target, python_bin=options.python) + + # Activate tps environment + tps_env = os.path.join(target, activate_env) + execfile(tps_env, dict(__file__=tps_env)) + + # Install TPS in environment + subprocess.check_call([os.path.join(target, python_env), + os.path.join(here, 'setup.py'), 'install']) + + # Get the path to tests and extensions directory by checking check where + # the tests and extensions directories are located + sync_dir = os.path.abspath(os.path.join(here, '..', '..', 'services', + 'sync')) + if os.path.exists(sync_dir): + testdir = os.path.join(sync_dir, 'tests', 'tps') + extdir = os.path.join(sync_dir, 'tps', 'extensions') + else: + testdir = os.path.join(here, 'tests') + extdir = os.path.join(here, 'extensions') + + update_configfile(os.path.join(here, 'config', 'config.json.in'), + os.path.join(target, 'config.json'), + replacements={ + '__TESTDIR__': testdir.replace('\\','/'), + '__EXTENSIONDIR__': extdir.replace('\\','/'), + '__FX_ACCOUNT_USERNAME__': options.username, + '__FX_ACCOUNT_PASSWORD__': options.password, + '__SYNC_ACCOUNT_USERNAME__': options.sync_username, + '__SYNC_ACCOUNT_PASSWORD__': options.sync_password, + '__SYNC_ACCOUNT_PASSPHRASE__': options.sync_passphrase}) + + if not (options.username and options.password): + print '\nFirefox Account credentials not specified.' + if not (options.sync_username and options.sync_password and options.passphrase): + print '\nFirefox Sync account credentials not specified.' + + # Print the user instructions + print usage_message.format(TARGET=target, + BIN_NAME=bin_name) + +if __name__ == "__main__": + main() diff --git a/testing/tps/pages/microsummary1.txt b/testing/tps/pages/microsummary1.txt new file mode 100644 index 000000000..9451d4b5c --- /dev/null +++ b/testing/tps/pages/microsummary1.txt @@ -0,0 +1 @@ +Static microsummary #1
diff --git a/testing/tps/pages/microsummary2.txt b/testing/tps/pages/microsummary2.txt new file mode 100644 index 000000000..7e3208504 --- /dev/null +++ b/testing/tps/pages/microsummary2.txt @@ -0,0 +1 @@ +Static microsummary #2
diff --git a/testing/tps/pages/microsummary3.txt b/testing/tps/pages/microsummary3.txt new file mode 100644 index 000000000..00ac62bbf --- /dev/null +++ b/testing/tps/pages/microsummary3.txt @@ -0,0 +1 @@ +Static microsummary #3
diff --git a/testing/tps/pages/page1.html b/testing/tps/pages/page1.html new file mode 100644 index 000000000..256c11c1c --- /dev/null +++ b/testing/tps/pages/page1.html @@ -0,0 +1,15 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 1</title> +</head> +<body> +<p> +Crossweave Test Page 1 +</p> +</body> +</html> + diff --git a/testing/tps/pages/page2.html b/testing/tps/pages/page2.html new file mode 100644 index 000000000..62b693ed5 --- /dev/null +++ b/testing/tps/pages/page2.html @@ -0,0 +1,15 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 2</title> +</head> +<body> +<p> +Crossweave Test Page 2 +</p> +</body> +</html> + diff --git a/testing/tps/pages/page3.html b/testing/tps/pages/page3.html new file mode 100644 index 000000000..2d0a981de --- /dev/null +++ b/testing/tps/pages/page3.html @@ -0,0 +1,15 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 3</title> +</head> +<body> +<p> +Crossweave Test Page 3 +</p> +</body> +</html> + diff --git a/testing/tps/pages/page4.html b/testing/tps/pages/page4.html new file mode 100644 index 000000000..b28f59d80 --- /dev/null +++ b/testing/tps/pages/page4.html @@ -0,0 +1,15 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 4</title> +</head> +<body> +<p> +Crossweave Test Page 4 +</p> +</body> +</html> + diff --git a/testing/tps/pages/page5.html b/testing/tps/pages/page5.html new file mode 100644 index 000000000..ffd4ce48b --- /dev/null +++ b/testing/tps/pages/page5.html @@ -0,0 +1,15 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head> +<title>Crossweave Test Page 5</title> +</head> +<body> +<p> +Crossweave Test Page 5 +</p> +</body> +</html> + diff --git a/testing/tps/setup.py b/testing/tps/setup.py new file mode 100644 index 000000000..19f109c10 --- /dev/null +++ b/testing/tps/setup.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup, find_packages +import sys + +version = '0.5' + +deps = ['httplib2 == 0.9.2', + 'mozfile == 1.2', + 'mozhttpd == 0.7', + 'mozinfo == 0.9', + 'mozinstall == 1.12', + 'mozprocess == 0.23', + 'mozprofile == 0.28', + 'mozrunner == 6.12', + 'mozversion == 1.4', + ] + +# we only support python 2.6+ right now +assert sys.version_info[0] == 2 +assert sys.version_info[1] >= 6 + +setup(name='tps', + version=version, + description='run automated multi-profile sync tests', + long_description="""\ +""", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='', + author='Mozilla Automation and Tools team', + author_email='tools@lists.mozilla.org', + url='https://developer.mozilla.org/en-US/docs/TPS', + license='MPL 2.0', + packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + include_package_data=True, + zip_safe=False, + install_requires=deps, + entry_points=""" + # -*- Entry points: -*- + [console_scripts] + runtps = tps.cli:main + """, + data_files=[ + ('tps', ['config/config.json.in']), + ], + ) diff --git a/testing/tps/tps/__init__.py b/testing/tps/tps/__init__.py new file mode 100644 index 000000000..4433ee9b8 --- /dev/null +++ b/testing/tps/tps/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from . firefoxrunner import TPSFirefoxRunner +from . testrunner import TPSTestRunner diff --git a/testing/tps/tps/cli.py b/testing/tps/tps/cli.py new file mode 100644 index 000000000..63e8db0d4 --- /dev/null +++ b/testing/tps/tps/cli.py @@ -0,0 +1,128 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import optparse +import os +import re +import sys +from threading import RLock + +from tps import TPSTestRunner + + +def main(): + parser = optparse.OptionParser() + parser.add_option('--binary', + action='store', + type='string', + dest='binary', + default=None, + help='path to the Firefox binary, specified either as ' + 'a local file or a url; if omitted, the PATH ' + 'will be searched;') + parser.add_option('--configfile', + action='store', + type='string', + dest='configfile', + default=None, + help='path to the config file to use default: %default]') + parser.add_option('--debug', + action='store_true', + dest='debug', + default=False, + help='run in debug mode') + parser.add_option('--ignore-unused-engines', + default=False, + action='store_true', + dest='ignore_unused_engines', + help='If defined, do not load unused engines in individual tests.' + ' Has no effect for pulse monitor.') + parser.add_option('--logfile', + action='store', + type='string', + dest='logfile', + default='tps.log', + help='path to the log file [default: %default]') + parser.add_option('--mobile', + action='store_true', + dest='mobile', + default=False, + help='run with mobile settings') + parser.add_option('--pulsefile', + action='store', + type='string', + dest='pulsefile', + default=None, + help='path to file containing a pulse message in ' + 'json format that you want to inject into the monitor') + parser.add_option('--resultfile', + action='store', + type='string', + dest='resultfile', + default='tps_result.json', + help='path to the result file [default: %default]') + parser.add_option('--testfile', + action='store', + type='string', + dest='testfile', + default='all_tests.json', + help='path to the test file to run [default: %default]') + parser.add_option('--stop-on-error', + action='store_true', + dest='stop_on_error', + help='stop running tests after the first failure') + (options, args) = parser.parse_args() + + configfile = options.configfile + if configfile is None: + virtual_env = os.environ.get('VIRTUAL_ENV') + if virtual_env: + configfile = os.path.join(virtual_env, 'config.json') + if configfile is None or not os.access(configfile, os.F_OK): + raise Exception('Unable to find config.json in a VIRTUAL_ENV; you must ' + 'specify a config file using the --configfile option') + + # load the config file + f = open(configfile, 'r') + configcontent = f.read() + f.close() + config = json.loads(configcontent) + testfile = os.path.join(config.get('testdir', ''), options.testfile) + + rlock = RLock() + + print 'using result file', options.resultfile + + extensionDir = config.get('extensiondir') + if not extensionDir or extensionDir == '__EXTENSIONDIR__': + extensionDir = os.path.join(os.getcwd(), '..', '..', + 'services', 'sync', 'tps', 'extensions') + else: + if sys.platform == 'win32': + # replace msys-style paths with proper Windows paths + m = re.match('^\/\w\/', extensionDir) + if m: + extensionDir = '%s:/%s' % (m.group(0)[1:2], extensionDir[3:]) + extensionDir = extensionDir.replace('/', '\\') + + TPS = TPSTestRunner(extensionDir, + binary=options.binary, + config=config, + debug=options.debug, + ignore_unused_engines=options.ignore_unused_engines, + logfile=options.logfile, + mobile=options.mobile, + resultfile=options.resultfile, + rlock=rlock, + testfile=testfile, + stop_on_error=options.stop_on_error, + ) + TPS.run_tests() + + if TPS.numfailed > 0 or TPS.numpassed == 0: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/testing/tps/tps/firefoxrunner.py b/testing/tps/tps/firefoxrunner.py new file mode 100644 index 000000000..3b7c143d8 --- /dev/null +++ b/testing/tps/tps/firefoxrunner.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy +import httplib2 +import os + +import mozfile +import mozinstall +from mozprofile import Profile +from mozrunner import FirefoxRunner + + +class TPSFirefoxRunner(object): + + PROCESS_TIMEOUT = 240 + + def __init__(self, binary): + if binary is not None and ('http://' in binary or 'ftp://' in binary): + self.url = binary + self.binary = None + else: + self.url = None + self.binary = binary + + self.installdir = None + + def __del__(self): + if self.installdir: + mozfile.remove(self.installdir, True) + + def download_url(self, url, dest=None): + h = httplib2.Http() + resp, content = h.request(url, 'GET') + if dest == None: + dest = os.path.basename(url) + + local = open(dest, 'wb') + local.write(content) + local.close() + return dest + + def download_build(self, installdir='downloadedbuild', appname='firefox'): + self.installdir = os.path.abspath(installdir) + buildName = os.path.basename(self.url) + pathToBuild = os.path.join(os.path.dirname(os.path.abspath(__file__)), + buildName) + + # delete the build if it already exists + if os.access(pathToBuild, os.F_OK): + os.remove(pathToBuild) + + # download the build + print 'downloading build' + self.download_url(self.url, pathToBuild) + + # install the build + print 'installing %s' % pathToBuild + mozfile.remove(self.installdir, True) + binary = mozinstall.install(src=pathToBuild, dest=self.installdir) + + # remove the downloaded archive + os.remove(pathToBuild) + + return binary + + def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None): + """Runs the given FirefoxRunner with the given Profile, waits + for completion, then returns the process exit code + """ + if profile is None: + profile = Profile() + self.profile = profile + + if self.binary is None and self.url: + self.binary = self.download_build() + + runner = FirefoxRunner(profile=self.profile, binary=self.binary, + env=env, cmdargs=args) + + runner.start(timeout=timeout) + return runner.wait() diff --git a/testing/tps/tps/phase.py b/testing/tps/tps/phase.py new file mode 100644 index 000000000..397c90796 --- /dev/null +++ b/testing/tps/tps/phase.py @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re + +class TPSTestPhase(object): + + lineRe = re.compile( + r'^(.*?)test phase (?P<matchphase>[^\s]+): (?P<matchstatus>.*)$') + + def __init__(self, phase, profile, testname, testpath, logfile, env, + firefoxRunner, logfn, ignore_unused_engines=False): + self.phase = phase + self.profile = profile + self.testname = str(testname) # this might be passed in as unicode + self.testpath = testpath + self.logfile = logfile + self.env = env + self.firefoxRunner = firefoxRunner + self.log = logfn + self.ignore_unused_engines = ignore_unused_engines + self._status = None + self.errline = '' + + @property + def status(self): + return self._status if self._status else 'unknown' + + def run(self): + # launch Firefox + args = [ '-tps', self.testpath, + '-tpsphase', self.phase, + '-tpslogfile', self.logfile ] + + if self.ignore_unused_engines: + args.append('--ignore-unused-engines') + + self.log('\nLaunching Firefox for phase %s with args %s\n' % + (self.phase, str(args))) + self.firefoxRunner.run(env=self.env, + args=args, + profile=self.profile) + + # parse the logfile and look for results from the current test phase + found_test = False + f = open(self.logfile, 'r') + for line in f: + + # skip to the part of the log file that deals with the test we're running + if not found_test: + if line.find('Running test %s' % self.testname) > -1: + found_test = True + else: + continue + + # look for the status of the current phase + match = self.lineRe.match(line) + if match: + if match.group('matchphase') == self.phase: + self._status = match.group('matchstatus') + break + + # set the status to FAIL if there is TPS error + if line.find('CROSSWEAVE ERROR: ') > -1 and not self._status: + self._status = 'FAIL' + self.errline = line[line.find('CROSSWEAVE ERROR: ') + len('CROSSWEAVE ERROR: '):] + + f.close() diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py new file mode 100644 index 000000000..8e5aee6c8 --- /dev/null +++ b/testing/tps/tps/testrunner.py @@ -0,0 +1,491 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import random +import re +import tempfile +import time +import traceback + +from mozhttpd import MozHttpd +import mozinfo +from mozprofile import Profile +import mozversion + +from .firefoxrunner import TPSFirefoxRunner +from .phase import TPSTestPhase + + +class TempFile(object): + """Class for temporary files that delete themselves when garbage-collected. + """ + + def __init__(self, prefix=None): + self.fd, self.filename = self.tmpfile = tempfile.mkstemp(prefix=prefix) + + def write(self, data): + if self.fd: + os.write(self.fd, data) + + def close(self): + if self.fd: + os.close(self.fd) + self.fd = None + + def cleanup(self): + if self.fd: + self.close() + if os.access(self.filename, os.F_OK): + os.remove(self.filename) + + __del__ = cleanup + + +class TPSTestRunner(object): + + extra_env = { + 'MOZ_CRASHREPORTER_DISABLE': '1', + 'GNOME_DISABLE_CRASH_DIALOG': '1', + 'XRE_NO_WINDOWS_CRASH_DIALOG': '1', + 'MOZ_NO_REMOTE': '1', + 'XPCOM_DEBUG_BREAK': 'warn', + } + + default_preferences = { + 'app.update.enabled': False, + 'browser.dom.window.dump.enabled': True, + 'browser.sessionstore.resume_from_crash': False, + 'browser.shell.checkDefaultBrowser': False, + 'browser.tabs.warnOnClose': False, + 'browser.warnOnQuit': False, + # Allow installing extensions dropped into the profile folder + 'extensions.autoDisableScopes': 10, + 'extensions.getAddons.get.url': 'http://127.0.0.1:4567/addons/api/%IDS%.xml', + # Our pretend addons server doesn't support metadata... + 'extensions.getAddons.cache.enabled': False, + 'extensions.install.requireSecureOrigin': False, + 'extensions.update.enabled': False, + # Don't open a dialog to show available add-on updates + 'extensions.update.notifyUser': False, + 'services.sync.firstSync': 'notReady', + 'services.sync.lastversion': '1.0', + 'toolkit.startup.max_resumed_crashes': -1, + # hrm - not sure what the release/beta channels will do? + 'xpinstall.signatures.required': False, + } + + debug_preferences = { + 'services.sync.log.appender.console': 'Trace', + 'services.sync.log.appender.dump': 'Trace', + 'services.sync.log.appender.file.level': 'Trace', + 'services.sync.log.appender.file.logOnSuccess': True, + 'services.sync.log.rootLogger': 'Trace', + 'services.sync.log.logger.addonutils': 'Trace', + 'services.sync.log.logger.declined': 'Trace', + 'services.sync.log.logger.service.main': 'Trace', + 'services.sync.log.logger.status': 'Trace', + 'services.sync.log.logger.authenticator': 'Trace', + 'services.sync.log.logger.network.resources': 'Trace', + 'services.sync.log.logger.service.jpakeclient': 'Trace', + 'services.sync.log.logger.engine.bookmarks': 'Trace', + 'services.sync.log.logger.engine.clients': 'Trace', + 'services.sync.log.logger.engine.forms': 'Trace', + 'services.sync.log.logger.engine.history': 'Trace', + 'services.sync.log.logger.engine.passwords': 'Trace', + 'services.sync.log.logger.engine.prefs': 'Trace', + 'services.sync.log.logger.engine.tabs': 'Trace', + 'services.sync.log.logger.engine.addons': 'Trace', + 'services.sync.log.logger.engine.apps': 'Trace', + 'services.sync.log.logger.identity': 'Trace', + 'services.sync.log.logger.userapi': 'Trace', + } + + syncVerRe = re.compile( + r'Sync version: (?P<syncversion>.*)\n') + ffVerRe = re.compile( + r'Firefox version: (?P<ffver>.*)\n') + ffBuildIDRe = re.compile( + r'Firefox buildid: (?P<ffbuildid>.*)\n') + + def __init__(self, extensionDir, + binary=None, + config=None, + debug=False, + ignore_unused_engines=False, + logfile='tps.log', + mobile=False, + rlock=None, + resultfile='tps_result.json', + testfile=None, + stop_on_error=False): + self.binary = binary + self.config = config if config else {} + self.debug = debug + self.extensions = [] + self.ignore_unused_engines = ignore_unused_engines + self.logfile = os.path.abspath(logfile) + self.mobile = mobile + self.rlock = rlock + self.resultfile = resultfile + self.testfile = testfile + self.stop_on_error = stop_on_error + + self.addonversion = None + self.branch = None + self.changeset = None + self.errorlogs = {} + self.extensionDir = extensionDir + self.firefoxRunner = None + self.nightly = False + self.numfailed = 0 + self.numpassed = 0 + self.postdata = {} + self.productversion = None + self.repo = None + self.tpsxpi = None + + @property + def mobile(self): + return self._mobile + + @mobile.setter + def mobile(self, value): + self._mobile = value + self.synctype = 'desktop' if not self._mobile else 'mobile' + + def log(self, msg, printToConsole=False): + """Appends a string to the logfile""" + + f = open(self.logfile, 'a') + f.write(msg) + f.close() + if printToConsole: + print msg + + def writeToResultFile(self, postdata, body=None, + sendTo=['crossweave@mozilla.com']): + """Writes results to test file""" + + results = {'results': []} + + if os.access(self.resultfile, os.F_OK): + f = open(self.resultfile, 'r') + results = json.loads(f.read()) + f.close() + + f = open(self.resultfile, 'w') + if body is not None: + postdata['body'] = body + if self.numpassed is not None: + postdata['numpassed'] = self.numpassed + if self.numfailed is not None: + postdata['numfailed'] = self.numfailed + if self.firefoxRunner and self.firefoxRunner.url: + postdata['firefoxrunnerurl'] = self.firefoxRunner.url + + postdata['sendTo'] = sendTo + results['results'].append(postdata) + f.write(json.dumps(results, indent=2)) + f.close() + + def _zip_add_file(self, zip, file, rootDir): + zip.write(os.path.join(rootDir, file), file) + + def _zip_add_dir(self, zip, dir, rootDir): + try: + zip.write(os.path.join(rootDir, dir), dir) + except: + # on some OS's, adding directory entries doesn't seem to work + pass + for root, dirs, files in os.walk(os.path.join(rootDir, dir)): + for f in files: + zip.write(os.path.join(root, f), os.path.join(dir, f)) + + def handle_phase_failure(self, profiles): + for profile in profiles: + self.log('\nDumping sync log for profile %s\n' % profiles[profile].profile) + for root, dirs, files in os.walk(os.path.join(profiles[profile].profile, 'weave', 'logs')): + for f in files: + weavelog = os.path.join(profiles[profile].profile, 'weave', 'logs', f) + if os.access(weavelog, os.F_OK): + with open(weavelog, 'r') as fh: + for line in fh: + possible_time = line[0:13] + if len(possible_time) == 13 and possible_time.isdigit(): + time_ms = int(possible_time) + formatted = time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime(time_ms / 1000)) + self.log('%s.%03d %s' % ( + formatted, time_ms % 1000, line[14:] )) + else: + self.log(line) + + def run_single_test(self, testdir, testname): + testpath = os.path.join(testdir, testname) + self.log("Running test %s\n" % testname, True) + + # Read and parse the test file, merge it with the contents of the config + # file, and write the combined output to a temporary file. + f = open(testpath, 'r') + testcontent = f.read() + f.close() + try: + test = json.loads(testcontent) + except: + test = json.loads(testcontent[testcontent.find('{'):testcontent.find('}') + 1]) + + self.preferences['tps.seconds_since_epoch'] = int(time.time()) + + # generate the profiles defined in the test, and a list of test phases + profiles = {} + phaselist = [] + for phase in test: + profilename = test[phase] + + # create the profile if necessary + if not profilename in profiles: + profiles[profilename] = Profile(preferences = self.preferences, + addons = self.extensions) + + # create the test phase + phaselist.append(TPSTestPhase( + phase, + profiles[profilename], + testname, + testpath, + self.logfile, + self.env, + self.firefoxRunner, + self.log, + ignore_unused_engines=self.ignore_unused_engines)) + + # sort the phase list by name + phaselist = sorted(phaselist, key=lambda phase: phase.phase) + + # run each phase in sequence, aborting at the first failure + failed = False + for phase in phaselist: + phase.run() + if phase.status != 'PASS': + failed = True + break; + + for profilename in profiles: + cleanup_phase = TPSTestPhase( + 'cleanup-' + profilename, + profiles[profilename], testname, + testpath, + self.logfile, + self.env, + self.firefoxRunner, + self.log) + + cleanup_phase.run() + if cleanup_phase.status != 'PASS': + failed = True + # Keep going to run the remaining cleanup phases. + + if failed: + self.handle_phase_failure(profiles) + + # grep the log for FF and sync versions + f = open(self.logfile) + logdata = f.read() + match = self.syncVerRe.search(logdata) + sync_version = match.group('syncversion') if match else 'unknown' + match = self.ffVerRe.search(logdata) + firefox_version = match.group('ffver') if match else 'unknown' + match = self.ffBuildIDRe.search(logdata) + firefox_buildid = match.group('ffbuildid') if match else 'unknown' + f.close() + if phase.status == 'PASS': + logdata = '' + else: + # we only care about the log data for this specific test + logdata = logdata[logdata.find('Running test %s' % (str(testname))):] + + result = { + 'PASS': lambda x: ('TEST-PASS', ''), + 'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()), + 'unknown': lambda x: ('TEST-UNEXPECTED-FAIL', 'test did not complete') + } [phase.status](phase.errline) + logstr = "\n%s | %s%s\n" % (result[0], testname, (' | %s' % result[1] if result[1] else '')) + + try: + repoinfo = mozversion.get_version(self.binary) + except: + repoinfo = {} + apprepo = repoinfo.get('application_repository', '') + appchangeset = repoinfo.get('application_changeset', '') + + # save logdata to a temporary file for posting to the db + tmplogfile = None + if logdata: + tmplogfile = TempFile(prefix='tps_log_') + tmplogfile.write(logdata) + tmplogfile.close() + self.errorlogs[testname] = tmplogfile + + resultdata = ({ 'productversion': { 'version': firefox_version, + 'buildid': firefox_buildid, + 'builddate': firefox_buildid[0:8], + 'product': 'Firefox', + 'repository': apprepo, + 'changeset': appchangeset, + }, + 'addonversion': { 'version': sync_version, + 'product': 'Firefox Sync' }, + 'name': testname, + 'message': result[1], + 'state': result[0], + 'logdata': logdata + }) + + self.log(logstr, True) + for phase in phaselist: + print "\t%s: %s" % (phase.phase, phase.status) + + return resultdata + + def update_preferences(self): + self.preferences = self.default_preferences.copy() + + if self.mobile: + self.preferences.update({'services.sync.client.type' : 'mobile'}) + + # If we are using legacy Sync, then set a dummy username to force the + # correct authentication type. Without this pref set to a value + # without an '@' character, Sync will initialize for FxA. + if self.config.get('auth_type', 'fx_account') != "fx_account": + self.preferences.update({'services.sync.username': "dummy"}) + + if self.debug: + self.preferences.update(self.debug_preferences) + + if 'preferences' in self.config: + self.preferences.update(self.config['preferences']) + + self.preferences['tps.config'] = json.dumps(self.config) + + def run_tests(self): + # delete the logfile if it already exists + if os.access(self.logfile, os.F_OK): + os.remove(self.logfile) + + # Copy the system env variables, and update them for custom settings + self.env = os.environ.copy() + self.env.update(self.extra_env) + + # Update preferences for custom settings + self.update_preferences() + + # Acquire a lock to make sure no other threads are running tests + # at the same time. + if self.rlock: + self.rlock.acquire() + + try: + # Create the Firefox runner, which will download and install the + # build, as needed. + if not self.firefoxRunner: + self.firefoxRunner = TPSFirefoxRunner(self.binary) + + # now, run the test group + self.run_test_group() + + except: + traceback.print_exc() + self.numpassed = 0 + self.numfailed = 1 + try: + self.writeToResultFile(self.postdata, + '<pre>%s</pre>' % traceback.format_exc()) + except: + traceback.print_exc() + else: + try: + + if self.numfailed > 0 or self.numpassed == 0: + To = self.config['email'].get('notificationlist') + else: + To = self.config['email'].get('passednotificationlist') + self.writeToResultFile(self.postdata, + sendTo=To) + except: + traceback.print_exc() + try: + self.writeToResultFile(self.postdata, + '<pre>%s</pre>' % traceback.format_exc()) + except: + traceback.print_exc() + + # release our lock + if self.rlock: + self.rlock.release() + + # dump out a summary of test results + print 'Test Summary\n' + for test in self.postdata.get('tests', {}): + print '%s | %s | %s' % (test['state'], test['name'], test['message']) + + def run_test_group(self): + self.results = [] + + # reset number of passed/failed tests + self.numpassed = 0 + self.numfailed = 0 + + # build our tps.xpi extension + self.extensions = [] + self.extensions.append(os.path.join(self.extensionDir, 'tps')) + self.extensions.append(os.path.join(self.extensionDir, "mozmill")) + + # build the test list + try: + f = open(self.testfile) + jsondata = f.read() + f.close() + testfiles = json.loads(jsondata) + testlist = testfiles['tests'] + except ValueError: + testlist = [os.path.basename(self.testfile)] + testdir = os.path.dirname(self.testfile) + + self.mozhttpd = MozHttpd(port=4567, docroot=testdir) + self.mozhttpd.start() + + # run each test, and save the results + for test in testlist: + result = self.run_single_test(testdir, test) + + if not self.productversion: + self.productversion = result['productversion'] + if not self.addonversion: + self.addonversion = result['addonversion'] + + self.results.append({'state': result['state'], + 'name': result['name'], + 'message': result['message'], + 'logdata': result['logdata']}) + if result['state'] == 'TEST-PASS': + self.numpassed += 1 + else: + self.numfailed += 1 + if self.stop_on_error: + print '\nTest failed with --stop-on-error specified; not running any more tests.\n' + break + + self.mozhttpd.stop() + + # generate the postdata we'll use to post the results to the db + self.postdata = { 'tests': self.results, + 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits), + 'testtype': 'crossweave', + 'productversion': self.productversion, + 'addonversion': self.addonversion, + 'synctype': self.synctype, + } |