summaryrefslogtreecommitdiffstats
path: root/testing/tps
diff options
context:
space:
mode:
Diffstat (limited to 'testing/tps')
-rw-r--r--testing/tps/.gitignore4
-rw-r--r--testing/tps/README42
-rw-r--r--testing/tps/config/config.json.in24
-rwxr-xr-xtesting/tps/create_venv.py194
-rw-r--r--testing/tps/pages/microsummary1.txt1
-rw-r--r--testing/tps/pages/microsummary2.txt1
-rw-r--r--testing/tps/pages/microsummary3.txt1
-rw-r--r--testing/tps/pages/page1.html15
-rw-r--r--testing/tps/pages/page2.html15
-rw-r--r--testing/tps/pages/page3.html15
-rw-r--r--testing/tps/pages/page4.html15
-rw-r--r--testing/tps/pages/page5.html15
-rw-r--r--testing/tps/setup.py48
-rw-r--r--testing/tps/tps/__init__.py6
-rw-r--r--testing/tps/tps/cli.py128
-rw-r--r--testing/tps/tps/firefoxrunner.py84
-rw-r--r--testing/tps/tps/phase.py69
-rw-r--r--testing/tps/tps/testrunner.py491
18 files changed, 1168 insertions, 0 deletions
diff --git a/testing/tps/.gitignore b/testing/tps/.gitignore
new file mode 100644
index 000000000..65f8b6e05
--- /dev/null
+++ b/testing/tps/.gitignore
@@ -0,0 +1,4 @@
+# These files are added by running the TPS test suite.
+build/
+dist/
+tps.egg-info/
diff --git a/testing/tps/README b/testing/tps/README
new file mode 100644
index 000000000..50280682e
--- /dev/null
+++ b/testing/tps/README
@@ -0,0 +1,42 @@
+TPS is a test automation framework for Firefox Sync. See
+https://developer.mozilla.org/en/TPS for documentation.
+
+Installation
+============
+
+TPS requires several packages to operate properly. To install TPS and
+required packages, use the INSTALL.sh script, provided:
+
+ ./INSTALL.sh /path/to/create/virtualenv
+
+This script will create a virtalenv and install TPS into it. TPS can then
+be run by activating the virtualenv and executing:
+
+ runtps --binary=/path/to/firefox
+
+
+Configuration
+=============
+To edit the TPS configuration, do not edit config/config.json.in in the tree.
+Instead, edit config.json inside your virtualenv; it will be located at
+something like:
+
+ (linux): /path/to/virtualenv/lib/python2.6/site-packages/tps-0.2.40-py2.6.egg/tps/config.json
+ (win): /path/to/virtualenv/Lib/site-packages/tps-0.2.40-py2.6.egg/tps/config.json
+
+
+Setting Up Test Accounts
+========================
+
+Firefox Accounts
+----------------
+To create a test account for using the Firefox Account authentication perform the
+following steps:
+
+1. Go to a URL like http://restmail.net/mail/%account_prefix%@restmail.net
+2. Go to https://accounts.firefox.com/signup?service=sync&context=fx_desktop_v1
+3. Sign in with the previous chosen email address and a password
+4. Go back to the Restmail URL, reload the page
+5. Search for the verification link and open that page
+
+Now you will be able to use your setup Firefox Account for Sync.
diff --git a/testing/tps/config/config.json.in b/testing/tps/config/config.json.in
new file mode 100644
index 000000000..53ccb016c
--- /dev/null
+++ b/testing/tps/config/config.json.in
@@ -0,0 +1,24 @@
+{
+ "sync_account": {
+ "username": "__SYNC_ACCOUNT_USERNAME__",
+ "password": "__SYNC_ACCOUNT_PASSWORD__",
+ "passphrase": "__SYNC_ACCOUNT_PASSPHRASE__"
+ },
+ "fx_account": {
+ "username": "__FX_ACCOUNT_USERNAME__",
+ "password": "__FX_ACCOUNT_PASSWORD__"
+ },
+ "email": {
+ "username": "crossweave@mozilla.com",
+ "password": "",
+ "passednotificationlist": ["crossweave@mozilla.com"],
+ "notificationlist": ["crossweave@mozilla.com"]
+ },
+ "auth_type": "fx_account",
+ "es": "localhost:9200",
+ "os": "Ubuntu",
+ "platform": "linux64",
+ "serverURL": null,
+ "extensiondir": "__EXTENSIONDIR__",
+ "testdir": "__TESTDIR__"
+}
diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py
new file mode 100755
index 000000000..3d5417f8a
--- /dev/null
+++ b/testing/tps/create_venv.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This scripts sets up a virtualenv and installs TPS into it.
+It's probably best to specify a path NOT inside the repo, otherwise
+all the virtualenv files will show up in e.g. hg status.
+"""
+
+import optparse
+import os
+import shutil
+import subprocess
+import sys
+import urllib2
+import zipfile
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+usage_message = """
+***********************************************************************
+
+To run TPS, activate the virtualenv using:
+ source {TARGET}/{BIN_NAME}
+
+To change your TPS config, please edit the file:
+ {TARGET}/config.json
+
+To execute tps use:
+ runtps --binary=/path/to/firefox
+
+See runtps --help for all options
+
+***********************************************************************
+"""
+
+# Link to the folder, which contains the zip archives of virtualenv
+URL_VIRTUALENV = 'https://codeload.github.com/pypa/virtualenv/zip/'
+VERSION_VIRTUALENV = '1.11.6'
+
+
+if sys.platform == 'win32':
+ bin_name = os.path.join('Scripts', 'activate.bat')
+ activate_env = os.path.join('Scripts', 'activate_this.py')
+ python_env = os.path.join('Scripts', 'python.exe')
+else:
+ bin_name = os.path.join('bin', 'activate')
+ activate_env = os.path.join('bin', 'activate_this.py')
+ python_env = os.path.join('bin', 'python')
+
+
+def download(url, target):
+ """Downloads the specified url to the given target."""
+ response = urllib2.urlopen(url)
+ with open(target, 'wb') as f:
+ f.write(response.read())
+
+ return target
+
+
+def setup_virtualenv(target, python_bin=None):
+ script_path = os.path.join(here, 'virtualenv-%s' % VERSION_VIRTUALENV,
+ 'virtualenv.py')
+
+ print 'Downloading virtualenv %s' % VERSION_VIRTUALENV
+ zip_path = download(URL_VIRTUALENV + VERSION_VIRTUALENV,
+ os.path.join(here, 'virtualenv.zip'))
+
+ try:
+ with zipfile.ZipFile(zip_path, 'r') as f:
+ f.extractall(here)
+
+ print 'Creating new virtual environment'
+ cmd_args = [sys.executable, script_path, target]
+
+ if python_bin:
+ cmd_args.extend(['-p', python_bin])
+
+ subprocess.check_call(cmd_args)
+ finally:
+ try:
+ os.remove(zip_path)
+ except OSError:
+ pass
+
+ shutil.rmtree(os.path.dirname(script_path), ignore_errors=True)
+
+
+def update_configfile(source, target, replacements):
+ lines = []
+
+ with open(source) as config:
+ for line in config:
+ for source_string, target_string in replacements.iteritems():
+ if target_string:
+ line = line.replace(source_string, target_string)
+ lines.append(line)
+
+ with open(target, 'w') as config:
+ for line in lines:
+ config.write(line)
+
+
+def main():
+ parser = optparse.OptionParser('Usage: %prog [options] path_to_venv')
+ parser.add_option('--password',
+ type='string',
+ dest='password',
+ metavar='FX_ACCOUNT_PASSWORD',
+ default=None,
+ help='The Firefox Account password.')
+ parser.add_option('-p', '--python',
+ type='string',
+ dest='python',
+ metavar='PYTHON_BIN',
+ default=None,
+ help='The Python interpreter to use.')
+ parser.add_option('--sync-passphrase',
+ type='string',
+ dest='sync_passphrase',
+ metavar='SYNC_ACCOUNT_PASSPHRASE',
+ default=None,
+ help='The old Firefox Sync account passphrase.')
+ parser.add_option('--sync-password',
+ type='string',
+ dest='sync_password',
+ metavar='SYNC_ACCOUNT_PASSWORD',
+ default=None,
+ help='The old Firefox Sync account password.')
+ parser.add_option('--sync-username',
+ type='string',
+ dest='sync_username',
+ metavar='SYNC_ACCOUNT_USERNAME',
+ default=None,
+ help='The old Firefox Sync account username.')
+ parser.add_option('--username',
+ type='string',
+ dest='username',
+ metavar='FX_ACCOUNT_USERNAME',
+ default=None,
+ help='The Firefox Account username.')
+
+ (options, args) = parser.parse_args(args=None, values=None)
+
+ if len(args) != 1:
+ parser.error('Path to the environment has to be specified')
+ target = args[0]
+ assert(target)
+
+ setup_virtualenv(target, python_bin=options.python)
+
+ # Activate tps environment
+ tps_env = os.path.join(target, activate_env)
+ execfile(tps_env, dict(__file__=tps_env))
+
+ # Install TPS in environment
+ subprocess.check_call([os.path.join(target, python_env),
+ os.path.join(here, 'setup.py'), 'install'])
+
+ # Get the path to tests and extensions directory by checking check where
+ # the tests and extensions directories are located
+ sync_dir = os.path.abspath(os.path.join(here, '..', '..', 'services',
+ 'sync'))
+ if os.path.exists(sync_dir):
+ testdir = os.path.join(sync_dir, 'tests', 'tps')
+ extdir = os.path.join(sync_dir, 'tps', 'extensions')
+ else:
+ testdir = os.path.join(here, 'tests')
+ extdir = os.path.join(here, 'extensions')
+
+ update_configfile(os.path.join(here, 'config', 'config.json.in'),
+ os.path.join(target, 'config.json'),
+ replacements={
+ '__TESTDIR__': testdir.replace('\\','/'),
+ '__EXTENSIONDIR__': extdir.replace('\\','/'),
+ '__FX_ACCOUNT_USERNAME__': options.username,
+ '__FX_ACCOUNT_PASSWORD__': options.password,
+ '__SYNC_ACCOUNT_USERNAME__': options.sync_username,
+ '__SYNC_ACCOUNT_PASSWORD__': options.sync_password,
+ '__SYNC_ACCOUNT_PASSPHRASE__': options.sync_passphrase})
+
+ if not (options.username and options.password):
+ print '\nFirefox Account credentials not specified.'
+ if not (options.sync_username and options.sync_password and options.passphrase):
+ print '\nFirefox Sync account credentials not specified.'
+
+ # Print the user instructions
+ print usage_message.format(TARGET=target,
+ BIN_NAME=bin_name)
+
+if __name__ == "__main__":
+ main()
diff --git a/testing/tps/pages/microsummary1.txt b/testing/tps/pages/microsummary1.txt
new file mode 100644
index 000000000..9451d4b5c
--- /dev/null
+++ b/testing/tps/pages/microsummary1.txt
@@ -0,0 +1 @@
+Static microsummary #1
diff --git a/testing/tps/pages/microsummary2.txt b/testing/tps/pages/microsummary2.txt
new file mode 100644
index 000000000..7e3208504
--- /dev/null
+++ b/testing/tps/pages/microsummary2.txt
@@ -0,0 +1 @@
+Static microsummary #2
diff --git a/testing/tps/pages/microsummary3.txt b/testing/tps/pages/microsummary3.txt
new file mode 100644
index 000000000..00ac62bbf
--- /dev/null
+++ b/testing/tps/pages/microsummary3.txt
@@ -0,0 +1 @@
+Static microsummary #3
diff --git a/testing/tps/pages/page1.html b/testing/tps/pages/page1.html
new file mode 100644
index 000000000..256c11c1c
--- /dev/null
+++ b/testing/tps/pages/page1.html
@@ -0,0 +1,15 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 1</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 1
+</p>
+</body>
+</html>
+
diff --git a/testing/tps/pages/page2.html b/testing/tps/pages/page2.html
new file mode 100644
index 000000000..62b693ed5
--- /dev/null
+++ b/testing/tps/pages/page2.html
@@ -0,0 +1,15 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 2</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 2
+</p>
+</body>
+</html>
+
diff --git a/testing/tps/pages/page3.html b/testing/tps/pages/page3.html
new file mode 100644
index 000000000..2d0a981de
--- /dev/null
+++ b/testing/tps/pages/page3.html
@@ -0,0 +1,15 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 3</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 3
+</p>
+</body>
+</html>
+
diff --git a/testing/tps/pages/page4.html b/testing/tps/pages/page4.html
new file mode 100644
index 000000000..b28f59d80
--- /dev/null
+++ b/testing/tps/pages/page4.html
@@ -0,0 +1,15 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 4</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 4
+</p>
+</body>
+</html>
+
diff --git a/testing/tps/pages/page5.html b/testing/tps/pages/page5.html
new file mode 100644
index 000000000..ffd4ce48b
--- /dev/null
+++ b/testing/tps/pages/page5.html
@@ -0,0 +1,15 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+<title>Crossweave Test Page 5</title>
+</head>
+<body>
+<p>
+Crossweave Test Page 5
+</p>
+</body>
+</html>
+
diff --git a/testing/tps/setup.py b/testing/tps/setup.py
new file mode 100644
index 000000000..19f109c10
--- /dev/null
+++ b/testing/tps/setup.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup, find_packages
+import sys
+
+version = '0.5'
+
+deps = ['httplib2 == 0.9.2',
+ 'mozfile == 1.2',
+ 'mozhttpd == 0.7',
+ 'mozinfo == 0.9',
+ 'mozinstall == 1.12',
+ 'mozprocess == 0.23',
+ 'mozprofile == 0.28',
+ 'mozrunner == 6.12',
+ 'mozversion == 1.4',
+ ]
+
+# we only support python 2.6+ right now
+assert sys.version_info[0] == 2
+assert sys.version_info[1] >= 6
+
+setup(name='tps',
+ version=version,
+ description='run automated multi-profile sync tests',
+ long_description="""\
+""",
+ classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+ keywords='',
+ author='Mozilla Automation and Tools team',
+ author_email='tools@lists.mozilla.org',
+ url='https://developer.mozilla.org/en-US/docs/TPS',
+ license='MPL 2.0',
+ packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+ include_package_data=True,
+ zip_safe=False,
+ install_requires=deps,
+ entry_points="""
+ # -*- Entry points: -*-
+ [console_scripts]
+ runtps = tps.cli:main
+ """,
+ data_files=[
+ ('tps', ['config/config.json.in']),
+ ],
+ )
diff --git a/testing/tps/tps/__init__.py b/testing/tps/tps/__init__.py
new file mode 100644
index 000000000..4433ee9b8
--- /dev/null
+++ b/testing/tps/tps/__init__.py
@@ -0,0 +1,6 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from . firefoxrunner import TPSFirefoxRunner
+from . testrunner import TPSTestRunner
diff --git a/testing/tps/tps/cli.py b/testing/tps/tps/cli.py
new file mode 100644
index 000000000..63e8db0d4
--- /dev/null
+++ b/testing/tps/tps/cli.py
@@ -0,0 +1,128 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import optparse
+import os
+import re
+import sys
+from threading import RLock
+
+from tps import TPSTestRunner
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option('--binary',
+ action='store',
+ type='string',
+ dest='binary',
+ default=None,
+ help='path to the Firefox binary, specified either as '
+ 'a local file or a url; if omitted, the PATH '
+ 'will be searched;')
+ parser.add_option('--configfile',
+ action='store',
+ type='string',
+ dest='configfile',
+ default=None,
+ help='path to the config file to use default: %default]')
+ parser.add_option('--debug',
+ action='store_true',
+ dest='debug',
+ default=False,
+ help='run in debug mode')
+ parser.add_option('--ignore-unused-engines',
+ default=False,
+ action='store_true',
+ dest='ignore_unused_engines',
+ help='If defined, do not load unused engines in individual tests.'
+ ' Has no effect for pulse monitor.')
+ parser.add_option('--logfile',
+ action='store',
+ type='string',
+ dest='logfile',
+ default='tps.log',
+ help='path to the log file [default: %default]')
+ parser.add_option('--mobile',
+ action='store_true',
+ dest='mobile',
+ default=False,
+ help='run with mobile settings')
+ parser.add_option('--pulsefile',
+ action='store',
+ type='string',
+ dest='pulsefile',
+ default=None,
+ help='path to file containing a pulse message in '
+ 'json format that you want to inject into the monitor')
+ parser.add_option('--resultfile',
+ action='store',
+ type='string',
+ dest='resultfile',
+ default='tps_result.json',
+ help='path to the result file [default: %default]')
+ parser.add_option('--testfile',
+ action='store',
+ type='string',
+ dest='testfile',
+ default='all_tests.json',
+ help='path to the test file to run [default: %default]')
+ parser.add_option('--stop-on-error',
+ action='store_true',
+ dest='stop_on_error',
+ help='stop running tests after the first failure')
+ (options, args) = parser.parse_args()
+
+ configfile = options.configfile
+ if configfile is None:
+ virtual_env = os.environ.get('VIRTUAL_ENV')
+ if virtual_env:
+ configfile = os.path.join(virtual_env, 'config.json')
+ if configfile is None or not os.access(configfile, os.F_OK):
+ raise Exception('Unable to find config.json in a VIRTUAL_ENV; you must '
+ 'specify a config file using the --configfile option')
+
+ # load the config file
+ f = open(configfile, 'r')
+ configcontent = f.read()
+ f.close()
+ config = json.loads(configcontent)
+ testfile = os.path.join(config.get('testdir', ''), options.testfile)
+
+ rlock = RLock()
+
+ print 'using result file', options.resultfile
+
+ extensionDir = config.get('extensiondir')
+ if not extensionDir or extensionDir == '__EXTENSIONDIR__':
+ extensionDir = os.path.join(os.getcwd(), '..', '..',
+ 'services', 'sync', 'tps', 'extensions')
+ else:
+ if sys.platform == 'win32':
+ # replace msys-style paths with proper Windows paths
+ m = re.match('^\/\w\/', extensionDir)
+ if m:
+ extensionDir = '%s:/%s' % (m.group(0)[1:2], extensionDir[3:])
+ extensionDir = extensionDir.replace('/', '\\')
+
+ TPS = TPSTestRunner(extensionDir,
+ binary=options.binary,
+ config=config,
+ debug=options.debug,
+ ignore_unused_engines=options.ignore_unused_engines,
+ logfile=options.logfile,
+ mobile=options.mobile,
+ resultfile=options.resultfile,
+ rlock=rlock,
+ testfile=testfile,
+ stop_on_error=options.stop_on_error,
+ )
+ TPS.run_tests()
+
+ if TPS.numfailed > 0 or TPS.numpassed == 0:
+ sys.exit(1)
+
+if __name__ == '__main__':
+ main()
diff --git a/testing/tps/tps/firefoxrunner.py b/testing/tps/tps/firefoxrunner.py
new file mode 100644
index 000000000..3b7c143d8
--- /dev/null
+++ b/testing/tps/tps/firefoxrunner.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import copy
+import httplib2
+import os
+
+import mozfile
+import mozinstall
+from mozprofile import Profile
+from mozrunner import FirefoxRunner
+
+
+class TPSFirefoxRunner(object):
+
+ PROCESS_TIMEOUT = 240
+
+ def __init__(self, binary):
+ if binary is not None and ('http://' in binary or 'ftp://' in binary):
+ self.url = binary
+ self.binary = None
+ else:
+ self.url = None
+ self.binary = binary
+
+ self.installdir = None
+
+ def __del__(self):
+ if self.installdir:
+ mozfile.remove(self.installdir, True)
+
+ def download_url(self, url, dest=None):
+ h = httplib2.Http()
+ resp, content = h.request(url, 'GET')
+ if dest == None:
+ dest = os.path.basename(url)
+
+ local = open(dest, 'wb')
+ local.write(content)
+ local.close()
+ return dest
+
+ def download_build(self, installdir='downloadedbuild', appname='firefox'):
+ self.installdir = os.path.abspath(installdir)
+ buildName = os.path.basename(self.url)
+ pathToBuild = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ buildName)
+
+ # delete the build if it already exists
+ if os.access(pathToBuild, os.F_OK):
+ os.remove(pathToBuild)
+
+ # download the build
+ print 'downloading build'
+ self.download_url(self.url, pathToBuild)
+
+ # install the build
+ print 'installing %s' % pathToBuild
+ mozfile.remove(self.installdir, True)
+ binary = mozinstall.install(src=pathToBuild, dest=self.installdir)
+
+ # remove the downloaded archive
+ os.remove(pathToBuild)
+
+ return binary
+
+ def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None):
+ """Runs the given FirefoxRunner with the given Profile, waits
+ for completion, then returns the process exit code
+ """
+ if profile is None:
+ profile = Profile()
+ self.profile = profile
+
+ if self.binary is None and self.url:
+ self.binary = self.download_build()
+
+ runner = FirefoxRunner(profile=self.profile, binary=self.binary,
+ env=env, cmdargs=args)
+
+ runner.start(timeout=timeout)
+ return runner.wait()
diff --git a/testing/tps/tps/phase.py b/testing/tps/tps/phase.py
new file mode 100644
index 000000000..397c90796
--- /dev/null
+++ b/testing/tps/tps/phase.py
@@ -0,0 +1,69 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+class TPSTestPhase(object):
+
+ lineRe = re.compile(
+ r'^(.*?)test phase (?P<matchphase>[^\s]+): (?P<matchstatus>.*)$')
+
+ def __init__(self, phase, profile, testname, testpath, logfile, env,
+ firefoxRunner, logfn, ignore_unused_engines=False):
+ self.phase = phase
+ self.profile = profile
+ self.testname = str(testname) # this might be passed in as unicode
+ self.testpath = testpath
+ self.logfile = logfile
+ self.env = env
+ self.firefoxRunner = firefoxRunner
+ self.log = logfn
+ self.ignore_unused_engines = ignore_unused_engines
+ self._status = None
+ self.errline = ''
+
+ @property
+ def status(self):
+ return self._status if self._status else 'unknown'
+
+ def run(self):
+ # launch Firefox
+ args = [ '-tps', self.testpath,
+ '-tpsphase', self.phase,
+ '-tpslogfile', self.logfile ]
+
+ if self.ignore_unused_engines:
+ args.append('--ignore-unused-engines')
+
+ self.log('\nLaunching Firefox for phase %s with args %s\n' %
+ (self.phase, str(args)))
+ self.firefoxRunner.run(env=self.env,
+ args=args,
+ profile=self.profile)
+
+ # parse the logfile and look for results from the current test phase
+ found_test = False
+ f = open(self.logfile, 'r')
+ for line in f:
+
+ # skip to the part of the log file that deals with the test we're running
+ if not found_test:
+ if line.find('Running test %s' % self.testname) > -1:
+ found_test = True
+ else:
+ continue
+
+ # look for the status of the current phase
+ match = self.lineRe.match(line)
+ if match:
+ if match.group('matchphase') == self.phase:
+ self._status = match.group('matchstatus')
+ break
+
+ # set the status to FAIL if there is TPS error
+ if line.find('CROSSWEAVE ERROR: ') > -1 and not self._status:
+ self._status = 'FAIL'
+ self.errline = line[line.find('CROSSWEAVE ERROR: ') + len('CROSSWEAVE ERROR: '):]
+
+ f.close()
diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py
new file mode 100644
index 000000000..8e5aee6c8
--- /dev/null
+++ b/testing/tps/tps/testrunner.py
@@ -0,0 +1,491 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import platform
+import random
+import re
+import tempfile
+import time
+import traceback
+
+from mozhttpd import MozHttpd
+import mozinfo
+from mozprofile import Profile
+import mozversion
+
+from .firefoxrunner import TPSFirefoxRunner
+from .phase import TPSTestPhase
+
+
+class TempFile(object):
+ """Class for temporary files that delete themselves when garbage-collected.
+ """
+
+ def __init__(self, prefix=None):
+ self.fd, self.filename = self.tmpfile = tempfile.mkstemp(prefix=prefix)
+
+ def write(self, data):
+ if self.fd:
+ os.write(self.fd, data)
+
+ def close(self):
+ if self.fd:
+ os.close(self.fd)
+ self.fd = None
+
+ def cleanup(self):
+ if self.fd:
+ self.close()
+ if os.access(self.filename, os.F_OK):
+ os.remove(self.filename)
+
+ __del__ = cleanup
+
+
+class TPSTestRunner(object):
+
+ extra_env = {
+ 'MOZ_CRASHREPORTER_DISABLE': '1',
+ 'GNOME_DISABLE_CRASH_DIALOG': '1',
+ 'XRE_NO_WINDOWS_CRASH_DIALOG': '1',
+ 'MOZ_NO_REMOTE': '1',
+ 'XPCOM_DEBUG_BREAK': 'warn',
+ }
+
+ default_preferences = {
+ 'app.update.enabled': False,
+ 'browser.dom.window.dump.enabled': True,
+ 'browser.sessionstore.resume_from_crash': False,
+ 'browser.shell.checkDefaultBrowser': False,
+ 'browser.tabs.warnOnClose': False,
+ 'browser.warnOnQuit': False,
+ # Allow installing extensions dropped into the profile folder
+ 'extensions.autoDisableScopes': 10,
+ 'extensions.getAddons.get.url': 'http://127.0.0.1:4567/addons/api/%IDS%.xml',
+ # Our pretend addons server doesn't support metadata...
+ 'extensions.getAddons.cache.enabled': False,
+ 'extensions.install.requireSecureOrigin': False,
+ 'extensions.update.enabled': False,
+ # Don't open a dialog to show available add-on updates
+ 'extensions.update.notifyUser': False,
+ 'services.sync.firstSync': 'notReady',
+ 'services.sync.lastversion': '1.0',
+ 'toolkit.startup.max_resumed_crashes': -1,
+ # hrm - not sure what the release/beta channels will do?
+ 'xpinstall.signatures.required': False,
+ }
+
+ debug_preferences = {
+ 'services.sync.log.appender.console': 'Trace',
+ 'services.sync.log.appender.dump': 'Trace',
+ 'services.sync.log.appender.file.level': 'Trace',
+ 'services.sync.log.appender.file.logOnSuccess': True,
+ 'services.sync.log.rootLogger': 'Trace',
+ 'services.sync.log.logger.addonutils': 'Trace',
+ 'services.sync.log.logger.declined': 'Trace',
+ 'services.sync.log.logger.service.main': 'Trace',
+ 'services.sync.log.logger.status': 'Trace',
+ 'services.sync.log.logger.authenticator': 'Trace',
+ 'services.sync.log.logger.network.resources': 'Trace',
+ 'services.sync.log.logger.service.jpakeclient': 'Trace',
+ 'services.sync.log.logger.engine.bookmarks': 'Trace',
+ 'services.sync.log.logger.engine.clients': 'Trace',
+ 'services.sync.log.logger.engine.forms': 'Trace',
+ 'services.sync.log.logger.engine.history': 'Trace',
+ 'services.sync.log.logger.engine.passwords': 'Trace',
+ 'services.sync.log.logger.engine.prefs': 'Trace',
+ 'services.sync.log.logger.engine.tabs': 'Trace',
+ 'services.sync.log.logger.engine.addons': 'Trace',
+ 'services.sync.log.logger.engine.apps': 'Trace',
+ 'services.sync.log.logger.identity': 'Trace',
+ 'services.sync.log.logger.userapi': 'Trace',
+ }
+
+ syncVerRe = re.compile(
+ r'Sync version: (?P<syncversion>.*)\n')
+ ffVerRe = re.compile(
+ r'Firefox version: (?P<ffver>.*)\n')
+ ffBuildIDRe = re.compile(
+ r'Firefox buildid: (?P<ffbuildid>.*)\n')
+
+ def __init__(self, extensionDir,
+ binary=None,
+ config=None,
+ debug=False,
+ ignore_unused_engines=False,
+ logfile='tps.log',
+ mobile=False,
+ rlock=None,
+ resultfile='tps_result.json',
+ testfile=None,
+ stop_on_error=False):
+ self.binary = binary
+ self.config = config if config else {}
+ self.debug = debug
+ self.extensions = []
+ self.ignore_unused_engines = ignore_unused_engines
+ self.logfile = os.path.abspath(logfile)
+ self.mobile = mobile
+ self.rlock = rlock
+ self.resultfile = resultfile
+ self.testfile = testfile
+ self.stop_on_error = stop_on_error
+
+ self.addonversion = None
+ self.branch = None
+ self.changeset = None
+ self.errorlogs = {}
+ self.extensionDir = extensionDir
+ self.firefoxRunner = None
+ self.nightly = False
+ self.numfailed = 0
+ self.numpassed = 0
+ self.postdata = {}
+ self.productversion = None
+ self.repo = None
+ self.tpsxpi = None
+
+ @property
+ def mobile(self):
+ return self._mobile
+
+ @mobile.setter
+ def mobile(self, value):
+ self._mobile = value
+ self.synctype = 'desktop' if not self._mobile else 'mobile'
+
+ def log(self, msg, printToConsole=False):
+ """Appends a string to the logfile"""
+
+ f = open(self.logfile, 'a')
+ f.write(msg)
+ f.close()
+ if printToConsole:
+ print msg
+
+ def writeToResultFile(self, postdata, body=None,
+ sendTo=['crossweave@mozilla.com']):
+ """Writes results to test file"""
+
+ results = {'results': []}
+
+ if os.access(self.resultfile, os.F_OK):
+ f = open(self.resultfile, 'r')
+ results = json.loads(f.read())
+ f.close()
+
+ f = open(self.resultfile, 'w')
+ if body is not None:
+ postdata['body'] = body
+ if self.numpassed is not None:
+ postdata['numpassed'] = self.numpassed
+ if self.numfailed is not None:
+ postdata['numfailed'] = self.numfailed
+ if self.firefoxRunner and self.firefoxRunner.url:
+ postdata['firefoxrunnerurl'] = self.firefoxRunner.url
+
+ postdata['sendTo'] = sendTo
+ results['results'].append(postdata)
+ f.write(json.dumps(results, indent=2))
+ f.close()
+
+ def _zip_add_file(self, zip, file, rootDir):
+ zip.write(os.path.join(rootDir, file), file)
+
+ def _zip_add_dir(self, zip, dir, rootDir):
+ try:
+ zip.write(os.path.join(rootDir, dir), dir)
+ except:
+ # on some OS's, adding directory entries doesn't seem to work
+ pass
+ for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
+ for f in files:
+ zip.write(os.path.join(root, f), os.path.join(dir, f))
+
+ def handle_phase_failure(self, profiles):
+ for profile in profiles:
+ self.log('\nDumping sync log for profile %s\n' % profiles[profile].profile)
+ for root, dirs, files in os.walk(os.path.join(profiles[profile].profile, 'weave', 'logs')):
+ for f in files:
+ weavelog = os.path.join(profiles[profile].profile, 'weave', 'logs', f)
+ if os.access(weavelog, os.F_OK):
+ with open(weavelog, 'r') as fh:
+ for line in fh:
+ possible_time = line[0:13]
+ if len(possible_time) == 13 and possible_time.isdigit():
+ time_ms = int(possible_time)
+ formatted = time.strftime('%Y-%m-%d %H:%M:%S',
+ time.localtime(time_ms / 1000))
+ self.log('%s.%03d %s' % (
+ formatted, time_ms % 1000, line[14:] ))
+ else:
+ self.log(line)
+
+ def run_single_test(self, testdir, testname):
+ testpath = os.path.join(testdir, testname)
+ self.log("Running test %s\n" % testname, True)
+
+ # Read and parse the test file, merge it with the contents of the config
+ # file, and write the combined output to a temporary file.
+ f = open(testpath, 'r')
+ testcontent = f.read()
+ f.close()
+ try:
+ test = json.loads(testcontent)
+ except:
+ test = json.loads(testcontent[testcontent.find('{'):testcontent.find('}') + 1])
+
+ self.preferences['tps.seconds_since_epoch'] = int(time.time())
+
+ # generate the profiles defined in the test, and a list of test phases
+ profiles = {}
+ phaselist = []
+ for phase in test:
+ profilename = test[phase]
+
+ # create the profile if necessary
+ if not profilename in profiles:
+ profiles[profilename] = Profile(preferences = self.preferences,
+ addons = self.extensions)
+
+ # create the test phase
+ phaselist.append(TPSTestPhase(
+ phase,
+ profiles[profilename],
+ testname,
+ testpath,
+ self.logfile,
+ self.env,
+ self.firefoxRunner,
+ self.log,
+ ignore_unused_engines=self.ignore_unused_engines))
+
+ # sort the phase list by name
+ phaselist = sorted(phaselist, key=lambda phase: phase.phase)
+
+ # run each phase in sequence, aborting at the first failure
+ failed = False
+ for phase in phaselist:
+ phase.run()
+ if phase.status != 'PASS':
+ failed = True
+ break;
+
+ for profilename in profiles:
+ cleanup_phase = TPSTestPhase(
+ 'cleanup-' + profilename,
+ profiles[profilename], testname,
+ testpath,
+ self.logfile,
+ self.env,
+ self.firefoxRunner,
+ self.log)
+
+ cleanup_phase.run()
+ if cleanup_phase.status != 'PASS':
+ failed = True
+ # Keep going to run the remaining cleanup phases.
+
+ if failed:
+ self.handle_phase_failure(profiles)
+
+ # grep the log for FF and sync versions
+ f = open(self.logfile)
+ logdata = f.read()
+ match = self.syncVerRe.search(logdata)
+ sync_version = match.group('syncversion') if match else 'unknown'
+ match = self.ffVerRe.search(logdata)
+ firefox_version = match.group('ffver') if match else 'unknown'
+ match = self.ffBuildIDRe.search(logdata)
+ firefox_buildid = match.group('ffbuildid') if match else 'unknown'
+ f.close()
+ if phase.status == 'PASS':
+ logdata = ''
+ else:
+ # we only care about the log data for this specific test
+ logdata = logdata[logdata.find('Running test %s' % (str(testname))):]
+
+ result = {
+ 'PASS': lambda x: ('TEST-PASS', ''),
+ 'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()),
+ 'unknown': lambda x: ('TEST-UNEXPECTED-FAIL', 'test did not complete')
+ } [phase.status](phase.errline)
+ logstr = "\n%s | %s%s\n" % (result[0], testname, (' | %s' % result[1] if result[1] else ''))
+
+ try:
+ repoinfo = mozversion.get_version(self.binary)
+ except:
+ repoinfo = {}
+ apprepo = repoinfo.get('application_repository', '')
+ appchangeset = repoinfo.get('application_changeset', '')
+
+ # save logdata to a temporary file for posting to the db
+ tmplogfile = None
+ if logdata:
+ tmplogfile = TempFile(prefix='tps_log_')
+ tmplogfile.write(logdata)
+ tmplogfile.close()
+ self.errorlogs[testname] = tmplogfile
+
+ resultdata = ({ 'productversion': { 'version': firefox_version,
+ 'buildid': firefox_buildid,
+ 'builddate': firefox_buildid[0:8],
+ 'product': 'Firefox',
+ 'repository': apprepo,
+ 'changeset': appchangeset,
+ },
+ 'addonversion': { 'version': sync_version,
+ 'product': 'Firefox Sync' },
+ 'name': testname,
+ 'message': result[1],
+ 'state': result[0],
+ 'logdata': logdata
+ })
+
+ self.log(logstr, True)
+ for phase in phaselist:
+ print "\t%s: %s" % (phase.phase, phase.status)
+
+ return resultdata
+
+ def update_preferences(self):
+ self.preferences = self.default_preferences.copy()
+
+ if self.mobile:
+ self.preferences.update({'services.sync.client.type' : 'mobile'})
+
+ # If we are using legacy Sync, then set a dummy username to force the
+ # correct authentication type. Without this pref set to a value
+ # without an '@' character, Sync will initialize for FxA.
+ if self.config.get('auth_type', 'fx_account') != "fx_account":
+ self.preferences.update({'services.sync.username': "dummy"})
+
+ if self.debug:
+ self.preferences.update(self.debug_preferences)
+
+ if 'preferences' in self.config:
+ self.preferences.update(self.config['preferences'])
+
+ self.preferences['tps.config'] = json.dumps(self.config)
+
+ def run_tests(self):
+ # delete the logfile if it already exists
+ if os.access(self.logfile, os.F_OK):
+ os.remove(self.logfile)
+
+ # Copy the system env variables, and update them for custom settings
+ self.env = os.environ.copy()
+ self.env.update(self.extra_env)
+
+ # Update preferences for custom settings
+ self.update_preferences()
+
+ # Acquire a lock to make sure no other threads are running tests
+ # at the same time.
+ if self.rlock:
+ self.rlock.acquire()
+
+ try:
+ # Create the Firefox runner, which will download and install the
+ # build, as needed.
+ if not self.firefoxRunner:
+ self.firefoxRunner = TPSFirefoxRunner(self.binary)
+
+ # now, run the test group
+ self.run_test_group()
+
+ except:
+ traceback.print_exc()
+ self.numpassed = 0
+ self.numfailed = 1
+ try:
+ self.writeToResultFile(self.postdata,
+ '<pre>%s</pre>' % traceback.format_exc())
+ except:
+ traceback.print_exc()
+ else:
+ try:
+
+ if self.numfailed > 0 or self.numpassed == 0:
+ To = self.config['email'].get('notificationlist')
+ else:
+ To = self.config['email'].get('passednotificationlist')
+ self.writeToResultFile(self.postdata,
+ sendTo=To)
+ except:
+ traceback.print_exc()
+ try:
+ self.writeToResultFile(self.postdata,
+ '<pre>%s</pre>' % traceback.format_exc())
+ except:
+ traceback.print_exc()
+
+ # release our lock
+ if self.rlock:
+ self.rlock.release()
+
+ # dump out a summary of test results
+ print 'Test Summary\n'
+ for test in self.postdata.get('tests', {}):
+ print '%s | %s | %s' % (test['state'], test['name'], test['message'])
+
+ def run_test_group(self):
+ self.results = []
+
+ # reset number of passed/failed tests
+ self.numpassed = 0
+ self.numfailed = 0
+
+ # build our tps.xpi extension
+ self.extensions = []
+ self.extensions.append(os.path.join(self.extensionDir, 'tps'))
+ self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
+
+ # build the test list
+ try:
+ f = open(self.testfile)
+ jsondata = f.read()
+ f.close()
+ testfiles = json.loads(jsondata)
+ testlist = testfiles['tests']
+ except ValueError:
+ testlist = [os.path.basename(self.testfile)]
+ testdir = os.path.dirname(self.testfile)
+
+ self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
+ self.mozhttpd.start()
+
+ # run each test, and save the results
+ for test in testlist:
+ result = self.run_single_test(testdir, test)
+
+ if not self.productversion:
+ self.productversion = result['productversion']
+ if not self.addonversion:
+ self.addonversion = result['addonversion']
+
+ self.results.append({'state': result['state'],
+ 'name': result['name'],
+ 'message': result['message'],
+ 'logdata': result['logdata']})
+ if result['state'] == 'TEST-PASS':
+ self.numpassed += 1
+ else:
+ self.numfailed += 1
+ if self.stop_on_error:
+ print '\nTest failed with --stop-on-error specified; not running any more tests.\n'
+ break
+
+ self.mozhttpd.stop()
+
+ # generate the postdata we'll use to post the results to the db
+ self.postdata = { 'tests': self.results,
+ 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits),
+ 'testtype': 'crossweave',
+ 'productversion': self.productversion,
+ 'addonversion': self.addonversion,
+ 'synctype': self.synctype,
+ }