summaryrefslogtreecommitdiffstats
path: root/testing/mochitest/runrobocop.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/mochitest/runrobocop.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/mochitest/runrobocop.py')
-rw-r--r--testing/mochitest/runrobocop.py587
1 files changed, 587 insertions, 0 deletions
diff --git a/testing/mochitest/runrobocop.py b/testing/mochitest/runrobocop.py
new file mode 100644
index 000000000..c95964e8d
--- /dev/null
+++ b/testing/mochitest/runrobocop.py
@@ -0,0 +1,587 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import shutil
+import sys
+import tempfile
+import traceback
+
+sys.path.insert(
+ 0, os.path.abspath(
+ os.path.realpath(
+ os.path.dirname(__file__))))
+
+from automation import Automation
+from remoteautomation import RemoteAutomation, fennecLogcatFilters
+from runtests import KeyValueParseError, MochitestDesktop, MessageLogger, parseKeyValue
+from mochitest_options import MochitestArgumentParser
+
+from manifestparser import TestManifest
+from manifestparser.filters import chunk_by_slice
+import mozdevice
+import mozinfo
+
+SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+
+
+class RobocopTestRunner(MochitestDesktop):
+ """
+ A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
+ based on the Robotium test framework. This harness leverages some functionality
+ from mochitest, for convenience.
+ """
+ auto = None
+ dm = None
+ # Some robocop tests run for >60 seconds without generating any output.
+ NO_OUTPUT_TIMEOUT = 180
+
+ def __init__(self, automation, devmgr, options):
+ """
+ Simple one-time initialization.
+ """
+ MochitestDesktop.__init__(self, options)
+
+ self.auto = automation
+ self.dm = devmgr
+ self.dm.default_timeout = 320
+ self.options = options
+ self.options.logFile = "robocop.log"
+ self.environment = self.auto.environment
+ self.deviceRoot = self.dm.getDeviceRoot()
+ self.remoteProfile = options.remoteTestRoot + "/profile"
+ self.remoteProfileCopy = options.remoteTestRoot + "/profile-copy"
+ self.auto.setRemoteProfile(self.remoteProfile)
+ self.remoteConfigFile = os.path.join(
+ self.deviceRoot, "robotium.config")
+ self.remoteLog = options.remoteLogFile
+ self.auto.setRemoteLog(self.remoteLog)
+ self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
+ self.remoteMozLog = os.path.join(options.remoteTestRoot, "mozlog")
+ self.auto.setServerInfo(
+ self.options.webServer, self.options.httpPort, self.options.sslPort)
+ self.localLog = options.logFile
+ self.localProfile = None
+ productPieces = self.options.remoteProductName.split('.')
+ if (productPieces is not None):
+ self.auto.setProduct(productPieces[0])
+ else:
+ self.auto.setProduct(self.options.remoteProductName)
+ self.auto.setAppName(self.options.remoteappname)
+ self.certdbNew = True
+ self.remoteCopyAvailable = True
+ self.passed = 0
+ self.failed = 0
+ self.todo = 0
+
+ def startup(self):
+ """
+ Second-stage initialization: One-time initialization which may require cleanup.
+ """
+ # Despite our efforts to clean up servers started by this script, in practice
+ # we still see infrequent cases where a process is orphaned and interferes
+ # with future tests, typically because the old server is keeping the port in use.
+ # Try to avoid those failures by checking for and killing orphan servers before
+ # trying to start new ones.
+ self.killNamedOrphans('ssltunnel')
+ self.killNamedOrphans('xpcshell')
+ self.auto.deleteANRs()
+ self.auto.deleteTombstones()
+ self.dm.killProcess(self.options.app.split('/')[-1])
+ self.dm.removeDir(self.remoteScreenshots)
+ self.dm.removeDir(self.remoteMozLog)
+ self.dm.mkDir(self.remoteMozLog)
+ self.dm.mkDir(os.path.dirname(self.options.remoteLogFile))
+ # Add Android version (SDK level) to mozinfo so that manifest entries
+ # can be conditional on android_version.
+ androidVersion = self.dm.shellCheckOutput(
+ ['getprop', 'ro.build.version.sdk'])
+ self.log.info(
+ "Android sdk version '%s'; will use this to filter manifests" %
+ str(androidVersion))
+ mozinfo.info['android_version'] = androidVersion
+ if (self.options.dm_trans == 'adb' and self.options.robocopApk):
+ self.dm._checkCmd(["install", "-r", self.options.robocopApk])
+ self.log.debug("Robocop APK %s installed" %
+ self.options.robocopApk)
+ # Display remote diagnostics; if running in mach, keep output terse.
+ if self.options.log_mach is None:
+ self.printDeviceInfo()
+ self.setupLocalPaths()
+ self.buildProfile()
+ # ignoreSSLTunnelExts is a workaround for bug 1109310
+ self.startServers(
+ self.options,
+ debuggerInfo=None,
+ ignoreSSLTunnelExts=True)
+ self.log.debug("Servers started")
+
+ def cleanup(self):
+ """
+ Cleanup at end of job run.
+ """
+ self.log.debug("Cleaning up...")
+ self.stopServers()
+ self.dm.killProcess(self.options.app.split('/')[-1])
+ blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
+ if blobberUploadDir:
+ self.log.debug("Pulling any remote moz logs and screenshots to %s." %
+ blobberUploadDir)
+ self.dm.getDirectory(self.remoteMozLog, blobberUploadDir)
+ self.dm.getDirectory(self.remoteScreenshots, blobberUploadDir)
+ MochitestDesktop.cleanup(self, self.options)
+ if self.localProfile:
+ os.system("rm -Rf %s" % self.localProfile)
+ self.dm.removeDir(self.remoteProfile)
+ self.dm.removeDir(self.remoteProfileCopy)
+ self.dm.removeDir(self.remoteScreenshots)
+ self.dm.removeDir(self.remoteMozLog)
+ self.dm.removeFile(self.remoteConfigFile)
+ if self.dm.fileExists(self.remoteLog):
+ self.dm.removeFile(self.remoteLog)
+ self.log.debug("Cleanup complete.")
+
+ def findPath(self, paths, filename=None):
+ for path in paths:
+ p = path
+ if filename:
+ p = os.path.join(p, filename)
+ if os.path.exists(self.getFullPath(p)):
+ return path
+ return None
+
+ def makeLocalAutomation(self):
+ localAutomation = Automation()
+ localAutomation.IS_WIN32 = False
+ localAutomation.IS_LINUX = False
+ localAutomation.IS_MAC = False
+ localAutomation.UNIXISH = False
+ hostos = sys.platform
+ if (hostos == 'mac' or hostos == 'darwin'):
+ localAutomation.IS_MAC = True
+ elif (hostos == 'linux' or hostos == 'linux2'):
+ localAutomation.IS_LINUX = True
+ localAutomation.UNIXISH = True
+ elif (hostos == 'win32' or hostos == 'win64'):
+ localAutomation.BIN_SUFFIX = ".exe"
+ localAutomation.IS_WIN32 = True
+ return localAutomation
+
+ def setupLocalPaths(self):
+ """
+ Setup xrePath and utilityPath and verify xpcshell.
+
+ This is similar to switchToLocalPaths in runtestsremote.py.
+ """
+ localAutomation = self.makeLocalAutomation()
+ paths = [
+ self.options.xrePath,
+ localAutomation.DIST_BIN,
+ self.auto._product,
+ os.path.join('..', self.auto._product)
+ ]
+ self.options.xrePath = self.findPath(paths)
+ if self.options.xrePath is None:
+ self.log.error(
+ "unable to find xulrunner path for %s, please specify with --xre-path" %
+ os.name)
+ sys.exit(1)
+ self.log.debug("using xre path %s" % self.options.xrePath)
+ xpcshell = "xpcshell"
+ if (os.name == "nt"):
+ xpcshell += ".exe"
+ if self.options.utilityPath:
+ paths = [self.options.utilityPath, self.options.xrePath]
+ else:
+ paths = [self.options.xrePath]
+ self.options.utilityPath = self.findPath(paths, xpcshell)
+ if self.options.utilityPath is None:
+ self.log.error(
+ "unable to find utility path for %s, please specify with --utility-path" %
+ os.name)
+ sys.exit(1)
+ self.log.debug("using utility path %s" % self.options.utilityPath)
+ xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
+ if localAutomation.elf_arm(xpcshell_path):
+ self.log.error('xpcshell at %s is an ARM binary; please use '
+ 'the --utility-path argument to specify the path '
+ 'to a desktop version.' % xpcshell_path)
+ sys.exit(1)
+ self.log.debug("xpcshell found at %s" % xpcshell_path)
+
+ def buildProfile(self):
+ """
+ Build a profile locally, keep it locally for use by servers and
+ push a copy to the remote profile-copy directory.
+
+ This is similar to buildProfile in runtestsremote.py.
+ """
+ self.options.extraPrefs.append('browser.search.suggest.enabled=true')
+ self.options.extraPrefs.append('browser.search.suggest.prompted=true')
+ self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
+ self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
+ self.options.extraPrefs.append('browser.snippets.enabled=false')
+ self.options.extraPrefs.append('browser.casting.enabled=true')
+ self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
+
+ # Override the telemetry init delay for integration testing.
+ self.options.extraPrefs.append('toolkit.telemetry.initDelay=1')
+
+ self.options.extensionsToExclude.extend([
+ 'mochikit@mozilla.org',
+ 'worker-test@mozilla.org.xpi',
+ 'workerbootstrap-test@mozilla.org.xpi',
+ 'indexedDB-test@mozilla.org.xpi',
+ ])
+
+ manifest = MochitestDesktop.buildProfile(self, self.options)
+ self.localProfile = self.options.profilePath
+ self.log.debug("Profile created at %s" % self.localProfile)
+ # some files are not needed for robocop; save time by not pushing
+ shutil.rmtree(os.path.join(self.localProfile, 'webapps'))
+ os.remove(os.path.join(self.localProfile, 'userChrome.css'))
+ try:
+ self.dm.pushDir(self.localProfile, self.remoteProfileCopy)
+ except mozdevice.DMError:
+ self.log.error(
+ "Automation Error: Unable to copy profile to device.")
+ raise
+
+ return manifest
+
+ def setupRemoteProfile(self):
+ """
+ Remove any remote profile and re-create it.
+ """
+ self.log.debug("Updating remote profile at %s" % self.remoteProfile)
+ self.dm.removeDir(self.remoteProfile)
+ if self.remoteCopyAvailable:
+ try:
+ self.dm.shellCheckOutput(
+ ['cp', '-r', self.remoteProfileCopy, self.remoteProfile],
+ root=True, timeout=60)
+ except mozdevice.DMError:
+ # For instance, cp is not available on some older versions of
+ # Android.
+ self.log.info(
+ "Unable to copy remote profile; falling back to push.")
+ self.remoteCopyAvailable = False
+ if not self.remoteCopyAvailable:
+ self.dm.pushDir(self.localProfile, self.remoteProfile)
+
+ def parseLocalLog(self):
+ """
+ Read and parse the local log file, noting any failures.
+ """
+ with open(self.localLog) as currentLog:
+ data = currentLog.readlines()
+ os.unlink(self.localLog)
+ start_found = False
+ end_found = False
+ fail_found = False
+ for line in data:
+ try:
+ message = json.loads(line)
+ if not isinstance(message, dict) or 'action' not in message:
+ continue
+ except ValueError:
+ continue
+ if message['action'] == 'test_end':
+ end_found = True
+ start_found = False
+ break
+ if start_found and not end_found:
+ if 'status' in message:
+ if 'expected' in message:
+ self.failed += 1
+ elif message['status'] == 'PASS':
+ self.passed += 1
+ elif message['status'] == 'FAIL':
+ self.todo += 1
+ if message['action'] == 'test_start':
+ start_found = True
+ if 'expected' in message:
+ fail_found = True
+ result = 0
+ if fail_found:
+ result = 1
+ if not end_found:
+ self.log.info(
+ "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
+ result = 1
+ return result
+
+ def logTestSummary(self):
+ """
+ Print a summary of all tests run to stdout, for treeherder parsing
+ (logging via self.log does not work here).
+ """
+ print("0 INFO TEST-START | Shutdown")
+ print("1 INFO Passed: %s" % (self.passed))
+ print("2 INFO Failed: %s" % (self.failed))
+ print("3 INFO Todo: %s" % (self.todo))
+ print("4 INFO SimpleTest FINISHED")
+ if self.failed > 0:
+ return 1
+ return 0
+
+ def printDeviceInfo(self, printLogcat=False):
+ """
+ Log remote device information and logcat (if requested).
+
+ This is similar to printDeviceInfo in runtestsremote.py
+ """
+ try:
+ if printLogcat:
+ logcat = self.dm.getLogcat(
+ filterOutRegexps=fennecLogcatFilters)
+ self.log.info(
+ '\n' +
+ ''.join(logcat).decode(
+ 'utf-8',
+ 'replace'))
+ self.log.info("Device info:")
+ devinfo = self.dm.getInfo()
+ for category in devinfo:
+ if type(devinfo[category]) is list:
+ self.log.info(" %s:" % category)
+ for item in devinfo[category]:
+ self.log.info(" %s" % item)
+ else:
+ self.log.info(" %s: %s" % (category, devinfo[category]))
+ self.log.info("Test root: %s" % self.dm.deviceRoot)
+ except mozdevice.DMError:
+ self.log.warning("Error getting device information")
+
+ def setupRobotiumConfig(self, browserEnv):
+ """
+ Create robotium.config and push it to the device.
+ """
+ fHandle = tempfile.NamedTemporaryFile(suffix='.config',
+ prefix='robotium-',
+ dir=os.getcwd(),
+ delete=False)
+ fHandle.write("profile=%s\n" % (self.remoteProfile))
+ fHandle.write("logfile=%s\n" % (self.options.remoteLogFile))
+ fHandle.write("host=http://mochi.test:8888/tests\n")
+ fHandle.write(
+ "rawhost=http://%s:%s/tests\n" %
+ (self.options.remoteWebServer, self.options.httpPort))
+ if browserEnv:
+ envstr = ""
+ delim = ""
+ for key, value in browserEnv.items():
+ try:
+ value.index(',')
+ self.log.error("setupRobotiumConfig: browserEnv - Found a ',' "
+ "in our value, unable to process value. key=%s,value=%s" %
+ (key, value))
+ self.log.error("browserEnv=%s" % browserEnv)
+ except ValueError:
+ envstr += "%s%s=%s" % (delim, key, value)
+ delim = ","
+ fHandle.write("envvars=%s\n" % envstr)
+ fHandle.close()
+ self.dm.removeFile(self.remoteConfigFile)
+ self.dm.pushFile(fHandle.name, self.remoteConfigFile)
+ os.unlink(fHandle.name)
+
+ def buildBrowserEnv(self):
+ """
+ Return an environment dictionary suitable for remote use.
+
+ This is similar to buildBrowserEnv in runtestsremote.py.
+ """
+ browserEnv = self.environment(
+ xrePath=None,
+ debugger=None)
+ # remove desktop environment not used on device
+ if "MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA" in browserEnv:
+ del browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"]
+ if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
+ del browserEnv["XPCOM_MEM_BLOAT_LOG"]
+ browserEnv["MOZ_LOG_FILE"] = os.path.join(
+ self.remoteMozLog,
+ self.mozLogName)
+
+ try:
+ browserEnv.update(
+ dict(
+ parseKeyValue(
+ self.options.environment,
+ context='--setenv')))
+ except KeyValueParseError as e:
+ self.log.error(str(e))
+ return None
+
+ return browserEnv
+
+ def runSingleTest(self, test):
+ """
+ Run the specified test.
+ """
+ self.log.debug("Running test %s" % test['name'])
+ self.mozLogName = "moz-%s.log" % test['name']
+ browserEnv = self.buildBrowserEnv()
+ self.setupRobotiumConfig(browserEnv)
+ self.setupRemoteProfile()
+ self.options.app = "am"
+ if self.options.autorun:
+ # This launches a test (using "am instrument") and instructs
+ # Fennec to /quit/ the browser (using Robocop:Quit) and to
+ # /finish/ all opened activities.
+ browserArgs = [
+ "instrument",
+ "-w",
+ "-e", "quit_and_finish", "1",
+ "-e", "deviceroot", self.deviceRoot,
+ "-e", "class",
+ "org.mozilla.gecko.tests.%s" % test['name'].split('/')[-1].split('.java')[0],
+ "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"]
+ else:
+ # This does not launch a test at all. It launches an activity
+ # that starts Fennec and then waits indefinitely, since cat
+ # never returns.
+ browserArgs = ["start", "-n",
+ "org.mozilla.roboexample.test/org.mozilla."
+ "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat"]
+ self.dm.default_timeout = sys.maxint # Forever.
+ self.log.info("")
+ self.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
+ (self.options.remoteWebServer, self.options.httpPort))
+ self.log.info("")
+ result = -1
+ log_result = -1
+ try:
+ self.dm.recordLogcat()
+ timeout = self.options.timeout
+ if not timeout:
+ timeout = self.NO_OUTPUT_TIMEOUT
+ result = self.auto.runApp(
+ None, browserEnv, "am", self.localProfile, browserArgs,
+ timeout=timeout, symbolsPath=self.options.symbolsPath)
+ self.log.debug("runApp completes with status %d" % result)
+ if result != 0:
+ self.log.error("runApp() exited with code %s" % result)
+ if self.dm.fileExists(self.remoteLog):
+ self.dm.getFile(self.remoteLog, self.localLog)
+ self.dm.removeFile(self.remoteLog)
+ self.log.debug("Remote log %s retrieved to %s" %
+ (self.remoteLog, self.localLog))
+ else:
+ self.log.warning(
+ "Unable to retrieve log file (%s) from remote device" %
+ self.remoteLog)
+ log_result = self.parseLocalLog()
+ if result != 0 or log_result != 0:
+ # Display remote diagnostics; if running in mach, keep output
+ # terse.
+ if self.options.log_mach is None:
+ self.printDeviceInfo(printLogcat=True)
+ except:
+ self.log.error(
+ "Automation Error: Exception caught while running tests")
+ traceback.print_exc()
+ result = 1
+ self.log.debug("Test %s completes with status %d (log status %d)" %
+ (test['name'], int(result), int(log_result)))
+ return result
+
+ def runTests(self):
+ self.startup()
+ if isinstance(self.options.manifestFile, TestManifest):
+ mp = self.options.manifestFile
+ else:
+ mp = TestManifest(strict=False)
+ mp.read(self.options.robocopIni)
+ filters = []
+ if self.options.totalChunks:
+ filters.append(
+ chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
+ robocop_tests = mp.active_tests(
+ exists=False, filters=filters, **mozinfo.info)
+ if not self.options.autorun:
+ # Force a single loop iteration. The iteration will start Fennec and
+ # the httpd server, but not actually run a test.
+ self.options.test_paths = [robocop_tests[0]['name']]
+ active_tests = []
+ for test in robocop_tests:
+ if self.options.test_paths and test['name'] not in self.options.test_paths:
+ continue
+ if 'disabled' in test:
+ self.log.info('TEST-INFO | skipping %s | %s' %
+ (test['name'], test['disabled']))
+ continue
+ active_tests.append(test)
+ self.log.suite_start([t['name'] for t in active_tests])
+ worstTestResult = None
+ for test in active_tests:
+ result = self.runSingleTest(test)
+ if worstTestResult is None or worstTestResult == 0:
+ worstTestResult = result
+ if worstTestResult is None:
+ self.log.warning(
+ "No tests run. Did you pass an invalid TEST_PATH?")
+ worstTestResult = 1
+ else:
+ print "INFO | runtests.py | Test summary: start."
+ logResult = self.logTestSummary()
+ print "INFO | runtests.py | Test summary: end."
+ if worstTestResult == 0:
+ worstTestResult = logResult
+ return worstTestResult
+
+
+def run_test_harness(parser, options):
+ parser.validate(options)
+
+ if options is None:
+ raise ValueError(
+ "Invalid options specified, use --help for a list of valid options")
+ message_logger = MessageLogger(logger=None)
+ process_args = {'messageLogger': message_logger}
+ auto = RemoteAutomation(None, "fennec", processArgs=process_args)
+ auto.setDeviceManager(options.dm)
+ runResult = -1
+ robocop = RobocopTestRunner(auto, options.dm, options)
+
+ # Check that Firefox is installed
+ expected = options.app.split('/')[-1]
+ installed = options.dm.shellCheckOutput(['pm', 'list', 'packages', expected])
+ if expected not in installed:
+ robocop.log.error("%s is not installed on this device" % expected)
+ return 1
+
+ try:
+ message_logger.logger = robocop.log
+ message_logger.buffering = False
+ robocop.message_logger = message_logger
+ robocop.log.debug("options=%s" % vars(options))
+ runResult = robocop.runTests()
+ except KeyboardInterrupt:
+ robocop.log.info("runrobocop.py | Received keyboard interrupt")
+ runResult = -1
+ except:
+ traceback.print_exc()
+ robocop.log.error(
+ "runrobocop.py | Received unexpected exception while running tests")
+ runResult = 1
+ finally:
+ try:
+ robocop.cleanup()
+ except mozdevice.DMError:
+ # ignore device error while cleaning up
+ pass
+ message_logger.finish()
+ return runResult
+
+
+def main(args=sys.argv[1:]):
+ parser = MochitestArgumentParser(app='android')
+ options = parser.parse_args(args)
+ return run_test_harness(parser, options)
+
+if __name__ == "__main__":
+ sys.exit(main())