diff options
Diffstat (limited to 'testing/mochitest/runrobocop.py')
| -rw-r--r-- | testing/mochitest/runrobocop.py | 587 |
1 files changed, 587 insertions, 0 deletions
diff --git a/testing/mochitest/runrobocop.py b/testing/mochitest/runrobocop.py new file mode 100644 index 000000000..c95964e8d --- /dev/null +++ b/testing/mochitest/runrobocop.py @@ -0,0 +1,587 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import shutil +import sys +import tempfile +import traceback + +sys.path.insert( + 0, os.path.abspath( + os.path.realpath( + os.path.dirname(__file__)))) + +from automation import Automation +from remoteautomation import RemoteAutomation, fennecLogcatFilters +from runtests import KeyValueParseError, MochitestDesktop, MessageLogger, parseKeyValue +from mochitest_options import MochitestArgumentParser + +from manifestparser import TestManifest +from manifestparser.filters import chunk_by_slice +import mozdevice +import mozinfo + +SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) + + +class RobocopTestRunner(MochitestDesktop): + """ + A test harness for Robocop. Robocop tests are UI tests for Firefox for Android, + based on the Robotium test framework. This harness leverages some functionality + from mochitest, for convenience. + """ + auto = None + dm = None + # Some robocop tests run for >60 seconds without generating any output. + NO_OUTPUT_TIMEOUT = 180 + + def __init__(self, automation, devmgr, options): + """ + Simple one-time initialization. + """ + MochitestDesktop.__init__(self, options) + + self.auto = automation + self.dm = devmgr + self.dm.default_timeout = 320 + self.options = options + self.options.logFile = "robocop.log" + self.environment = self.auto.environment + self.deviceRoot = self.dm.getDeviceRoot() + self.remoteProfile = options.remoteTestRoot + "/profile" + self.remoteProfileCopy = options.remoteTestRoot + "/profile-copy" + self.auto.setRemoteProfile(self.remoteProfile) + self.remoteConfigFile = os.path.join( + self.deviceRoot, "robotium.config") + self.remoteLog = options.remoteLogFile + self.auto.setRemoteLog(self.remoteLog) + self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots" + self.remoteMozLog = os.path.join(options.remoteTestRoot, "mozlog") + self.auto.setServerInfo( + self.options.webServer, self.options.httpPort, self.options.sslPort) + self.localLog = options.logFile + self.localProfile = None + productPieces = self.options.remoteProductName.split('.') + if (productPieces is not None): + self.auto.setProduct(productPieces[0]) + else: + self.auto.setProduct(self.options.remoteProductName) + self.auto.setAppName(self.options.remoteappname) + self.certdbNew = True + self.remoteCopyAvailable = True + self.passed = 0 + self.failed = 0 + self.todo = 0 + + def startup(self): + """ + Second-stage initialization: One-time initialization which may require cleanup. + """ + # Despite our efforts to clean up servers started by this script, in practice + # we still see infrequent cases where a process is orphaned and interferes + # with future tests, typically because the old server is keeping the port in use. + # Try to avoid those failures by checking for and killing orphan servers before + # trying to start new ones. + self.killNamedOrphans('ssltunnel') + self.killNamedOrphans('xpcshell') + self.auto.deleteANRs() + self.auto.deleteTombstones() + self.dm.killProcess(self.options.app.split('/')[-1]) + self.dm.removeDir(self.remoteScreenshots) + self.dm.removeDir(self.remoteMozLog) + self.dm.mkDir(self.remoteMozLog) + self.dm.mkDir(os.path.dirname(self.options.remoteLogFile)) + # Add Android version (SDK level) to mozinfo so that manifest entries + # can be conditional on android_version. + androidVersion = self.dm.shellCheckOutput( + ['getprop', 'ro.build.version.sdk']) + self.log.info( + "Android sdk version '%s'; will use this to filter manifests" % + str(androidVersion)) + mozinfo.info['android_version'] = androidVersion + if (self.options.dm_trans == 'adb' and self.options.robocopApk): + self.dm._checkCmd(["install", "-r", self.options.robocopApk]) + self.log.debug("Robocop APK %s installed" % + self.options.robocopApk) + # Display remote diagnostics; if running in mach, keep output terse. + if self.options.log_mach is None: + self.printDeviceInfo() + self.setupLocalPaths() + self.buildProfile() + # ignoreSSLTunnelExts is a workaround for bug 1109310 + self.startServers( + self.options, + debuggerInfo=None, + ignoreSSLTunnelExts=True) + self.log.debug("Servers started") + + def cleanup(self): + """ + Cleanup at end of job run. + """ + self.log.debug("Cleaning up...") + self.stopServers() + self.dm.killProcess(self.options.app.split('/')[-1]) + blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) + if blobberUploadDir: + self.log.debug("Pulling any remote moz logs and screenshots to %s." % + blobberUploadDir) + self.dm.getDirectory(self.remoteMozLog, blobberUploadDir) + self.dm.getDirectory(self.remoteScreenshots, blobberUploadDir) + MochitestDesktop.cleanup(self, self.options) + if self.localProfile: + os.system("rm -Rf %s" % self.localProfile) + self.dm.removeDir(self.remoteProfile) + self.dm.removeDir(self.remoteProfileCopy) + self.dm.removeDir(self.remoteScreenshots) + self.dm.removeDir(self.remoteMozLog) + self.dm.removeFile(self.remoteConfigFile) + if self.dm.fileExists(self.remoteLog): + self.dm.removeFile(self.remoteLog) + self.log.debug("Cleanup complete.") + + def findPath(self, paths, filename=None): + for path in paths: + p = path + if filename: + p = os.path.join(p, filename) + if os.path.exists(self.getFullPath(p)): + return path + return None + + def makeLocalAutomation(self): + localAutomation = Automation() + localAutomation.IS_WIN32 = False + localAutomation.IS_LINUX = False + localAutomation.IS_MAC = False + localAutomation.UNIXISH = False + hostos = sys.platform + if (hostos == 'mac' or hostos == 'darwin'): + localAutomation.IS_MAC = True + elif (hostos == 'linux' or hostos == 'linux2'): + localAutomation.IS_LINUX = True + localAutomation.UNIXISH = True + elif (hostos == 'win32' or hostos == 'win64'): + localAutomation.BIN_SUFFIX = ".exe" + localAutomation.IS_WIN32 = True + return localAutomation + + def setupLocalPaths(self): + """ + Setup xrePath and utilityPath and verify xpcshell. + + This is similar to switchToLocalPaths in runtestsremote.py. + """ + localAutomation = self.makeLocalAutomation() + paths = [ + self.options.xrePath, + localAutomation.DIST_BIN, + self.auto._product, + os.path.join('..', self.auto._product) + ] + self.options.xrePath = self.findPath(paths) + if self.options.xrePath is None: + self.log.error( + "unable to find xulrunner path for %s, please specify with --xre-path" % + os.name) + sys.exit(1) + self.log.debug("using xre path %s" % self.options.xrePath) + xpcshell = "xpcshell" + if (os.name == "nt"): + xpcshell += ".exe" + if self.options.utilityPath: + paths = [self.options.utilityPath, self.options.xrePath] + else: + paths = [self.options.xrePath] + self.options.utilityPath = self.findPath(paths, xpcshell) + if self.options.utilityPath is None: + self.log.error( + "unable to find utility path for %s, please specify with --utility-path" % + os.name) + sys.exit(1) + self.log.debug("using utility path %s" % self.options.utilityPath) + xpcshell_path = os.path.join(self.options.utilityPath, xpcshell) + if localAutomation.elf_arm(xpcshell_path): + self.log.error('xpcshell at %s is an ARM binary; please use ' + 'the --utility-path argument to specify the path ' + 'to a desktop version.' % xpcshell_path) + sys.exit(1) + self.log.debug("xpcshell found at %s" % xpcshell_path) + + def buildProfile(self): + """ + Build a profile locally, keep it locally for use by servers and + push a copy to the remote profile-copy directory. + + This is similar to buildProfile in runtestsremote.py. + """ + self.options.extraPrefs.append('browser.search.suggest.enabled=true') + self.options.extraPrefs.append('browser.search.suggest.prompted=true') + self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') + self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false') + self.options.extraPrefs.append('browser.snippets.enabled=false') + self.options.extraPrefs.append('browser.casting.enabled=true') + self.options.extraPrefs.append('extensions.autoupdate.enabled=false') + + # Override the telemetry init delay for integration testing. + self.options.extraPrefs.append('toolkit.telemetry.initDelay=1') + + self.options.extensionsToExclude.extend([ + 'mochikit@mozilla.org', + 'worker-test@mozilla.org.xpi', + 'workerbootstrap-test@mozilla.org.xpi', + 'indexedDB-test@mozilla.org.xpi', + ]) + + manifest = MochitestDesktop.buildProfile(self, self.options) + self.localProfile = self.options.profilePath + self.log.debug("Profile created at %s" % self.localProfile) + # some files are not needed for robocop; save time by not pushing + shutil.rmtree(os.path.join(self.localProfile, 'webapps')) + os.remove(os.path.join(self.localProfile, 'userChrome.css')) + try: + self.dm.pushDir(self.localProfile, self.remoteProfileCopy) + except mozdevice.DMError: + self.log.error( + "Automation Error: Unable to copy profile to device.") + raise + + return manifest + + def setupRemoteProfile(self): + """ + Remove any remote profile and re-create it. + """ + self.log.debug("Updating remote profile at %s" % self.remoteProfile) + self.dm.removeDir(self.remoteProfile) + if self.remoteCopyAvailable: + try: + self.dm.shellCheckOutput( + ['cp', '-r', self.remoteProfileCopy, self.remoteProfile], + root=True, timeout=60) + except mozdevice.DMError: + # For instance, cp is not available on some older versions of + # Android. + self.log.info( + "Unable to copy remote profile; falling back to push.") + self.remoteCopyAvailable = False + if not self.remoteCopyAvailable: + self.dm.pushDir(self.localProfile, self.remoteProfile) + + def parseLocalLog(self): + """ + Read and parse the local log file, noting any failures. + """ + with open(self.localLog) as currentLog: + data = currentLog.readlines() + os.unlink(self.localLog) + start_found = False + end_found = False + fail_found = False + for line in data: + try: + message = json.loads(line) + if not isinstance(message, dict) or 'action' not in message: + continue + except ValueError: + continue + if message['action'] == 'test_end': + end_found = True + start_found = False + break + if start_found and not end_found: + if 'status' in message: + if 'expected' in message: + self.failed += 1 + elif message['status'] == 'PASS': + self.passed += 1 + elif message['status'] == 'FAIL': + self.todo += 1 + if message['action'] == 'test_start': + start_found = True + if 'expected' in message: + fail_found = True + result = 0 + if fail_found: + result = 1 + if not end_found: + self.log.info( + "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)") + result = 1 + return result + + def logTestSummary(self): + """ + Print a summary of all tests run to stdout, for treeherder parsing + (logging via self.log does not work here). + """ + print("0 INFO TEST-START | Shutdown") + print("1 INFO Passed: %s" % (self.passed)) + print("2 INFO Failed: %s" % (self.failed)) + print("3 INFO Todo: %s" % (self.todo)) + print("4 INFO SimpleTest FINISHED") + if self.failed > 0: + return 1 + return 0 + + def printDeviceInfo(self, printLogcat=False): + """ + Log remote device information and logcat (if requested). + + This is similar to printDeviceInfo in runtestsremote.py + """ + try: + if printLogcat: + logcat = self.dm.getLogcat( + filterOutRegexps=fennecLogcatFilters) + self.log.info( + '\n' + + ''.join(logcat).decode( + 'utf-8', + 'replace')) + self.log.info("Device info:") + devinfo = self.dm.getInfo() + for category in devinfo: + if type(devinfo[category]) is list: + self.log.info(" %s:" % category) + for item in devinfo[category]: + self.log.info(" %s" % item) + else: + self.log.info(" %s: %s" % (category, devinfo[category])) + self.log.info("Test root: %s" % self.dm.deviceRoot) + except mozdevice.DMError: + self.log.warning("Error getting device information") + + def setupRobotiumConfig(self, browserEnv): + """ + Create robotium.config and push it to the device. + """ + fHandle = tempfile.NamedTemporaryFile(suffix='.config', + prefix='robotium-', + dir=os.getcwd(), + delete=False) + fHandle.write("profile=%s\n" % (self.remoteProfile)) + fHandle.write("logfile=%s\n" % (self.options.remoteLogFile)) + fHandle.write("host=http://mochi.test:8888/tests\n") + fHandle.write( + "rawhost=http://%s:%s/tests\n" % + (self.options.remoteWebServer, self.options.httpPort)) + if browserEnv: + envstr = "" + delim = "" + for key, value in browserEnv.items(): + try: + value.index(',') + self.log.error("setupRobotiumConfig: browserEnv - Found a ',' " + "in our value, unable to process value. key=%s,value=%s" % + (key, value)) + self.log.error("browserEnv=%s" % browserEnv) + except ValueError: + envstr += "%s%s=%s" % (delim, key, value) + delim = "," + fHandle.write("envvars=%s\n" % envstr) + fHandle.close() + self.dm.removeFile(self.remoteConfigFile) + self.dm.pushFile(fHandle.name, self.remoteConfigFile) + os.unlink(fHandle.name) + + def buildBrowserEnv(self): + """ + Return an environment dictionary suitable for remote use. + + This is similar to buildBrowserEnv in runtestsremote.py. + """ + browserEnv = self.environment( + xrePath=None, + debugger=None) + # remove desktop environment not used on device + if "MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA" in browserEnv: + del browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"] + if "XPCOM_MEM_BLOAT_LOG" in browserEnv: + del browserEnv["XPCOM_MEM_BLOAT_LOG"] + browserEnv["MOZ_LOG_FILE"] = os.path.join( + self.remoteMozLog, + self.mozLogName) + + try: + browserEnv.update( + dict( + parseKeyValue( + self.options.environment, + context='--setenv'))) + except KeyValueParseError as e: + self.log.error(str(e)) + return None + + return browserEnv + + def runSingleTest(self, test): + """ + Run the specified test. + """ + self.log.debug("Running test %s" % test['name']) + self.mozLogName = "moz-%s.log" % test['name'] + browserEnv = self.buildBrowserEnv() + self.setupRobotiumConfig(browserEnv) + self.setupRemoteProfile() + self.options.app = "am" + if self.options.autorun: + # This launches a test (using "am instrument") and instructs + # Fennec to /quit/ the browser (using Robocop:Quit) and to + # /finish/ all opened activities. + browserArgs = [ + "instrument", + "-w", + "-e", "quit_and_finish", "1", + "-e", "deviceroot", self.deviceRoot, + "-e", "class", + "org.mozilla.gecko.tests.%s" % test['name'].split('/')[-1].split('.java')[0], + "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"] + else: + # This does not launch a test at all. It launches an activity + # that starts Fennec and then waits indefinitely, since cat + # never returns. + browserArgs = ["start", "-n", + "org.mozilla.roboexample.test/org.mozilla." + "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat"] + self.dm.default_timeout = sys.maxint # Forever. + self.log.info("") + self.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % + (self.options.remoteWebServer, self.options.httpPort)) + self.log.info("") + result = -1 + log_result = -1 + try: + self.dm.recordLogcat() + timeout = self.options.timeout + if not timeout: + timeout = self.NO_OUTPUT_TIMEOUT + result = self.auto.runApp( + None, browserEnv, "am", self.localProfile, browserArgs, + timeout=timeout, symbolsPath=self.options.symbolsPath) + self.log.debug("runApp completes with status %d" % result) + if result != 0: + self.log.error("runApp() exited with code %s" % result) + if self.dm.fileExists(self.remoteLog): + self.dm.getFile(self.remoteLog, self.localLog) + self.dm.removeFile(self.remoteLog) + self.log.debug("Remote log %s retrieved to %s" % + (self.remoteLog, self.localLog)) + else: + self.log.warning( + "Unable to retrieve log file (%s) from remote device" % + self.remoteLog) + log_result = self.parseLocalLog() + if result != 0 or log_result != 0: + # Display remote diagnostics; if running in mach, keep output + # terse. + if self.options.log_mach is None: + self.printDeviceInfo(printLogcat=True) + except: + self.log.error( + "Automation Error: Exception caught while running tests") + traceback.print_exc() + result = 1 + self.log.debug("Test %s completes with status %d (log status %d)" % + (test['name'], int(result), int(log_result))) + return result + + def runTests(self): + self.startup() + if isinstance(self.options.manifestFile, TestManifest): + mp = self.options.manifestFile + else: + mp = TestManifest(strict=False) + mp.read(self.options.robocopIni) + filters = [] + if self.options.totalChunks: + filters.append( + chunk_by_slice(self.options.thisChunk, self.options.totalChunks)) + robocop_tests = mp.active_tests( + exists=False, filters=filters, **mozinfo.info) + if not self.options.autorun: + # Force a single loop iteration. The iteration will start Fennec and + # the httpd server, but not actually run a test. + self.options.test_paths = [robocop_tests[0]['name']] + active_tests = [] + for test in robocop_tests: + if self.options.test_paths and test['name'] not in self.options.test_paths: + continue + if 'disabled' in test: + self.log.info('TEST-INFO | skipping %s | %s' % + (test['name'], test['disabled'])) + continue + active_tests.append(test) + self.log.suite_start([t['name'] for t in active_tests]) + worstTestResult = None + for test in active_tests: + result = self.runSingleTest(test) + if worstTestResult is None or worstTestResult == 0: + worstTestResult = result + if worstTestResult is None: + self.log.warning( + "No tests run. Did you pass an invalid TEST_PATH?") + worstTestResult = 1 + else: + print "INFO | runtests.py | Test summary: start." + logResult = self.logTestSummary() + print "INFO | runtests.py | Test summary: end." + if worstTestResult == 0: + worstTestResult = logResult + return worstTestResult + + +def run_test_harness(parser, options): + parser.validate(options) + + if options is None: + raise ValueError( + "Invalid options specified, use --help for a list of valid options") + message_logger = MessageLogger(logger=None) + process_args = {'messageLogger': message_logger} + auto = RemoteAutomation(None, "fennec", processArgs=process_args) + auto.setDeviceManager(options.dm) + runResult = -1 + robocop = RobocopTestRunner(auto, options.dm, options) + + # Check that Firefox is installed + expected = options.app.split('/')[-1] + installed = options.dm.shellCheckOutput(['pm', 'list', 'packages', expected]) + if expected not in installed: + robocop.log.error("%s is not installed on this device" % expected) + return 1 + + try: + message_logger.logger = robocop.log + message_logger.buffering = False + robocop.message_logger = message_logger + robocop.log.debug("options=%s" % vars(options)) + runResult = robocop.runTests() + except KeyboardInterrupt: + robocop.log.info("runrobocop.py | Received keyboard interrupt") + runResult = -1 + except: + traceback.print_exc() + robocop.log.error( + "runrobocop.py | Received unexpected exception while running tests") + runResult = 1 + finally: + try: + robocop.cleanup() + except mozdevice.DMError: + # ignore device error while cleaning up + pass + message_logger.finish() + return runResult + + +def main(args=sys.argv[1:]): + parser = MochitestArgumentParser(app='android') + options = parser.parse_args(args) + return run_test_harness(parser, options) + +if __name__ == "__main__": + sys.exit(main()) |