summaryrefslogtreecommitdiffstats
path: root/layout/tools/reftest/remotereftest.py
diff options
context:
space:
mode:
Diffstat (limited to 'layout/tools/reftest/remotereftest.py')
-rw-r--r--layout/tools/reftest/remotereftest.py412
1 files changed, 412 insertions, 0 deletions
diff --git a/layout/tools/reftest/remotereftest.py b/layout/tools/reftest/remotereftest.py
new file mode 100644
index 000000000..f871c7586
--- /dev/null
+++ b/layout/tools/reftest/remotereftest.py
@@ -0,0 +1,412 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import os
+import time
+import tempfile
+import traceback
+import urllib2
+
+import mozdevice
+import mozinfo
+from automation import Automation
+from remoteautomation import RemoteAutomation, fennecLogcatFilters
+
+from output import OutputHandler
+from runreftest import RefTest, ReftestResolver
+import reftestcommandline
+
+# We need to know our current directory so that we can serve our test files from it.
+SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+
+
+class RemoteReftestResolver(ReftestResolver):
+ def absManifestPath(self, path):
+ script_abs_path = os.path.join(SCRIPT_DIRECTORY, path)
+ if os.path.exists(script_abs_path):
+ rv = script_abs_path
+ elif os.path.exists(os.path.abspath(path)):
+ rv = os.path.abspath(path)
+ else:
+ print >> sys.stderr, "Could not find manifest %s" % script_abs_path
+ sys.exit(1)
+ return os.path.normpath(rv)
+
+ def manifestURL(self, options, path):
+ # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
+ # It's possible for this url to have a leading "..", but reftest.js will fix that up
+ relPath = os.path.relpath(path, SCRIPT_DIRECTORY)
+ return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath)
+
+
+class ReftestServer:
+ """ Web server used to serve Reftests, for closer fidelity to the real web.
+ It is virtually identical to the server used in mochitest and will only
+ be used for running reftests remotely.
+ Bug 581257 has been filed to refactor this wrapper around httpd.js into
+ it's own class and use it in both remote and non-remote testing. """
+
+ def __init__(self, automation, options, scriptDir):
+ self.automation = automation
+ self._utilityPath = options.utilityPath
+ self._xrePath = options.xrePath
+ self._profileDir = options.serverProfilePath
+ self.webServer = options.remoteWebServer
+ self.httpPort = options.httpPort
+ self.scriptDir = scriptDir
+ self.pidFile = options.pidFile
+ self._httpdPath = os.path.abspath(options.httpdPath)
+ self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort }
+
+ def start(self):
+ "Run the Refest server, returning the process ID of the server."
+
+ env = self.automation.environment(xrePath = self._xrePath)
+ env["XPCOM_DEBUG_BREAK"] = "warn"
+ if self.automation.IS_WIN32:
+ env["PATH"] = env["PATH"] + ";" + self._xrePath
+
+ args = ["-g", self._xrePath,
+ "-v", "170",
+ "-f", os.path.join(self._httpdPath, "httpd.js"),
+ "-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = '%(port)s'; const _SERVER_ADDR ='%(server)s';" %
+ {"profile" : self._profileDir.replace('\\', '\\\\'), "port" : self.httpPort, "server" : self.webServer },
+ "-f", os.path.join(self.scriptDir, "server.js")]
+
+ xpcshell = os.path.join(self._utilityPath,
+ "xpcshell" + self.automation.BIN_SUFFIX)
+
+ if not os.access(xpcshell, os.F_OK):
+ raise Exception('xpcshell not found at %s' % xpcshell)
+ if self.automation.elf_arm(xpcshell):
+ raise Exception('xpcshell at %s is an ARM binary; please use '
+ 'the --utility-path argument to specify the path '
+ 'to a desktop version.' % xpcshell)
+
+ self._process = self.automation.Process([xpcshell] + args, env = env)
+ pid = self._process.pid
+ if pid < 0:
+ print "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server."
+ return 2
+ self.automation.log.info("INFO | remotereftests.py | Server pid: %d", pid)
+
+ if (self.pidFile != ""):
+ f = open(self.pidFile + ".xpcshell.pid", 'w')
+ f.write("%s" % pid)
+ f.close()
+
+ def ensureReady(self, timeout):
+ assert timeout >= 0
+
+ aliveFile = os.path.join(self._profileDir, "server_alive.txt")
+ i = 0
+ while i < timeout:
+ if os.path.exists(aliveFile):
+ break
+ time.sleep(1)
+ i += 1
+ else:
+ print "TEST-UNEXPECTED-FAIL | remotereftests.py | Timed out while waiting for server startup."
+ self.stop()
+ return 1
+
+ def stop(self):
+ if hasattr(self, '_process'):
+ try:
+ c = urllib2.urlopen(self.shutdownURL)
+ c.read()
+ c.close()
+
+ rtncode = self._process.poll()
+ if (rtncode == None):
+ self._process.terminate()
+ except:
+ self._process.kill()
+
+class RemoteReftest(RefTest):
+ use_marionette = False
+ remoteApp = ''
+ resolver_cls = RemoteReftestResolver
+
+ def __init__(self, automation, devicemanager, options, scriptDir):
+ RefTest.__init__(self)
+ self.automation = automation
+ self._devicemanager = devicemanager
+ self.scriptDir = scriptDir
+ self.remoteApp = options.app
+ self.remoteProfile = options.remoteProfile
+ self.remoteTestRoot = options.remoteTestRoot
+ self.remoteLogFile = options.remoteLogFile
+ self.localLogName = options.localLogName
+ self.pidFile = options.pidFile
+ if self.automation.IS_DEBUG_BUILD:
+ self.SERVER_STARTUP_TIMEOUT = 180
+ else:
+ self.SERVER_STARTUP_TIMEOUT = 90
+ self.automation.deleteANRs()
+ self.automation.deleteTombstones()
+
+ self._populate_logger(options)
+ outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath)
+ # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest
+ # MessageLogger object to re-use this code path.
+ outputHandler.write = outputHandler.__call__
+ self.automation._processArgs['messageLogger'] = outputHandler
+
+ def findPath(self, paths, filename = None):
+ for path in paths:
+ p = path
+ if filename:
+ p = os.path.join(p, filename)
+ if os.path.exists(self.getFullPath(p)):
+ return path
+ return None
+
+ def startWebServer(self, options):
+ """ Create the webserver on the host and start it up """
+ remoteXrePath = options.xrePath
+ remoteUtilityPath = options.utilityPath
+ localAutomation = Automation()
+ localAutomation.IS_WIN32 = False
+ localAutomation.IS_LINUX = False
+ localAutomation.IS_MAC = False
+ localAutomation.UNIXISH = False
+ hostos = sys.platform
+ if (hostos == 'mac' or hostos == 'darwin'):
+ localAutomation.IS_MAC = True
+ elif (hostos == 'linux' or hostos == 'linux2'):
+ localAutomation.IS_LINUX = True
+ localAutomation.UNIXISH = True
+ elif (hostos == 'win32' or hostos == 'win64'):
+ localAutomation.BIN_SUFFIX = ".exe"
+ localAutomation.IS_WIN32 = True
+
+ paths = [options.xrePath, localAutomation.DIST_BIN, self.automation._product, os.path.join('..', self.automation._product)]
+ options.xrePath = self.findPath(paths)
+ if options.xrePath == None:
+ print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
+ return 1
+ paths.append("bin")
+ paths.append(os.path.join("..", "bin"))
+
+ xpcshell = "xpcshell"
+ if (os.name == "nt"):
+ xpcshell += ".exe"
+
+ if (options.utilityPath):
+ paths.insert(0, options.utilityPath)
+ options.utilityPath = self.findPath(paths, xpcshell)
+ if options.utilityPath == None:
+ print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
+ return 1
+
+ options.serverProfilePath = tempfile.mkdtemp()
+ self.server = ReftestServer(localAutomation, options, self.scriptDir)
+ retVal = self.server.start()
+ if retVal:
+ return retVal
+ retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
+ if retVal:
+ return retVal
+
+ options.xrePath = remoteXrePath
+ options.utilityPath = remoteUtilityPath
+ return 0
+
+ def stopWebServer(self, options):
+ self.server.stop()
+
+ def createReftestProfile(self, options, manifest):
+ profile = RefTest.createReftestProfile(self,
+ options,
+ manifest,
+ server=options.remoteWebServer,
+ port=options.httpPort)
+ profileDir = profile.profile
+
+ prefs = {}
+ prefs["app.update.url.android"] = ""
+ prefs["browser.firstrun.show.localepicker"] = False
+ prefs["reftest.remote"] = True
+ prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
+
+ prefs["layout.css.devPixelsPerPx"] = "1.0"
+ # Because Fennec is a little wacky (see bug 1156817) we need to load the
+ # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
+ prefs["apz.allow_zooming"] = False
+
+ # Set the extra prefs.
+ profile.set_preferences(prefs)
+
+ try:
+ self._devicemanager.pushDir(profileDir, options.remoteProfile)
+ self._devicemanager.chmodDir(options.remoteProfile)
+ except mozdevice.DMError:
+ print "Automation Error: Failed to copy profiledir to device"
+ raise
+
+ return profile
+
+ def copyExtraFilesToProfile(self, options, profile):
+ profileDir = profile.profile
+ RefTest.copyExtraFilesToProfile(self, options, profile)
+ try:
+ self._devicemanager.pushDir(profileDir, options.remoteProfile)
+ self._devicemanager.chmodDir(options.remoteProfile)
+ except mozdevice.DMError:
+ print "Automation Error: Failed to copy extra files to device"
+ raise
+
+ def printDeviceInfo(self, printLogcat=False):
+ try:
+ if printLogcat:
+ logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
+ print ''.join(logcat)
+ print "Device info:"
+ devinfo = self._devicemanager.getInfo()
+ for category in devinfo:
+ if type(devinfo[category]) is list:
+ print " %s:" % category
+ for item in devinfo[category]:
+ print " %s" % item
+ else:
+ print " %s: %s" % (category, devinfo[category])
+ print "Test root: %s" % self._devicemanager.deviceRoot
+ except mozdevice.DMError:
+ print "WARNING: Error getting device information"
+
+ def environment(self, **kwargs):
+ return self.automation.environment(**kwargs)
+
+ def buildBrowserEnv(self, options, profileDir):
+ browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
+ # remove desktop environment not used on device
+ if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
+ del browserEnv["XPCOM_MEM_BLOAT_LOG"]
+ return browserEnv
+
+ def runApp(self, profile, binary, cmdargs, env,
+ timeout=None, debuggerInfo=None,
+ symbolsPath=None, options=None,
+ valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None):
+ status = self.automation.runApp(None, env,
+ binary,
+ profile.profile,
+ cmdargs,
+ utilityPath=options.utilityPath,
+ xrePath=options.xrePath,
+ debuggerInfo=debuggerInfo,
+ symbolsPath=symbolsPath,
+ timeout=timeout)
+ return status
+
+ def cleanup(self, profileDir):
+ # Pull results back from device
+ if self.remoteLogFile and \
+ self._devicemanager.fileExists(self.remoteLogFile):
+ self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
+ else:
+ print "WARNING: Unable to retrieve log file (%s) from remote " \
+ "device" % self.remoteLogFile
+ self._devicemanager.removeDir(self.remoteProfile)
+ self._devicemanager.removeDir(self.remoteTestRoot)
+ RefTest.cleanup(self, profileDir)
+ if (self.pidFile != ""):
+ try:
+ os.remove(self.pidFile)
+ os.remove(self.pidFile + ".xpcshell.pid")
+ except:
+ print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
+
+
+def run_test_harness(parser, options):
+ if options.dm_trans == 'sut' and options.deviceIP == None:
+ print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option"
+ return 1
+
+ dm_args = {
+ 'deviceRoot': options.remoteTestRoot,
+ 'host': options.deviceIP,
+ 'port': options.devicePort,
+ }
+
+ dm_cls = mozdevice.DroidSUT
+ if options.dm_trans == 'adb':
+ dm_args['adbPath'] = options.adb_path
+ if not dm_args['host']:
+ dm_args['deviceSerial'] = options.deviceSerial
+ dm_cls = mozdevice.DroidADB
+
+ try:
+ dm = dm_cls(**dm_args)
+ except mozdevice.DMError:
+ traceback.print_exc()
+ print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state."
+ return 1
+
+ automation = RemoteAutomation(None)
+ automation.setDeviceManager(dm)
+
+ if options.remoteProductName:
+ automation.setProduct(options.remoteProductName)
+
+ # Set up the defaults and ensure options are set
+ parser.validate_remote(options, automation)
+
+ # Check that Firefox is installed
+ expected = options.app.split('/')[-1]
+ installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected])
+ if expected not in installed:
+ print "%s is not installed on this device" % expected
+ return 1
+
+ automation.setAppName(options.app)
+ automation.setRemoteProfile(options.remoteProfile)
+ automation.setRemoteLog(options.remoteLogFile)
+ reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
+ parser.validate(options, reftest)
+
+ if mozinfo.info['debug']:
+ print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout
+ options.timeout = 600
+
+ # Hack in a symbolic link for jsreftest
+ os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
+
+ # Start the webserver
+ retVal = reftest.startWebServer(options)
+ if retVal:
+ return retVal
+
+ procName = options.app.split('/')[-1]
+ if (dm.processExist(procName)):
+ dm.killProcess(procName)
+
+ if options.printDeviceInfo:
+ reftest.printDeviceInfo()
+
+#an example manifest name to use on the cli
+# manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
+ retVal = 0
+ try:
+ dm.recordLogcat()
+ retVal = reftest.runTests(options.tests, options)
+ except:
+ print "Automation Error: Exception caught while running tests"
+ traceback.print_exc()
+ retVal = 1
+
+ reftest.stopWebServer(options)
+
+ if options.printDeviceInfo:
+ reftest.printDeviceInfo(printLogcat=True)
+
+ return retVal
+
+
+if __name__ == "__main__":
+ parser = reftestcommandline.RemoteArgumentsParser()
+ options = parser.parse_args()
+ sys.exit(run_test_harness(parser, options))