# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import with_statement import logging import os import re import select import signal import subprocess import sys import tempfile from datetime import datetime, timedelta SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) sys.path.insert(0, SCRIPT_DIR) # -------------------------------------------------------------- # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900 # These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR here = os.path.dirname(os.path.realpath(__file__)) mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase')) if os.path.isdir(mozbase): for package in os.listdir(mozbase): package_path = os.path.join(mozbase, package) if package_path not in sys.path: sys.path.append(package_path) import mozcrash from mozscreenshot import printstatus, dump_screen # --------------------------------------------------------------- _DEFAULT_PREFERENCE_FILE = os.path.join(SCRIPT_DIR, 'prefs_general.js') _DEFAULT_APPS_FILE = os.path.join(SCRIPT_DIR, 'webapps_mochitest.json') _DEFAULT_WEB_SERVER = "127.0.0.1" _DEFAULT_HTTP_PORT = 8888 _DEFAULT_SSL_PORT = 4443 _DEFAULT_WEBSOCKET_PORT = 9988 # from nsIPrincipal.idl _APP_STATUS_NOT_INSTALLED = 0 _APP_STATUS_INSTALLED = 1 _APP_STATUS_PRIVILEGED = 2 _APP_STATUS_CERTIFIED = 3 #expand _DIST_BIN = __XPC_BIN_PATH__ #expand _IS_WIN32 = len("__WIN32__") != 0 #expand _IS_MAC = __IS_MAC__ != 0 #expand _IS_LINUX = __IS_LINUX__ != 0 #ifdef IS_CYGWIN #expand _IS_CYGWIN = __IS_CYGWIN__ == 1 #else _IS_CYGWIN = False #endif #expand _BIN_SUFFIX = __BIN_SUFFIX__ #expand _DEFAULT_APP = "./" + __BROWSER_PATH__ #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__ #expand _IS_TEST_BUILD = __IS_TEST_BUILD__ #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__ #expand _CRASHREPORTER = __CRASHREPORTER__ == 1 #expand _IS_ASAN = __IS_ASAN__ == 1 if _IS_WIN32: import ctypes, ctypes.wintypes, time, msvcrt else: import errno def resetGlobalLog(log): while _log.handlers: _log.removeHandler(_log.handlers[0]) handler = logging.StreamHandler(log) _log.setLevel(logging.INFO) _log.addHandler(handler) # We use the logging system here primarily because it'll handle multiple # threads, which is needed to process the output of the server and application # processes simultaneously. _log = logging.getLogger() resetGlobalLog(sys.stdout) ################# # PROFILE SETUP # ################# class Automation(object): """ Runs the browser from a script, and provides useful utilities for setting up the browser environment. """ DIST_BIN = _DIST_BIN IS_WIN32 = _IS_WIN32 IS_MAC = _IS_MAC IS_LINUX = _IS_LINUX IS_CYGWIN = _IS_CYGWIN BIN_SUFFIX = _BIN_SUFFIX UNIXISH = not IS_WIN32 and not IS_MAC DEFAULT_APP = _DEFAULT_APP CERTS_SRC_DIR = _CERTS_SRC_DIR IS_TEST_BUILD = _IS_TEST_BUILD IS_DEBUG_BUILD = _IS_DEBUG_BUILD CRASHREPORTER = _CRASHREPORTER IS_ASAN = _IS_ASAN # timeout, in seconds DEFAULT_TIMEOUT = 60.0 DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT def __init__(self): self.log = _log self.lastTestSeen = "automation.py" self.haveDumpedScreen = False def setServerInfo(self, webServer = _DEFAULT_WEB_SERVER, httpPort = _DEFAULT_HTTP_PORT, sslPort = _DEFAULT_SSL_PORT, webSocketPort = _DEFAULT_WEBSOCKET_PORT): self.webServer = webServer self.httpPort = httpPort self.sslPort = sslPort self.webSocketPort = webSocketPort @property def __all__(self): return [ "UNIXISH", "IS_WIN32", "IS_MAC", "log", "runApp", "Process", "DIST_BIN", "DEFAULT_APP", "CERTS_SRC_DIR", "environment", "IS_TEST_BUILD", "IS_DEBUG_BUILD", "DEFAULT_TIMEOUT", ] class Process(subprocess.Popen): """ Represents our view of a subprocess. It adds a kill() method which allows it to be stopped explicitly. """ def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): _log.info("INFO | automation.py | Launching: %s", subprocess.list2cmdline(args)) subprocess.Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags) self.log = _log def kill(self): if Automation().IS_WIN32: import platform pid = "%i" % self.pid if platform.release() == "2000": # Windows 2000 needs 'kill.exe' from the #'Windows 2000 Resource Kit tools'. (See bug 475455.) try: subprocess.Popen(["kill", "-f", pid]).wait() except: self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid) else: # Windows XP and later. subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait() else: os.kill(self.pid, signal.SIGKILL) def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None): if xrePath == None: xrePath = self.DIST_BIN if env == None: env = dict(os.environ) ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath)) dmdLibrary = None preloadEnvVar = None if self.UNIXISH or self.IS_MAC: envVar = "LD_LIBRARY_PATH" preloadEnvVar = "LD_PRELOAD" if self.IS_MAC: envVar = "DYLD_LIBRARY_PATH" dmdLibrary = "libdmd.dylib" else: # unixish env['MOZILLA_FIVE_HOME'] = xrePath dmdLibrary = "libdmd.so" if envVar in env: ldLibraryPath = ldLibraryPath + ":" + env[envVar] env[envVar] = ldLibraryPath elif self.IS_WIN32: env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath) dmdLibrary = "dmd.dll" preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" if dmdPath and dmdLibrary and preloadEnvVar: env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) if crashreporter and not debugger: env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' else: env['MOZ_CRASHREPORTER_DISABLE'] = '1' # Crash on non-local network connections by default. # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily # enable non-local connections for the purposes of local testing. Don't # override the user's choice here. See bug 1049688. env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1') env['GNOME_DISABLE_CRASH_DIALOG'] = '1' env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' # Set WebRTC logging in case it is not set yet env.setdefault('MOZ_LOG', 'signaling:3,mtransport:4,DataChannel:4,jsep:4,MediaPipelineFactory:4') env.setdefault('R_LOG_LEVEL', '6') env.setdefault('R_LOG_DESTINATION', 'stderr') env.setdefault('R_LOG_VERBOSE', '1') # ASan specific environment stuff if self.IS_ASAN and (self.IS_LINUX or self.IS_MAC): # Symbolizer support llvmsym = os.path.join(xrePath, "llvm-symbolizer") if os.path.isfile(llvmsym): env["ASAN_SYMBOLIZER_PATH"] = llvmsym self.log.info("INFO | automation.py | ASan using symbolizer at %s", llvmsym) else: self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Failed to find ASan symbolizer at %s", llvmsym) try: totalMemory = int(os.popen("free").readlines()[1].split()[1]) # Only 4 GB RAM or less available? Use custom ASan options to reduce # the amount of resources required to do the tests. Standard options # will otherwise lead to OOM conditions on the current test slaves. if totalMemory <= 1024 * 1024 * 4: self.log.info("INFO | automation.py | ASan running in low-memory configuration") env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5" else: self.log.info("INFO | automation.py | ASan running in default memory configuration") except OSError,err: self.log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror) except: self.log.info("Failed determine available memory, disabling ASan low-memory configuration") return env def killPid(self, pid): try: os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM)) except WindowsError: self.log.info("Failed to kill process %d." % pid) if IS_WIN32: PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe GetLastError = ctypes.windll.kernel32.GetLastError def readWithTimeout(self, f, timeout): """ Try to read a line of output from the file object |f|. |f| must be a pipe, like the |stdout| member of a subprocess.Popen object created with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout| is True if the read timed out, and False otherwise. If no output is received within |timeout| seconds, returns a blank line. """ if timeout is None: timeout = 0 x = msvcrt.get_osfhandle(f.fileno()) l = ctypes.c_long() done = time.time() + timeout buffer = "" while timeout == 0 or time.time() < done: if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0: err = self.GetLastError() if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE return ('', False) else: self.log.error("readWithTimeout got error: %d", err) # read a character at a time, checking for eol. Return once we get there. index = 0 while index < l.value: char = f.read(1) buffer += char if char == '\n': return (buffer, False) index = index + 1 time.sleep(0.01) return (buffer, True) def isPidAlive(self, pid): STILL_ACTIVE = 259 PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) if not pHandle: return False pExitCode = ctypes.wintypes.DWORD() ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode)) ctypes.windll.kernel32.CloseHandle(pHandle) return pExitCode.value == STILL_ACTIVE else: def readWithTimeout(self, f, timeout): """Try to read a line of output from the file object |f|. If no output is received within |timeout| seconds, return a blank line. Returns a tuple (line, did_timeout), where |did_timeout| is True if the read timed out, and False otherwise.""" (r, w, e) = select.select([f], [], [], timeout) if len(r) == 0: return ('', True) return (f.readline(), False) def isPidAlive(self, pid): try: # kill(pid, 0) checks for a valid PID without actually sending a signal # The method throws OSError if the PID is invalid, which we catch below. os.kill(pid, 0) # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if # the process terminates before we get to this point. wpid, wstatus = os.waitpid(pid, os.WNOHANG) return wpid == 0 except OSError, err: # Catch the errors we might expect from os.kill/os.waitpid, # and re-raise any others if err.errno == errno.ESRCH or err.errno == errno.ECHILD: return False raise def dumpScreen(self, utilityPath): if self.haveDumpedScreen: self.log.info("Not taking screenshot here: see the one that was previously logged") return self.haveDumpedScreen = True; dump_screen(utilityPath, self.log) def killAndGetStack(self, processPID, utilityPath, debuggerInfo): """Kill the process, preferrably in a way that gets us a stack trace. Also attempts to obtain a screenshot before killing the process.""" if not debuggerInfo: self.dumpScreen(utilityPath) self.killAndGetStackNoScreenshot(processPID, utilityPath, debuggerInfo) def killAndGetStackNoScreenshot(self, processPID, utilityPath, debuggerInfo): """Kill the process, preferrably in a way that gets us a stack trace.""" if self.CRASHREPORTER and not debuggerInfo: if not self.IS_WIN32: # ABRT will get picked up by Breakpad's signal handler os.kill(processPID, signal.SIGABRT) return else: # We should have a "crashinject" program in our utility path crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe")) if os.path.exists(crashinject): status = subprocess.Popen([crashinject, str(processPID)]).wait() printstatus("crashinject", status) if status == 0: return self.log.info("Can't trigger Breakpad, just killing process") self.killPid(processPID) def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, outputHandler=None): """ Look for timeout or crashes and return the status after the process terminates """ stackFixerFunction = None didTimeout = False hitMaxTime = False if proc.stdout is None: self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection") else: logsource = proc.stdout if self.IS_DEBUG_BUILD and symbolsPath and os.path.exists(symbolsPath): # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files) # This method is preferred for Tinderbox builds, since native symbols may have been stripped. sys.path.insert(0, utilityPath) import fix_stack_using_bpsyms as stackFixerModule stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath) del sys.path[0] elif self.IS_DEBUG_BUILD and self.IS_MAC: # Run each line through a function in fix_macosx_stack.py (uses atos) sys.path.insert(0, utilityPath) import fix_macosx_stack as stackFixerModule stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line) del sys.path[0] elif self.IS_DEBUG_BUILD and self.IS_LINUX: # Run each line through a function in fix_linux_stack.py (uses addr2line) # This method is preferred for developer machines, so we don't have to run "make buildsymbols". sys.path.insert(0, utilityPath) import fix_linux_stack as stackFixerModule stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line) del sys.path[0] # With metro browser runs this script launches the metro test harness which launches the browser. # The metro test harness hands back the real browser process id via log output which we need to # pick up on and parse out. This variable tracks the real browser process id if we find it. browserProcessId = -1 (line, didTimeout) = self.readWithTimeout(logsource, timeout) while line != "" and not didTimeout: if stackFixerFunction: line = stackFixerFunction(line) if outputHandler is None: self.log.info(line.rstrip().decode("UTF-8", "ignore")) else: outputHandler(line) if "TEST-START" in line and "|" in line: self.lastTestSeen = line.split("|")[1].strip() if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line: self.dumpScreen(utilityPath) (line, didTimeout) = self.readWithTimeout(logsource, timeout) if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime): # Kill the application. hitMaxTime = True self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime)) self.log.error("Force-terminating active process(es)."); self.killAndGetStack(proc.pid, utilityPath, debuggerInfo) if didTimeout: if line: self.log.info(line.rstrip().decode("UTF-8", "ignore")) self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout)) self.log.error("Force-terminating active process(es)."); if browserProcessId == -1: browserProcessId = proc.pid self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo) status = proc.wait() printstatus("Main app process", status) if status == 0: self.lastTestSeen = "Main app process exited normally" if status != 0 and not didTimeout and not hitMaxTime: self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status) return status def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs): """ build the application command line """ cmd = os.path.abspath(app) if self.IS_MAC and os.path.exists(cmd + "-bin"): # Prefer 'app-bin' in case 'app' is a shell script. # We can remove this hack once bug 673899 etc are fixed. cmd += "-bin" args = [] if debuggerInfo: args.extend(debuggerInfo.args) args.append(cmd) cmd = os.path.abspath(debuggerInfo.path) if self.IS_MAC: args.append("-foreground") if self.IS_CYGWIN: profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"") else: profileDirectory = profileDir + "/" args.extend(("-no-remote", "-profile", profileDirectory)) if testURL is not None: args.append((testURL)) args.extend(extraArgs) return cmd, args def checkForZombies(self, processLog, utilityPath, debuggerInfo): """ Look for hung processes """ if not os.path.exists(processLog): self.log.info('Automation Error: PID log not found: %s', processLog) # Whilst no hung process was found, the run should still display as a failure return True foundZombie = False self.log.info('INFO | zombiecheck | Reading PID log: %s', processLog) processList = [] pidRE = re.compile(r'launched child process (\d+)$') processLogFD = open(processLog) for line in processLogFD: self.log.info(line.rstrip()) m = pidRE.search(line) if m: processList.append(int(m.group(1))) processLogFD.close() for processPID in processList: self.log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID) if self.isPidAlive(processPID): foundZombie = True self.log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID) self.killAndGetStack(processPID, utilityPath, debuggerInfo) return foundZombie def checkForCrashes(self, minidumpDir, symbolsPath): return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen) def runApp(self, testURL, env, app, profileDir, extraArgs, utilityPath = None, xrePath = None, certPath = None, debuggerInfo = None, symbolsPath = None, timeout = -1, maxTime = None, onLaunch = None, detectShutdownLeaks = False, screenshotOnFail=False, testPath=None, bisectChunk=None, valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, outputHandler=None): """ Run the app, log the duration it took to execute, return the status code. Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds. """ if utilityPath == None: utilityPath = self.DIST_BIN if xrePath == None: xrePath = self.DIST_BIN if certPath == None: certPath = self.CERTS_SRC_DIR if timeout == -1: timeout = self.DEFAULT_TIMEOUT # copy env so we don't munge the caller's environment env = dict(env); env["NO_EM_RESTART"] = "1" tmpfd, processLog = tempfile.mkstemp(suffix='pidlog') os.close(tmpfd) env["MOZ_PROCESS_LOG"] = processLog cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs) startTime = datetime.now() if debuggerInfo and debuggerInfo.interactive: # If an interactive debugger is attached, don't redirect output, # don't use timeouts, and don't capture ctrl-c. timeout = None maxTime = None outputPipe = None signal.signal(signal.SIGINT, lambda sigid, frame: None) else: outputPipe = subprocess.PIPE self.lastTestSeen = "automation.py" proc = self.Process([cmd] + args, env = self.environment(env, xrePath = xrePath, crashreporter = not debuggerInfo), stdout = outputPipe, stderr = subprocess.STDOUT) self.log.info("INFO | automation.py | Application pid: %d", proc.pid) if onLaunch is not None: # Allow callers to specify an onLaunch callback to be fired after the # app is launched. onLaunch() status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, outputHandler=outputHandler) self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime)) # Do a final check for zombie child processes. zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo) crashed = self.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath) if crashed or zombieProcesses: status = 1 if os.path.exists(processLog): os.unlink(processLog) return status def elf_arm(self, filename): data = open(filename, 'rb').read(20) return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM