diff options
Diffstat (limited to 'testing/mozbase/mozprocess')
41 files changed, 4939 insertions, 0 deletions
diff --git a/testing/mozbase/mozprocess/mozprocess/__init__.py b/testing/mozbase/mozprocess/mozprocess/__init__.py new file mode 100644 index 000000000..0b238c2b2 --- /dev/null +++ b/testing/mozbase/mozprocess/mozprocess/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from .processhandler import * diff --git a/testing/mozbase/mozprocess/mozprocess/processhandler.py b/testing/mozbase/mozprocess/mozprocess/processhandler.py new file mode 100644 index 000000000..661a7820e --- /dev/null +++ b/testing/mozbase/mozprocess/mozprocess/processhandler.py @@ -0,0 +1,1079 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import signal +import subprocess +import sys +import threading +import time +import traceback +from Queue import Queue, Empty +from datetime import datetime + +__all__ = ['ProcessHandlerMixin', 'ProcessHandler', 'LogOutput', + 'StoreOutput', 'StreamOutput'] + +# Set the MOZPROCESS_DEBUG environment variable to 1 to see some debugging output +MOZPROCESS_DEBUG = os.getenv("MOZPROCESS_DEBUG") + +# We dont use mozinfo because it is expensive to import, see bug 933558. +isWin = os.name == "nt" +isPosix = os.name == "posix" # includes MacOS X + +if isWin: + from ctypes import sizeof, addressof, c_ulong, byref, WinError, c_longlong + from . import winprocess + from .qijo import JobObjectAssociateCompletionPortInformation,\ + JOBOBJECT_ASSOCIATE_COMPLETION_PORT, JobObjectExtendedLimitInformation,\ + JOBOBJECT_BASIC_LIMIT_INFORMATION, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, IO_COUNTERS + + +class ProcessHandlerMixin(object): + """ + A class for launching and manipulating local processes. + + :param cmd: command to run. May be a string or a list. If specified as a list, the first + element will be interpreted as the command, and all additional elements will be interpreted + as arguments to that command. + :param args: list of arguments to pass to the command (defaults to None). Must not be set when + `cmd` is specified as a list. + :param cwd: working directory for command (defaults to None). + :param env: is the environment to use for the process (defaults to os.environ). + :param ignore_children: causes system to ignore child processes when True, + defaults to False (which tracks child processes). + :param kill_on_timeout: when True, the process will be killed when a timeout is reached. + When False, the caller is responsible for killing the process. + Failure to do so could cause a call to wait() to hang indefinitely. (Defaults to True.) + :param processOutputLine: function or list of functions to be called for + each line of output produced by the process (defaults to an empty + list). + :param processStderrLine: function or list of functions to be called + for each line of error output - stderr - produced by the process + (defaults to an empty list). If this is not specified, stderr lines + will be sent to the *processOutputLine* callbacks. + :param onTimeout: function or list of functions to be called when the process times out. + :param onFinish: function or list of functions to be called when the process terminates + normally without timing out. + :param kwargs: additional keyword args to pass directly into Popen. + + NOTE: Child processes will be tracked by default. If for any reason + we are unable to track child processes and ignore_children is set to False, + then we will fall back to only tracking the root process. The fallback + will be logged. + """ + + class Process(subprocess.Popen): + """ + Represents our view of a subprocess. + It adds a kill() method which allows it to be stopped explicitly. + """ + + MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY = 180 + MAX_PROCESS_KILL_DELAY = 30 + TIMEOUT_BEFORE_SIGKILL = 1.0 + + def __init__(self, + args, + bufsize=0, + executable=None, + stdin=None, + stdout=None, + stderr=None, + preexec_fn=None, + close_fds=False, + shell=False, + cwd=None, + env=None, + universal_newlines=False, + startupinfo=None, + creationflags=0, + ignore_children=False): + + # Parameter for whether or not we should attempt to track child processes + self._ignore_children = ignore_children + + if not self._ignore_children and not isWin: + # Set the process group id for linux systems + # Sets process group id to the pid of the parent process + # NOTE: This prevents you from using preexec_fn and managing + # child processes, TODO: Ideally, find a way around this + def setpgidfn(): + os.setpgid(0, 0) + preexec_fn = setpgidfn + + try: + subprocess.Popen.__init__(self, args, bufsize, executable, + stdin, stdout, stderr, + preexec_fn, close_fds, + shell, cwd, env, + universal_newlines, startupinfo, creationflags) + except OSError: + print >> sys.stderr, args + raise + + def debug(self, msg): + if not MOZPROCESS_DEBUG: + return + thread = threading.current_thread().name + print("DBG::MOZPROC PID:{} ({}) | {}".format(self.pid, thread, msg)) + + def __del__(self, _maxint=sys.maxint): + if isWin: + handle = getattr(self, '_handle', None) + if handle: + if hasattr(self, '_internal_poll'): + self._internal_poll(_deadstate=_maxint) + else: + self.poll(_deadstate=sys.maxint) + if handle or self._job or self._io_port: + self._cleanup() + else: + subprocess.Popen.__del__(self) + + def kill(self, sig=None): + if isWin: + if not self._ignore_children and self._handle and self._job: + self.debug("calling TerminateJobObject") + winprocess.TerminateJobObject(self._job, winprocess.ERROR_CONTROL_C_EXIT) + self.returncode = winprocess.GetExitCodeProcess(self._handle) + elif self._handle: + self.debug("calling TerminateProcess") + try: + winprocess.TerminateProcess(self._handle, winprocess.ERROR_CONTROL_C_EXIT) + except: + traceback.print_exc() + raise OSError("Could not terminate process") + finally: + winprocess.GetExitCodeProcess(self._handle) + self._cleanup() + else: + def send_sig(sig): + pid = self.detached_pid or self.pid + if not self._ignore_children: + try: + os.killpg(pid, sig) + except BaseException as e: + # Error 3 is a "no such process" failure, which is fine because the + # application might already have been terminated itself. Any other + # error would indicate a problem in killing the process. + if getattr(e, "errno", None) != 3: + print >> sys.stderr, "Could not terminate process: %s" % self.pid + raise + else: + os.kill(pid, sig) + + if sig is None and isPosix: + # ask the process for termination and wait a bit + send_sig(signal.SIGTERM) + limit = time.time() + self.TIMEOUT_BEFORE_SIGKILL + while time.time() <= limit: + if self.poll() is not None: + # process terminated nicely + break + time.sleep(0.02) + else: + # process did not terminate - send SIGKILL to force + send_sig(signal.SIGKILL) + else: + # a signal was explicitly set or not posix + send_sig(sig or signal.SIGKILL) + + self.returncode = self.wait() + self._cleanup() + return self.returncode + + def poll(self): + """ Popen.poll + Check if child process has terminated. Set and return returncode attribute. + """ + # If we have a handle, the process is alive + if isWin and getattr(self, '_handle', None): + return None + + return subprocess.Popen.poll(self) + + def wait(self): + """ Popen.wait + Called to wait for a running process to shut down and return + its exit code + Returns the main process's exit code + """ + # This call will be different for each OS + self.returncode = self._wait() + self._cleanup() + return self.returncode + + """ Private Members of Process class """ + + if isWin: + # Redefine the execute child so that we can track process groups + def _execute_child(self, *args_tuple): + # workaround for bug 950894 + if sys.hexversion < 0x02070600: # prior to 2.7.6 + (args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, startupinfo, + creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = args_tuple + to_close = set() + else: # 2.7.6 and later + (args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, startupinfo, + creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = args_tuple + if not isinstance(args, basestring): + args = subprocess.list2cmdline(args) + + # Always or in the create new process group + creationflags |= winprocess.CREATE_NEW_PROCESS_GROUP + + if startupinfo is None: + startupinfo = winprocess.STARTUPINFO() + + if None not in (p2cread, c2pwrite, errwrite): + startupinfo.dwFlags |= winprocess.STARTF_USESTDHANDLES + startupinfo.hStdInput = int(p2cread) + startupinfo.hStdOutput = int(c2pwrite) + startupinfo.hStdError = int(errwrite) + if shell: + startupinfo.dwFlags |= winprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = winprocess.SW_HIDE + comspec = os.environ.get("COMSPEC", "cmd.exe") + args = comspec + " /c " + args + + # Determine if we can create a job or create nested jobs. + can_create_job = winprocess.CanCreateJobObject() + can_nest_jobs = self._can_nest_jobs() + + # Ensure we write a warning message if we are falling back + if not (can_create_job or can_nest_jobs) and not self._ignore_children: + # We can't create job objects AND the user wanted us to + # Warn the user about this. + print >> sys.stderr, \ + "ProcessManager UNABLE to use job objects to manage child processes" + + # set process creation flags + creationflags |= winprocess.CREATE_SUSPENDED + creationflags |= winprocess.CREATE_UNICODE_ENVIRONMENT + if can_create_job: + creationflags |= winprocess.CREATE_BREAKAWAY_FROM_JOB + if not (can_create_job or can_nest_jobs): + # Since we've warned, we just log info here to inform you + # of the consequence of setting ignore_children = True + print "ProcessManager NOT managing child processes" + + # create the process + hp, ht, pid, tid = winprocess.CreateProcess( + executable, args, + None, None, # No special security + 1, # Must inherit handles! + creationflags, + winprocess.EnvironmentBlock(env), + cwd, startupinfo) + self._child_created = True + self._handle = hp + self._thread = ht + self.pid = pid + self.tid = tid + + if not self._ignore_children and (can_create_job or can_nest_jobs): + try: + # We create a new job for this process, so that we can kill + # the process and any sub-processes + # Create the IO Completion Port + self._io_port = winprocess.CreateIoCompletionPort() + self._job = winprocess.CreateJobObject() + + # Now associate the io comp port and the job object + joacp = JOBOBJECT_ASSOCIATE_COMPLETION_PORT(winprocess.COMPKEY_JOBOBJECT, + self._io_port) + winprocess.SetInformationJobObject( + self._job, + JobObjectAssociateCompletionPortInformation, + addressof(joacp), + sizeof(joacp) + ) + + # Allow subprocesses to break away from us - necessary for + # flash with protected mode + jbli = JOBOBJECT_BASIC_LIMIT_INFORMATION( + c_longlong(0), # per process time limit (ignored) + c_longlong(0), # per job user time limit (ignored) + winprocess.JOB_OBJECT_LIMIT_BREAKAWAY_OK, + 0, # min working set (ignored) + 0, # max working set (ignored) + 0, # active process limit (ignored) + None, # affinity (ignored) + 0, # Priority class (ignored) + 0, # Scheduling class (ignored) + ) + + iocntr = IO_COUNTERS() + jeli = JOBOBJECT_EXTENDED_LIMIT_INFORMATION( + jbli, # basic limit info struct + iocntr, # io_counters (ignored) + 0, # process mem limit (ignored) + 0, # job mem limit (ignored) + 0, # peak process limit (ignored) + 0) # peak job limit (ignored) + + winprocess.SetInformationJobObject(self._job, + JobObjectExtendedLimitInformation, + addressof(jeli), + sizeof(jeli) + ) + + # Assign the job object to the process + winprocess.AssignProcessToJobObject(self._job, int(hp)) + + # It's overkill, but we use Queue to signal between threads + # because it handles errors more gracefully than event or condition. + self._process_events = Queue() + + # Spin up our thread for managing the IO Completion Port + self._procmgrthread = threading.Thread(target=self._procmgr) + except: + print >> sys.stderr, """Exception trying to use job objects; +falling back to not using job objects for managing child processes""" + tb = traceback.format_exc() + print >> sys.stderr, tb + # Ensure no dangling handles left behind + self._cleanup_job_io_port() + else: + self._job = None + + winprocess.ResumeThread(int(ht)) + if getattr(self, '_procmgrthread', None): + self._procmgrthread.start() + ht.Close() + + for i in (p2cread, c2pwrite, errwrite): + if i is not None: + i.Close() + + # Per: + # https://msdn.microsoft.com/en-us/library/windows/desktop/hh448388%28v=vs.85%29.aspx + # Nesting jobs came in with windows versions starting with 6.2 according to the table + # on this page: + # https://msdn.microsoft.com/en-us/library/ms724834%28v=vs.85%29.aspx + def _can_nest_jobs(self): + winver = sys.getwindowsversion() + return (winver.major > 6 or + winver.major == 6 and winver.minor >= 2) + + # Windows Process Manager - watches the IO Completion Port and + # keeps track of child processes + def _procmgr(self): + if not (self._io_port) or not (self._job): + return + + try: + self._poll_iocompletion_port() + except KeyboardInterrupt: + raise KeyboardInterrupt + + def _poll_iocompletion_port(self): + # Watch the IO Completion port for status + self._spawned_procs = {} + countdowntokill = 0 + + self.debug("start polling IO completion port") + + while True: + msgid = c_ulong(0) + compkey = c_ulong(0) + pid = c_ulong(0) + portstatus = winprocess.GetQueuedCompletionStatus(self._io_port, + byref(msgid), + byref(compkey), + byref(pid), + 5000) + + # If the countdowntokill has been activated, we need to check + # if we should start killing the children or not. + if countdowntokill != 0: + diff = datetime.now() - countdowntokill + # Arbitrarily wait 3 minutes for windows to get its act together + # Windows sometimes takes a small nap between notifying the + # IO Completion port and actually killing the children, and we + # don't want to mistake that situation for the situation of an unexpected + # parent abort (which is what we're looking for here). + if diff.seconds > self.MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY: + print >> sys.stderr, \ + "WARNING | IO Completion Port failed to signal process shutdown" + print >> sys.stderr, \ + "Parent process %s exited with children alive:" % self.pid + print >> sys.stderr, \ + "PIDS: %s" % ', '.join([str(i) for i in self._spawned_procs]) + print >> sys.stderr, \ + "Attempting to kill them, but no guarantee of success" + + self.kill() + self._process_events.put({self.pid: 'FINISHED'}) + break + + if not portstatus: + # Check to see what happened + errcode = winprocess.GetLastError() + if errcode == winprocess.ERROR_ABANDONED_WAIT_0: + # Then something has killed the port, break the loop + print >> sys.stderr, "IO Completion Port unexpectedly closed" + self._process_events.put({self.pid: 'FINISHED'}) + break + elif errcode == winprocess.WAIT_TIMEOUT: + # Timeouts are expected, just keep on polling + continue + else: + print >> sys.stderr, \ + "Error Code %s trying to query IO Completion Port, " \ + "exiting" % errcode + raise WinError(errcode) + break + + if compkey.value == winprocess.COMPKEY_TERMINATE.value: + self.debug("compkeyterminate detected") + # Then we're done + break + + # Check the status of the IO Port and do things based on it + if compkey.value == winprocess.COMPKEY_JOBOBJECT.value: + if msgid.value == winprocess.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO: + # No processes left, time to shut down + # Signal anyone waiting on us that it is safe to shut down + self.debug("job object msg active processes zero") + self._process_events.put({self.pid: 'FINISHED'}) + break + elif msgid.value == winprocess.JOB_OBJECT_MSG_NEW_PROCESS: + # New Process started + # Add the child proc to our list in case our parent flakes out on us + # without killing everything. + if pid.value != self.pid: + self._spawned_procs[pid.value] = 1 + self.debug("new process detected with pid value: %s" % pid.value) + elif msgid.value == winprocess.JOB_OBJECT_MSG_EXIT_PROCESS: + self.debug("process id %s exited normally" % pid.value) + # One process exited normally + if pid.value == self.pid and len(self._spawned_procs) > 0: + # Parent process dying, start countdown timer + countdowntokill = datetime.now() + elif pid.value in self._spawned_procs: + # Child Process died remove from list + del(self._spawned_procs[pid.value]) + elif msgid.value == winprocess.JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS: + # One process existed abnormally + self.debug("process id %s exited abnormally" % pid.value) + if pid.value == self.pid and len(self._spawned_procs) > 0: + # Parent process dying, start countdown timer + countdowntokill = datetime.now() + elif pid.value in self._spawned_procs: + # Child Process died remove from list + del self._spawned_procs[pid.value] + else: + # We don't care about anything else + self.debug("We got a message %s" % msgid.value) + pass + + def _wait(self): + # First, check to see if the process is still running + if self._handle: + self.returncode = winprocess.GetExitCodeProcess(self._handle) + else: + # Dude, the process is like totally dead! + return self.returncode + + threadalive = False + if hasattr(self, "_procmgrthread"): + threadalive = self._procmgrthread.is_alive() + if self._job and threadalive and threading.current_thread() != self._procmgrthread: + self.debug("waiting with IO completion port") + # Then we are managing with IO Completion Ports + # wait on a signal so we know when we have seen the last + # process come through. + # We use queues to synchronize between the thread and this + # function because events just didn't have robust enough error + # handling on pre-2.7 versions + try: + # timeout is the max amount of time the procmgr thread will wait for + # child processes to shutdown before killing them with extreme prejudice. + item = self._process_events.get( + timeout=self.MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY + + self.MAX_PROCESS_KILL_DELAY) + if item[self.pid] == 'FINISHED': + self.debug("received 'FINISHED' from _procmgrthread") + self._process_events.task_done() + except: + traceback.print_exc() + raise OSError("IO Completion Port failed to signal process shutdown") + finally: + if self._handle: + self.returncode = winprocess.GetExitCodeProcess(self._handle) + self._cleanup() + + else: + # Not managing with job objects, so all we can reasonably do + # is call waitforsingleobject and hope for the best + self.debug("waiting without IO completion port") + + if not self._ignore_children: + self.debug("NOT USING JOB OBJECTS!!!") + # First, make sure we have not already ended + if self.returncode != winprocess.STILL_ACTIVE: + self._cleanup() + return self.returncode + + rc = None + if self._handle: + rc = winprocess.WaitForSingleObject(self._handle, -1) + + if rc == winprocess.WAIT_TIMEOUT: + # The process isn't dead, so kill it + print "Timed out waiting for process to close, attempting TerminateProcess" + self.kill() + elif rc == winprocess.WAIT_OBJECT_0: + # We caught WAIT_OBJECT_0, which indicates all is well + print "Single process terminated successfully" + self.returncode = winprocess.GetExitCodeProcess(self._handle) + else: + # An error occured we should probably throw + rc = winprocess.GetLastError() + if rc: + raise WinError(rc) + + self._cleanup() + + return self.returncode + + def _cleanup_job_io_port(self): + """ Do the job and IO port cleanup separately because there are + cases where we want to clean these without killing _handle + (i.e. if we fail to create the job object in the first place) + """ + if getattr(self, '_job') and self._job != winprocess.INVALID_HANDLE_VALUE: + self._job.Close() + self._job = None + else: + # If windows already freed our handle just set it to none + # (saw this intermittently while testing) + self._job = None + + if getattr(self, '_io_port', None) and \ + self._io_port != winprocess.INVALID_HANDLE_VALUE: + self._io_port.Close() + self._io_port = None + else: + self._io_port = None + + if getattr(self, '_procmgrthread', None): + self._procmgrthread = None + + def _cleanup(self): + self._cleanup_job_io_port() + if self._thread and self._thread != winprocess.INVALID_HANDLE_VALUE: + self._thread.Close() + self._thread = None + else: + self._thread = None + + if self._handle and self._handle != winprocess.INVALID_HANDLE_VALUE: + self._handle.Close() + self._handle = None + else: + self._handle = None + + elif isPosix: + + def _wait(self): + """ Haven't found any reason to differentiate between these platforms + so they all use the same wait callback. If it is necessary to + craft different styles of wait, then a new _wait method + could be easily implemented. + """ + + if not self._ignore_children: + try: + # os.waitpid return value: + # > [...] a tuple containing its pid and exit status + # > indication: a 16-bit number, whose low byte is the + # > signal number that killed the process, and whose + # > high byte is the exit status (if the signal number + # > is zero) + # - http://docs.python.org/2/library/os.html#os.wait + status = os.waitpid(self.pid, 0)[1] + + # For consistency, format status the same as subprocess' + # returncode attribute + if status > 255: + return status >> 8 + return -status + except OSError as e: + if getattr(e, "errno", None) != 10: + # Error 10 is "no child process", which could indicate normal + # close + print >> sys.stderr, \ + "Encountered error waiting for pid to close: %s" % e + raise + + return self.returncode + + else: + # For non-group wait, call base class + subprocess.Popen.wait(self) + return self.returncode + + def _cleanup(self): + pass + + else: + # An unrecognized platform, we will call the base class for everything + print >> sys.stderr, \ + "Unrecognized platform, process groups may not be managed properly" + + def _wait(self): + self.returncode = subprocess.Popen.wait(self) + return self.returncode + + def _cleanup(self): + pass + + def __init__(self, + cmd, + args=None, + cwd=None, + env=None, + ignore_children=False, + kill_on_timeout=True, + processOutputLine=(), + processStderrLine=(), + onTimeout=(), + onFinish=(), + **kwargs): + self.cmd = cmd + self.args = args + self.cwd = cwd + self.didTimeout = False + self._ignore_children = ignore_children + self.keywordargs = kwargs + self.read_buffer = '' + + if env is None: + env = os.environ.copy() + self.env = env + + # handlers + def to_callable_list(arg): + if callable(arg): + arg = [arg] + return CallableList(arg) + + processOutputLine = to_callable_list(processOutputLine) + processStderrLine = to_callable_list(processStderrLine) + onTimeout = to_callable_list(onTimeout) + onFinish = to_callable_list(onFinish) + + def on_timeout(): + self.didTimeout = True + if kill_on_timeout: + self.kill() + onTimeout.insert(0, on_timeout) + + self._stderr = subprocess.STDOUT + if processStderrLine: + self._stderr = subprocess.PIPE + self.reader = ProcessReader(stdout_callback=processOutputLine, + stderr_callback=processStderrLine, + finished_callback=onFinish, + timeout_callback=onTimeout) + + # It is common for people to pass in the entire array with the cmd and + # the args together since this is how Popen uses it. Allow for that. + if isinstance(self.cmd, list): + if self.args is not None: + raise TypeError("cmd and args must not both be lists") + (self.cmd, self.args) = (self.cmd[0], self.cmd[1:]) + elif self.args is None: + self.args = [] + + @property + def timedOut(self): + """True if the process has timed out.""" + return self.didTimeout + + @property + def commandline(self): + """the string value of the command line (command + args)""" + return subprocess.list2cmdline([self.cmd] + self.args) + + def run(self, timeout=None, outputTimeout=None): + """ + Starts the process. + + If timeout is not None, the process will be allowed to continue for + that number of seconds before being killed. If the process is killed + due to a timeout, the onTimeout handler will be called. + + If outputTimeout is not None, the process will be allowed to continue + for that number of seconds without producing any output before + being killed. + """ + self.didTimeout = False + + # default arguments + args = dict(stdout=subprocess.PIPE, + stderr=self._stderr, + cwd=self.cwd, + env=self.env, + ignore_children=self._ignore_children) + + # build process arguments + args.update(self.keywordargs) + + # launch the process + self.proc = self.Process([self.cmd] + self.args, **args) + + if isPosix: + # Keep track of the initial process group in case the process detaches itself + self.proc.pgid = os.getpgid(self.proc.pid) + self.proc.detached_pid = None + + self.processOutput(timeout=timeout, outputTimeout=outputTimeout) + + def kill(self, sig=None): + """ + Kills the managed process. + + If you created the process with 'ignore_children=False' (the + default) then it will also also kill all child processes spawned by + it. If you specified 'ignore_children=True' when creating the + process, only the root process will be killed. + + Note that this does not manage any state, save any output etc, + it immediately kills the process. + + :param sig: Signal used to kill the process, defaults to SIGKILL + (has no effect on Windows) + """ + if not hasattr(self, 'proc'): + raise RuntimeError("Calling kill() on a non started process is not" + " allowed.") + self.proc.kill(sig=sig) + + # When we kill the the managed process we also have to wait for the + # reader thread to be finished. Otherwise consumers would have to assume + # that it still has not completely shutdown. + return self.wait() + + def poll(self): + """Check if child process has terminated + + Returns the current returncode value: + - None if the process hasn't terminated yet + - A negative number if the process was killed by signal N (Unix only) + - '0' if the process ended without failures + + """ + # Ensure that we first check for the reader status. Otherwise + # we might mark the process as finished while output is still getting + # processed. + if not hasattr(self, 'proc'): + raise RuntimeError("Calling poll() on a non started process is not" + " allowed.") + elif self.reader.is_alive(): + return None + elif hasattr(self.proc, "returncode"): + return self.proc.returncode + else: + return self.proc.poll() + + def processOutput(self, timeout=None, outputTimeout=None): + """ + Handle process output until the process terminates or times out. + + If timeout is not None, the process will be allowed to continue for + that number of seconds before being killed. + + If outputTimeout is not None, the process will be allowed to continue + for that number of seconds without producing any output before + being killed. + """ + # this method is kept for backward compatibility + if not hasattr(self, 'proc'): + self.run(timeout=timeout, outputTimeout=outputTimeout) + # self.run will call this again + return + if not self.reader.is_alive(): + self.reader.timeout = timeout + self.reader.output_timeout = outputTimeout + self.reader.start(self.proc) + + def wait(self, timeout=None): + """ + Waits until all output has been read and the process is + terminated. + + If timeout is not None, will return after timeout seconds. + This timeout only causes the wait function to return and + does not kill the process. + + Returns the process exit code value: + - None if the process hasn't terminated yet + - A negative number if the process was killed by signal N (Unix only) + - '0' if the process ended without failures + + """ + if self.reader.thread and self.reader.thread is not threading.current_thread(): + # Thread.join() blocks the main thread until the reader thread is finished + # wake up once a second in case a keyboard interrupt is sent + count = 0 + while self.reader.is_alive(): + self.reader.thread.join(timeout=1) + count += 1 + if timeout and count > timeout: + return None + + self.returncode = self.proc.wait() + return self.returncode + + # TODO Remove this method when consumers have been fixed + def waitForFinish(self, timeout=None): + print >> sys.stderr, "MOZPROCESS WARNING: ProcessHandler.waitForFinish() is deprecated, " \ + "use ProcessHandler.wait() instead" + return self.wait(timeout=timeout) + + @property + def pid(self): + return self.proc.pid + + def check_for_detached(self, new_pid): + """Check if the current process has been detached and mark it appropriately. + + In case of application restarts the process can spawn itself into a new process group. + From now on the process can no longer be tracked by mozprocess anymore and has to be + marked as detached. If the consumer of mozprocess still knows the new process id it could + check for the detached state. + + new_pid is the new process id of the child process. + """ + if not self.proc: + return + + if isPosix: + new_pgid = None + try: + new_pgid = os.getpgid(new_pid) + except OSError as e: + # Do not consume errors except "No such process" + if e.errno != 3: + raise + + if new_pgid and new_pgid != self.proc.pgid: + self.proc.detached_pid = new_pid + print >> sys.stdout, \ + 'Child process with id "%s" has been marked as detached because it is no ' \ + 'longer in the managed process group. Keeping reference to the process id ' \ + '"%s" which is the new child process.' % (self.pid, new_pid) + + +class CallableList(list): + + def __call__(self, *args, **kwargs): + for e in self: + e(*args, **kwargs) + + def __add__(self, lst): + return CallableList(list.__add__(self, lst)) + + +class ProcessReader(object): + + def __init__(self, stdout_callback=None, stderr_callback=None, + finished_callback=None, timeout_callback=None, + timeout=None, output_timeout=None): + self.stdout_callback = stdout_callback or (lambda line: True) + self.stderr_callback = stderr_callback or (lambda line: True) + self.finished_callback = finished_callback or (lambda: True) + self.timeout_callback = timeout_callback or (lambda: True) + self.timeout = timeout + self.output_timeout = output_timeout + self.thread = None + + def _create_stream_reader(self, name, stream, queue, callback): + thread = threading.Thread(name=name, + target=self._read_stream, + args=(stream, queue, callback)) + thread.daemon = True + thread.start() + return thread + + def _read_stream(self, stream, queue, callback): + while True: + line = stream.readline() + if not line: + break + queue.put((line, callback)) + stream.close() + + def start(self, proc): + queue = Queue() + stdout_reader = None + if proc.stdout: + stdout_reader = self._create_stream_reader('ProcessReaderStdout', + proc.stdout, + queue, + self.stdout_callback) + stderr_reader = None + if proc.stderr and proc.stderr != proc.stdout: + stderr_reader = self._create_stream_reader('ProcessReaderStderr', + proc.stderr, + queue, + self.stderr_callback) + self.thread = threading.Thread(name='ProcessReader', + target=self._read, + args=(stdout_reader, + stderr_reader, + queue)) + self.thread.daemon = True + self.thread.start() + + def _read(self, stdout_reader, stderr_reader, queue): + start_time = time.time() + timed_out = False + timeout = self.timeout + if timeout is not None: + timeout += start_time + output_timeout = self.output_timeout + if output_timeout is not None: + output_timeout += start_time + + while (stdout_reader and stdout_reader.is_alive()) \ + or (stderr_reader and stderr_reader.is_alive()): + has_line = True + try: + line, callback = queue.get(True, 0.02) + except Empty: + has_line = False + now = time.time() + if not has_line: + if output_timeout is not None and now > output_timeout: + timed_out = True + break + else: + if output_timeout is not None: + output_timeout = now + self.output_timeout + callback(line.rstrip()) + if timeout is not None and now > timeout: + timed_out = True + break + # process remaining lines to read + while not queue.empty(): + line, callback = queue.get(False) + callback(line.rstrip()) + if timed_out: + self.timeout_callback() + if stdout_reader: + stdout_reader.join() + if stderr_reader: + stderr_reader.join() + if not timed_out: + self.finished_callback() + + def is_alive(self): + if self.thread: + return self.thread.is_alive() + return False + +# default output handlers +# these should be callables that take the output line + + +class StoreOutput(object): + """accumulate stdout""" + + def __init__(self): + self.output = [] + + def __call__(self, line): + self.output.append(line) + + +class StreamOutput(object): + """pass output to a stream and flush""" + + def __init__(self, stream): + self.stream = stream + + def __call__(self, line): + try: + self.stream.write(line + '\n') + except UnicodeDecodeError: + # TODO: Workaround for bug #991866 to make sure we can display when + # when normal UTF-8 display is failing + self.stream.write(line.decode('iso8859-1') + '\n') + self.stream.flush() + + +class LogOutput(StreamOutput): + """pass output to a file""" + + def __init__(self, filename): + self.file_obj = open(filename, 'a') + StreamOutput.__init__(self, self.file_obj) + + def __del__(self): + if self.file_obj is not None: + self.file_obj.close() + + +# front end class with the default handlers + + +class ProcessHandler(ProcessHandlerMixin): + """ + Convenience class for handling processes with default output handlers. + + By default, all output is sent to stdout. This can be disabled by setting + the *stream* argument to None. + + If processOutputLine keyword argument is specified the function or the + list of functions specified by this argument will be called for each line + of output; the output will not be written to stdout automatically then + if stream is True (the default). + + If storeOutput==True, the output produced by the process will be saved + as self.output. + + If logfile is not None, the output produced by the process will be + appended to the given file. + """ + + def __init__(self, cmd, logfile=None, stream=True, storeOutput=True, + **kwargs): + kwargs.setdefault('processOutputLine', []) + if callable(kwargs['processOutputLine']): + kwargs['processOutputLine'] = [kwargs['processOutputLine']] + + if logfile: + logoutput = LogOutput(logfile) + kwargs['processOutputLine'].append(logoutput) + + if stream is True: + # Print to standard output only if no outputline provided + if not kwargs['processOutputLine']: + kwargs['processOutputLine'].append(StreamOutput(sys.stdout)) + elif stream: + streamoutput = StreamOutput(stream) + kwargs['processOutputLine'].append(streamoutput) + + self.output = None + if storeOutput: + storeoutput = StoreOutput() + self.output = storeoutput.output + kwargs['processOutputLine'].append(storeoutput) + + ProcessHandlerMixin.__init__(self, cmd, **kwargs) diff --git a/testing/mozbase/mozprocess/mozprocess/qijo.py b/testing/mozbase/mozprocess/mozprocess/qijo.py new file mode 100644 index 000000000..ce23909fa --- /dev/null +++ b/testing/mozbase/mozprocess/mozprocess/qijo.py @@ -0,0 +1,166 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from ctypes import ( + c_void_p, + POINTER, + sizeof, + Structure, + windll, + WinError, + WINFUNCTYPE, + addressof, + c_size_t, + c_ulong +) + +from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LARGE_INTEGER + +LPVOID = c_void_p +LPDWORD = POINTER(DWORD) +SIZE_T = c_size_t +ULONG_PTR = POINTER(c_ulong) + +# A ULONGLONG is a 64-bit unsigned integer. +# Thus there are 8 bytes in a ULONGLONG. +# XXX why not import c_ulonglong ? +ULONGLONG = BYTE * 8 + + +class IO_COUNTERS(Structure): + # The IO_COUNTERS struct is 6 ULONGLONGs. + # TODO: Replace with non-dummy fields. + _fields_ = [('dummy', ULONGLONG * 6)] + + +class JOBOBJECT_BASIC_ACCOUNTING_INFORMATION(Structure): + _fields_ = [('TotalUserTime', LARGE_INTEGER), + ('TotalKernelTime', LARGE_INTEGER), + ('ThisPeriodTotalUserTime', LARGE_INTEGER), + ('ThisPeriodTotalKernelTime', LARGE_INTEGER), + ('TotalPageFaultCount', DWORD), + ('TotalProcesses', DWORD), + ('ActiveProcesses', DWORD), + ('TotalTerminatedProcesses', DWORD)] + + +class JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION(Structure): + _fields_ = [('BasicInfo', JOBOBJECT_BASIC_ACCOUNTING_INFORMATION), + ('IoInfo', IO_COUNTERS)] + + +# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx +class JOBOBJECT_BASIC_LIMIT_INFORMATION(Structure): + _fields_ = [('PerProcessUserTimeLimit', LARGE_INTEGER), + ('PerJobUserTimeLimit', LARGE_INTEGER), + ('LimitFlags', DWORD), + ('MinimumWorkingSetSize', SIZE_T), + ('MaximumWorkingSetSize', SIZE_T), + ('ActiveProcessLimit', DWORD), + ('Affinity', ULONG_PTR), + ('PriorityClass', DWORD), + ('SchedulingClass', DWORD) + ] + + +class JOBOBJECT_ASSOCIATE_COMPLETION_PORT(Structure): + _fields_ = [('CompletionKey', c_ulong), + ('CompletionPort', HANDLE)] + + +# see http://msdn.microsoft.com/en-us/library/ms684156%28VS.85%29.aspx +class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(Structure): + _fields_ = [('BasicLimitInformation', JOBOBJECT_BASIC_LIMIT_INFORMATION), + ('IoInfo', IO_COUNTERS), + ('ProcessMemoryLimit', SIZE_T), + ('JobMemoryLimit', SIZE_T), + ('PeakProcessMemoryUsed', SIZE_T), + ('PeakJobMemoryUsed', SIZE_T)] + +# These numbers below come from: +# http://msdn.microsoft.com/en-us/library/ms686216%28v=vs.85%29.aspx +JobObjectAssociateCompletionPortInformation = 7 +JobObjectBasicAndIoAccountingInformation = 8 +JobObjectExtendedLimitInformation = 9 + + +class JobObjectInfo(object): + mapping = {'JobObjectBasicAndIoAccountingInformation': 8, + 'JobObjectExtendedLimitInformation': 9, + 'JobObjectAssociateCompletionPortInformation': 7} + structures = { + 7: JOBOBJECT_ASSOCIATE_COMPLETION_PORT, + 8: JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION, + 9: JOBOBJECT_EXTENDED_LIMIT_INFORMATION + } + + def __init__(self, _class): + if isinstance(_class, basestring): + assert _class in self.mapping, \ + 'Class should be one of %s; you gave %s' % (self.mapping, _class) + _class = self.mapping[_class] + assert _class in self.structures, \ + 'Class should be one of %s; you gave %s' % (self.structures, _class) + self.code = _class + self.info = self.structures[_class]() + + +QueryInformationJobObjectProto = WINFUNCTYPE( + BOOL, # Return type + HANDLE, # hJob + DWORD, # JobObjectInfoClass + LPVOID, # lpJobObjectInfo + DWORD, # cbJobObjectInfoLength + LPDWORD # lpReturnLength +) + +QueryInformationJobObjectFlags = ( + (1, 'hJob'), + (1, 'JobObjectInfoClass'), + (1, 'lpJobObjectInfo'), + (1, 'cbJobObjectInfoLength'), + (1, 'lpReturnLength', None) +) + +_QueryInformationJobObject = QueryInformationJobObjectProto( + ('QueryInformationJobObject', windll.kernel32), + QueryInformationJobObjectFlags +) + + +class SubscriptableReadOnlyStruct(object): + + def __init__(self, struct): + self._struct = struct + + def _delegate(self, name): + result = getattr(self._struct, name) + if isinstance(result, Structure): + return SubscriptableReadOnlyStruct(result) + return result + + def __getitem__(self, name): + match = [fname for fname, ftype in self._struct._fields_ + if fname == name] + if match: + return self._delegate(name) + raise KeyError(name) + + def __getattr__(self, name): + return self._delegate(name) + + +def QueryInformationJobObject(hJob, JobObjectInfoClass): + jobinfo = JobObjectInfo(JobObjectInfoClass) + result = _QueryInformationJobObject( + hJob=hJob, + JobObjectInfoClass=jobinfo.code, + lpJobObjectInfo=addressof(jobinfo.info), + cbJobObjectInfoLength=sizeof(jobinfo.info) + ) + if not result: + raise WinError() + return SubscriptableReadOnlyStruct(jobinfo.info) diff --git a/testing/mozbase/mozprocess/mozprocess/winprocess.py b/testing/mozbase/mozprocess/mozprocess/winprocess.py new file mode 100644 index 000000000..b748f8f30 --- /dev/null +++ b/testing/mozbase/mozprocess/mozprocess/winprocess.py @@ -0,0 +1,479 @@ +# A module to expose various thread/process/job related structures and +# methods from kernel32 +# +# The MIT License +# +# Copyright (c) 2003-2004 by Peter Astrand <astrand@lysator.liu.se> +# +# Additions and modifications written by Benjamin Smedberg +# <benjamin@smedbergs.us> are Copyright (c) 2006 by the Mozilla Foundation +# <http://www.mozilla.org/> +# +# More Modifications +# Copyright (c) 2006-2007 by Mike Taylor <bear@code-bear.com> +# Copyright (c) 2007-2008 by Mikeal Rogers <mikeal@mozilla.com> +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of the +# author not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import absolute_import, unicode_literals + +import sys +import subprocess + +from ctypes import c_void_p, POINTER, sizeof, Structure, windll, WinError, WINFUNCTYPE, c_ulong +from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPCWSTR, LPWSTR, UINT, WORD +from .qijo import QueryInformationJobObject + +LPVOID = c_void_p +LPBYTE = POINTER(BYTE) +LPDWORD = POINTER(DWORD) +LPBOOL = POINTER(BOOL) +LPULONG = POINTER(c_ulong) + + +def ErrCheckBool(result, func, args): + """errcheck function for Windows functions that return a BOOL True + on success""" + if not result: + raise WinError() + return args + + +# AutoHANDLE + +class AutoHANDLE(HANDLE): + """Subclass of HANDLE which will call CloseHandle() on deletion.""" + + CloseHandleProto = WINFUNCTYPE(BOOL, HANDLE) + CloseHandle = CloseHandleProto(("CloseHandle", windll.kernel32)) + CloseHandle.errcheck = ErrCheckBool + + def Close(self): + if self.value and self.value != HANDLE(-1).value: + self.CloseHandle(self) + self.value = 0 + + def __del__(self): + self.Close() + + def __int__(self): + return self.value + + +def ErrCheckHandle(result, func, args): + """errcheck function for Windows functions that return a HANDLE.""" + if not result: + raise WinError() + return AutoHANDLE(result) + +# PROCESS_INFORMATION structure + + +class PROCESS_INFORMATION(Structure): + _fields_ = [("hProcess", HANDLE), + ("hThread", HANDLE), + ("dwProcessID", DWORD), + ("dwThreadID", DWORD)] + + def __init__(self): + Structure.__init__(self) + + self.cb = sizeof(self) + +LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION) + +# STARTUPINFO structure + + +class STARTUPINFO(Structure): + _fields_ = [("cb", DWORD), + ("lpReserved", LPWSTR), + ("lpDesktop", LPWSTR), + ("lpTitle", LPWSTR), + ("dwX", DWORD), + ("dwY", DWORD), + ("dwXSize", DWORD), + ("dwYSize", DWORD), + ("dwXCountChars", DWORD), + ("dwYCountChars", DWORD), + ("dwFillAttribute", DWORD), + ("dwFlags", DWORD), + ("wShowWindow", WORD), + ("cbReserved2", WORD), + ("lpReserved2", LPBYTE), + ("hStdInput", HANDLE), + ("hStdOutput", HANDLE), + ("hStdError", HANDLE) + ] +LPSTARTUPINFO = POINTER(STARTUPINFO) + +SW_HIDE = 0 + +STARTF_USESHOWWINDOW = 0x01 +STARTF_USESIZE = 0x02 +STARTF_USEPOSITION = 0x04 +STARTF_USECOUNTCHARS = 0x08 +STARTF_USEFILLATTRIBUTE = 0x10 +STARTF_RUNFULLSCREEN = 0x20 +STARTF_FORCEONFEEDBACK = 0x40 +STARTF_FORCEOFFFEEDBACK = 0x80 +STARTF_USESTDHANDLES = 0x100 + +# EnvironmentBlock + + +class EnvironmentBlock: + """An object which can be passed as the lpEnv parameter of CreateProcess. + It is initialized with a dictionary.""" + + def __init__(self, env): + if not env: + self._as_parameter_ = None + else: + values = [] + fs_encoding = sys.getfilesystemencoding() or 'mbcs' + for k, v in env.iteritems(): + if isinstance(k, bytes): + k = k.decode(fs_encoding, 'replace') + if isinstance(v, bytes): + v = v.decode(fs_encoding, 'replace') + values.append("{}={}".format(k, v)) + values.append("") + self._as_parameter_ = LPCWSTR("\0".join(values)) + +# Error Messages we need to watch for go here +# See: http://msdn.microsoft.com/en-us/library/ms681388%28v=vs.85%29.aspx +ERROR_ABANDONED_WAIT_0 = 735 + +# GetLastError() +GetLastErrorProto = WINFUNCTYPE(DWORD) # Return Type +GetLastErrorFlags = () +GetLastError = GetLastErrorProto(("GetLastError", windll.kernel32), GetLastErrorFlags) + +# CreateProcess() + +CreateProcessProto = WINFUNCTYPE(BOOL, # Return type + LPCWSTR, # lpApplicationName + LPWSTR, # lpCommandLine + LPVOID, # lpProcessAttributes + LPVOID, # lpThreadAttributes + BOOL, # bInheritHandles + DWORD, # dwCreationFlags + LPVOID, # lpEnvironment + LPCWSTR, # lpCurrentDirectory + LPSTARTUPINFO, # lpStartupInfo + LPPROCESS_INFORMATION # lpProcessInformation + ) + +CreateProcessFlags = ((1, "lpApplicationName", None), + (1, "lpCommandLine"), + (1, "lpProcessAttributes", None), + (1, "lpThreadAttributes", None), + (1, "bInheritHandles", True), + (1, "dwCreationFlags", 0), + (1, "lpEnvironment", None), + (1, "lpCurrentDirectory", None), + (1, "lpStartupInfo"), + (2, "lpProcessInformation")) + + +def ErrCheckCreateProcess(result, func, args): + ErrCheckBool(result, func, args) + # return a tuple (hProcess, hThread, dwProcessID, dwThreadID) + pi = args[9] + return AutoHANDLE(pi.hProcess), AutoHANDLE(pi.hThread), pi.dwProcessID, pi.dwThreadID + +CreateProcess = CreateProcessProto(("CreateProcessW", windll.kernel32), + CreateProcessFlags) +CreateProcess.errcheck = ErrCheckCreateProcess + +# flags for CreateProcess +CREATE_BREAKAWAY_FROM_JOB = 0x01000000 +CREATE_DEFAULT_ERROR_MODE = 0x04000000 +CREATE_NEW_CONSOLE = 0x00000010 +CREATE_NEW_PROCESS_GROUP = 0x00000200 +CREATE_NO_WINDOW = 0x08000000 +CREATE_SUSPENDED = 0x00000004 +CREATE_UNICODE_ENVIRONMENT = 0x00000400 + +# Flags for IOCompletion ports (some of these would probably be defined if +# we used the win32 extensions for python, but we don't want to do that if we +# can help it. +INVALID_HANDLE_VALUE = HANDLE(-1) # From winbase.h + +# Self Defined Constants for IOPort <--> Job Object communication +COMPKEY_TERMINATE = c_ulong(0) +COMPKEY_JOBOBJECT = c_ulong(1) + +# flags for job limit information +# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx +JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800 +JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x00001000 + +# Flags for Job Object Completion Port Message IDs from winnt.h +# See also: http://msdn.microsoft.com/en-us/library/ms684141%28v=vs.85%29.aspx +JOB_OBJECT_MSG_END_OF_JOB_TIME = 1 +JOB_OBJECT_MSG_END_OF_PROCESS_TIME = 2 +JOB_OBJECT_MSG_ACTIVE_PROCESS_LIMIT = 3 +JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO = 4 +JOB_OBJECT_MSG_NEW_PROCESS = 6 +JOB_OBJECT_MSG_EXIT_PROCESS = 7 +JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS = 8 +JOB_OBJECT_MSG_PROCESS_MEMORY_LIMIT = 9 +JOB_OBJECT_MSG_JOB_MEMORY_LIMIT = 10 + +# See winbase.h +DEBUG_ONLY_THIS_PROCESS = 0x00000002 +DEBUG_PROCESS = 0x00000001 +DETACHED_PROCESS = 0x00000008 + +# GetQueuedCompletionPortStatus - +# http://msdn.microsoft.com/en-us/library/aa364986%28v=vs.85%29.aspx +GetQueuedCompletionStatusProto = WINFUNCTYPE(BOOL, # Return Type + HANDLE, # Completion Port + LPDWORD, # Msg ID + LPULONG, # Completion Key + # PID Returned from the call (may be null) + LPULONG, + DWORD) # milliseconds to wait +GetQueuedCompletionStatusFlags = ((1, "CompletionPort", INVALID_HANDLE_VALUE), + (1, "lpNumberOfBytes", None), + (1, "lpCompletionKey", None), + (1, "lpPID", None), + (1, "dwMilliseconds", 0)) +GetQueuedCompletionStatus = GetQueuedCompletionStatusProto(("GetQueuedCompletionStatus", + windll.kernel32), + GetQueuedCompletionStatusFlags) + +# CreateIOCompletionPort +# Note that the completion key is just a number, not a pointer. +CreateIoCompletionPortProto = WINFUNCTYPE(HANDLE, # Return Type + HANDLE, # File Handle + HANDLE, # Existing Completion Port + c_ulong, # Completion Key + DWORD) # Number of Threads + +CreateIoCompletionPortFlags = ((1, "FileHandle", INVALID_HANDLE_VALUE), + (1, "ExistingCompletionPort", 0), + (1, "CompletionKey", c_ulong(0)), + (1, "NumberOfConcurrentThreads", 0)) +CreateIoCompletionPort = CreateIoCompletionPortProto(("CreateIoCompletionPort", + windll.kernel32), + CreateIoCompletionPortFlags) +CreateIoCompletionPort.errcheck = ErrCheckHandle + +# SetInformationJobObject +SetInformationJobObjectProto = WINFUNCTYPE(BOOL, # Return Type + HANDLE, # Job Handle + DWORD, # Type of Class next param is + LPVOID, # Job Object Class + DWORD) # Job Object Class Length + +SetInformationJobObjectProtoFlags = ((1, "hJob", None), + (1, "JobObjectInfoClass", None), + (1, "lpJobObjectInfo", None), + (1, "cbJobObjectInfoLength", 0)) +SetInformationJobObject = SetInformationJobObjectProto(("SetInformationJobObject", + windll.kernel32), + SetInformationJobObjectProtoFlags) +SetInformationJobObject.errcheck = ErrCheckBool + +# CreateJobObject() +CreateJobObjectProto = WINFUNCTYPE(HANDLE, # Return type + LPVOID, # lpJobAttributes + LPCWSTR # lpName + ) + +CreateJobObjectFlags = ((1, "lpJobAttributes", None), + (1, "lpName", None)) + +CreateJobObject = CreateJobObjectProto(("CreateJobObjectW", windll.kernel32), + CreateJobObjectFlags) +CreateJobObject.errcheck = ErrCheckHandle + +# AssignProcessToJobObject() + +AssignProcessToJobObjectProto = WINFUNCTYPE(BOOL, # Return type + HANDLE, # hJob + HANDLE # hProcess + ) +AssignProcessToJobObjectFlags = ((1, "hJob"), + (1, "hProcess")) +AssignProcessToJobObject = AssignProcessToJobObjectProto( + ("AssignProcessToJobObject", windll.kernel32), + AssignProcessToJobObjectFlags) +AssignProcessToJobObject.errcheck = ErrCheckBool + +# GetCurrentProcess() +# because os.getPid() is way too easy +GetCurrentProcessProto = WINFUNCTYPE(HANDLE # Return type + ) +GetCurrentProcessFlags = () +GetCurrentProcess = GetCurrentProcessProto( + ("GetCurrentProcess", windll.kernel32), + GetCurrentProcessFlags) +GetCurrentProcess.errcheck = ErrCheckHandle + +# IsProcessInJob() +try: + IsProcessInJobProto = WINFUNCTYPE(BOOL, # Return type + HANDLE, # Process Handle + HANDLE, # Job Handle + LPBOOL # Result + ) + IsProcessInJobFlags = ((1, "ProcessHandle"), + (1, "JobHandle", HANDLE(0)), + (2, "Result")) + IsProcessInJob = IsProcessInJobProto( + ("IsProcessInJob", windll.kernel32), + IsProcessInJobFlags) + IsProcessInJob.errcheck = ErrCheckBool +except AttributeError: + # windows 2k doesn't have this API + def IsProcessInJob(process): + return False + + +# ResumeThread() + +def ErrCheckResumeThread(result, func, args): + if result == -1: + raise WinError() + + return args + +ResumeThreadProto = WINFUNCTYPE(DWORD, # Return type + HANDLE # hThread + ) +ResumeThreadFlags = ((1, "hThread"),) +ResumeThread = ResumeThreadProto(("ResumeThread", windll.kernel32), + ResumeThreadFlags) +ResumeThread.errcheck = ErrCheckResumeThread + +# TerminateProcess() + +TerminateProcessProto = WINFUNCTYPE(BOOL, # Return type + HANDLE, # hProcess + UINT # uExitCode + ) +TerminateProcessFlags = ((1, "hProcess"), + (1, "uExitCode", 127)) +TerminateProcess = TerminateProcessProto( + ("TerminateProcess", windll.kernel32), + TerminateProcessFlags) +TerminateProcess.errcheck = ErrCheckBool + +# TerminateJobObject() + +TerminateJobObjectProto = WINFUNCTYPE(BOOL, # Return type + HANDLE, # hJob + UINT # uExitCode + ) +TerminateJobObjectFlags = ((1, "hJob"), + (1, "uExitCode", 127)) +TerminateJobObject = TerminateJobObjectProto( + ("TerminateJobObject", windll.kernel32), + TerminateJobObjectFlags) +TerminateJobObject.errcheck = ErrCheckBool + +# WaitForSingleObject() + +WaitForSingleObjectProto = WINFUNCTYPE(DWORD, # Return type + HANDLE, # hHandle + DWORD, # dwMilliseconds + ) +WaitForSingleObjectFlags = ((1, "hHandle"), + (1, "dwMilliseconds", -1)) +WaitForSingleObject = WaitForSingleObjectProto( + ("WaitForSingleObject", windll.kernel32), + WaitForSingleObjectFlags) + +# http://msdn.microsoft.com/en-us/library/ms681381%28v=vs.85%29.aspx +INFINITE = -1 +WAIT_TIMEOUT = 0x0102 +WAIT_OBJECT_0 = 0x0 +WAIT_ABANDONED = 0x0080 + +# http://msdn.microsoft.com/en-us/library/ms683189%28VS.85%29.aspx +STILL_ACTIVE = 259 + +# Used when we terminate a process. +ERROR_CONTROL_C_EXIT = 0x23c + +# GetExitCodeProcess() + +GetExitCodeProcessProto = WINFUNCTYPE(BOOL, # Return type + HANDLE, # hProcess + LPDWORD, # lpExitCode + ) +GetExitCodeProcessFlags = ((1, "hProcess"), + (2, "lpExitCode")) +GetExitCodeProcess = GetExitCodeProcessProto( + ("GetExitCodeProcess", windll.kernel32), + GetExitCodeProcessFlags) +GetExitCodeProcess.errcheck = ErrCheckBool + + +def CanCreateJobObject(): + currentProc = GetCurrentProcess() + if IsProcessInJob(currentProc): + jobinfo = QueryInformationJobObject(HANDLE(0), 'JobObjectExtendedLimitInformation') + limitflags = jobinfo['BasicLimitInformation']['LimitFlags'] + return bool(limitflags & JOB_OBJECT_LIMIT_BREAKAWAY_OK) or \ + bool(limitflags & JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK) + else: + return True + +# testing functions + + +def parent(): + print 'Starting parent' + currentProc = GetCurrentProcess() + if IsProcessInJob(currentProc): + print >> sys.stderr, "You should not be in a job object to test" + sys.exit(1) + assert CanCreateJobObject() + print 'File: %s' % __file__ + command = [sys.executable, __file__, '-child'] + print 'Running command: %s' % command + process = subprocess.Popen(command) + process.kill() + code = process.returncode + print 'Child code: %s' % code + assert code == 127 + + +def child(): + print 'Starting child' + currentProc = GetCurrentProcess() + injob = IsProcessInJob(currentProc) + print "Is in a job?: %s" % injob + can_create = CanCreateJobObject() + print 'Can create job?: %s' % can_create + process = subprocess.Popen('c:\\windows\\notepad.exe') + assert process._job + jobinfo = QueryInformationJobObject(process._job, 'JobObjectExtendedLimitInformation') + print 'Job info: %s' % jobinfo + limitflags = jobinfo['BasicLimitInformation']['LimitFlags'] + print 'LimitFlags: %s' % limitflags + process.kill() diff --git a/testing/mozbase/mozprocess/setup.py b/testing/mozbase/mozprocess/setup.py new file mode 100644 index 000000000..fc8225a12 --- /dev/null +++ b/testing/mozbase/mozprocess/setup.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_VERSION = '0.23' + +setup(name='mozprocess', + version=PACKAGE_VERSION, + description="Mozilla-authored process handling", + long_description='see http://mozbase.readthedocs.org/', + classifiers=['Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + keywords='mozilla', + author='Mozilla Automation and Tools team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL 2.0', + packages=['mozprocess'], + include_package_data=True, + zip_safe=False, + install_requires=['mozinfo'], + entry_points=""" + # -*- Entry points: -*- + """, + ) diff --git a/testing/mozbase/mozprocess/tests/Makefile b/testing/mozbase/mozprocess/tests/Makefile new file mode 100644 index 000000000..ea7163b00 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/Makefile @@ -0,0 +1,55 @@ +# +# mozprocess proclaunch tests Makefile +# + +# include rules for platform determination +include iniparser/platform.mk + +ifeq ($(WIN32), 1) +# Win 32 +CC = cl +LINK = link +CFLAGS = //Od //I "iniparser" //D "WIN32" //D "_WIN32" //D "_DEBUG" //D "_CONSOLE" //D "_UNICODE" //D "UNICODE" //Gm //EHsc //RTC1 //MDd //W3 //nologo //c //ZI //TC +LFLAGS = //OUT:"proclaunch.exe" //INCREMENTAL //LIBPATH:"iniparser\\" //NOLOGO //DEBUG //SUBSYSTEM:CONSOLE //DYNAMICBASE //NXCOMPAT //ERRORREPORT:PROMPT iniparser.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib +RM = rm -f + +all: iniparser proclaunch + +iniparser: + $(MAKE) -C iniparser + +proclaunch.obj: proclaunch.c + @(echo "compiling proclaunch; platform: $(UNAME), WIN32: $(WIN32)") + $(CC) $(CFLAGS) proclaunch.c + +proclaunch: proclaunch.obj + $(LINK) $(LFLAGS) proclaunch.obj + +clean: + $(RM) proclaunch.exe proclaunch.obj +else +# *nix/Mac +LFLAGS = -L.. -liniparser +AR = ar +ARFLAGS = rcv +RM = rm -f +CC = gcc +ifeq ($(UNAME), Linux) +CFLAGS = -g -v -Iiniparser +else +CFLAGS = -g -v -arch i386 -Iiniparser +endif + +all: libiniparser.a proclaunch + +libiniparser.a: + $(MAKE) -C iniparser + +proclaunch: proclaunch.c + @(echo "compiling proclaunch; platform: $(UNAME), WIN32: $(WIN32)") + $(CC) $(CFLAGS) -o proclaunch proclaunch.c -Iiniparser -Liniparser -liniparser + +clean: + $(RM) proclaunch + $(MAKE) -C iniparser clean +endif diff --git a/testing/mozbase/mozprocess/tests/infinite_loop.py b/testing/mozbase/mozprocess/tests/infinite_loop.py new file mode 100644 index 000000000..e38e425e0 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/infinite_loop.py @@ -0,0 +1,18 @@ +import threading +import time +import sys +import signal + +if 'deadlock' in sys.argv: + lock = threading.Lock() + + def trap(sig, frame): + lock.acquire() + + # get the lock once + lock.acquire() + # and take it again on SIGTERM signal: deadlock. + signal.signal(signal.SIGTERM, trap) + +while 1: + time.sleep(1) diff --git a/testing/mozbase/mozprocess/tests/iniparser/AUTHORS b/testing/mozbase/mozprocess/tests/iniparser/AUTHORS new file mode 100644 index 000000000..d5a3f6b2e --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/AUTHORS @@ -0,0 +1,6 @@ +Author: Nicolas Devillard <ndevilla@free.fr> + +This tiny library has received countless contributions and I have +not kept track of all the people who contributed. Let them be thanked +for their ideas, code, suggestions, corrections, enhancements! + diff --git a/testing/mozbase/mozprocess/tests/iniparser/INSTALL b/testing/mozbase/mozprocess/tests/iniparser/INSTALL new file mode 100644 index 000000000..a5b05d0e2 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/INSTALL @@ -0,0 +1,15 @@ + +iniParser installation instructions +----------------------------------- + +- Modify the Makefile to suit your environment. +- Type 'make' to make the library. +- Type 'make check' to make the test program. +- Type 'test/iniexample' to launch the test program. +- Type 'test/parse' to launch torture tests. + + + +Enjoy! +N. Devillard +Wed Mar 2 21:14:17 CET 2011 diff --git a/testing/mozbase/mozprocess/tests/iniparser/LICENSE b/testing/mozbase/mozprocess/tests/iniparser/LICENSE new file mode 100644 index 000000000..5a3a80bab --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) 2000-2011 by Nicolas Devillard. +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + diff --git a/testing/mozbase/mozprocess/tests/iniparser/Makefile b/testing/mozbase/mozprocess/tests/iniparser/Makefile new file mode 100644 index 000000000..48c86f9d6 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/Makefile @@ -0,0 +1,85 @@ +# +# iniparser Makefile +# + +# source files +SRCS = iniparser.c \ + dictionary.c + +# include rules for platform determination +include platform.mk + +# flags for the the various systems +ifeq ($(UNAME), Linux) + # Compiler settings + CC = gcc + AR = ar + ARFLAGS = rcv + SHLD = ${CC} ${CFLAGS} + CFLAGS = -O2 -fPIC -Wall -ansi -pedantic + LDSHFLAGS = -shared -Wl,-Bsymbolic -Wl,-rpath -Wl,/usr/lib -Wl,-rpath,/usr/lib + LDFLAGS = -Wl,-rpath -Wl,/usr/lib -Wl,-rpath,/usr/lib +endif +ifeq ($(UNAME), Darwin) + # Compiler settings + CC = gcc + # Ar settings to build the library + AR = ar + ARFLAGS = rcv + SHLD = libtool + CFLAGS = -v -arch i386 -fPIC -Wall -ansi -pedantic + LDFLAGS = -arch_only i386 +endif +ifeq ($(WIN32), 1) + CC = cl + CFLAGS = //Od //D "_WIN32" //D "WIN32" //D "_CONSOLE" //D "_CRT_SECURE_NO_WARNINGS" //D "_UNICODE" //D "UNICODE" //Gm //EHsc //RTC1 //MDd //W3 //nologo //c //ZI //TC + LDFLAGS = //OUT:"iniparser.lib" //NOLOGO + LINK = lib + RM = rm -f +endif + +# windows build rules +ifeq ($(WIN32), 1) + +COMPILE.c = $(CC) $(CFLAGS) -c +OBJS = $(SRCS:.c=.obj) + +all: iniparser.obj dictionary.obj iniparser.lib + +iniparser.obj: dictionary.obj + @($(CC) $(CFLAGS) iniparser.c) + +dictionary.obj: + @(echo "compiling dictionary; WIN32: $(WIN32); platform: $(UNAME)") + @($(CC) $(CFLAGS) dictionary.c) + +iniparser.lib: dictionary.obj iniparser.obj + @(echo "linking $(OBJS)") + @($(LINK) $(LDFLAGS) $(OBJS)) +else + +# *nix (and Mac) build rules +RM = rm -f +COMPILE.c = $(CC) $(CFLAGS) -c +OBJS = $(SRCS:.c=.o) + +all: libiniparser.a libiniparser.so + +.c.o: + @(echo "platform: $(UNAME), WIN32=$(WIN32); compiling $< ...") + @($(COMPILE.c) -o $@ $<) + +libiniparser.a: $(OBJS) + @($(AR) $(ARFLAGS) libiniparser.a $(OBJS)) + +ifeq ($(UNAME), Linux) +libiniparser.so: $(OBJS) + @$(SHLD) $(LDSHFLAGS) -o $@.0 $(OBJS) $(LDFLAGS) +else +libiniparser.so: $(OBJS) + @$(SHLD) -o $@.0 $(LDFLAGS) $(OBJS) +endif +endif + +clean: + $(RM) $(OBJS) libiniparser.* diff --git a/testing/mozbase/mozprocess/tests/iniparser/README b/testing/mozbase/mozprocess/tests/iniparser/README new file mode 100644 index 000000000..af2a5c38f --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/README @@ -0,0 +1,12 @@ + +Welcome to iniParser -- version 3.0 +released 02 Mar 2011 + +This modules offers parsing of ini files from the C level. +See a complete documentation in HTML format, from this directory +open the file html/index.html with any HTML-capable browser. + +Enjoy! + +N.Devillard +Wed Mar 2 21:46:14 CET 2011 diff --git a/testing/mozbase/mozprocess/tests/iniparser/dictionary.c b/testing/mozbase/mozprocess/tests/iniparser/dictionary.c new file mode 100644 index 000000000..da41d9b2e --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/dictionary.c @@ -0,0 +1,407 @@ +/*-------------------------------------------------------------------------*/ +/** + @file dictionary.c + @author N. Devillard + @date Sep 2007 + @version $Revision: 1.27 $ + @brief Implements a dictionary for string variables. + + This module implements a simple dictionary object, i.e. a list + of string/string associations. This object is useful to store e.g. + informations retrieved from a configuration file (ini files). +*/ +/*--------------------------------------------------------------------------*/ + +/* + $Id: dictionary.c,v 1.27 2007-11-23 21:39:18 ndevilla Exp $ + $Revision: 1.27 $ +*/ +/*--------------------------------------------------------------------------- + Includes + ---------------------------------------------------------------------------*/ +#include "dictionary.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +/** Maximum value size for integers and doubles. */ +#define MAXVALSZ 1024 + +/** Minimal allocated number of entries in a dictionary */ +#define DICTMINSZ 128 + +/** Invalid key token */ +#define DICT_INVALID_KEY ((char*)-1) + +/*--------------------------------------------------------------------------- + Private functions + ---------------------------------------------------------------------------*/ + +/* Doubles the allocated size associated to a pointer */ +/* 'size' is the current allocated size. */ +static void * mem_double(void * ptr, int size) +{ + void * newptr ; + + newptr = calloc(2*size, 1); + if (newptr==NULL) { + return NULL ; + } + memcpy(newptr, ptr, size); + free(ptr); + return newptr ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Duplicate a string + @param s String to duplicate + @return Pointer to a newly allocated string, to be freed with free() + + This is a replacement for strdup(). This implementation is provided + for systems that do not have it. + */ +/*--------------------------------------------------------------------------*/ +static char * xstrdup(char * s) +{ + char * t ; + if (!s) + return NULL ; + t = malloc(strlen(s)+1) ; + if (t) { + strcpy(t,s); + } + return t ; +} + +/*--------------------------------------------------------------------------- + Function codes + ---------------------------------------------------------------------------*/ +/*-------------------------------------------------------------------------*/ +/** + @brief Compute the hash key for a string. + @param key Character string to use for key. + @return 1 unsigned int on at least 32 bits. + + This hash function has been taken from an Article in Dr Dobbs Journal. + This is normally a collision-free function, distributing keys evenly. + The key is stored anyway in the struct so that collision can be avoided + by comparing the key itself in last resort. + */ +/*--------------------------------------------------------------------------*/ +unsigned dictionary_hash(char * key) +{ + int len ; + unsigned hash ; + int i ; + + len = strlen(key); + for (hash=0, i=0 ; i<len ; i++) { + hash += (unsigned)key[i] ; + hash += (hash<<10); + hash ^= (hash>>6) ; + } + hash += (hash <<3); + hash ^= (hash >>11); + hash += (hash <<15); + return hash ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Create a new dictionary object. + @param size Optional initial size of the dictionary. + @return 1 newly allocated dictionary objet. + + This function allocates a new dictionary object of given size and returns + it. If you do not know in advance (roughly) the number of entries in the + dictionary, give size=0. + */ +/*--------------------------------------------------------------------------*/ +dictionary * dictionary_new(int size) +{ + dictionary * d ; + + /* If no size was specified, allocate space for DICTMINSZ */ + if (size<DICTMINSZ) size=DICTMINSZ ; + + if (!(d = (dictionary *)calloc(1, sizeof(dictionary)))) { + return NULL; + } + d->size = size ; + d->val = (char **)calloc(size, sizeof(char*)); + d->key = (char **)calloc(size, sizeof(char*)); + d->hash = (unsigned int *)calloc(size, sizeof(unsigned)); + return d ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Delete a dictionary object + @param d dictionary object to deallocate. + @return void + + Deallocate a dictionary object and all memory associated to it. + */ +/*--------------------------------------------------------------------------*/ +void dictionary_del(dictionary * d) +{ + int i ; + + if (d==NULL) return ; + for (i=0 ; i<d->size ; i++) { + if (d->key[i]!=NULL) + free(d->key[i]); + if (d->val[i]!=NULL) + free(d->val[i]); + } + free(d->val); + free(d->key); + free(d->hash); + free(d); + return ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get a value from a dictionary. + @param d dictionary object to search. + @param key Key to look for in the dictionary. + @param def Default value to return if key not found. + @return 1 pointer to internally allocated character string. + + This function locates a key in a dictionary and returns a pointer to its + value, or the passed 'def' pointer if no such key can be found in + dictionary. The returned character pointer points to data internal to the + dictionary object, you should not try to free it or modify it. + */ +/*--------------------------------------------------------------------------*/ +char * dictionary_get(dictionary * d, char * key, char * def) +{ + unsigned hash ; + int i ; + + hash = dictionary_hash(key); + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + /* Compare hash */ + if (hash==d->hash[i]) { + /* Compare string, to avoid hash collisions */ + if (!strcmp(key, d->key[i])) { + return d->val[i] ; + } + } + } + return def ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Set a value in a dictionary. + @param d dictionary object to modify. + @param key Key to modify or add. + @param val Value to add. + @return int 0 if Ok, anything else otherwise + + If the given key is found in the dictionary, the associated value is + replaced by the provided one. If the key cannot be found in the + dictionary, it is added to it. + + It is Ok to provide a NULL value for val, but NULL values for the dictionary + or the key are considered as errors: the function will return immediately + in such a case. + + Notice that if you dictionary_set a variable to NULL, a call to + dictionary_get will return a NULL value: the variable will be found, and + its value (NULL) is returned. In other words, setting the variable + content to NULL is equivalent to deleting the variable from the + dictionary. It is not possible (in this implementation) to have a key in + the dictionary without value. + + This function returns non-zero in case of failure. + */ +/*--------------------------------------------------------------------------*/ +int dictionary_set(dictionary * d, char * key, char * val) +{ + int i ; + unsigned hash ; + + if (d==NULL || key==NULL) return -1 ; + + /* Compute hash for this key */ + hash = dictionary_hash(key) ; + /* Find if value is already in dictionary */ + if (d->n>0) { + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + if (hash==d->hash[i]) { /* Same hash value */ + if (!strcmp(key, d->key[i])) { /* Same key */ + /* Found a value: modify and return */ + if (d->val[i]!=NULL) + free(d->val[i]); + d->val[i] = val ? xstrdup(val) : NULL ; + /* Value has been modified: return */ + return 0 ; + } + } + } + } + /* Add a new value */ + /* See if dictionary needs to grow */ + if (d->n==d->size) { + + /* Reached maximum size: reallocate dictionary */ + d->val = (char **)mem_double(d->val, d->size * sizeof(char*)) ; + d->key = (char **)mem_double(d->key, d->size * sizeof(char*)) ; + d->hash = (unsigned int *)mem_double(d->hash, d->size * sizeof(unsigned)) ; + if ((d->val==NULL) || (d->key==NULL) || (d->hash==NULL)) { + /* Cannot grow dictionary */ + return -1 ; + } + /* Double size */ + d->size *= 2 ; + } + + /* Insert key in the first empty slot */ + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) { + /* Add key here */ + break ; + } + } + /* Copy key */ + d->key[i] = xstrdup(key); + d->val[i] = val ? xstrdup(val) : NULL ; + d->hash[i] = hash; + d->n ++ ; + return 0 ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Delete a key in a dictionary + @param d dictionary object to modify. + @param key Key to remove. + @return void + + This function deletes a key in a dictionary. Nothing is done if the + key cannot be found. + */ +/*--------------------------------------------------------------------------*/ +void dictionary_unset(dictionary * d, char * key) +{ + unsigned hash ; + int i ; + + if (key == NULL) { + return; + } + + hash = dictionary_hash(key); + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + /* Compare hash */ + if (hash==d->hash[i]) { + /* Compare string, to avoid hash collisions */ + if (!strcmp(key, d->key[i])) { + /* Found key */ + break ; + } + } + } + if (i>=d->size) + /* Key not found */ + return ; + + free(d->key[i]); + d->key[i] = NULL ; + if (d->val[i]!=NULL) { + free(d->val[i]); + d->val[i] = NULL ; + } + d->hash[i] = 0 ; + d->n -- ; + return ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Dump a dictionary to an opened file pointer. + @param d Dictionary to dump + @param f Opened file pointer. + @return void + + Dumps a dictionary onto an opened file pointer. Key pairs are printed out + as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as + output file pointers. + */ +/*--------------------------------------------------------------------------*/ +void dictionary_dump(dictionary * d, FILE * out) +{ + int i ; + + if (d==NULL || out==NULL) return ; + if (d->n<1) { + fprintf(out, "empty dictionary\n"); + return ; + } + for (i=0 ; i<d->size ; i++) { + if (d->key[i]) { + fprintf(out, "%20s\t[%s]\n", + d->key[i], + d->val[i] ? d->val[i] : "UNDEF"); + } + } + return ; +} + + +/* Test code */ +#ifdef TESTDIC +#define NVALS 20000 +int main(int argc, char *argv[]) +{ + dictionary * d ; + char * val ; + int i ; + char cval[90] ; + + /* Allocate dictionary */ + printf("allocating...\n"); + d = dictionary_new(0); + + /* Set values in dictionary */ + printf("setting %d values...\n", NVALS); + for (i=0 ; i<NVALS ; i++) { + sprintf(cval, "%04d", i); + dictionary_set(d, cval, "salut"); + } + printf("getting %d values...\n", NVALS); + for (i=0 ; i<NVALS ; i++) { + sprintf(cval, "%04d", i); + val = dictionary_get(d, cval, DICT_INVALID_KEY); + if (val==DICT_INVALID_KEY) { + printf("cannot get value for key [%s]\n", cval); + } + } + printf("unsetting %d values...\n", NVALS); + for (i=0 ; i<NVALS ; i++) { + sprintf(cval, "%04d", i); + dictionary_unset(d, cval); + } + if (d->n != 0) { + printf("error deleting values\n"); + } + printf("deallocating...\n"); + dictionary_del(d); + return 0 ; +} +#endif +/* vim: set ts=4 et sw=4 tw=75 */ diff --git a/testing/mozbase/mozprocess/tests/iniparser/dictionary.h b/testing/mozbase/mozprocess/tests/iniparser/dictionary.h new file mode 100644 index 000000000..e340a82d0 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/dictionary.h @@ -0,0 +1,176 @@ + +/*-------------------------------------------------------------------------*/ +/** + @file dictionary.h + @author N. Devillard + @date Sep 2007 + @version $Revision: 1.12 $ + @brief Implements a dictionary for string variables. + + This module implements a simple dictionary object, i.e. a list + of string/string associations. This object is useful to store e.g. + informations retrieved from a configuration file (ini files). +*/ +/*--------------------------------------------------------------------------*/ + +/* + $Id: dictionary.h,v 1.12 2007-11-23 21:37:00 ndevilla Exp $ + $Author: ndevilla $ + $Date: 2007-11-23 21:37:00 $ + $Revision: 1.12 $ +*/ + +#ifndef _DICTIONARY_H_ +#define _DICTIONARY_H_ + +/*--------------------------------------------------------------------------- + Includes + ---------------------------------------------------------------------------*/ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +/*--------------------------------------------------------------------------- + New types + ---------------------------------------------------------------------------*/ + + +/*-------------------------------------------------------------------------*/ +/** + @brief Dictionary object + + This object contains a list of string/string associations. Each + association is identified by a unique string key. Looking up values + in the dictionary is speeded up by the use of a (hopefully collision-free) + hash function. + */ +/*-------------------------------------------------------------------------*/ +typedef struct _dictionary_ { + int n ; /** Number of entries in dictionary */ + int size ; /** Storage size */ + char ** val ; /** List of string values */ + char ** key ; /** List of string keys */ + unsigned * hash ; /** List of hash values for keys */ +} dictionary ; + + +/*--------------------------------------------------------------------------- + Function prototypes + ---------------------------------------------------------------------------*/ + +/*-------------------------------------------------------------------------*/ +/** + @brief Compute the hash key for a string. + @param key Character string to use for key. + @return 1 unsigned int on at least 32 bits. + + This hash function has been taken from an Article in Dr Dobbs Journal. + This is normally a collision-free function, distributing keys evenly. + The key is stored anyway in the struct so that collision can be avoided + by comparing the key itself in last resort. + */ +/*--------------------------------------------------------------------------*/ +unsigned dictionary_hash(char * key); + +/*-------------------------------------------------------------------------*/ +/** + @brief Create a new dictionary object. + @param size Optional initial size of the dictionary. + @return 1 newly allocated dictionary objet. + + This function allocates a new dictionary object of given size and returns + it. If you do not know in advance (roughly) the number of entries in the + dictionary, give size=0. + */ +/*--------------------------------------------------------------------------*/ +dictionary * dictionary_new(int size); + +/*-------------------------------------------------------------------------*/ +/** + @brief Delete a dictionary object + @param d dictionary object to deallocate. + @return void + + Deallocate a dictionary object and all memory associated to it. + */ +/*--------------------------------------------------------------------------*/ +void dictionary_del(dictionary * vd); + +/*-------------------------------------------------------------------------*/ +/** + @brief Get a value from a dictionary. + @param d dictionary object to search. + @param key Key to look for in the dictionary. + @param def Default value to return if key not found. + @return 1 pointer to internally allocated character string. + + This function locates a key in a dictionary and returns a pointer to its + value, or the passed 'def' pointer if no such key can be found in + dictionary. The returned character pointer points to data internal to the + dictionary object, you should not try to free it or modify it. + */ +/*--------------------------------------------------------------------------*/ +char * dictionary_get(dictionary * d, char * key, char * def); + + +/*-------------------------------------------------------------------------*/ +/** + @brief Set a value in a dictionary. + @param d dictionary object to modify. + @param key Key to modify or add. + @param val Value to add. + @return int 0 if Ok, anything else otherwise + + If the given key is found in the dictionary, the associated value is + replaced by the provided one. If the key cannot be found in the + dictionary, it is added to it. + + It is Ok to provide a NULL value for val, but NULL values for the dictionary + or the key are considered as errors: the function will return immediately + in such a case. + + Notice that if you dictionary_set a variable to NULL, a call to + dictionary_get will return a NULL value: the variable will be found, and + its value (NULL) is returned. In other words, setting the variable + content to NULL is equivalent to deleting the variable from the + dictionary. It is not possible (in this implementation) to have a key in + the dictionary without value. + + This function returns non-zero in case of failure. + */ +/*--------------------------------------------------------------------------*/ +int dictionary_set(dictionary * vd, char * key, char * val); + +/*-------------------------------------------------------------------------*/ +/** + @brief Delete a key in a dictionary + @param d dictionary object to modify. + @param key Key to remove. + @return void + + This function deletes a key in a dictionary. Nothing is done if the + key cannot be found. + */ +/*--------------------------------------------------------------------------*/ +void dictionary_unset(dictionary * d, char * key); + + +/*-------------------------------------------------------------------------*/ +/** + @brief Dump a dictionary to an opened file pointer. + @param d Dictionary to dump + @param f Opened file pointer. + @return void + + Dumps a dictionary onto an opened file pointer. Key pairs are printed out + as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as + output file pointers. + */ +/*--------------------------------------------------------------------------*/ +void dictionary_dump(dictionary * d, FILE * out); + +#endif diff --git a/testing/mozbase/mozprocess/tests/iniparser/iniparser.c b/testing/mozbase/mozprocess/tests/iniparser/iniparser.c new file mode 100644 index 000000000..02a23b755 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/iniparser.c @@ -0,0 +1,648 @@ + +/*-------------------------------------------------------------------------*/ +/** + @file iniparser.c + @author N. Devillard + @date Sep 2007 + @version 3.0 + @brief Parser for ini files. +*/ +/*--------------------------------------------------------------------------*/ +/* + $Id: iniparser.c,v 2.19 2011-03-02 20:15:13 ndevilla Exp $ + $Revision: 2.19 $ + $Date: 2011-03-02 20:15:13 $ +*/ +/*---------------------------- Includes ------------------------------------*/ +#include <ctype.h> +#include "iniparser.h" + +/*---------------------------- Defines -------------------------------------*/ +#define ASCIILINESZ (1024) +#define INI_INVALID_KEY ((char*)-1) + +/*--------------------------------------------------------------------------- + Private to this module + ---------------------------------------------------------------------------*/ +/** + * This enum stores the status for each parsed line (internal use only). + */ +typedef enum _line_status_ { + LINE_UNPROCESSED, + LINE_ERROR, + LINE_EMPTY, + LINE_COMMENT, + LINE_SECTION, + LINE_VALUE +} line_status ; + +/*-------------------------------------------------------------------------*/ +/** + @brief Convert a string to lowercase. + @param s String to convert. + @return ptr to statically allocated string. + + This function returns a pointer to a statically allocated string + containing a lowercased version of the input string. Do not free + or modify the returned string! Since the returned string is statically + allocated, it will be modified at each function call (not re-entrant). + */ +/*--------------------------------------------------------------------------*/ +static char * strlwc(char * s) +{ + static char l[ASCIILINESZ+1]; + int i ; + + if (s==NULL) return NULL ; + memset(l, 0, ASCIILINESZ+1); + i=0 ; + while (s[i] && i<ASCIILINESZ) { + l[i] = (char)tolower((int)s[i]); + i++ ; + } + l[ASCIILINESZ]=(char)0; + return l ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Remove blanks at the beginning and the end of a string. + @param s String to parse. + @return ptr to statically allocated string. + + This function returns a pointer to a statically allocated string, + which is identical to the input string, except that all blank + characters at the end and the beg. of the string have been removed. + Do not free or modify the returned string! Since the returned string + is statically allocated, it will be modified at each function call + (not re-entrant). + */ +/*--------------------------------------------------------------------------*/ +static char * strstrip(char * s) +{ + static char l[ASCIILINESZ+1]; + char * last ; + + if (s==NULL) return NULL ; + + while (isspace((int)*s) && *s) s++; + memset(l, 0, ASCIILINESZ+1); + strcpy(l, s); + last = l + strlen(l); + while (last > l) { + if (!isspace((int)*(last-1))) + break ; + last -- ; + } + *last = (char)0; + return (char*)l ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get number of sections in a dictionary + @param d Dictionary to examine + @return int Number of sections found in dictionary + + This function returns the number of sections found in a dictionary. + The test to recognize sections is done on the string stored in the + dictionary: a section name is given as "section" whereas a key is + stored as "section:key", thus the test looks for entries that do not + contain a colon. + + This clearly fails in the case a section name contains a colon, but + this should simply be avoided. + + This function returns -1 in case of error. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_getnsec(dictionary * d) +{ + int i ; + int nsec ; + + if (d==NULL) return -1 ; + nsec=0 ; + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + if (strchr(d->key[i], ':')==NULL) { + nsec ++ ; + } + } + return nsec ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get name for section n in a dictionary. + @param d Dictionary to examine + @param n Section number (from 0 to nsec-1). + @return Pointer to char string + + This function locates the n-th section in a dictionary and returns + its name as a pointer to a string statically allocated inside the + dictionary. Do not free or modify the returned string! + + This function returns NULL in case of error. + */ +/*--------------------------------------------------------------------------*/ +char * iniparser_getsecname(dictionary * d, int n) +{ + int i ; + int foundsec ; + + if (d==NULL || n<0) return NULL ; + foundsec=0 ; + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + if (strchr(d->key[i], ':')==NULL) { + foundsec++ ; + if (foundsec>n) + break ; + } + } + if (foundsec<=n) { + return NULL ; + } + return d->key[i] ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Dump a dictionary to an opened file pointer. + @param d Dictionary to dump. + @param f Opened file pointer to dump to. + @return void + + This function prints out the contents of a dictionary, one element by + line, onto the provided file pointer. It is OK to specify @c stderr + or @c stdout as output files. This function is meant for debugging + purposes mostly. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_dump(dictionary * d, FILE * f) +{ + int i ; + + if (d==NULL || f==NULL) return ; + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + if (d->val[i]!=NULL) { + fprintf(f, "[%s]=[%s]\n", d->key[i], d->val[i]); + } else { + fprintf(f, "[%s]=UNDEF\n", d->key[i]); + } + } + return ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Save a dictionary to a loadable ini file + @param d Dictionary to dump + @param f Opened file pointer to dump to + @return void + + This function dumps a given dictionary into a loadable ini file. + It is Ok to specify @c stderr or @c stdout as output files. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_dump_ini(dictionary * d, FILE * f) +{ + int i, j ; + char keym[ASCIILINESZ+1]; + int nsec ; + char * secname ; + int seclen ; + + if (d==NULL || f==NULL) return ; + + nsec = iniparser_getnsec(d); + if (nsec<1) { + /* No section in file: dump all keys as they are */ + for (i=0 ; i<d->size ; i++) { + if (d->key[i]==NULL) + continue ; + fprintf(f, "%s = %s\n", d->key[i], d->val[i]); + } + return ; + } + for (i=0 ; i<nsec ; i++) { + secname = iniparser_getsecname(d, i) ; + seclen = (int)strlen(secname); + fprintf(f, "\n[%s]\n", secname); + sprintf(keym, "%s:", secname); + for (j=0 ; j<d->size ; j++) { + if (d->key[j]==NULL) + continue ; + if (!strncmp(d->key[j], keym, seclen+1)) { + fprintf(f, + "%-30s = %s\n", + d->key[j]+seclen+1, + d->val[j] ? d->val[j] : ""); + } + } + } + fprintf(f, "\n"); + return ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key + @param d Dictionary to search + @param key Key string to look for + @param def Default value to return if key not found. + @return pointer to statically allocated character string + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the pointer passed as 'def' is returned. + The returned char pointer is pointing to a string allocated in + the dictionary, do not free or modify it. + */ +/*--------------------------------------------------------------------------*/ +char * iniparser_getstring(dictionary * d, char * key, char * def) +{ + char * lc_key ; + char * sval ; + + if (d==NULL || key==NULL) + return def ; + + lc_key = strlwc(key); + sval = dictionary_get(d, lc_key, def); + return sval ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key, convert to an int + @param d Dictionary to search + @param key Key string to look for + @param notfound Value to return in case of error + @return integer + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the notfound value is returned. + + Supported values for integers include the usual C notation + so decimal, octal (starting with 0) and hexadecimal (starting with 0x) + are supported. Examples: + + "42" -> 42 + "042" -> 34 (octal -> decimal) + "0x42" -> 66 (hexa -> decimal) + + Warning: the conversion may overflow in various ways. Conversion is + totally outsourced to strtol(), see the associated man page for overflow + handling. + + Credits: Thanks to A. Becker for suggesting strtol() + */ +/*--------------------------------------------------------------------------*/ +int iniparser_getint(dictionary * d, char * key, int notfound) +{ + char * str ; + + str = iniparser_getstring(d, key, INI_INVALID_KEY); + if (str==INI_INVALID_KEY) return notfound ; + return (int)strtol(str, NULL, 0); +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key, convert to a double + @param d Dictionary to search + @param key Key string to look for + @param notfound Value to return in case of error + @return double + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the notfound value is returned. + */ +/*--------------------------------------------------------------------------*/ +double iniparser_getdouble(dictionary * d, char * key, double notfound) +{ + char * str ; + + str = iniparser_getstring(d, key, INI_INVALID_KEY); + if (str==INI_INVALID_KEY) return notfound ; + return atof(str); +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key, convert to a boolean + @param d Dictionary to search + @param key Key string to look for + @param notfound Value to return in case of error + @return integer + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the notfound value is returned. + + A true boolean is found if one of the following is matched: + + - A string starting with 'y' + - A string starting with 'Y' + - A string starting with 't' + - A string starting with 'T' + - A string starting with '1' + + A false boolean is found if one of the following is matched: + + - A string starting with 'n' + - A string starting with 'N' + - A string starting with 'f' + - A string starting with 'F' + - A string starting with '0' + + The notfound value returned if no boolean is identified, does not + necessarily have to be 0 or 1. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_getboolean(dictionary * d, char * key, int notfound) +{ + char * c ; + int ret ; + + c = iniparser_getstring(d, key, INI_INVALID_KEY); + if (c==INI_INVALID_KEY) return notfound ; + if (c[0]=='y' || c[0]=='Y' || c[0]=='1' || c[0]=='t' || c[0]=='T') { + ret = 1 ; + } else if (c[0]=='n' || c[0]=='N' || c[0]=='0' || c[0]=='f' || c[0]=='F') { + ret = 0 ; + } else { + ret = notfound ; + } + return ret; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Finds out if a given entry exists in a dictionary + @param ini Dictionary to search + @param entry Name of the entry to look for + @return integer 1 if entry exists, 0 otherwise + + Finds out if a given entry exists in the dictionary. Since sections + are stored as keys with NULL associated values, this is the only way + of querying for the presence of sections in a dictionary. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_find_entry( + dictionary * ini, + char * entry +) +{ + int found=0 ; + if (iniparser_getstring(ini, entry, INI_INVALID_KEY)!=INI_INVALID_KEY) { + found = 1 ; + } + return found ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Set an entry in a dictionary. + @param ini Dictionary to modify. + @param entry Entry to modify (entry name) + @param val New value to associate to the entry. + @return int 0 if Ok, -1 otherwise. + + If the given entry can be found in the dictionary, it is modified to + contain the provided value. If it cannot be found, -1 is returned. + It is Ok to set val to NULL. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_set(dictionary * ini, char * entry, char * val) +{ + return dictionary_set(ini, strlwc(entry), val) ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Delete an entry in a dictionary + @param ini Dictionary to modify + @param entry Entry to delete (entry name) + @return void + + If the given entry can be found, it is deleted from the dictionary. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_unset(dictionary * ini, char * entry) +{ + dictionary_unset(ini, strlwc(entry)); +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Load a single line from an INI file + @param input_line Input line, may be concatenated multi-line input + @param section Output space to store section + @param key Output space to store key + @param value Output space to store value + @return line_status value + */ +/*--------------------------------------------------------------------------*/ +static line_status iniparser_line( + char * input_line, + char * section, + char * key, + char * value) +{ + line_status sta ; + char line[ASCIILINESZ+1]; + int len ; + + strcpy(line, strstrip(input_line)); + len = (int)strlen(line); + + sta = LINE_UNPROCESSED ; + if (len<1) { + /* Empty line */ + sta = LINE_EMPTY ; + } else if (line[0]=='#' || line[0]==';') { + /* Comment line */ + sta = LINE_COMMENT ; + } else if (line[0]=='[' && line[len-1]==']') { + /* Section name */ + sscanf(line, "[%[^]]", section); + strcpy(section, strstrip(section)); + strcpy(section, strlwc(section)); + sta = LINE_SECTION ; + } else if (sscanf (line, "%[^=] = \"%[^\"]\"", key, value) == 2 + || sscanf (line, "%[^=] = '%[^\']'", key, value) == 2 + || sscanf (line, "%[^=] = %[^;#]", key, value) == 2) { + /* Usual key=value, with or without comments */ + strcpy(key, strstrip(key)); + strcpy(key, strlwc(key)); + strcpy(value, strstrip(value)); + /* + * sscanf cannot handle '' or "" as empty values + * this is done here + */ + if (!strcmp(value, "\"\"") || (!strcmp(value, "''"))) { + value[0]=0 ; + } + sta = LINE_VALUE ; + } else if (sscanf(line, "%[^=] = %[;#]", key, value)==2 + || sscanf(line, "%[^=] %[=]", key, value) == 2) { + /* + * Special cases: + * key= + * key=; + * key=# + */ + strcpy(key, strstrip(key)); + strcpy(key, strlwc(key)); + value[0]=0 ; + sta = LINE_VALUE ; + } else { + /* Generate syntax error */ + sta = LINE_ERROR ; + } + return sta ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Parse an ini file and return an allocated dictionary object + @param ininame Name of the ini file to read. + @return Pointer to newly allocated dictionary + + This is the parser for ini files. This function is called, providing + the name of the file to be read. It returns a dictionary object that + should not be accessed directly, but through accessor functions + instead. + + The returned dictionary must be freed using iniparser_freedict(). + */ +/*--------------------------------------------------------------------------*/ +dictionary * iniparser_load(char * ininame) +{ + FILE * in ; + + char line [ASCIILINESZ+1] ; + char section [ASCIILINESZ+1] ; + char key [ASCIILINESZ+1] ; + char tmp [ASCIILINESZ+1] ; + char val [ASCIILINESZ+1] ; + + int last=0 ; + int len ; + int lineno=0 ; + int errs=0; + + dictionary * dict ; + + if ((in=fopen(ininame, "r"))==NULL) { + fprintf(stderr, "iniparser: cannot open %s\n", ininame); + return NULL ; + } + + dict = dictionary_new(0) ; + if (!dict) { + fclose(in); + return NULL ; + } + + memset(line, 0, ASCIILINESZ); + memset(section, 0, ASCIILINESZ); + memset(key, 0, ASCIILINESZ); + memset(val, 0, ASCIILINESZ); + last=0 ; + + while (fgets(line+last, ASCIILINESZ-last, in)!=NULL) { + lineno++ ; + len = (int)strlen(line)-1; + if (len==0) + continue; + /* Safety check against buffer overflows */ + if (line[len]!='\n') { + fprintf(stderr, + "iniparser: input line too long in %s (%d)\n", + ininame, + lineno); + dictionary_del(dict); + fclose(in); + return NULL ; + } + /* Get rid of \n and spaces at end of line */ + while ((len>=0) && + ((line[len]=='\n') || (isspace(line[len])))) { + line[len]=0 ; + len-- ; + } + /* Detect multi-line */ + if (line[len]=='\\') { + /* Multi-line value */ + last=len ; + continue ; + } else { + last=0 ; + } + switch (iniparser_line(line, section, key, val)) { + case LINE_EMPTY: + case LINE_COMMENT: + break ; + + case LINE_SECTION: + errs = dictionary_set(dict, section, NULL); + break ; + + case LINE_VALUE: + sprintf(tmp, "%s:%s", section, key); + errs = dictionary_set(dict, tmp, val) ; + break ; + + case LINE_ERROR: + fprintf(stderr, "iniparser: syntax error in %s (%d):\n", + ininame, + lineno); + fprintf(stderr, "-> %s\n", line); + errs++ ; + break; + + default: + break ; + } + memset(line, 0, ASCIILINESZ); + last=0; + if (errs<0) { + fprintf(stderr, "iniparser: memory allocation failure\n"); + break ; + } + } + if (errs) { + dictionary_del(dict); + dict = NULL ; + } + fclose(in); + return dict ; +} + +/*-------------------------------------------------------------------------*/ +/** + @brief Free all memory associated to an ini dictionary + @param d Dictionary to free + @return void + + Free all memory associated to an ini dictionary. + It is mandatory to call this function before the dictionary object + gets out of the current context. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_freedict(dictionary * d) +{ + dictionary_del(d); +} + +/* vim: set ts=4 et sw=4 tw=75 */ diff --git a/testing/mozbase/mozprocess/tests/iniparser/iniparser.h b/testing/mozbase/mozprocess/tests/iniparser/iniparser.h new file mode 100644 index 000000000..e3468b2c9 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/iniparser.h @@ -0,0 +1,273 @@ + +/*-------------------------------------------------------------------------*/ +/** + @file iniparser.h + @author N. Devillard + @date Sep 2007 + @version 3.0 + @brief Parser for ini files. +*/ +/*--------------------------------------------------------------------------*/ + +/* + $Id: iniparser.h,v 1.26 2011-03-02 20:15:13 ndevilla Exp $ + $Revision: 1.26 $ +*/ + +#ifndef _INIPARSER_H_ +#define _INIPARSER_H_ + +/*--------------------------------------------------------------------------- + Includes + ---------------------------------------------------------------------------*/ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* + * The following #include is necessary on many Unixes but not Linux. + * It is not needed for Windows platforms. + * Uncomment it if needed. + */ +/* #include <unistd.h> */ + +#include "dictionary.h" + +/*-------------------------------------------------------------------------*/ +/** + @brief Get number of sections in a dictionary + @param d Dictionary to examine + @return int Number of sections found in dictionary + + This function returns the number of sections found in a dictionary. + The test to recognize sections is done on the string stored in the + dictionary: a section name is given as "section" whereas a key is + stored as "section:key", thus the test looks for entries that do not + contain a colon. + + This clearly fails in the case a section name contains a colon, but + this should simply be avoided. + + This function returns -1 in case of error. + */ +/*--------------------------------------------------------------------------*/ + +int iniparser_getnsec(dictionary * d); + + +/*-------------------------------------------------------------------------*/ +/** + @brief Get name for section n in a dictionary. + @param d Dictionary to examine + @param n Section number (from 0 to nsec-1). + @return Pointer to char string + + This function locates the n-th section in a dictionary and returns + its name as a pointer to a string statically allocated inside the + dictionary. Do not free or modify the returned string! + + This function returns NULL in case of error. + */ +/*--------------------------------------------------------------------------*/ + +char * iniparser_getsecname(dictionary * d, int n); + + +/*-------------------------------------------------------------------------*/ +/** + @brief Save a dictionary to a loadable ini file + @param d Dictionary to dump + @param f Opened file pointer to dump to + @return void + + This function dumps a given dictionary into a loadable ini file. + It is Ok to specify @c stderr or @c stdout as output files. + */ +/*--------------------------------------------------------------------------*/ + +void iniparser_dump_ini(dictionary * d, FILE * f); + +/*-------------------------------------------------------------------------*/ +/** + @brief Dump a dictionary to an opened file pointer. + @param d Dictionary to dump. + @param f Opened file pointer to dump to. + @return void + + This function prints out the contents of a dictionary, one element by + line, onto the provided file pointer. It is OK to specify @c stderr + or @c stdout as output files. This function is meant for debugging + purposes mostly. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_dump(dictionary * d, FILE * f); + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key + @param d Dictionary to search + @param key Key string to look for + @param def Default value to return if key not found. + @return pointer to statically allocated character string + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the pointer passed as 'def' is returned. + The returned char pointer is pointing to a string allocated in + the dictionary, do not free or modify it. + */ +/*--------------------------------------------------------------------------*/ +char * iniparser_getstring(dictionary * d, char * key, char * def); + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key, convert to an int + @param d Dictionary to search + @param key Key string to look for + @param notfound Value to return in case of error + @return integer + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the notfound value is returned. + + Supported values for integers include the usual C notation + so decimal, octal (starting with 0) and hexadecimal (starting with 0x) + are supported. Examples: + + - "42" -> 42 + - "042" -> 34 (octal -> decimal) + - "0x42" -> 66 (hexa -> decimal) + + Warning: the conversion may overflow in various ways. Conversion is + totally outsourced to strtol(), see the associated man page for overflow + handling. + + Credits: Thanks to A. Becker for suggesting strtol() + */ +/*--------------------------------------------------------------------------*/ +int iniparser_getint(dictionary * d, char * key, int notfound); + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key, convert to a double + @param d Dictionary to search + @param key Key string to look for + @param notfound Value to return in case of error + @return double + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the notfound value is returned. + */ +/*--------------------------------------------------------------------------*/ +double iniparser_getdouble(dictionary * d, char * key, double notfound); + +/*-------------------------------------------------------------------------*/ +/** + @brief Get the string associated to a key, convert to a boolean + @param d Dictionary to search + @param key Key string to look for + @param notfound Value to return in case of error + @return integer + + This function queries a dictionary for a key. A key as read from an + ini file is given as "section:key". If the key cannot be found, + the notfound value is returned. + + A true boolean is found if one of the following is matched: + + - A string starting with 'y' + - A string starting with 'Y' + - A string starting with 't' + - A string starting with 'T' + - A string starting with '1' + + A false boolean is found if one of the following is matched: + + - A string starting with 'n' + - A string starting with 'N' + - A string starting with 'f' + - A string starting with 'F' + - A string starting with '0' + + The notfound value returned if no boolean is identified, does not + necessarily have to be 0 or 1. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_getboolean(dictionary * d, char * key, int notfound); + + +/*-------------------------------------------------------------------------*/ +/** + @brief Set an entry in a dictionary. + @param ini Dictionary to modify. + @param entry Entry to modify (entry name) + @param val New value to associate to the entry. + @return int 0 if Ok, -1 otherwise. + + If the given entry can be found in the dictionary, it is modified to + contain the provided value. If it cannot be found, -1 is returned. + It is Ok to set val to NULL. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_set(dictionary * ini, char * entry, char * val); + + +/*-------------------------------------------------------------------------*/ +/** + @brief Delete an entry in a dictionary + @param ini Dictionary to modify + @param entry Entry to delete (entry name) + @return void + + If the given entry can be found, it is deleted from the dictionary. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_unset(dictionary * ini, char * entry); + +/*-------------------------------------------------------------------------*/ +/** + @brief Finds out if a given entry exists in a dictionary + @param ini Dictionary to search + @param entry Name of the entry to look for + @return integer 1 if entry exists, 0 otherwise + + Finds out if a given entry exists in the dictionary. Since sections + are stored as keys with NULL associated values, this is the only way + of querying for the presence of sections in a dictionary. + */ +/*--------------------------------------------------------------------------*/ +int iniparser_find_entry(dictionary * ini, char * entry) ; + +/*-------------------------------------------------------------------------*/ +/** + @brief Parse an ini file and return an allocated dictionary object + @param ininame Name of the ini file to read. + @return Pointer to newly allocated dictionary + + This is the parser for ini files. This function is called, providing + the name of the file to be read. It returns a dictionary object that + should not be accessed directly, but through accessor functions + instead. + + The returned dictionary must be freed using iniparser_freedict(). + */ +/*--------------------------------------------------------------------------*/ +dictionary * iniparser_load(char * ininame); + +/*-------------------------------------------------------------------------*/ +/** + @brief Free all memory associated to an ini dictionary + @param d Dictionary to free + @return void + + Free all memory associated to an ini dictionary. + It is mandatory to call this function before the dictionary object + gets out of the current context. + */ +/*--------------------------------------------------------------------------*/ +void iniparser_freedict(dictionary * d); + +#endif diff --git a/testing/mozbase/mozprocess/tests/iniparser/platform.mk b/testing/mozbase/mozprocess/tests/iniparser/platform.mk new file mode 100644 index 000000000..bff0296fe --- /dev/null +++ b/testing/mozbase/mozprocess/tests/iniparser/platform.mk @@ -0,0 +1,8 @@ +# System platform + +# determine if windows +WIN32 := 0 +UNAME := $(shell uname -s) +ifneq (,$(findstring MINGW32_NT,$(UNAME))) +WIN32 = 1 +endif diff --git a/testing/mozbase/mozprocess/tests/manifest.ini b/testing/mozbase/mozprocess/tests/manifest.ini new file mode 100644 index 000000000..d869952e3 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/manifest.ini @@ -0,0 +1,18 @@ +# does not currently work on windows +# see https://bugzilla.mozilla.org/show_bug.cgi?id=790765#c51 + +[DEFAULT] +# bug https://bugzilla.mozilla.org/show_bug.cgi?id=778267#c26 +skip-if = (os == "win") + +[test_mozprocess.py] +disabled = bug 877864 +[test_mozprocess_kill.py] +[test_mozprocess_kill_broad_wait.py] +disabled = bug 921632 +[test_mozprocess_misc.py] +[test_mozprocess_poll.py] +[test_mozprocess_wait.py] +[test_mozprocess_output.py] +[test_mozprocess_params.py] +[test_process_reader.py] diff --git a/testing/mozbase/mozprocess/tests/proccountfive.py b/testing/mozbase/mozprocess/tests/proccountfive.py new file mode 100644 index 000000000..5ec74b32a --- /dev/null +++ b/testing/mozbase/mozprocess/tests/proccountfive.py @@ -0,0 +1,2 @@ +for i in range(0, 5): + print i diff --git a/testing/mozbase/mozprocess/tests/process_normal_broad_python.ini b/testing/mozbase/mozprocess/tests/process_normal_broad_python.ini new file mode 100644 index 000000000..28109cb31 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_normal_broad_python.ini @@ -0,0 +1,30 @@ +; Generate a Broad Process Tree +; This generates a Tree of the form: +; +; main +; \_ c1 +; | \_ c2 +; | \_ c2 +; | \_ c2 +; | \_ c2 +; | \_ c2 +; | +; \_ c1 +; | \_ c2 +; | \_ c2 +; | \_ c2 +; | \_ c2 +; | \_ c2 +; | +; \_ ... 23 more times + +[main] +children=25*c1 +maxtime=10 + +[c1] +children=5*c2 +maxtime=10 + +[c2] +maxtime=5 diff --git a/testing/mozbase/mozprocess/tests/process_normal_deep_python.ini b/testing/mozbase/mozprocess/tests/process_normal_deep_python.ini new file mode 100644 index 000000000..ef9809f6a --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_normal_deep_python.ini @@ -0,0 +1,65 @@ +; Deep Process Tree +; Should generate a process tree of the form: +; +; main +; \_ c2 +; | \_ c5 +; | | \_ c6 +; | | \_ c7 +; | | \_ c8 +; | | \_ c1 +; | | \_ c4 +; | \_ c5 +; | \_ c6 +; | \_ c7 +; | \_ c8 +; | \_ c1 +; | \_ c4 +; \_ c2 +; | \_ c5 +; | | \_ c6 +; | | \_ c7 +; | | \_ c8 +; | | \_ c1 +; | | \_ c4 +; | \_ c5 +; | \_ c6 +; | \_ c7 +; | \_ c8 +; | \_ c1 +; | \_ c4 +; \_ c1 +; | \_ c4 +; \_ c1 +; \_ c4 + +[main] +children=2*c1, 2*c2 +maxtime=20 + +[c1] +children=c4 +maxtime=20 + +[c2] +children=2*c5 +maxtime=20 + +[c4] +maxtime=20 + +[c5] +children=c6 +maxtime=20 + +[c6] +children=c7 +maxtime=20 + +[c7] +children=c8 +maxtime=20 + +[c8] +children=c1 +maxtime=20 diff --git a/testing/mozbase/mozprocess/tests/process_normal_finish.ini b/testing/mozbase/mozprocess/tests/process_normal_finish.ini new file mode 100644 index 000000000..c4468de49 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_normal_finish.ini @@ -0,0 +1,11 @@ +[main] +children=c1,c2 +maxtime=60 + +[c1] +children=2 +maxtime=60 + +[c2] +children=0 +maxtime=30 diff --git a/testing/mozbase/mozprocess/tests/process_normal_finish_no_process_group.ini b/testing/mozbase/mozprocess/tests/process_normal_finish_no_process_group.ini new file mode 100644 index 000000000..2b0f1f9a4 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_normal_finish_no_process_group.ini @@ -0,0 +1,2 @@ +[main] +maxtime=10 diff --git a/testing/mozbase/mozprocess/tests/process_normal_finish_python.ini b/testing/mozbase/mozprocess/tests/process_normal_finish_python.ini new file mode 100644 index 000000000..4519c7083 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_normal_finish_python.ini @@ -0,0 +1,17 @@ +; Generates a normal process tree +; Tree is of the form: +; main +; \_ c1 +; \_ c2 + +[main] +children=c1,c2 +maxtime=10 + +[c1] +children=c2 +maxtime=5 + +[c2] +maxtime=5 + diff --git a/testing/mozbase/mozprocess/tests/process_waittimeout.ini b/testing/mozbase/mozprocess/tests/process_waittimeout.ini new file mode 100644 index 000000000..77cbf2e39 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_waittimeout.ini @@ -0,0 +1,11 @@ +[main] +children=c1,c2 +maxtime=300 + +[c1] +children=2 +maxtime=300 + +[c2] +children=3 +maxtime=300 diff --git a/testing/mozbase/mozprocess/tests/process_waittimeout_10s.ini b/testing/mozbase/mozprocess/tests/process_waittimeout_10s.ini new file mode 100644 index 000000000..59d2d76ff --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_waittimeout_10s.ini @@ -0,0 +1,8 @@ +[main] +children=c1 +maxtime=10 + +[c1] +children=2 +maxtime=5 + diff --git a/testing/mozbase/mozprocess/tests/process_waittimeout_10s_python.ini b/testing/mozbase/mozprocess/tests/process_waittimeout_10s_python.ini new file mode 100644 index 000000000..abf8d6a4e --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_waittimeout_10s_python.ini @@ -0,0 +1,16 @@ +; Generate a normal process tree +; Tree is of the form: +; main +; \_ c1 +; \_ c2 + +[main] +children=c1 +maxtime=10 + +[c1] +children=2*c2 +maxtime=5 + +[c2] +maxtime=5 diff --git a/testing/mozbase/mozprocess/tests/process_waittimeout_python.ini b/testing/mozbase/mozprocess/tests/process_waittimeout_python.ini new file mode 100644 index 000000000..5800267d1 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/process_waittimeout_python.ini @@ -0,0 +1,16 @@ +; Generates a normal process tree +; Tree is of the form: +; main +; \_ c1 +; \_ c2 + +[main] +children=2*c1 +maxtime=300 + +[c1] +children=2*c2 +maxtime=300 + +[c2] +maxtime=300 diff --git a/testing/mozbase/mozprocess/tests/proclaunch.c b/testing/mozbase/mozprocess/tests/proclaunch.c new file mode 100644 index 000000000..05c564c79 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/proclaunch.c @@ -0,0 +1,156 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <stdio.h> +#include <stdlib.h> +#include "iniparser.h" + +#ifdef _WIN32 +#include <windows.h> +#include <tchar.h> + +extern int iniparser_getint(dictionary *d, char *key, int notfound); +extern char *iniparser_getstring(dictionary *d, char *key, char *def); + +// This is the windows launcher function +int launchWindows(int children, int maxtime) { + _TCHAR cmdline[50]; + STARTUPINFO startup; + PROCESS_INFORMATION procinfo; + BOOL rv = 0; + + _stprintf(cmdline, _T("proclaunch.exe %d %d"), children, maxtime); + ZeroMemory(&startup, sizeof(STARTUPINFO)); + startup.cb = sizeof(STARTUPINFO); + + ZeroMemory(&procinfo, sizeof(PROCESS_INFORMATION)); + + printf("Launching process!\n"); + rv = CreateProcess(NULL, + cmdline, + NULL, + NULL, + FALSE, + 0, + NULL, + NULL, + &startup, + &procinfo); + + if (!rv) { + DWORD dw = GetLastError(); + printf("error: %d\n", dw); + } + CloseHandle(procinfo.hProcess); + CloseHandle(procinfo.hThread); + return 0; +} +#endif + +int main(int argc, char **argv) { + int children = 0; + int maxtime = 0; + int passedtime = 0; + dictionary *dict = NULL; + + // Command line handling + if (argc == 1 || (0 == strcmp(argv[1], "-h")) || (0 == strcmp(argv[1], "--help"))) { + printf("ProcLauncher takes an ini file. Specify the ini file as the only\n"); + printf("parameter of the command line:\n"); + printf("proclauncher my.ini\n\n"); + printf("The ini file has the form:\n"); + printf("[main]\n"); + printf("children=child1,child2 ; These comma separated values are sections\n"); + printf("maxtime=60 ; Max time this process lives\n"); + printf("[child1] ; Here is a child section\n"); + printf("children=3 ; You can have grandchildren: this spawns 3 of them for child1\n"); + printf("maxtime=30 ; Max time, note it's in seconds. If this time\n"); + printf(" ; is > main:maxtime then the child process will be\n"); + printf(" ; killed when the parent exits. Also, grandchildren\n"); + printf("[child2] ; inherit this maxtime and can't change it.\n"); + printf("maxtime=25 ; You can call these sections whatever you want\n"); + printf("children=0 ; as long as you reference them in a children attribute\n"); + printf("....\n"); + return 0; + } else if (argc == 2) { + // This is ini file mode: + // proclauncher <inifile> + dict = iniparser_load(argv[1]); + + } else if (argc == 3) { + // Then we've been called in child process launching mode: + // proclauncher <children> <maxtime> + children = atoi(argv[1]); + maxtime = atoi(argv[2]); + } + + if (dict) { + /* Dict operation */ + char *childlist = iniparser_getstring(dict, "main:children", NULL); + maxtime = iniparser_getint(dict, (char*)"main:maxtime", 10);; + if (childlist) { + int c = 0, m = 10; + char childkey[50], maxkey[50]; + char cmd[25]; + char *token = strtok(childlist, ","); + + while (token) { + // Reset defaults + memset(childkey, 0, 50); + memset(maxkey, 0, 50); + memset(cmd, 0, 25); + c = 0; + m = 10; + + sprintf(childkey, "%s:children", token); + sprintf(maxkey, "%s:maxtime", token); + c = iniparser_getint(dict, childkey, 0); + m = iniparser_getint(dict, maxkey, 10); + + // Launch the child process + #ifdef _WIN32 + launchWindows(c, m); + #else + sprintf(cmd, "./proclaunch %d %d &", c, m); + system(cmd); + #endif + + // Get the next child entry + token = strtok(NULL, ","); + } + } + iniparser_freedict(dict); + } else { + // Child Process operation - put on your recursive thinking cap + char cmd[25]; + // This is launching grandchildren, there are no great grandchildren, so we + // pass in a 0 for the children to spawn. + #ifdef _WIN32 + while(children > 0) { + launchWindows(0, maxtime); + children--; + } + #else + sprintf(cmd, "./proclaunch %d %d &", 0, maxtime); + printf("Launching child process: %s\n", cmd); + while (children > 0) { + system(cmd); + children--; + } + #endif + } + + /* Now we have launched all the children. Let's wait for max time before returning + This does pseudo busy waiting just to appear active */ + while (passedtime < maxtime) { +#ifdef _WIN32 + Sleep(1000); +#else + sleep(1); +#endif + passedtime++; + } + exit(0); + return 0; +} diff --git a/testing/mozbase/mozprocess/tests/proclaunch.py b/testing/mozbase/mozprocess/tests/proclaunch.py new file mode 100644 index 000000000..ad06a23a1 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/proclaunch.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +import argparse +import collections +import ConfigParser +import multiprocessing +import time + +ProcessNode = collections.namedtuple('ProcessNode', ['maxtime', 'children']) + + +class ProcessLauncher(object): + + """ Create and Launch process trees specified by a '.ini' file + + Typical .ini file accepted by this class : + + [main] + children=c1, 1*c2, 4*c3 + maxtime=10 + + [c1] + children= 2*c2, c3 + maxtime=20 + + [c2] + children=3*c3 + maxtime=5 + + [c3] + maxtime=3 + + This generates a process tree of the form: + [main] + |---[c1] + | |---[c2] + | | |---[c3] + | | |---[c3] + | | |---[c3] + | | + | |---[c2] + | | |---[c3] + | | |---[c3] + | | |---[c3] + | | + | |---[c3] + | + |---[c2] + | |---[c3] + | |---[c3] + | |---[c3] + | + |---[c3] + |---[c3] + |---[c3] + + Caveat: The section names cannot contain a '*'(asterisk) or a ','(comma) + character as these are used as delimiters for parsing. + """ + + # Unit time for processes in seconds + UNIT_TIME = 1 + + def __init__(self, manifest, verbose=False): + """ + Parses the manifest and stores the information about the process tree + in a format usable by the class. + + Raises IOError if : + - The path does not exist + - The file cannot be read + Raises ConfigParser.*Error if: + - Files does not contain section headers + - File cannot be parsed because of incorrect specification + + :param manifest: Path to the manifest file that contains the + configuration for the process tree to be launched + :verbose: Print the process start and end information. + Genrates a lot of output. Disabled by default. + """ + + self.verbose = verbose + + # Children is a dictionary used to store information from the, + # Configuration file in a more usable format. + # Key : string contain the name of child process + # Value : A Named tuple of the form (max_time, (list of child processes of Key)) + # Where each child process is a list of type: [count to run, name of child] + self.children = {} + + cfgparser = ConfigParser.ConfigParser() + + if not cfgparser.read(manifest): + raise IOError('The manifest %s could not be found/opened', manifest) + + sections = cfgparser.sections() + for section in sections: + # Maxtime is a mandatory option + # ConfigParser.NoOptionError is raised if maxtime does not exist + if '*' in section or ',' in section: + raise ConfigParser.ParsingError( + "%s is not a valid section name. " + "Section names cannot contain a '*' or ','." % section) + m_time = cfgparser.get(section, 'maxtime') + try: + m_time = int(m_time) + except ValueError: + raise ValueError('Expected maxtime to be an integer, specified %s' % m_time) + + # No children option implies there are no further children + # Leaving the children option blank is an error. + try: + c = cfgparser.get(section, 'children') + if not c: + # If children is an empty field, assume no children + children = None + + else: + # Tokenize chilren field, ignore empty strings + children = [[y.strip() for y in x.strip().split('*', 1)] + for x in c.split(',') if x] + try: + for i, child in enumerate(children): + # No multiplicate factor infront of a process implies 1 + if len(child) == 1: + children[i] = [1, child[0]] + else: + children[i][0] = int(child[0]) + + if children[i][1] not in sections: + raise ConfigParser.ParsingError( + 'No section corresponding to child %s' % child[1]) + except ValueError: + raise ValueError( + 'Expected process count to be an integer, specified %s' % child[0]) + + except ConfigParser.NoOptionError: + children = None + pn = ProcessNode(maxtime=m_time, + children=children) + self.children[section] = pn + + def run(self): + """ + This function launches the process tree. + """ + self._run('main', 0) + + def _run(self, proc_name, level): + """ + Runs the process specified by the section-name `proc_name` in the manifest file. + Then makes calls to launch the child processes of `proc_name` + + :param proc_name: File name of the manifest as a string. + :param level: Depth of the current process in the tree. + """ + if proc_name not in self.children.keys(): + raise IOError("%s is not a valid process" % proc_name) + + maxtime = self.children[proc_name].maxtime + if self.verbose: + print "%sLaunching %s for %d*%d seconds" % (" " * level, + proc_name, + maxtime, + self.UNIT_TIME) + + while self.children[proc_name].children: + child = self.children[proc_name].children.pop() + + count, child_proc = child + for i in range(count): + p = multiprocessing.Process(target=self._run, args=(child[1], level + 1)) + p.start() + + self._launch(maxtime) + if self.verbose: + print "%sFinished %s" % (" " * level, proc_name) + + def _launch(self, running_time): + """ + Create and launch a process and idles for the time specified by + `running_time` + + :param running_time: Running time of the process in seconds. + """ + elapsed_time = 0 + + while elapsed_time < running_time: + time.sleep(self.UNIT_TIME) + elapsed_time += self.UNIT_TIME + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument("manifest", help="Specify the configuration .ini file") + args = parser.parse_args() + + proclaunch = ProcessLauncher(args.manifest) + proclaunch.run() diff --git a/testing/mozbase/mozprocess/tests/procnonewline.py b/testing/mozbase/mozprocess/tests/procnonewline.py new file mode 100644 index 000000000..428a02bcb --- /dev/null +++ b/testing/mozbase/mozprocess/tests/procnonewline.py @@ -0,0 +1,3 @@ +import sys +print "this is a newline" +sys.stdout.write("this has NO newline") diff --git a/testing/mozbase/mozprocess/tests/proctest.py b/testing/mozbase/mozprocess/tests/proctest.py new file mode 100644 index 000000000..62ccf940c --- /dev/null +++ b/testing/mozbase/mozprocess/tests/proctest.py @@ -0,0 +1,52 @@ +import os +import sys +import unittest +import psutil + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.proclaunch = os.path.join(here, "proclaunch.py") + cls.python = sys.executable + + def determine_status(self, proc, isalive=False, expectedfail=()): + """ + Use to determine if the situation has failed. + Parameters: + proc -- the processhandler instance + isalive -- Use True to indicate we pass if the process exists; however, by default + the test will pass if the process does not exist (isalive == False) + expectedfail -- Defaults to [], used to indicate a list of fields + that are expected to fail + """ + returncode = proc.proc.returncode + didtimeout = proc.didTimeout + detected = psutil.pid_exists(proc.pid) + output = '' + # ProcessHandler has output when store_output is set to True in the constructor + # (this is the default) + if getattr(proc, 'output'): + output = proc.output + + if 'returncode' in expectedfail: + self.assertTrue(returncode, "Detected an unexpected return code of: %s" % returncode) + elif isalive: + self.assertEqual(returncode, None, "Detected not None return code of: %s" % returncode) + else: + self.assertNotEqual(returncode, None, "Detected unexpected None return code of") + + if 'didtimeout' in expectedfail: + self.assertTrue(didtimeout, "Detected that process didn't time out") + else: + self.assertTrue(not didtimeout, "Detected that process timed out") + + if isalive: + self.assertTrue(detected, "Detected process is not running, " + "process output: %s" % output) + else: + self.assertTrue(not detected, "Detected process is still running, " + "process output: %s" % output) diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess.py b/testing/mozbase/mozprocess/tests/test_mozprocess.py new file mode 100644 index 000000000..bf8ba194c --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import subprocess +import sys +import unittest +import proctest +from mozprocess import processhandler + +here = os.path.dirname(os.path.abspath(__file__)) + + +def make_proclaunch(aDir): + """ + Makes the proclaunch executable. + Params: + aDir - the directory in which to issue the make commands + Returns: + the path to the proclaunch executable that is generated + """ + + if sys.platform == "win32": + exepath = os.path.join(aDir, "proclaunch.exe") + else: + exepath = os.path.join(aDir, "proclaunch") + + # remove the launcher, if it already exists + # otherwise, if the make fails you may not notice + if os.path.exists(exepath): + os.remove(exepath) + + # Ideally make should take care of both calls through recursion, but since it doesn't, + # on windows anyway (to file?), let's just call out both targets explicitly. + for command in [["make", "-C", "iniparser"], + ["make"]]: + process = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=aDir) + stdout, stderr = process.communicate() + if process.returncode: + # SomethingBadHappen; print all the things + print "%s: exit %d" % (command, process.returncode) + print "stdout:\n%s" % stdout + print "stderr:\n%s" % stderr + raise subprocess.CalledProcessError(process.returncode, command, stdout) + + # ensure the launcher now exists + if not os.path.exists(exepath): + raise AssertionError("proclaunch executable '%s' " + "does not exist (sys.platform=%s)" % (exepath, sys.platform)) + return exepath + + +class ProcTest(proctest.ProcTest): + + # whether to remove created files on exit + cleanup = os.environ.get('CLEANUP', 'true').lower() in ('1', 'true') + + @classmethod + def setUpClass(cls): + cls.proclaunch = make_proclaunch(here) + + @classmethod + def tearDownClass(cls): + del cls.proclaunch + if not cls.cleanup: + return + files = [('proclaunch',), + ('proclaunch.exe',), + ('iniparser', 'dictionary.o'), + ('iniparser', 'iniparser.lib'), + ('iniparser', 'iniparser.o'), + ('iniparser', 'libiniparser.a'), + ('iniparser', 'libiniparser.so.0'), + ] + files = [os.path.join(here, *path) for path in files] + errors = [] + for path in files: + if os.path.exists(path): + try: + os.remove(path) + except OSError as e: + errors.append(str(e)) + if errors: + raise OSError("Error(s) encountered tearing down " + "%s.%s:\n%s" % (cls.__module__, cls.__name__, '\n'.join(errors))) + + def test_process_normal_finish(self): + """Process is started, runs to completion while we wait for it""" + + p = processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"], + cwd=here) + p.run() + p.wait() + + self.determine_status(p) + + def test_commandline_no_args(self): + """Command line is reported correctly when no arguments are specified""" + p = processhandler.ProcessHandler(self.proclaunch, cwd=here) + self.assertEqual(p.commandline, self.proclaunch) + + def test_commandline_overspecified(self): + """Command line raises an exception when the arguments are specified ambiguously""" + err = None + try: + processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"], + args=["1", "2", "3"], + cwd=here) + except TypeError, e: + err = e + + self.assertTrue(err) + + def test_commandline_from_list(self): + """Command line is reported correctly when command and arguments are specified in a list""" + p = processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"], + cwd=here) + self.assertEqual(p.commandline, self.proclaunch + ' process_normal_finish.ini') + + def test_commandline_over_specified(self): + """Command line raises an exception when the arguments are specified ambiguously""" + err = None + try: + processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"], + args=["1", "2", "3"], + cwd=here) + except TypeError, e: + err = e + + self.assertTrue(err) + + def test_commandline_from_args(self): + """Command line is reported correctly when arguments are specified in a dedicated list""" + p = processhandler.ProcessHandler(self.proclaunch, + args=["1", "2", "3"], + cwd=here) + self.assertEqual(p.commandline, self.proclaunch + ' 1 2 3') + + def test_process_wait(self): + """Process is started runs to completion while we wait indefinitely""" + + p = processhandler.ProcessHandler([self.proclaunch, + "process_waittimeout_10s.ini"], + cwd=here) + p.run() + p.wait() + + self.determine_status(p) + + def test_process_timeout(self): + """ Process is started, runs but we time out waiting on it + to complete + """ + p = processhandler.ProcessHandler([self.proclaunch, "process_waittimeout.ini"], + cwd=here) + p.run(timeout=10) + p.wait() + + self.determine_status(p, False, ['returncode', 'didtimeout']) + + def test_process_timeout_no_kill(self): + """ Process is started, runs but we time out waiting on it + to complete. Process should not be killed. + """ + p = None + + def timeout_handler(): + self.assertEqual(p.proc.poll(), None) + p.kill() + p = processhandler.ProcessHandler([self.proclaunch, "process_waittimeout.ini"], + cwd=here, + onTimeout=(timeout_handler,), + kill_on_timeout=False) + p.run(timeout=1) + p.wait() + self.assertTrue(p.didTimeout) + + self.determine_status(p, False, ['returncode', 'didtimeout']) + + def test_process_waittimeout(self): + """ + Process is started, then wait is called and times out. + Process is still running and didn't timeout + """ + p = processhandler.ProcessHandler([self.proclaunch, + "process_waittimeout_10s.ini"], + cwd=here) + + p.run() + p.wait(timeout=5) + + self.determine_status(p, True, ()) + + def test_process_waitnotimeout(self): + """ Process is started, runs to completion before our wait times out + """ + p = processhandler.ProcessHandler([self.proclaunch, + "process_waittimeout_10s.ini"], + cwd=here) + p.run(timeout=30) + p.wait() + + self.determine_status(p) + + def test_process_kill(self): + """Process is started, we kill it""" + + p = processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"], + cwd=here) + p.run() + p.kill() + + self.determine_status(p) + + def test_process_output_twice(self): + """ + Process is started, then processOutput is called a second time explicitly + """ + p = processhandler.ProcessHandler([self.proclaunch, + "process_waittimeout_10s.ini"], + cwd=here) + + p.run() + p.processOutput(timeout=5) + p.wait() + + self.determine_status(p, False, ()) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_kill.py b/testing/mozbase/mozprocess/tests/test_mozprocess_kill.py new file mode 100644 index 000000000..36dbc95cc --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_kill.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +import os +import time +import unittest +import proctest +import signal +from mozprocess import processhandler + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTestKill(proctest.ProcTest): + """ Class to test various process tree killing scenatios """ + + def test_kill_before_run(self): + """Process is not started, and kill() is called""" + + p = processhandler.ProcessHandler([self.python, '-V']) + self.assertRaises(RuntimeError, p.kill) + + def test_process_kill(self): + """Process is started, we kill it""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + p.run() + p.kill() + + self.determine_status(p, expectedfail=('returncode',)) + + def test_process_kill_deep(self): + """Process is started, we kill it, we use a deep process tree""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_deep_python.ini"], + cwd=here) + p.run() + p.kill() + + self.determine_status(p, expectedfail=('returncode',)) + + def test_process_kill_deep_wait(self): + """Process is started, we use a deep process tree, we let it spawn + for a bit, we kill it""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_deep_python.ini"], + cwd=here) + p.run() + # Let the tree spawn a bit, before attempting to kill + time.sleep(3) + p.kill() + + self.determine_status(p, expectedfail=('returncode',)) + + def test_process_kill_broad(self): + """Process is started, we kill it, we use a broad process tree""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_broad_python.ini"], + cwd=here) + p.run() + p.kill() + + self.determine_status(p, expectedfail=('returncode',)) + + @unittest.skipUnless(processhandler.isPosix, "posix only") + def test_process_kill_with_sigterm(self): + script = os.path.join(here, 'infinite_loop.py') + p = processhandler.ProcessHandler([self.python, script]) + + p.run() + p.kill() + + self.assertEquals(p.proc.returncode, -signal.SIGTERM) + + @unittest.skipUnless(processhandler.isPosix, "posix only") + def test_process_kill_with_sigint_if_needed(self): + script = os.path.join(here, 'infinite_loop.py') + p = processhandler.ProcessHandler([self.python, script, 'deadlock']) + + p.run() + time.sleep(1) + p.kill() + + self.assertEquals(p.proc.returncode, -signal.SIGKILL) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_kill_broad_wait.py b/testing/mozbase/mozprocess/tests/test_mozprocess_kill_broad_wait.py new file mode 100644 index 000000000..cc8cef978 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_kill_broad_wait.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import os +import time +import unittest +import proctest +from mozprocess import processhandler + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTestKill(proctest.ProcTest): + """ Class to test various process tree killing scenatios """ + + # This test should ideally be a part of test_mozprocess_kill.py + # It has been separated for the purpose of tempporarily disabling it. + # See https://bugzilla.mozilla.org/show_bug.cgi?id=921632 + def test_process_kill_broad_wait(self): + """Process is started, we use a broad process tree, we let it spawn + for a bit, we kill it""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_broad_python.ini"], + cwd=here) + p.run() + # Let the tree spawn a bit, before attempting to kill + time.sleep(3) + p.kill() + + self.determine_status(p, expectedfail=('returncode',)) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_misc.py b/testing/mozbase/mozprocess/tests/test_mozprocess_misc.py new file mode 100644 index 000000000..7a7c690d1 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_misc.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import unittest +import proctest +from mozprocess import processhandler + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTestMisc(proctest.ProcTest): + """ Class to test misc operations """ + + def test_process_output_twice(self): + """ + Process is started, then processOutput is called a second time explicitly + """ + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_waittimeout_10s_python.ini"], + cwd=here) + + p.run() + p.processOutput(timeout=5) + p.wait() + + self.determine_status(p, False, ()) + + def test_unicode_in_environment(self): + env = { + 'FOOBAR': 'ʘ', + } + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here, env=env) + # passes if no exceptions are raised + p.run() + p.wait() + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_output.py b/testing/mozbase/mozprocess/tests/test_mozprocess_output.py new file mode 100644 index 000000000..e9ad26620 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_output.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +import io +import os +import unittest +import proctest +from mozprocess import processhandler + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTestOutput(proctest.ProcTest): + """ Class to test operations related to output handling """ + + def test_process_output_nonewline(self): + """ + Process is started, outputs data with no newline + """ + p = processhandler.ProcessHandler([self.python, "procnonewline.py"], + cwd=here) + + p.run() + p.processOutput(timeout=5) + p.wait() + + self.determine_status(p, False, ()) + + def test_stream_process_output(self): + """ + Process output stream does not buffer + """ + expected = '\n'.join([str(n) for n in range(0, 10)]) + + stream = io.BytesIO() + buf = io.BufferedRandom(stream) + + p = processhandler.ProcessHandler([self.python, "proccountfive.py"], + cwd=here, + stream=buf) + + p.run() + p.wait() + for i in range(5, 10): + stream.write(str(i) + '\n') + + buf.flush() + self.assertEquals(stream.getvalue().strip(), expected) + + # make sure mozprocess doesn't close the stream + # since mozprocess didn't create it + self.assertFalse(buf.closed) + buf.close() + + self.determine_status(p, False, ()) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_params.py b/testing/mozbase/mozprocess/tests/test_mozprocess_params.py new file mode 100644 index 000000000..d4d1f00f3 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_params.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from mozprocess import processhandler + + +class ParamTests(unittest.TestCase): + + def test_process_outputline_handler(self): + """Parameter processOutputLine is accepted with a single function""" + def output(line): + print("output " + str(line)) + err = None + try: + processhandler.ProcessHandler(['ls', '-l'], processOutputLine=output) + except (TypeError, AttributeError) as e: + err = e + self.assertFalse(err) + + def test_process_outputline_handler_list(self): + """Parameter processOutputLine is accepted with a list of functions""" + def output(line): + print("output " + str(line)) + err = None + try: + processhandler.ProcessHandler(['ls', '-l'], processOutputLine=[output]) + except (TypeError, AttributeError) as e: + err = e + self.assertFalse(err) + + def test_process_ontimeout_handler(self): + """Parameter onTimeout is accepted with a single function""" + def timeout(): + print("timeout!") + err = None + try: + processhandler.ProcessHandler(['sleep', '2'], onTimeout=timeout) + except (TypeError, AttributeError) as e: + err = e + self.assertFalse(err) + + def test_process_ontimeout_handler_list(self): + """Parameter onTimeout is accepted with a list of functions""" + def timeout(): + print("timeout!") + err = None + try: + processhandler.ProcessHandler(['sleep', '2'], onTimeout=[timeout]) + except (TypeError, AttributeError) as e: + err = e + self.assertFalse(err) + + def test_process_onfinish_handler(self): + """Parameter onFinish is accepted with a single function""" + def finish(): + print("finished!") + err = None + try: + processhandler.ProcessHandler(['sleep', '1'], onFinish=finish) + except (TypeError, AttributeError) as e: + err = e + self.assertFalse(err) + + def test_process_onfinish_handler_list(self): + """Parameter onFinish is accepted with a list of functions""" + def finish(): + print("finished!") + err = None + try: + processhandler.ProcessHandler(['sleep', '1'], onFinish=[finish]) + except (TypeError, AttributeError) as e: + err = e + self.assertFalse(err) + + +def main(): + unittest.main() + +if __name__ == '__main__': + main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_poll.py b/testing/mozbase/mozprocess/tests/test_mozprocess_poll.py new file mode 100644 index 000000000..a1ae070aa --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_poll.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +import os +import signal +import unittest + +from mozprocess import processhandler + +import proctest + + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTestPoll(proctest.ProcTest): + """ Class to test process poll """ + + def test_poll_before_run(self): + """Process is not started, and poll() is called""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + self.assertRaises(RuntimeError, p.poll) + + def test_poll_while_running(self): + """Process is started, and poll() is called""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + p.run() + returncode = p.poll() + + self.assertEqual(returncode, None) + + self.determine_status(p, True) + p.kill() + + def test_poll_after_kill(self): + """Process is killed, and poll() is called""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + p.run() + returncode = p.kill() + + # We killed the process, so the returncode should be < 0 + self.assertLess(returncode, 0) + self.assertEqual(returncode, p.poll()) + + self.determine_status(p) + + def test_poll_after_kill_no_process_group(self): + """Process (no group) is killed, and poll() is called""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_no_process_group.ini"], + cwd=here, + ignore_children=True + ) + p.run() + returncode = p.kill() + + # We killed the process, so the returncode should be < 0 + self.assertLess(returncode, 0) + self.assertEqual(returncode, p.poll()) + + self.determine_status(p) + + def test_poll_after_double_kill(self): + """Process is killed twice, and poll() is called""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + p.run() + p.kill() + returncode = p.kill() + + # We killed the process, so the returncode should be < 0 + self.assertLess(returncode, 0) + self.assertEqual(returncode, p.poll()) + + self.determine_status(p) + + def test_poll_after_external_kill(self): + """Process is killed externally, and poll() is called""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + p.run() + os.kill(p.pid, signal.SIGTERM) + returncode = p.wait() + + # We killed the process, so the returncode should be < 0 + self.assertEqual(returncode, -signal.SIGTERM) + self.assertEqual(returncode, p.poll()) + + self.determine_status(p) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_mozprocess_wait.py b/testing/mozbase/mozprocess/tests/test_mozprocess_wait.py new file mode 100644 index 000000000..df9e753ee --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_mozprocess_wait.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +import os +import unittest +import proctest +import mozinfo +from mozprocess import processhandler + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ProcTestWait(proctest.ProcTest): + """ Class to test process waits and timeouts """ + + def test_normal_finish(self): + """Process is started, runs to completion while we wait for it""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_normal_finish_python.ini"], + cwd=here) + p.run() + p.wait() + + self.determine_status(p) + + def test_wait(self): + """Process is started runs to completion while we wait indefinitely""" + + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_waittimeout_10s_python.ini"], + cwd=here) + p.run() + p.wait() + + self.determine_status(p) + + def test_timeout(self): + """ Process is started, runs but we time out waiting on it + to complete + """ + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_waittimeout_python.ini"], + cwd=here) + p.run(timeout=10) + p.wait() + + if mozinfo.isUnix: + # process was killed, so returncode should be negative + self.assertLess(p.proc.returncode, 0) + + self.determine_status(p, False, ['returncode', 'didtimeout']) + + def test_waittimeout(self): + """ + Process is started, then wait is called and times out. + Process is still running and didn't timeout + """ + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_waittimeout_10s_python.ini"], + cwd=here) + + p.run() + p.wait(timeout=5) + + self.determine_status(p, True, ()) + + def test_waitnotimeout(self): + """ Process is started, runs to completion before our wait times out + """ + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_waittimeout_10s_python.ini"], + cwd=here) + p.run(timeout=30) + p.wait() + + self.determine_status(p) + + def test_wait_twice_after_kill(self): + """Bug 968718: Process is started and stopped. wait() twice afterward.""" + p = processhandler.ProcessHandler([self.python, self.proclaunch, + "process_waittimeout_python.ini"], + cwd=here) + p.run() + p.kill() + returncode1 = p.wait() + returncode2 = p.wait() + + self.determine_status(p) + + self.assertLess(returncode2, 0, + 'Negative returncode expected, got "%s"' % returncode2) + self.assertEqual(returncode1, returncode2, + 'Expected both returncodes of wait() to be equal') + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozprocess/tests/test_process_reader.py b/testing/mozbase/mozprocess/tests/test_process_reader.py new file mode 100644 index 000000000..0cf84d9a4 --- /dev/null +++ b/testing/mozbase/mozprocess/tests/test_process_reader.py @@ -0,0 +1,101 @@ +import unittest +import subprocess +import sys +from mozprocess.processhandler import ProcessReader, StoreOutput + + +def run_python(str_code, stdout=subprocess.PIPE, stderr=subprocess.PIPE): + cmd = [sys.executable, '-c', str_code] + return subprocess.Popen(cmd, stdout=stdout, stderr=stderr) + + +class TestProcessReader(unittest.TestCase): + + def setUp(self): + self.out = StoreOutput() + self.err = StoreOutput() + self.finished = False + + def on_finished(): + self.finished = True + self.timeout = False + + def on_timeout(): + self.timeout = True + self.reader = ProcessReader(stdout_callback=self.out, + stderr_callback=self.err, + finished_callback=on_finished, + timeout_callback=on_timeout) + + def test_stdout_callback(self): + proc = run_python('print 1; print 2') + self.reader.start(proc) + self.reader.thread.join() + + self.assertEqual(self.out.output, ['1', '2']) + self.assertEqual(self.err.output, []) + + def test_stderr_callback(self): + proc = run_python('import sys; sys.stderr.write("hello world\\n")') + self.reader.start(proc) + self.reader.thread.join() + + self.assertEqual(self.out.output, []) + self.assertEqual(self.err.output, ['hello world']) + + def test_stdout_and_stderr_callbacks(self): + proc = run_python('import sys; sys.stderr.write("hello world\\n"); print 1; print 2') + self.reader.start(proc) + self.reader.thread.join() + + self.assertEqual(self.out.output, ['1', '2']) + self.assertEqual(self.err.output, ['hello world']) + + def test_finished_callback(self): + self.assertFalse(self.finished) + proc = run_python('') + self.reader.start(proc) + self.reader.thread.join() + self.assertTrue(self.finished) + + def test_timeout(self): + self.reader.timeout = 0.05 + self.assertFalse(self.timeout) + proc = run_python('import time; time.sleep(0.1)') + self.reader.start(proc) + self.reader.thread.join() + self.assertTrue(self.timeout) + self.assertFalse(self.finished) + + def test_output_timeout(self): + self.reader.output_timeout = 0.05 + self.assertFalse(self.timeout) + proc = run_python('import time; time.sleep(0.1)') + self.reader.start(proc) + self.reader.thread.join() + self.assertTrue(self.timeout) + self.assertFalse(self.finished) + + def test_read_without_eol(self): + proc = run_python('import sys; sys.stdout.write("1")') + self.reader.start(proc) + self.reader.thread.join() + self.assertEqual(self.out.output, ['1']) + + def test_read_with_strange_eol(self): + proc = run_python('import sys; sys.stdout.write("1\\r\\r\\r\\n")') + self.reader.start(proc) + self.reader.thread.join() + self.assertEqual(self.out.output, ['1']) + + def test_mixed_stdout_stderr(self): + proc = run_python('import sys; sys.stderr.write("hello world\\n"); print 1; print 2', + stderr=subprocess.STDOUT) + self.reader.start(proc) + self.reader.thread.join() + + self.assertEqual(sorted(self.out.output), sorted(['1', '2', 'hello world'])) + self.assertEqual(self.err.output, []) + +if __name__ == '__main__': + unittest.main() |