diff options
Diffstat (limited to 'python/mach')
39 files changed, 4146 insertions, 0 deletions
diff --git a/python/mach/README.rst b/python/mach/README.rst new file mode 100644 index 000000000..7c2e00bec --- /dev/null +++ b/python/mach/README.rst @@ -0,0 +1,13 @@ +==== +mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +To learn more, read the docs in ``docs/``. diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh new file mode 100644 index 000000000..e4b151f24 --- /dev/null +++ b/python/mach/bash-completion.sh @@ -0,0 +1,29 @@ +function _mach() +{ + local cur cmds c subcommand + COMPREPLY=() + + # Load the list of commands + cmds=`"${COMP_WORDS[0]}" mach-commands` + + # Look for the subcommand. + cur="${COMP_WORDS[COMP_CWORD]}" + subcommand="" + c=1 + while [ $c -lt $COMP_CWORD ]; do + word="${COMP_WORDS[c]}" + for cmd in $cmds; do + if [ "$cmd" = "$word" ]; then + subcommand="$word" + fi + done + c=$((++c)) + done + + if [[ "$subcommand" == "help" || -z "$subcommand" ]]; then + COMPREPLY=( $(compgen -W "$cmds" -- ${cur}) ) + fi + + return 0 +} +complete -o default -F _mach mach diff --git a/python/mach/docs/commands.rst b/python/mach/docs/commands.rst new file mode 100644 index 000000000..af2973dd7 --- /dev/null +++ b/python/mach/docs/commands.rst @@ -0,0 +1,145 @@ +.. _mach_commands: + +===================== +Implementing Commands +===================== + +Mach commands are defined via Python decorators. + +All the relevant decorators are defined in the *mach.decorators* module. +The important decorators are as follows: + +:py:func:`CommandProvider <mach.decorators.CommandProvider>` + A class decorator that denotes that a class contains mach + commands. The decorator takes no arguments. + +:py:func:`Command <mach.decorators.Command>` + A method decorator that denotes that the method should be called when + the specified command is requested. The decorator takes a command name + as its first argument and a number of additional arguments to + configure the behavior of the command. + +:py:func:`CommandArgument <mach.decorators.CommandArgument>` + A method decorator that defines an argument to the command. Its + arguments are essentially proxied to ArgumentParser.add_argument() + +:py:func:`SubCommand <mach.decorators.SubCommand>` + A method decorator that denotes that the method should be a + sub-command to an existing ``@Command``. The decorator takes the + parent command name as its first argument and the sub-command name + as its second argument. + + ``@CommandArgument`` can be used on ``@SubCommand`` instances just + like they can on ``@Command`` instances. + +Classes with the ``@CommandProvider`` decorator **must** have an +``__init__`` method that accepts 1 or 2 arguments. If it accepts 2 +arguments, the 2nd argument will be a +:py:class:`mach.base.CommandContext` instance. + +Here is a complete example: + +.. code-block:: python + + from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + ) + + @CommandProvider + class MyClass(object): + @Command('doit', help='Do ALL OF THE THINGS.') + @CommandArgument('--force', '-f', action='store_true', + help='Force doing it.') + def doit(self, force=False): + # Do stuff here. + +When the module is loaded, the decorators tell mach about all handlers. +When mach runs, it takes the assembled metadata from these handlers and +hooks it up to the command line driver. Under the hood, arguments passed +to the decorators are being used to help mach parse command arguments, +formulate arguments to the methods, etc. See the documentation in the +:py:mod:`mach.base` module for more. + +The Python modules defining mach commands do not need to live inside the +main mach source tree. + +Conditionally Filtering Commands +================================ + +Sometimes it might only make sense to run a command given a certain +context. For example, running tests only makes sense if the product +they are testing has been built, and said build is available. To make +sure a command is only runnable from within a correct context, you can +define a series of conditions on the +:py:func:`Command <mach.decorators.Command>` decorator. + +A condition is simply a function that takes an instance of the +:py:func:`mach.decorators.CommandProvider` class as an argument, and +returns ``True`` or ``False``. If any of the conditions defined on a +command return ``False``, the command will not be runnable. The +docstring of a condition function is used in error messages, to explain +why the command cannot currently be run. + +Here is an example: + +.. code-block:: python + + from mach.decorators import ( + CommandProvider, + Command, + ) + + def build_available(cls): + """The build needs to be available.""" + return cls.build_path is not None + + @CommandProvider + class MyClass(MachCommandBase): + def __init__(self, build_path=None): + self.build_path = build_path + + @Command('run_tests', conditions=[build_available]) + def run_tests(self): + # Do stuff here. + +It is important to make sure that any state needed by the condition is +available to instances of the command provider. + +By default all commands without any conditions applied will be runnable, +but it is possible to change this behaviour by setting +``require_conditions`` to ``True``: + +.. code-block:: python + + m = mach.main.Mach() + m.require_conditions = True + +Minimizing Code in Commands +=========================== + +Mach command modules, classes, and methods work best when they are +minimal dispatchers. The reason is import bloat. Currently, the mach +core needs to import every Python file potentially containing mach +commands for every command invocation. If you have dozens of commands or +commands in modules that import a lot of Python code, these imports +could slow mach down and waste memory. + +It is thus recommended that mach modules, classes, and methods do as +little work as possible. Ideally the module should only import from +the :py:mod:`mach` package. If you need external modules, you should +import them from within the command method. + +To keep code size small, the body of a command method should be limited +to: + +1. Obtaining user input (parsing arguments, prompting, etc) +2. Calling into some other Python package +3. Formatting output + +Of course, these recommendations can be ignored if you want to risk +slower performance. + +In the future, the mach driver may cache the dispatching information or +have it intelligently loaded to facilitate lazy loading. diff --git a/python/mach/docs/driver.rst b/python/mach/docs/driver.rst new file mode 100644 index 000000000..022ebe657 --- /dev/null +++ b/python/mach/docs/driver.rst @@ -0,0 +1,51 @@ +.. _mach_driver: + +======= +Drivers +======= + +Entry Points +============ + +It is possible to use setuptools' entry points to load commands +directly from python packages. A mach entry point is a function which +returns a list of files or directories containing mach command +providers. e.g.: + +.. code-block:: python + + def list_providers(): + providers = [] + here = os.path.abspath(os.path.dirname(__file__)) + for p in os.listdir(here): + if p.endswith('.py'): + providers.append(os.path.join(here, p)) + return providers + +See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins +for more information on creating an entry point. To search for entry +point plugins, you can call +:py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.: + +.. code-block:: python + + mach.load_commands_from_entry_point("mach.external.providers") + +Adding Global Arguments +======================= + +Arguments to mach commands are usually command-specific. However, +mach ships with a handful of global arguments that apply to all +commands. + +It is possible to extend the list of global arguments. In your +*mach driver*, simply call +:py:meth:`mach.main.Mach.add_global_argument`. e.g.: + +.. code-block:: python + + mach = mach.main.Mach(os.getcwd()) + + # Will allow --example to be specified on every mach command. + mach.add_global_argument('--example', action='store_true', + help='Demonstrate an example global argument.') diff --git a/python/mach/docs/index.rst b/python/mach/docs/index.rst new file mode 100644 index 000000000..cd2056333 --- /dev/null +++ b/python/mach/docs/index.rst @@ -0,0 +1,75 @@ +==== +mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +Features +======== + +On a high level, mach is similar to using argparse with subparsers (for +command handling). When you dig deeper, mach offers a number of +additional features: + +Distributed command definitions + With optparse/argparse, you have to define your commands on a central + parser instance. With mach, you annotate your command methods with + decorators and mach finds and dispatches to them automatically. + +Command categories + Mach commands can be grouped into categories when displayed in help. + This is currently not possible with argparse. + +Logging management + Mach provides a facility for logging (both classical text and + structured) that is available to any command handler. + +Settings files + Mach provides a facility for reading settings from an ini-like file + format. + +Components +========== + +Mach is conceptually composed of the following components: + +core + The mach core is the core code powering mach. This is a Python package + that contains all the business logic that makes mach work. The mach + core is common to all mach deployments. + +commands + These are what mach dispatches to. Commands are simply Python methods + registered as command names. The set of commands is unique to the + environment mach is deployed in. + +driver + The *driver* is the entry-point to mach. It is simply an executable + script that loads the mach core, tells it where commands can be found, + then asks the mach core to handle the current request. The driver is + unique to the deployed environment. But, it's usually based on an + example from this source tree. + +Project State +============= + +mach was originally written as a command dispatching framework to aid +Firefox development. While the code is mostly generic, there are still +some pieces that closely tie it to Mozilla/Firefox. The goal is for +these to eventually be removed and replaced with generic features so +mach is suitable for anybody to use. Until then, mach may not be the +best fit for you. + +.. toctree:: + :maxdepth: 1 + + commands + driver + logging + settings diff --git a/python/mach/docs/logging.rst b/python/mach/docs/logging.rst new file mode 100644 index 000000000..ff245cf03 --- /dev/null +++ b/python/mach/docs/logging.rst @@ -0,0 +1,100 @@ +.. _mach_logging: + +======= +Logging +======= + +Mach configures a built-in logging facility so commands can easily log +data. + +What sets the logging facility apart from most loggers you've seen is +that it encourages structured logging. Instead of conventional logging +where simple strings are logged, the internal logging mechanism logs all +events with the following pieces of information: + +* A string *action* +* A dict of log message fields +* A formatting string + +Essentially, instead of assembling a human-readable string at +logging-time, you create an object holding all the pieces of data that +will constitute your logged event. For each unique type of logged event, +you assign an *action* name. + +Depending on how logging is configured, your logged event could get +written a couple of different ways. + +JSON Logging +============ + +Where machines are the intended target of the logging data, a JSON +logger is configured. The JSON logger assembles an array consisting of +the following elements: + +* Decimal wall clock time in seconds since UNIX epoch +* String *action* of message +* Object with structured message data + +The JSON-serialized array is written to a configured file handle. +Consumers of this logging stream can just perform a readline() then feed +that into a JSON deserializer to reconstruct the original logged +message. They can key off the *action* element to determine how to +process individual events. There is no need to invent a parser. +Convenient, isn't it? + +Logging for Humans +================== + +Where humans are the intended consumer of a log message, the structured +log message are converted to more human-friendly form. This is done by +utilizing the *formatting* string provided at log time. The logger +simply calls the *format* method of the formatting string, passing the +dict containing the message's fields. + +When *mach* is used in a terminal that supports it, the logging facility +also supports terminal features such as colorization. This is done +automatically in the logging layer - there is no need to control this at +logging time. + +In addition, messages intended for humans typically prepends every line +with the time passed since the application started. + +Logging HOWTO +============= + +Structured logging piggybacks on top of Python's built-in logging +infrastructure provided by the *logging* package. We accomplish this by +taking advantage of *logging.Logger.log()*'s *extra* argument. To this +argument, we pass a dict with the fields *action* and *params*. These +are the string *action* and dict of message fields, respectively. The +formatting string is passed as the *msg* argument, like normal. + +If you were logging to a logger directly, you would do something like: + +.. code-block:: python + + logger.log(logging.INFO, 'My name is {name}', + extra={'action': 'my_name', 'params': {'name': 'Gregory'}}) + +The JSON logging would produce something like:: + + [1339985554.306338, "my_name", {"name": "Gregory"}] + +Human logging would produce something like:: + + 0.52 My name is Gregory + +Since there is a lot of complexity using logger.log directly, it is +recommended to go through a wrapping layer that hides part of the +complexity for you. The easiest way to do this is by utilizing the +LoggingMixin: + +.. code-block:: python + + import logging + from mach.mixin.logging import LoggingMixin + + class MyClass(LoggingMixin): + def foo(self): + self.log(logging.INFO, 'foo_start', {'bar': True}, + 'Foo performed. Bar: {bar}') diff --git a/python/mach/docs/settings.rst b/python/mach/docs/settings.rst new file mode 100644 index 000000000..b51dc54a2 --- /dev/null +++ b/python/mach/docs/settings.rst @@ -0,0 +1,140 @@ +.. _mach_settings: + +======== +Settings +======== + +Mach can read settings in from a set of configuration files. These +configuration files are either named ``machrc`` or ``.machrc`` and +are specified by the bootstrap script. In mozilla-central, these files +can live in ``~/.mozbuild`` and/or ``topsrcdir``. + +Settings can be specified anywhere, and used both by mach core or +individual commands. + + +Core Settings +============= + +These settings are implemented by mach core. + +* alias - Create a command alias. This is useful if you want to alias a command to something else, optionally including some defaults. It can either be used to create an entire new command, or provide defaults for an existing one. For example: + +.. parsed-literal:: + + [alias] + mochitest = mochitest -f browser + browser-test = mochitest -f browser + + +Defining Settings +================= + +Settings need to be explicitly defined, along with their type, +otherwise mach will throw when trying to access them. + +To define settings, use the :func:`~decorators.SettingsProvider` +decorator in an existing mach command module. E.g: + +.. code-block:: python + + from mach.decorators import SettingsProvider + + @SettingsProvider + class ArbitraryClassName(object): + config_settings = [ + ('foo.bar', 'string'), + ('foo.baz', 'int', 0, set([0,1,2])), + ] + +``@SettingsProvider``'s must specify a variable called ``config_settings`` +that returns a list of tuples. Alternatively, it can specify a function +called ``config_settings`` that returns a list of tuples. + +Each tuple is of the form: + +.. code-block:: python + + ('<section>.<option>', '<type>', default, extra) + +``type`` is a string and can be one of: +string, boolean, int, pos_int, path + +``default`` is optional, and provides a default value in case none was +specified by any of the configuration files. + +``extra`` is also optional and is a dict containing additional key/value +pairs to add to the setting's metadata. The following keys may be specified +in the ``extra`` dict: + * ``choices`` - A set of allowed values for the setting. + +Wildcards +--------- + +Sometimes a section should allow arbitrarily defined options from the user, such +as the ``alias`` section mentioned above. To define a section like this, use ``*`` +as the option name. For example: + +.. parsed-literal:: + + ('foo.*', 'string') + +This allows configuration files like this: + +.. parsed-literal:: + + [foo] + arbitrary1 = some string + arbitrary2 = some other string + + +Documenting Settings +==================== + +All settings must at least be documented in the en_US locale. Otherwise, +running ``mach settings`` will raise. Mach uses gettext to perform localization. + +A handy command exists to generate the localization files: + +.. parsed-literal:: + + mach settings locale-gen <section> + +You'll be prompted to add documentation for all options in section with the +en_US locale. To add documentation in another locale, pass in ``--locale``. + + +Accessing Settings +================== + +Now that the settings are defined and documented, they're accessible from +individual mach commands if the command receives a context in its constructor. +For example: + +.. code-block:: python + + from mach.decorators import ( + Command, + CommandProvider, + SettingsProvider, + ) + + @SettingsProvider + class ExampleSettings(object): + config_settings = [ + ('a.b', 'string', 'default'), + ('foo.bar', 'string'), + ('foo.baz', 'int', 0, {'choices': set([0,1,2])}), + ] + + @CommandProvider + class Commands(object): + def __init__(self, context): + self.settings = context.settings + + @Command('command', category='misc', + description='Prints a setting') + def command(self): + print(self.settings.a.b) + for option in self.settings.foo: + print(self.settings.foo[option]) diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/mach/mach/__init__.py diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py new file mode 100644 index 000000000..3556dc6e5 --- /dev/null +++ b/python/mach/mach/base.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + + +class CommandContext(object): + """Holds run-time state so it can easily be passed to command providers.""" + def __init__(self, cwd=None, settings=None, log_manager=None, + commands=None, **kwargs): + self.cwd = cwd + self.settings = settings + self.log_manager = log_manager + self.commands = commands + + for k,v in kwargs.items(): + setattr(self, k, v) + + +class MachError(Exception): + """Base class for all errors raised by mach itself.""" + + +class NoCommandError(MachError): + """No command was passed into mach.""" + + +class UnknownCommandError(MachError): + """Raised when we attempted to execute an unknown command.""" + + def __init__(self, command, verb, suggested_commands=None): + MachError.__init__(self) + + self.command = command + self.verb = verb + self.suggested_commands = suggested_commands or [] + +class UnrecognizedArgumentError(MachError): + """Raised when an unknown argument is passed to mach.""" + + def __init__(self, command, arguments): + MachError.__init__(self) + + self.command = command + self.arguments = arguments diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/mach/mach/commands/__init__.py diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py new file mode 100644 index 000000000..cce85f859 --- /dev/null +++ b/python/mach/mach/commands/commandinfo.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, + CommandArgument, +) + + +@CommandProvider +class BuiltinCommands(object): + def __init__(self, context): + self.context = context + + @property + def command_keys(self): + # NOTE 'REMOVED' is a function in testing/mochitest/mach_commands.py + return (k for k, v in self.context.commands.command_handlers.items() + if not v.conditions or v.conditions[0].__name__ != 'REMOVED') + + @Command('mach-commands', category='misc', + description='List all mach commands.') + def commands(self): + print("\n".join(self.command_keys)) + + @Command('mach-debug-commands', category='misc', + description='Show info about available mach commands.') + @CommandArgument('match', metavar='MATCH', default=None, nargs='?', + help='Only display commands containing given substring.') + def debug_commands(self, match=None): + import inspect + + handlers = self.context.commands.command_handlers + for command in sorted(self.command_keys): + if match and match not in command: + continue + + handler = handlers[command] + cls = handler.cls + method = getattr(cls, getattr(handler, 'method')) + + print(command) + print('=' * len(command)) + print('') + print('File: %s' % inspect.getsourcefile(method)) + print('Class: %s' % cls.__name__) + print('Method: %s' % handler.method) + print('') + diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py new file mode 100644 index 000000000..d5bb807d8 --- /dev/null +++ b/python/mach/mach/commands/settings.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +from textwrap import TextWrapper + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + SubCommand, +) + +POLIB_NOT_FOUND = """ +Could not detect the 'polib' package on the local system. +Please run: + + pip install polib +""".lstrip() + + +@CommandProvider +class Settings(object): + """Interact with settings for mach. + + Currently, we only provide functionality to view what settings are + available. In the future, this module will be used to modify settings, help + people create configs via a wizard, etc. + """ + def __init__(self, context): + self._settings = context.settings + + @Command('settings', category='devenv', + description='Show available config settings.') + @CommandArgument('-l', '--list', dest='short', action='store_true', + help='Show settings in a concise list') + def settings(self, short=None): + """List available settings.""" + if short: + for section in sorted(self._settings): + for option in sorted(self._settings[section]._settings): + short, full = self._settings.option_help(section, option) + print('%s.%s -- %s' % (section, option, short)) + return + + wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ') + for section in sorted(self._settings): + print('[%s]' % section) + + for option in sorted(self._settings[section]._settings): + short, full = self._settings.option_help(section, option) + print(wrapper.fill(full)) + + if option != '*': + print(';%s =' % option) + print('') + + @SubCommand('settings', 'locale-gen', + description='Generate or update gettext .po and .mo locale files.') + @CommandArgument('sections', nargs='*', + help='A list of strings in the form of either <section> or ' + '<section>.<option> to translate. By default, prompt to ' + 'translate all applicable options.') + @CommandArgument('--locale', default='en_US', + help='Locale to generate, defaults to en_US.') + @CommandArgument('--overwrite', action='store_true', default=False, + help='Overwrite pre-existing entries in .po files.') + def locale_gen(self, sections, **kwargs): + try: + import polib + except ImportError: + print(POLIB_NOT_FOUND) + return 1 + + self.was_prompted = False + + sections = sections or self._settings + for section in sections: + self._gen_section(section, **kwargs) + + if not self.was_prompted: + print("All specified options already have an {} translation. " + "To overwrite existing translations, pass --overwrite." + .format(kwargs['locale'])) + + def _gen_section(self, section, **kwargs): + if '.' in section: + section, option = section.split('.') + return self._gen_option(section, option, **kwargs) + + for option in sorted(self._settings[section]._settings): + self._gen_option(section, option, **kwargs) + + def _gen_option(self, section, option, locale, overwrite): + import polib + + meta = self._settings[section]._settings[option] + + localedir = os.path.join(meta['localedir'], locale, 'LC_MESSAGES') + if not os.path.isdir(localedir): + os.makedirs(localedir) + + path = os.path.join(localedir, '{}.po'.format(section)) + if os.path.isfile(path): + po = polib.pofile(path) + else: + po = polib.POFile() + + optionid = '{}.{}'.format(section, option) + for name in ('short', 'full'): + msgid = '{}.{}'.format(optionid, name) + entry = po.find(msgid) + if not entry: + entry = polib.POEntry(msgid=msgid) + po.append(entry) + + if entry in po.translated_entries() and not overwrite: + continue + + self.was_prompted = True + + msgstr = raw_input("Translate {} to {}:\n" + .format(msgid, locale)) + entry.msgstr = msgstr + + if self.was_prompted: + mopath = os.path.join(localedir, '{}.mo'.format(section)) + po.save(path) + po.save_as_mofile(mopath) diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py new file mode 100644 index 000000000..26c9a4482 --- /dev/null +++ b/python/mach/mach/config.py @@ -0,0 +1,461 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +r""" +This file defines classes for representing config data/settings. + +Config data is modeled as key-value pairs. Keys are grouped together into named +sections. Individual config settings (options) have metadata associated with +them. This metadata includes type, default value, valid values, etc. + +The main interface to config data is the ConfigSettings class. 1 or more +ConfigProvider classes are associated with ConfigSettings and define what +settings are available. + +Descriptions of individual config options can be translated to multiple +languages using gettext. Each option has associated with it a domain and locale +directory. By default, the domain is the section the option is in and the +locale directory is the "locale" directory beneath the directory containing the +module that defines it. + +People implementing ConfigProvider instances are expected to define a complete +gettext .po and .mo file for the en_US locale. The |mach settings locale-gen| +command can be used to populate these files. +""" + +from __future__ import absolute_import, unicode_literals + +import collections +import gettext +import os +import sys +from functools import wraps + +if sys.version_info[0] == 3: + from configparser import RawConfigParser, NoSectionError + str_type = str +else: + from ConfigParser import RawConfigParser, NoSectionError + str_type = basestring + + +TRANSLATION_NOT_FOUND = """ +No translation files detected for {section}, there must at least be a +translation for the 'en_US' locale. To generate these files, run: + + mach settings locale-gen {section} +""".lstrip() + + +class ConfigException(Exception): + pass + + +class ConfigType(object): + """Abstract base class for config values.""" + + @staticmethod + def validate(value): + """Validates a Python value conforms to this type. + + Raises a TypeError or ValueError if it doesn't conform. Does not do + anything if the value is valid. + """ + + @staticmethod + def from_config(config, section, option): + """Obtain the value of this type from a RawConfigParser. + + Receives a RawConfigParser instance, a str section name, and the str + option in that section to retrieve. + + The implementation may assume the option exists in the RawConfigParser + instance. + + Implementations are not expected to validate the value. But, they + should return the appropriate Python type. + """ + + @staticmethod + def to_config(value): + return value + + +class StringType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, str_type): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +class BooleanType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, bool): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getboolean(section, option) + + @staticmethod + def to_config(value): + return 'true' if value else 'false' + + +class IntegerType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getint(section, option) + + +class PositiveIntegerType(IntegerType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + if value < 0: + raise ValueError() + + +class PathType(StringType): + @staticmethod + def validate(value): + if not isinstance(value, str_type): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +TYPE_CLASSES = { + 'string': StringType, + 'boolean': BooleanType, + 'int': IntegerType, + 'pos_int': PositiveIntegerType, + 'path': PathType, +} + + +class DefaultValue(object): + pass + + +def reraise_attribute_error(func): + """Used to make sure __getattr__ wrappers around __getitem__ + raise AttributeError instead of KeyError. + """ + @wraps(func) + def _(*args, **kwargs): + try: + return func(*args, **kwargs) + except KeyError: + exc_class, exc, tb = sys.exc_info() + raise AttributeError().__class__, exc, tb + return _ + + +class ConfigSettings(collections.Mapping): + """Interface for configuration settings. + + This is the main interface to the configuration. + + A configuration is a collection of sections. Each section contains + key-value pairs. + + When an instance is created, the caller first registers ConfigProvider + instances with it. This tells the ConfigSettings what individual settings + are available and defines extra metadata associated with those settings. + This is used for validation, etc. + + Once ConfigProvider instances are registered, a config is populated. It can + be loaded from files or populated by hand. + + ConfigSettings instances are accessed like dictionaries or by using + attributes. e.g. the section "foo" is accessed through either + settings.foo or settings['foo']. + + Sections are modeled by the ConfigSection class which is defined inside + this one. They look just like dicts or classes with attributes. To access + the "bar" option in the "foo" section: + + value = settings.foo.bar + value = settings['foo']['bar'] + value = settings.foo['bar'] + + Assignment is similar: + + settings.foo.bar = value + settings['foo']['bar'] = value + settings['foo'].bar = value + + You can even delete user-assigned values: + + del settings.foo.bar + del settings['foo']['bar'] + + If there is a default, it will be returned. + + When settings are mutated, they are validated against the registered + providers. Setting unknown settings or setting values to illegal values + will result in exceptions being raised. + """ + + class ConfigSection(collections.MutableMapping, object): + """Represents an individual config section.""" + def __init__(self, config, name, settings): + object.__setattr__(self, '_config', config) + object.__setattr__(self, '_name', name) + object.__setattr__(self, '_settings', settings) + + wildcard = any(s == '*' for s in self._settings) + object.__setattr__(self, '_wildcard', wildcard) + + @property + def options(self): + try: + return self._config.options(self._name) + except NoSectionError: + return [] + + def get_meta(self, option): + if option in self._settings: + return self._settings[option] + if self._wildcard: + return self._settings['*'] + raise KeyError('Option not registered with provider: %s' % option) + + def _validate(self, option, value): + meta = self.get_meta(option) + meta['type_cls'].validate(value) + + if 'choices' in meta and value not in meta['choices']: + raise ValueError("Value '%s' must be one of: %s" % ( + value, ', '.join(sorted(meta['choices'])))) + + # MutableMapping interface + def __len__(self): + return len(self.options) + + def __iter__(self): + return iter(self.options) + + def __contains__(self, k): + return self._config.has_option(self._name, k) + + def __getitem__(self, k): + meta = self.get_meta(k) + + if self._config.has_option(self._name, k): + v = meta['type_cls'].from_config(self._config, self._name, k) + else: + v = meta.get('default', DefaultValue) + + if v == DefaultValue: + raise KeyError('No default value registered: %s' % k) + + self._validate(k, v) + return v + + def __setitem__(self, k, v): + self._validate(k, v) + meta = self.get_meta(k) + + if not self._config.has_section(self._name): + self._config.add_section(self._name) + + self._config.set(self._name, k, meta['type_cls'].to_config(v)) + + def __delitem__(self, k): + self._config.remove_option(self._name, k) + + # Prune empty sections. + if not len(self._config.options(self._name)): + self._config.remove_section(self._name) + + @reraise_attribute_error + def __getattr__(self, k): + return self.__getitem__(k) + + @reraise_attribute_error + def __setattr__(self, k, v): + self.__setitem__(k, v) + + @reraise_attribute_error + def __delattr__(self, k): + self.__delitem__(k) + + + def __init__(self): + self._config = RawConfigParser() + + self._settings = {} + self._sections = {} + self._finalized = False + self.loaded_files = set() + + def load_file(self, filename): + self.load_files([filename]) + + def load_files(self, filenames): + """Load a config from files specified by their paths. + + Files are loaded in the order given. Subsequent files will overwrite + values from previous files. If a file does not exist, it will be + ignored. + """ + filtered = [f for f in filenames if os.path.exists(f)] + + fps = [open(f, 'rt') for f in filtered] + self.load_fps(fps) + self.loaded_files.update(set(filtered)) + for fp in fps: + fp.close() + + def load_fps(self, fps): + """Load config data by reading file objects.""" + + for fp in fps: + self._config.readfp(fp) + + def write(self, fh): + """Write the config to a file object.""" + self._config.write(fh) + + @classmethod + def _format_metadata(cls, provider, section, option, type_cls, + default=DefaultValue, extra=None): + """Formats and returns the metadata for a setting. + + Each setting must have: + + section -- str section to which the setting belongs. This is how + settings are grouped. + + option -- str id for the setting. This must be unique within the + section it appears. + + type -- a ConfigType-derived type defining the type of the setting. + + Each setting has the following optional parameters: + + default -- The default value for the setting. If None (the default) + there is no default. + + extra -- A dict of additional key/value pairs to add to the + setting metadata. + """ + if isinstance(type_cls, basestring): + type_cls = TYPE_CLASSES[type_cls] + + meta = { + 'short': '%s.short' % option, + 'full': '%s.full' % option, + 'type_cls': type_cls, + 'domain': section, + 'localedir': provider.config_settings_locale_directory, + } + + if default != DefaultValue: + meta['default'] = default + + if extra: + meta.update(extra) + + return meta + + def register_provider(self, provider): + """Register a SettingsProvider with this settings interface.""" + + if self._finalized: + raise ConfigException('Providers cannot be registered after finalized.') + + settings = provider.config_settings + if callable(settings): + settings = settings() + + config_settings = collections.defaultdict(dict) + for setting in settings: + section, option = setting[0].split('.') + + if option in config_settings[section]: + raise ConfigException('Setting has already been registered: %s.%s' % ( + section, option)) + + meta = self._format_metadata(provider, section, option, *setting[1:]) + config_settings[section][option] = meta + + for section_name, settings in config_settings.items(): + section = self._settings.get(section_name, {}) + + for k, v in settings.items(): + if k in section: + raise ConfigException('Setting already registered: %s.%s' % + section_name, k) + + section[k] = v + + self._settings[section_name] = section + + def option_help(self, section, option): + """Obtain the translated help messages for an option.""" + + meta = self[section].get_meta(option) + + # Providers should always have an en_US translation. If they don't, + # they are coded wrong and this will raise. + default = gettext.translation(meta['domain'], meta['localedir'], + ['en_US']) + + t = gettext.translation(meta['domain'], meta['localedir'], + fallback=True) + t.add_fallback(default) + + short = t.ugettext('%s.%s.short' % (section, option)) + full = t.ugettext('%s.%s.full' % (section, option)) + + return (short, full) + + def _finalize(self): + if self._finalized: + return + + for section, settings in self._settings.items(): + s = ConfigSettings.ConfigSection(self._config, section, settings) + self._sections[section] = s + + self._finalized = True + + # Mapping interface. + def __len__(self): + return len(self._settings) + + def __iter__(self): + self._finalize() + + return iter(self._sections.keys()) + + def __contains__(self, k): + return k in self._settings + + def __getitem__(self, k): + self._finalize() + + return self._sections[k] + + # Allow attribute access because it looks nice. + @reraise_attribute_error + def __getattr__(self, k): + return self.__getitem__(k) diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py new file mode 100644 index 000000000..6c1713d7f --- /dev/null +++ b/python/mach/mach/decorators.py @@ -0,0 +1,353 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import argparse +import collections +import inspect +import os +import types + +from .base import MachError +from .registrar import Registrar + + +class _MachCommand(object): + """Container for mach command metadata. + + Mach commands contain lots of attributes. This class exists to capture them + in a sane way so tuples, etc aren't used instead. + """ + __slots__ = ( + # Content from decorator arguments to define the command. + 'name', + 'subcommand', + 'category', + 'description', + 'conditions', + '_parser', + 'arguments', + 'argument_group_names', + + # Describes how dispatch is performed. + + # The Python class providing the command. This is the class type not + # an instance of the class. Mach will instantiate a new instance of + # the class if the command is executed. + 'cls', + + # Whether the __init__ method of the class should receive a mach + # context instance. This should only affect the mach driver and how + # it instantiates classes. + 'pass_context', + + # The name of the method providing the command. In other words, this + # is the str name of the attribute on the class type corresponding to + # the name of the function. + 'method', + + # Dict of string to _MachCommand defining sub-commands for this + # command. + 'subcommand_handlers', + ) + + def __init__(self, name=None, subcommand=None, category=None, + description=None, conditions=None, parser=None): + self.name = name + self.subcommand = subcommand + self.category = category + self.description = description + self.conditions = conditions or [] + self._parser = parser + self.arguments = [] + self.argument_group_names = [] + + self.cls = None + self.pass_context = None + self.method = None + self.subcommand_handlers = {} + + @property + def parser(self): + # Creating CLI parsers at command dispatch time can be expensive. Make + # it possible to lazy load them by using functions. + if callable(self._parser): + self._parser = self._parser() + + return self._parser + + @property + def docstring(self): + return self.cls.__dict__[self.method].__doc__ + + def __ior__(self, other): + if not isinstance(other, _MachCommand): + raise ValueError('can only operate on _MachCommand instances') + + for a in self.__slots__: + if not getattr(self, a): + setattr(self, a, getattr(other, a)) + + return self + + +def CommandProvider(cls): + """Class decorator to denote that it provides subcommands for Mach. + + When this decorator is present, mach looks for commands being defined by + methods inside the class. + """ + + # The implementation of this decorator relies on the parse-time behavior of + # decorators. When the module is imported, the method decorators (like + # @Command and @CommandArgument) are called *before* this class decorator. + # The side-effect of the method decorators is to store specifically-named + # attributes on the function types. We just scan over all functions in the + # class looking for the side-effects of the method decorators. + + # Tell mach driver whether to pass context argument to __init__. + pass_context = False + + if inspect.ismethod(cls.__init__): + spec = inspect.getargspec(cls.__init__) + + if len(spec.args) > 2: + msg = 'Mach @CommandProvider class %s implemented incorrectly. ' + \ + '__init__() must take 1 or 2 arguments. From %s' + msg = msg % (cls.__name__, inspect.getsourcefile(cls)) + raise MachError(msg) + + if len(spec.args) == 2: + pass_context = True + + seen_commands = set() + + # We scan __dict__ because we only care about the classes own attributes, + # not inherited ones. If we did inherited attributes, we could potentially + # define commands multiple times. We also sort keys so commands defined in + # the same class are grouped in a sane order. + for attr in sorted(cls.__dict__.keys()): + value = cls.__dict__[attr] + + if not isinstance(value, types.FunctionType): + continue + + command = getattr(value, '_mach_command', None) + if not command: + continue + + # Ignore subcommands for now: we handle them later. + if command.subcommand: + continue + + seen_commands.add(command.name) + + if not command.conditions and Registrar.require_conditions: + continue + + msg = 'Mach command \'%s\' implemented incorrectly. ' + \ + 'Conditions argument must take a list ' + \ + 'of functions. Found %s instead.' + + if not isinstance(command.conditions, collections.Iterable): + msg = msg % (command.name, type(command.conditions)) + raise MachError(msg) + + for c in command.conditions: + if not hasattr(c, '__call__'): + msg = msg % (command.name, type(c)) + raise MachError(msg) + + command.cls = cls + command.method = attr + command.pass_context = pass_context + + Registrar.register_command_handler(command) + + # Now do another pass to get sub-commands. We do this in two passes so + # we can check the parent command existence without having to hold + # state and reconcile after traversal. + for attr in sorted(cls.__dict__.keys()): + value = cls.__dict__[attr] + + if not isinstance(value, types.FunctionType): + continue + + command = getattr(value, '_mach_command', None) + if not command: + continue + + # It is a regular command. + if not command.subcommand: + continue + + if command.name not in seen_commands: + raise MachError('Command referenced by sub-command does not ' + 'exist: %s' % command.name) + + if command.name not in Registrar.command_handlers: + continue + + command.cls = cls + command.method = attr + command.pass_context = pass_context + parent = Registrar.command_handlers[command.name] + + if parent._parser: + raise MachError('cannot declare sub commands against a command ' + 'that has a parser installed: %s' % command) + if command.subcommand in parent.subcommand_handlers: + raise MachError('sub-command already defined: %s' % command.subcommand) + + parent.subcommand_handlers[command.subcommand] = command + + return cls + + +class Command(object): + """Decorator for functions or methods that provide a mach command. + + The decorator accepts arguments that define basic attributes of the + command. The following arguments are recognized: + + category -- The string category to which this command belongs. Mach's + help will group commands by category. + + description -- A brief description of what the command does. + + parser -- an optional argparse.ArgumentParser instance or callable + that returns an argparse.ArgumentParser instance to use as the + basis for the command arguments. + + For example: + + @Command('foo', category='misc', description='Run the foo action') + def foo(self): + pass + """ + def __init__(self, name, **kwargs): + self._mach_command = _MachCommand(name=name, **kwargs) + + def __call__(self, func): + if not hasattr(func, '_mach_command'): + func._mach_command = _MachCommand() + + func._mach_command |= self._mach_command + + return func + +class SubCommand(object): + """Decorator for functions or methods that provide a sub-command. + + Mach commands can have sub-commands. e.g. ``mach command foo`` or + ``mach command bar``. Each sub-command has its own parser and is + effectively its own mach command. + + The decorator accepts arguments that define basic attributes of the + sub command: + + command -- The string of the command this sub command should be + attached to. + + subcommand -- The string name of the sub command to register. + + description -- A textual description for this sub command. + """ + def __init__(self, command, subcommand, description=None): + self._mach_command = _MachCommand(name=command, subcommand=subcommand, + description=description) + + def __call__(self, func): + if not hasattr(func, '_mach_command'): + func._mach_command = _MachCommand() + + func._mach_command |= self._mach_command + + return func + +class CommandArgument(object): + """Decorator for additional arguments to mach subcommands. + + This decorator should be used to add arguments to mach commands. Arguments + to the decorator are proxied to ArgumentParser.add_argument(). + + For example: + + @Command('foo', help='Run the foo action') + @CommandArgument('-b', '--bar', action='store_true', default=False, + help='Enable bar mode.') + def foo(self): + pass + """ + def __init__(self, *args, **kwargs): + if kwargs.get('nargs') == argparse.REMAINDER: + # These are the assertions we make in dispatcher.py about + # those types of CommandArguments. + assert len(args) == 1 + assert all(k in ('default', 'nargs', 'help', 'group') for k in kwargs) + self._command_args = (args, kwargs) + + def __call__(self, func): + if not hasattr(func, '_mach_command'): + func._mach_command = _MachCommand() + + func._mach_command.arguments.insert(0, self._command_args) + + return func + + +class CommandArgumentGroup(object): + """Decorator for additional argument groups to mach commands. + + This decorator should be used to add arguments groups to mach commands. + Arguments to the decorator are proxied to + ArgumentParser.add_argument_group(). + + For example: + + @Command('foo', helps='Run the foo action') + @CommandArgumentGroup('group1') + @CommandArgument('-b', '--bar', group='group1', action='store_true', + default=False, help='Enable bar mode.') + def foo(self): + pass + + The name should be chosen so that it makes sense as part of the phrase + 'Command Arguments for <name>' because that's how it will be shown in the + help message. + """ + def __init__(self, group_name): + self._group_name = group_name + + def __call__(self, func): + if not hasattr(func, '_mach_command'): + func._mach_command = _MachCommand() + + func._mach_command.argument_group_names.insert(0, self._group_name) + + return func + + +def SettingsProvider(cls): + """Class decorator to denote that this class provides Mach settings. + + When this decorator is encountered, the underlying class will automatically + be registered with the Mach registrar and will (likely) be hooked up to the + mach driver. + """ + if not hasattr(cls, 'config_settings'): + raise MachError('@SettingsProvider must contain a config_settings attribute. It ' + 'may either be a list of tuples, or a callable that returns a list ' + 'of tuples. Each tuple must be of the form:\n' + '(<section>.<option>, <type_cls>, <default>, <choices>)\n' + 'as specified by ConfigSettings._format_metadata.') + + if not hasattr(cls, 'config_settings_locale_directory'): + cls_dir = os.path.dirname(inspect.getfile(cls)) + cls.config_settings_locale_directory = os.path.join(cls_dir, 'locale') + + Registrar.register_settings_provider(cls) + return cls + diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py new file mode 100644 index 000000000..5604d981d --- /dev/null +++ b/python/mach/mach/dispatcher.py @@ -0,0 +1,453 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import argparse +import difflib +import shlex +import sys + +from operator import itemgetter + +from .base import ( + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, +) +from .decorators import SettingsProvider + + +@SettingsProvider +class DispatchSettings(): + config_settings = [ + ('alias.*', 'string'), + ] + + +class CommandFormatter(argparse.HelpFormatter): + """Custom formatter to format just a subcommand.""" + + def add_usage(self, *args): + pass + + +class CommandAction(argparse.Action): + """An argparse action that handles mach commands. + + This class is essentially a reimplementation of argparse's sub-parsers + feature. We first tried to use sub-parsers. However, they were missing + features like grouping of commands (http://bugs.python.org/issue14037). + + The way this works involves light magic and a partial understanding of how + argparse works. + + Arguments registered with an argparse.ArgumentParser have an action + associated with them. An action is essentially a class that when called + does something with the encountered argument(s). This class is one of those + action classes. + + An instance of this class is created doing something like: + + parser.add_argument('command', action=CommandAction, registrar=r) + + Note that a mach.registrar.Registrar instance is passed in. The Registrar + holds information on all the mach commands that have been registered. + + When this argument is registered with the ArgumentParser, an instance of + this class is instantiated. One of the subtle but important things it does + is tell the argument parser that it's interested in *all* of the remaining + program arguments. So, when the ArgumentParser calls this action, we will + receive the command name plus all of its arguments. + + For more, read the docs in __call__. + """ + def __init__(self, option_strings, dest, required=True, default=None, + registrar=None, context=None): + # A proper API would have **kwargs here. However, since we are a little + # hacky, we intentionally omit it as a way of detecting potentially + # breaking changes with argparse's implementation. + # + # In a similar vein, default is passed in but is not needed, so we drop + # it. + argparse.Action.__init__(self, option_strings, dest, required=required, + help=argparse.SUPPRESS, nargs=argparse.REMAINDER) + + self._mach_registrar = registrar + self._context = context + + def __call__(self, parser, namespace, values, option_string=None): + """This is called when the ArgumentParser has reached our arguments. + + Since we always register ourselves with nargs=argparse.REMAINDER, + values should be a list of remaining arguments to parse. The first + argument should be the name of the command to invoke and all remaining + arguments are arguments for that command. + + The gist of the flow is that we look at the command being invoked. If + it's *help*, we handle that specially (because argparse's default help + handler isn't satisfactory). Else, we create a new, independent + ArgumentParser instance for just the invoked command (based on the + information contained in the command registrar) and feed the arguments + into that parser. We then merge the results with the main + ArgumentParser. + """ + if namespace.help: + # -h or --help is in the global arguments. + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif values: + command = values[0].lower() + args = values[1:] + if command == 'help': + if args and args[0] not in ['-h', '--help']: + # Make sure args[0] is indeed a command. + self._handle_command_help(parser, args[0]) + else: + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif '-h' in args or '--help' in args: + # -h or --help is in the command arguments. + if '--' in args: + # -- is in command arguments + if '-h' in args[:args.index('--')] or '--help' in args[:args.index('--')]: + # Honor -h or --help only if it appears before -- + self._handle_command_help(parser, command) + sys.exit(0) + else: + self._handle_command_help(parser, command) + sys.exit(0) + else: + raise NoCommandError() + + # First see if the this is a user-defined alias + if command in self._context.settings.alias: + alias = self._context.settings.alias[command] + defaults = shlex.split(alias) + command = defaults.pop(0) + args = defaults + args + + if command not in self._mach_registrar.command_handlers: + # Try to find similar commands, may raise UnknownCommandError. + command = self._suggest_command(command) + + handler = self._mach_registrar.command_handlers.get(command) + + usage = '%(prog)s [global arguments] ' + command + \ + ' [command arguments]' + + subcommand = None + + # If there are sub-commands, parse the intent out immediately. + if handler.subcommand_handlers and args: + if len(args) == 1 and args[0] in ('help', '--help'): + self._handle_subcommand_main_help(parser, handler) + sys.exit(0) + # mach <command> help <subcommand> + elif len(args) == 2 and args[0] == 'help': + subcommand = args[1] + subhandler = handler.subcommand_handlers[subcommand] + self._handle_subcommand_help(parser, command, subcommand, subhandler) + sys.exit(0) + # We are running a sub command. + elif args[0] in handler.subcommand_handlers: + subcommand = args[0] + handler = handler.subcommand_handlers[subcommand] + usage = '%(prog)s [global arguments] ' + command + ' ' + \ + subcommand + ' [command arguments]' + args.pop(0) + + # We create a new parser, populate it with the command's arguments, + # then feed all remaining arguments to it, merging the results + # with ourselves. This is essentially what argparse subparsers + # do. + + parser_args = { + 'add_help': False, + 'usage': usage, + } + + remainder = None + + if handler.parser: + subparser = handler.parser + subparser.context = self._context + for arg in subparser._actions[:]: + if arg.nargs == argparse.REMAINDER: + subparser._actions.remove(arg) + remainder = (arg.dest,), {'default': arg.default, + 'nargs': arg.nargs, + 'help': arg.help} + else: + subparser = argparse.ArgumentParser(**parser_args) + + for arg in handler.arguments: + # Remove our group keyword; it's not needed here. + group_name = arg[1].get('group') + if group_name: + del arg[1]['group'] + + if arg[1].get('nargs') == argparse.REMAINDER: + # parse_known_args expects all argparse.REMAINDER ('...') + # arguments to be all stuck together. Instead, we want them to + # pick any extra argument, wherever they are. + # Assume a limited CommandArgument for those arguments. + assert len(arg[0]) == 1 + assert all(k in ('default', 'nargs', 'help') for k in arg[1]) + remainder = arg + else: + subparser.add_argument(*arg[0], **arg[1]) + + # We define the command information on the main parser result so as to + # not interfere with arguments passed to the command. + setattr(namespace, 'mach_handler', handler) + setattr(namespace, 'command', command) + setattr(namespace, 'subcommand', subcommand) + + command_namespace, extra = subparser.parse_known_args(args) + setattr(namespace, 'command_args', command_namespace) + if remainder: + (name,), options = remainder + # parse_known_args usefully puts all arguments after '--' in + # extra, but also puts '--' there. We don't want to pass it down + # to the command handler. Note that if multiple '--' are on the + # command line, only the first one is removed, so that subsequent + # ones are passed down. + if '--' in extra: + extra.remove('--') + + # Commands with argparse.REMAINDER arguments used to force the + # other arguments to be '+' prefixed. If a user now passes such + # an argument, if will silently end up in extra. So, check if any + # of the allowed arguments appear in a '+' prefixed form, and error + # out if that's the case. + for args, _ in handler.arguments: + for arg in args: + arg = arg.replace('-', '+', 1) + if arg in extra: + raise UnrecognizedArgumentError(command, [arg]) + + if extra: + setattr(command_namespace, name, extra) + else: + setattr(command_namespace, name, options.get('default', [])) + elif extra and handler.cls.__name__ != 'DeprecatedCommands': + raise UnrecognizedArgumentError(command, extra) + + def _handle_main_help(self, parser, verbose): + # Since we don't need full sub-parser support for the main help output, + # we create groups in the ArgumentParser and populate each group with + # arguments corresponding to command names. This has the side-effect + # that argparse renders it nicely. + r = self._mach_registrar + disabled_commands = [] + + cats = [(k, v[2]) for k, v in r.categories.items()] + sorted_cats = sorted(cats, key=itemgetter(1), reverse=True) + for category, priority in sorted_cats: + group = None + + for command in sorted(r.commands_by_category[category]): + handler = r.command_handlers[command] + + # Instantiate a handler class to see if it should be filtered + # out for the current context or not. Condition functions can be + # applied to the command's decorator. + if handler.conditions: + if handler.pass_context: + instance = handler.cls(self._context) + else: + instance = handler.cls() + + is_filtered = False + for c in handler.conditions: + if not c(instance): + is_filtered = True + break + if is_filtered: + description = handler.description + disabled_command = {'command': command, 'description': description} + disabled_commands.append(disabled_command) + continue + + if group is None: + title, description, _priority = r.categories[category] + group = parser.add_argument_group(title, description) + + description = handler.description + group.add_argument(command, help=description, + action='store_true') + + if disabled_commands and 'disabled' in r.categories: + title, description, _priority = r.categories['disabled'] + group = parser.add_argument_group(title, description) + if verbose: + for c in disabled_commands: + group.add_argument(c['command'], help=c['description'], + action='store_true') + + parser.print_help() + + def _populate_command_group(self, parser, handler, group): + extra_groups = {} + for group_name in handler.argument_group_names: + group_full_name = 'Command Arguments for ' + group_name + extra_groups[group_name] = \ + parser.add_argument_group(group_full_name) + + for arg in handler.arguments: + # Apply our group keyword. + group_name = arg[1].get('group') + if group_name: + del arg[1]['group'] + group = extra_groups[group_name] + group.add_argument(*arg[0], **arg[1]) + + def _handle_command_help(self, parser, command): + handler = self._mach_registrar.command_handlers.get(command) + + if not handler: + raise UnknownCommandError(command, 'query') + + if handler.subcommand_handlers: + self._handle_subcommand_main_help(parser, handler) + return + + # This code is worth explaining. Because we are doing funky things with + # argument registration to allow the same option in both global and + # command arguments, we can't simply put all arguments on the same + # parser instance because argparse would complain. We can't register an + # argparse subparser here because it won't properly show help for + # global arguments. So, we employ a strategy similar to command + # execution where we construct a 2nd, independent ArgumentParser for + # just the command data then supplement the main help's output with + # this 2nd parser's. We use a custom formatter class to ignore some of + # the help output. + parser_args = { + 'formatter_class': CommandFormatter, + 'add_help': False, + } + + if handler.parser: + c_parser = handler.parser + c_parser.context = self._context + c_parser.formatter_class = NoUsageFormatter + # Accessing _action_groups is a bit shady. We are highly dependent + # on the argparse implementation not changing. We fail fast to + # detect upstream changes so we can intelligently react to them. + group = c_parser._action_groups[1] + + # By default argparse adds two groups called "positional arguments" + # and "optional arguments". We want to rename these to reflect standard + # mach terminology. + c_parser._action_groups[0].title = 'Command Parameters' + c_parser._action_groups[1].title = 'Command Arguments' + + if not handler.description: + handler.description = c_parser.description + c_parser.description = None + else: + c_parser = argparse.ArgumentParser(**parser_args) + group = c_parser.add_argument_group('Command Arguments') + + self._populate_command_group(c_parser, handler, group) + + # Set the long help of the command to the docstring (if present) or + # the command decorator description argument (if present). + if handler.docstring: + parser.description = format_docstring(handler.docstring) + elif handler.description: + parser.description = handler.description + + parser.usage = '%(prog)s [global arguments] ' + command + \ + ' [command arguments]' + + # This is needed to preserve line endings in the description field, + # which may be populated from a docstring. + parser.formatter_class = argparse.RawDescriptionHelpFormatter + parser.print_help() + print('') + c_parser.print_help() + + def _handle_subcommand_main_help(self, parser, handler): + parser.usage = '%(prog)s [global arguments] ' + handler.name + \ + ' subcommand [subcommand arguments]' + group = parser.add_argument_group('Sub Commands') + + for subcommand, subhandler in sorted(handler.subcommand_handlers.iteritems()): + group.add_argument(subcommand, help=subhandler.description, + action='store_true') + + if handler.docstring: + parser.description = format_docstring(handler.docstring) + + parser.formatter_class = argparse.RawDescriptionHelpFormatter + + parser.print_help() + + def _handle_subcommand_help(self, parser, command, subcommand, handler): + parser.usage = '%(prog)s [global arguments] ' + command + \ + ' ' + subcommand + ' [command arguments]' + + c_parser = argparse.ArgumentParser(add_help=False, + formatter_class=CommandFormatter) + group = c_parser.add_argument_group('Sub Command Arguments') + self._populate_command_group(c_parser, handler, group) + + if handler.docstring: + parser.description = format_docstring(handler.docstring) + + parser.formatter_class = argparse.RawDescriptionHelpFormatter + + parser.print_help() + print('') + c_parser.print_help() + + def _suggest_command(self, command): + # Make sure we don't suggest any deprecated commands. + names = [h.name for h in self._mach_registrar.command_handlers.values() + if h.cls.__name__ != 'DeprecatedCommands'] + # We first try to look for a valid command that is very similar to the given command. + suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8) + # If we find more than one matching command, or no command at all, + # we give command suggestions instead (with a lower matching threshold). + # All commands that start with the given command (for instance: + # 'mochitest-plain', 'mochitest-chrome', etc. for 'mochitest-') + # are also included. + if len(suggested_commands) != 1: + suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5)) + suggested_commands |= {cmd for cmd in names if cmd.startswith(command)} + raise UnknownCommandError(command, 'run', suggested_commands) + sys.stderr.write("We're assuming the '%s' command is '%s' and we're " + "executing it for you.\n\n" % (command, suggested_commands[0])) + return suggested_commands[0] + + +class NoUsageFormatter(argparse.HelpFormatter): + def _format_usage(self, *args, **kwargs): + return "" + + +def format_docstring(docstring): + """Format a raw docstring into something suitable for presentation. + + This function is based on the example function in PEP-0257. + """ + if not docstring: + return '' + lines = docstring.expandtabs().splitlines() + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + return '\n'.join(trimmed) diff --git a/python/mach/mach/locale/en_US/LC_MESSAGES/alias.mo b/python/mach/mach/locale/en_US/LC_MESSAGES/alias.mo Binary files differnew file mode 100644 index 000000000..663180841 --- /dev/null +++ b/python/mach/mach/locale/en_US/LC_MESSAGES/alias.mo diff --git a/python/mach/mach/locale/en_US/LC_MESSAGES/alias.po b/python/mach/mach/locale/en_US/LC_MESSAGES/alias.po new file mode 100644 index 000000000..8ea0e2f86 --- /dev/null +++ b/python/mach/mach/locale/en_US/LC_MESSAGES/alias.po @@ -0,0 +1,9 @@ +# +msgid "" +msgstr "" + +msgid "alias.*.short" +msgstr "Create a command alias" + +msgid "alias.*.full" +msgstr "Create a command alias of the form `<alias> = <command> <args>`." diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py new file mode 100644 index 000000000..729e6cb3d --- /dev/null +++ b/python/mach/mach/logging.py @@ -0,0 +1,256 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains logging functionality for mach. It essentially provides +# support for a structured logging framework built on top of Python's built-in +# logging framework. + +from __future__ import absolute_import, unicode_literals + +try: + import blessings +except ImportError: + blessings = None + +import json +import logging +import sys +import time + + +def format_seconds(total): + """Format number of seconds to MM:SS.DD form.""" + + minutes, seconds = divmod(total, 60) + + return '%2d:%05.2f' % (minutes, seconds) + + +class ConvertToStructuredFilter(logging.Filter): + """Filter that converts unstructured records into structured ones.""" + def filter(self, record): + if hasattr(record, 'action') and hasattr(record, 'params'): + return True + + record.action = 'unstructured' + record.params = {'msg': record.getMessage()} + record.msg = '{msg}' + + return True + + +class StructuredJSONFormatter(logging.Formatter): + """Log formatter that writes a structured JSON entry.""" + + def format(self, record): + action = getattr(record, 'action', 'UNKNOWN') + params = getattr(record, 'params', {}) + + return json.dumps([record.created, action, params]) + + +class StructuredHumanFormatter(logging.Formatter): + """Log formatter that writes structured messages for humans. + + It is important that this formatter never be added to a logger that + produces unstructured/classic log messages. If it is, the call to format() + could fail because the string could contain things (like JSON) that look + like formatting character sequences. + + Because of this limitation, format() will fail with a KeyError if an + unstructured record is passed or if the structured message is malformed. + """ + def __init__(self, start_time, write_interval=False, write_times=True): + self.start_time = start_time + self.write_interval = write_interval + self.write_times = write_times + self.last_time = None + + def format(self, record): + f = record.msg.format(**record.params) + + if not self.write_times: + return f + + elapsed = self._time(record) + + return '%s %s' % (format_seconds(elapsed), f) + + def _time(self, record): + t = record.created - self.start_time + + if self.write_interval and self.last_time is not None: + t = record.created - self.last_time + + self.last_time = record.created + + return t + + +class StructuredTerminalFormatter(StructuredHumanFormatter): + """Log formatter for structured messages writing to a terminal.""" + + def set_terminal(self, terminal): + self.terminal = terminal + + def format(self, record): + f = record.msg.format(**record.params) + + if not self.write_times: + return f + + t = self.terminal.blue(format_seconds(self._time(record))) + + return '%s %s' % (t, self._colorize(f)) + + def _colorize(self, s): + if not self.terminal: + return s + + result = s + + reftest = s.startswith('REFTEST ') + if reftest: + s = s[8:] + + if s.startswith('TEST-PASS'): + result = self.terminal.green(s[0:9]) + s[9:] + elif s.startswith('TEST-UNEXPECTED'): + result = self.terminal.red(s[0:20]) + s[20:] + elif s.startswith('TEST-START'): + result = self.terminal.yellow(s[0:10]) + s[10:] + elif s.startswith('TEST-INFO'): + result = self.terminal.yellow(s[0:9]) + s[9:] + + if reftest: + result = 'REFTEST ' + result + + return result + + +class LoggingManager(object): + """Holds and controls global logging state. + + An application should instantiate one of these and configure it as needed. + + This class provides a mechanism to configure the output of logging data + both from mach and from the overall logging system (e.g. from other + modules). + """ + + def __init__(self): + self.start_time = time.time() + + self.json_handlers = [] + self.terminal_handler = None + self.terminal_formatter = None + + self.root_logger = logging.getLogger() + self.root_logger.setLevel(logging.DEBUG) + + # Installing NullHandler on the root logger ensures that *all* log + # messages have at least one handler. This prevents Python from + # complaining about "no handlers could be found for logger XXX." + self.root_logger.addHandler(logging.NullHandler()) + + self.mach_logger = logging.getLogger('mach') + self.mach_logger.setLevel(logging.DEBUG) + + self.structured_filter = ConvertToStructuredFilter() + + self.structured_loggers = [self.mach_logger] + + self._terminal = None + + @property + def terminal(self): + if not self._terminal and blessings: + # Sometimes blessings fails to set up the terminal. In that case, + # silently fail. + try: + terminal = blessings.Terminal(stream=sys.stdout) + + if terminal.is_a_tty: + self._terminal = terminal + except Exception: + pass + + return self._terminal + + def add_json_handler(self, fh): + """Enable JSON logging on the specified file object.""" + + # Configure the consumer of structured messages. + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(StructuredJSONFormatter()) + handler.setLevel(logging.DEBUG) + + # And hook it up. + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.json_handlers.append(handler) + + def add_terminal_logging(self, fh=sys.stdout, level=logging.INFO, + write_interval=False, write_times=True): + """Enable logging to the terminal.""" + + formatter = StructuredHumanFormatter(self.start_time, + write_interval=write_interval, write_times=write_times) + + if self.terminal: + formatter = StructuredTerminalFormatter(self.start_time, + write_interval=write_interval, write_times=write_times) + formatter.set_terminal(self.terminal) + + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(formatter) + handler.setLevel(level) + + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + self.terminal_formatter = formatter + + def replace_terminal_handler(self, handler): + """Replace the installed terminal handler. + + Returns the old handler or None if none was configured. + If the new handler is None, removes any existing handler and disables + logging to the terminal. + """ + old = self.terminal_handler + + if old: + for logger in self.structured_loggers: + logger.removeHandler(old) + + if handler: + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + + return old + + def enable_unstructured(self): + """Enable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.addFilter(self.structured_filter) + self.root_logger.addHandler(self.terminal_handler) + + def disable_unstructured(self): + """Disable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.removeFilter(self.structured_filter) + self.root_logger.removeHandler(self.terminal_handler) + + def register_structured_logger(self, logger): + """Register a structured logger. + + This needs to be called for all structured loggers that don't chain up + to the mach logger in order for their output to be captured. + """ + self.structured_loggers.append(logger) diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py new file mode 100644 index 000000000..dec3d7dfd --- /dev/null +++ b/python/mach/mach/main.py @@ -0,0 +1,594 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides functionality for the command-line build tool +# (mach). It is packaged as a module because everything is a library. + +from __future__ import absolute_import, print_function, unicode_literals +from collections import Iterable + +import argparse +import codecs +import imp +import logging +import os +import sys +import traceback +import uuid + +from .base import ( + CommandContext, + MachError, + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, +) + +from .decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from .config import ConfigSettings +from .dispatcher import CommandAction +from .logging import LoggingManager +from .registrar import Registrar + + + +MACH_ERROR = r''' +The error occurred in mach itself. This is likely a bug in mach itself or a +fundamental problem with a loaded module. + +Please consider filing a bug against mach by going to the URL: + + https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=mach + +'''.lstrip() + +ERROR_FOOTER = r''' +If filing a bug, please include the full output of mach, including this error +message. + +The details of the failure are as follows: +'''.lstrip() + +COMMAND_ERROR = r''' +The error occurred in the implementation of the invoked mach command. + +This should never occur and is likely a bug in the implementation of that +command. Consider filing a bug for this issue. +'''.lstrip() + +MODULE_ERROR = r''' +The error occurred in code that was called by the mach command. This is either +a bug in the called code itself or in the way that mach is calling it. + +You should consider filing a bug for this issue. +'''.lstrip() + +NO_COMMAND_ERROR = r''' +It looks like you tried to run mach without a command. + +Run |mach help| to show a list of commands. +'''.lstrip() + +UNKNOWN_COMMAND_ERROR = r''' +It looks like you are trying to %s an unknown mach command: %s +%s +Run |mach help| to show a list of commands. +'''.lstrip() + +SUGGESTED_COMMANDS_MESSAGE = r''' +Did you want to %s any of these commands instead: %s? +''' + +UNRECOGNIZED_ARGUMENT_ERROR = r''' +It looks like you passed an unrecognized argument into mach. + +The %s command does not accept the arguments: %s +'''.lstrip() + +INVALID_ENTRY_POINT = r''' +Entry points should return a list of command providers or directories +containing command providers. The following entry point is invalid: + + %s + +You are seeing this because there is an error in an external module attempting +to implement a mach command. Please fix the error, or uninstall the module from +your system. +'''.lstrip() + +class ArgumentParser(argparse.ArgumentParser): + """Custom implementation argument parser to make things look pretty.""" + + def error(self, message): + """Custom error reporter to give more helpful text on bad commands.""" + if not message.startswith('argument command: invalid choice'): + argparse.ArgumentParser.error(self, message) + assert False + + print('Invalid command specified. The list of commands is below.\n') + self.print_help() + sys.exit(1) + + def format_help(self): + text = argparse.ArgumentParser.format_help(self) + + # Strip out the silly command list that would preceed the pretty list. + # + # Commands: + # {foo,bar} + # foo Do foo. + # bar Do bar. + search = 'Commands:\n {' + start = text.find(search) + + if start != -1: + end = text.find('}\n', start) + assert end != -1 + + real_start = start + len('Commands:\n') + real_end = end + len('}\n') + + text = text[0:real_start] + text[real_end:] + + return text + + +class ContextWrapper(object): + def __init__(self, context, handler): + object.__setattr__(self, '_context', context) + object.__setattr__(self, '_handler', handler) + + def __getattribute__(self, key): + try: + return getattr(object.__getattribute__(self, '_context'), key) + except AttributeError as e: + try: + ret = object.__getattribute__(self, '_handler')(self, key) + except (AttributeError, TypeError): + # TypeError is in case the handler comes from old code not + # taking a key argument. + raise e + setattr(self, key, ret) + return ret + + def __setattr__(self, key, value): + setattr(object.__getattribute__(self, '_context'), key, value) + + +@CommandProvider +class Mach(object): + """Main mach driver type. + + This type is responsible for holding global mach state and dispatching + a command from arguments. + + The following attributes may be assigned to the instance to influence + behavior: + + populate_context_handler -- If defined, it must be a callable. The + callable signature is the following: + populate_context_handler(context, key=None) + It acts as a fallback getter for the mach.base.CommandContext + instance. + This allows to augment the context instance with arbitrary data + for use in command handlers. + For backwards compatibility, it is also called before command + dispatch without a key, allowing the context handler to add + attributes to the context instance. + + require_conditions -- If True, commands that do not have any condition + functions applied will be skipped. Defaults to False. + + settings_paths -- A list of files or directories in which to search + for settings files to load. + + """ + + USAGE = """%(prog)s [global arguments] command [command arguments] + +mach (German for "do") is the main interface to the Mozilla build system and +common developer tasks. + +You tell mach the command you want to perform and it does it for you. + +Some common commands are: + + %(prog)s build Build/compile the source tree. + %(prog)s help Show full help, including the list of all commands. + +To see more help for a specific command, run: + + %(prog)s help <command> +""" + + def __init__(self, cwd): + assert os.path.isdir(cwd) + + self.cwd = cwd + self.log_manager = LoggingManager() + self.logger = logging.getLogger(__name__) + self.settings = ConfigSettings() + self.settings_paths = [] + + if 'MACHRC' in os.environ: + self.settings_paths.append(os.environ['MACHRC']) + + self.log_manager.register_structured_logger(self.logger) + self.global_arguments = [] + self.populate_context_handler = None + + def add_global_argument(self, *args, **kwargs): + """Register a global argument with the argument parser. + + Arguments are proxied to ArgumentParser.add_argument() + """ + + self.global_arguments.append((args, kwargs)) + + def load_commands_from_directory(self, path): + """Scan for mach commands from modules in a directory. + + This takes a path to a directory, loads the .py files in it, and + registers and found mach command providers with this mach instance. + """ + for f in sorted(os.listdir(path)): + if not f.endswith('.py') or f == '__init__.py': + continue + + full_path = os.path.join(path, f) + module_name = 'mach.commands.%s' % f[0:-3] + + self.load_commands_from_file(full_path, module_name=module_name) + + def load_commands_from_file(self, path, module_name=None): + """Scan for mach commands from a file. + + This takes a path to a file and loads it as a Python module under the + module name specified. If no name is specified, a random one will be + chosen. + """ + if module_name is None: + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if b'mach.commands' not in sys.modules: + mod = imp.new_module(b'mach.commands') + sys.modules[b'mach.commands'] = mod + + module_name = 'mach.commands.%s' % uuid.uuid1().get_hex() + + imp.load_source(module_name, path) + + def load_commands_from_entry_point(self, group='mach.providers'): + """Scan installed packages for mach command provider entry points. An + entry point is a function that returns a list of paths to files or + directories containing command providers. + + This takes an optional group argument which specifies the entry point + group to use. If not specified, it defaults to 'mach.providers'. + """ + try: + import pkg_resources + except ImportError: + print("Could not find setuptools, ignoring command entry points", + file=sys.stderr) + return + + for entry in pkg_resources.iter_entry_points(group=group, name=None): + paths = entry.load()() + if not isinstance(paths, Iterable): + print(INVALID_ENTRY_POINT % entry) + sys.exit(1) + + for path in paths: + if os.path.isfile(path): + self.load_commands_from_file(path) + elif os.path.isdir(path): + self.load_commands_from_directory(path) + else: + print("command provider '%s' does not exist" % path) + + def define_category(self, name, title, description, priority=50): + """Provide a description for a named command category.""" + + Registrar.register_category(name, title, description, priority) + + @property + def require_conditions(self): + return Registrar.require_conditions + + @require_conditions.setter + def require_conditions(self, value): + Registrar.require_conditions = value + + def run(self, argv, stdin=None, stdout=None, stderr=None): + """Runs mach with arguments provided from the command line. + + Returns the integer exit code that should be used. 0 means success. All + other values indicate failure. + """ + + # If no encoding is defined, we default to UTF-8 because without this + # Python 2.7 will assume the default encoding of ASCII. This will blow + # up with UnicodeEncodeError as soon as it encounters a non-ASCII + # character in a unicode instance. We simply install a wrapper around + # the streams and restore once we have finished. + stdin = sys.stdin if stdin is None else stdin + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + + orig_stdin = sys.stdin + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + sys.stdin = stdin + sys.stdout = stdout + sys.stderr = stderr + + orig_env = dict(os.environ) + + try: + if stdin.encoding is None: + sys.stdin = codecs.getreader('utf-8')(stdin) + + if stdout.encoding is None: + sys.stdout = codecs.getwriter('utf-8')(stdout) + + if stderr.encoding is None: + sys.stderr = codecs.getwriter('utf-8')(stderr) + + # Allow invoked processes (which may not have a handle on the + # original stdout file descriptor) to know if the original stdout + # is a TTY. This provides a mechanism to allow said processes to + # enable emitting code codes, for example. + if os.isatty(orig_stdout.fileno()): + os.environ[b'MACH_STDOUT_ISATTY'] = b'1' + + return self._run(argv) + except KeyboardInterrupt: + print('mach interrupted by signal or user action. Stopping.') + return 1 + + except Exception as e: + # _run swallows exceptions in invoked handlers and converts them to + # a proper exit code. So, the only scenario where we should get an + # exception here is if _run itself raises. If _run raises, that's a + # bug in mach (or a loaded command module being silly) and thus + # should be reported differently. + self._print_error_header(argv, sys.stdout) + print(MACH_ERROR) + + exc_type, exc_value, exc_tb = sys.exc_info() + stack = traceback.extract_tb(exc_tb) + + self._print_exception(sys.stdout, exc_type, exc_value, stack) + + return 1 + + finally: + os.environ.clear() + os.environ.update(orig_env) + + sys.stdin = orig_stdin + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + def _run(self, argv): + # Load settings as early as possible so things in dispatcher.py + # can use them. + for provider in Registrar.settings_providers: + self.settings.register_provider(provider) + self.load_settings(self.settings_paths) + + context = CommandContext(cwd=self.cwd, + settings=self.settings, log_manager=self.log_manager, + commands=Registrar) + + if self.populate_context_handler: + self.populate_context_handler(context) + context = ContextWrapper(context, self.populate_context_handler) + + parser = self.get_argument_parser(context) + + if not len(argv): + # We don't register the usage until here because if it is globally + # registered, argparse always prints it. This is not desired when + # running with --help. + parser.usage = Mach.USAGE + parser.print_usage() + return 0 + + try: + args = parser.parse_args(argv) + except NoCommandError: + print(NO_COMMAND_ERROR) + return 1 + except UnknownCommandError as e: + suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else '' + print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message)) + return 1 + except UnrecognizedArgumentError as e: + print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, + ' '.join(e.arguments))) + return 1 + + # Add JSON logging to a file if requested. + if args.logfile: + self.log_manager.add_json_handler(args.logfile) + + # Up the logging level if requested. + log_level = logging.INFO + if args.verbose: + log_level = logging.DEBUG + + self.log_manager.register_structured_logger(logging.getLogger('mach')) + + write_times = True + if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ: + write_times = False + + # Always enable terminal logging. The log manager figures out if we are + # actually in a TTY or are a pipe and does the right thing. + self.log_manager.add_terminal_logging(level=log_level, + write_interval=args.log_interval, write_times=write_times) + + if args.settings_file: + # Argument parsing has already happened, so settings that apply + # to command line handling (e.g alias, defaults) will be ignored. + self.load_settings(args.settings_file) + + if not hasattr(args, 'mach_handler'): + raise MachError('ArgumentParser result missing mach handler info.') + + handler = getattr(args, 'mach_handler') + + try: + return Registrar._run_command_handler(handler, context=context, + debug_command=args.debug_command, **vars(args.command_args)) + except KeyboardInterrupt as ki: + raise ki + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + + # The first two frames are us and are never used. + stack = traceback.extract_tb(exc_tb)[2:] + + # If we have nothing on the stack, the exception was raised as part + # of calling the @Command method itself. This likely means a + # mismatch between @CommandArgument and arguments to the method. + # e.g. there exists a @CommandArgument without the corresponding + # argument on the method. We handle that here until the module + # loader grows the ability to validate better. + if not len(stack): + print(COMMAND_ERROR) + self._print_exception(sys.stdout, exc_type, exc_value, + traceback.extract_tb(exc_tb)) + return 1 + + # Split the frames into those from the module containing the + # command and everything else. + command_frames = [] + other_frames = [] + + initial_file = stack[0][0] + + for frame in stack: + if frame[0] == initial_file: + command_frames.append(frame) + else: + other_frames.append(frame) + + # If the exception was in the module providing the command, it's + # likely the bug is in the mach command module, not something else. + # If there are other frames, the bug is likely not the mach + # command's fault. + self._print_error_header(argv, sys.stdout) + + if len(other_frames): + print(MODULE_ERROR) + else: + print(COMMAND_ERROR) + + self._print_exception(sys.stdout, exc_type, exc_value, stack) + + return 1 + + def log(self, level, action, params, format_str): + """Helper method to record a structured log event.""" + self.logger.log(level, format_str, + extra={'action': action, 'params': params}) + + def _print_error_header(self, argv, fh): + fh.write('Error running mach:\n\n') + fh.write(' ') + fh.write(repr(argv)) + fh.write('\n\n') + + def _print_exception(self, fh, exc_type, exc_value, stack): + fh.write(ERROR_FOOTER) + fh.write('\n') + + for l in traceback.format_exception_only(exc_type, exc_value): + fh.write(l) + + fh.write('\n') + for l in traceback.format_list(stack): + fh.write(l) + + def load_settings(self, paths): + """Load the specified settings files. + + If a directory is specified, the following basenames will be + searched for in this order: + + machrc, .machrc + """ + if isinstance(paths, basestring): + paths = [paths] + + valid_names = ('machrc', '.machrc') + def find_in_dir(base): + if os.path.isfile(base): + return base + + for name in valid_names: + path = os.path.join(base, name) + if os.path.isfile(path): + return path + + files = map(find_in_dir, self.settings_paths) + files = filter(bool, files) + + self.settings.load_files(files) + + def get_argument_parser(self, context): + """Returns an argument parser for the command-line interface.""" + + parser = ArgumentParser(add_help=False, + usage='%(prog)s [global arguments] command [command arguments]') + + # Order is important here as it dictates the order the auto-generated + # help messages are printed. + global_group = parser.add_argument_group('Global Arguments') + + global_group.add_argument('-v', '--verbose', dest='verbose', + action='store_true', default=False, + help='Print verbose output.') + global_group.add_argument('-l', '--log-file', dest='logfile', + metavar='FILENAME', type=argparse.FileType('ab'), + help='Filename to write log data to.') + global_group.add_argument('--log-interval', dest='log_interval', + action='store_true', default=False, + help='Prefix log line with interval from last message rather ' + 'than relative time. Note that this is NOT execution time ' + 'if there are parallel operations.') + suppress_log_by_default = False + if 'INSIDE_EMACS' in os.environ: + suppress_log_by_default = True + global_group.add_argument('--log-no-times', dest='log_no_times', + action='store_true', default=suppress_log_by_default, + help='Do not prefix log lines with times. By default, mach will ' + 'prefix each output line with the time since command start.') + global_group.add_argument('-h', '--help', dest='help', + action='store_true', default=False, + help='Show this help message.') + global_group.add_argument('--debug-command', action='store_true', + help='Start a Python debugger when command is dispatched.') + global_group.add_argument('--settings', dest='settings_file', + metavar='FILENAME', default=None, + help='Path to settings file.') + + for args, kwargs in self.global_arguments: + global_group.add_argument(*args, **kwargs) + + # We need to be last because CommandAction swallows all remaining + # arguments and argparse parses arguments in the order they were added. + parser.add_argument('command', action=CommandAction, + registrar=Registrar, context=context) + + return parser diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/mach/mach/mixin/__init__.py diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py new file mode 100644 index 000000000..5c37b54f1 --- /dev/null +++ b/python/mach/mach/mixin/logging.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging + + +class LoggingMixin(object): + """Provides functionality to control logging.""" + + def populate_logger(self, name=None): + """Ensure this class instance has a logger associated with it. + + Users of this mixin that call log() will need to ensure self._logger is + a logging.Logger instance before they call log(). This function ensures + self._logger is defined by populating it if it isn't. + """ + if hasattr(self, '_logger'): + return + + if name is None: + name = '.'.join([self.__module__, self.__class__.__name__]) + + self._logger = logging.getLogger(name) + + def log(self, level, action, params, format_str): + """Log a structured log event. + + A structured log event consists of a logging level, a string action, a + dictionary of attributes, and a formatting string. + + The logging level is one of the logging.* constants, such as + logging.INFO. + + The action string is essentially the enumeration of the event. Each + different type of logged event should have a different action. + + The params dict is the metadata constituting the logged event. + + The formatting string is used to convert the structured message back to + human-readable format. Conversion back to human-readable form is + performed by calling format() on this string, feeding into it the dict + of attributes constituting the event. + + Example Usage + ------------- + + self.log(logging.DEBUG, 'login', {'username': 'johndoe'}, + 'User login: {username}') + """ + self._logger.log(level, format_str, + extra={'action': action, 'params': params}) + diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py new file mode 100644 index 000000000..a6d3a2a1b --- /dev/null +++ b/python/mach/mach/mixin/process.py @@ -0,0 +1,175 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides mixins to perform process execution. + +from __future__ import absolute_import, unicode_literals + +import logging +import os +import subprocess +import sys + +from mozprocess.processhandler import ProcessHandlerMixin + +from .logging import LoggingMixin + + +# Perform detection of operating system environment. This is used by command +# execution. We only do this once to save redundancy. Yes, this can fail module +# loading. That is arguably OK. +if 'SHELL' in os.environ: + _current_shell = os.environ['SHELL'] +elif 'MOZILLABUILD' in os.environ: + _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' +elif 'COMSPEC' in os.environ: + _current_shell = os.environ['COMSPEC'] +else: + raise Exception('Could not detect environment shell!') + +_in_msys = False + +if os.environ.get('MSYSTEM', None) in ('MINGW32', 'MINGW64'): + _in_msys = True + + if not _current_shell.lower().endswith('.exe'): + _current_shell += '.exe' + + +class ProcessExecutionMixin(LoggingMixin): + """Mix-in that provides process execution functionality.""" + + def run_process(self, args=None, cwd=None, append_env=None, + explicit_env=None, log_name=None, log_level=logging.INFO, + line_handler=None, require_unix_environment=False, + ensure_exit_code=0, ignore_children=False, pass_thru=False): + """Runs a single process to completion. + + Takes a list of arguments to run where the first item is the + executable. Runs the command in the specified directory and + with optional environment variables. + + append_env -- Dict of environment variables to append to the current + set of environment variables. + explicit_env -- Dict of environment variables to set for the new + process. Any existing environment variables will be ignored. + + require_unix_environment if True will ensure the command is executed + within a UNIX environment. Basically, if we are on Windows, it will + execute the command via an appropriate UNIX-like shell. + + ignore_children is proxied to mozprocess's ignore_children. + + ensure_exit_code is used to ensure the exit code of a process matches + what is expected. If it is an integer, we raise an Exception if the + exit code does not match this value. If it is True, we ensure the exit + code is 0. If it is False, we don't perform any exit code validation. + + pass_thru is a special execution mode where the child process inherits + this process's standard file handles (stdin, stdout, stderr) as well as + additional file descriptors. It should be used for interactive processes + where buffering from mozprocess could be an issue. pass_thru does not + use mozprocess. Therefore, arguments like log_name, line_handler, + and ignore_children have no effect. + """ + args = self._normalize_command(args, require_unix_environment) + + self.log(logging.INFO, 'new_process', {'args': ' '.join(args)}, '{args}') + + def handleLine(line): + # Converts str to unicode on Python 2 and bytes to str on Python 3. + if isinstance(line, bytes): + line = line.decode(sys.stdout.encoding or 'utf-8', 'replace') + + if line_handler: + line_handler(line) + + if not log_name: + return + + self.log(log_level, log_name, {'line': line.rstrip()}, '{line}') + + use_env = {} + if explicit_env: + use_env = explicit_env + else: + use_env.update(os.environ) + + if append_env: + use_env.update(append_env) + + self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}') + + # There is a bug in subprocess where it doesn't like unicode types in + # environment variables. Here, ensure all unicode are converted to + # binary. utf-8 is our globally assumed default. If the caller doesn't + # want UTF-8, they shouldn't pass in a unicode instance. + normalized_env = {} + for k, v in use_env.items(): + if isinstance(k, unicode): + k = k.encode('utf-8', 'strict') + + if isinstance(v, unicode): + v = v.encode('utf-8', 'strict') + + normalized_env[k] = v + + use_env = normalized_env + + if pass_thru: + proc = subprocess.Popen(args, cwd=cwd, env=use_env) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as + # a result of Ctrl+C, proc.wait() will return a status code, and, + # we get out of the loop. If it doesn't, like e.g. gdb, we continue + # waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + else: + p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, + processOutputLine=[handleLine], universal_newlines=True, + ignore_children=ignore_children) + p.run() + p.processOutput() + status = p.wait() + + if ensure_exit_code is False: + return status + + if ensure_exit_code is True: + ensure_exit_code = 0 + + if status != ensure_exit_code: + raise Exception('Process executed with non-0 exit code %d: %s' % (status, args)) + + return status + + def _normalize_command(self, args, require_unix_environment): + """Adjust command arguments to run in the necessary environment. + + This exists mainly to facilitate execution of programs requiring a *NIX + shell when running on Windows. The caller specifies whether a shell + environment is required. If it is and we are running on Windows but + aren't running in the UNIX-like msys environment, then we rewrite the + command to execute via a shell. + """ + assert isinstance(args, list) and len(args) + + if not require_unix_environment or not _in_msys: + return args + + # Always munge Windows-style into Unix style for the command. + prog = args[0].replace('\\', '/') + + # PyMake removes the C: prefix. But, things seem to work here + # without it. Not sure what that's about. + + # We run everything through the msys shell. We need to use + # '-c' and pass all the arguments as one argument because that is + # how sh works. + cline = subprocess.list2cmdline([prog] + args[1:]) + return [_current_shell, '-c', cline] diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py new file mode 100644 index 000000000..63c9099c0 --- /dev/null +++ b/python/mach/mach/registrar.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from .base import MachError + +INVALID_COMMAND_CONTEXT = r''' +It looks like you tried to run a mach command from an invalid context. The %s +command failed to meet the following conditions: %s + +Run |mach help| to show a list of all commands available to the current context. +'''.lstrip() + + +class MachRegistrar(object): + """Container for mach command and config providers.""" + + def __init__(self): + self.command_handlers = {} + self.commands_by_category = {} + self.settings_providers = set() + self.categories = {} + self.require_conditions = False + + def register_command_handler(self, handler): + name = handler.name + + if not handler.category: + raise MachError('Cannot register a mach command without a ' + 'category: %s' % name) + + if handler.category not in self.categories: + raise MachError('Cannot register a command to an undefined ' + 'category: %s -> %s' % (name, handler.category)) + + self.command_handlers[name] = handler + self.commands_by_category[handler.category].add(name) + + def register_settings_provider(self, cls): + self.settings_providers.add(cls) + + def register_category(self, name, title, description, priority=50): + self.categories[name] = (title, description, priority) + self.commands_by_category[name] = set() + + @classmethod + def _condition_failed_message(cls, name, conditions): + msg = ['\n'] + for c in conditions: + part = [' %s' % c.__name__] + if c.__doc__ is not None: + part.append(c.__doc__) + msg.append(' - '.join(part)) + return INVALID_COMMAND_CONTEXT % (name, '\n'.join(msg)) + + def _run_command_handler(self, handler, context=None, debug_command=False, **kwargs): + cls = handler.cls + + if handler.pass_context and not context: + raise Exception('mach command class requires context.') + + if context: + prerun = getattr(context, 'pre_dispatch_handler', None) + if prerun: + prerun(context, handler, args=kwargs) + + if handler.pass_context: + instance = cls(context) + else: + instance = cls() + + if handler.conditions: + fail_conditions = [] + for c in handler.conditions: + if not c(instance): + fail_conditions.append(c) + + if fail_conditions: + print(self._condition_failed_message(handler.name, fail_conditions)) + return 1 + + fn = getattr(instance, handler.method) + + if debug_command: + import pdb + result = pdb.runcall(fn, **kwargs) + else: + result = fn(**kwargs) + + result = result or 0 + assert isinstance(result, (int, long)) + + if context: + postrun = getattr(context, 'post_dispatch_handler', None) + if postrun: + postrun(context, handler, args=kwargs) + + return result + + def dispatch(self, name, context=None, argv=None, subcommand=None, **kwargs): + """Dispatch/run a command. + + Commands can use this to call other commands. + """ + handler = self.command_handlers[name] + + if subcommand: + handler = handler.subcommand_handlers[subcommand] + + if handler.parser: + parser = handler.parser + + # save and restore existing defaults so **kwargs don't persist across + # subsequent invocations of Registrar.dispatch() + old_defaults = parser._defaults.copy() + parser.set_defaults(**kwargs) + kwargs, _ = parser.parse_known_args(argv or []) + kwargs = vars(kwargs) + parser._defaults = old_defaults + + return self._run_command_handler(handler, context=context, **kwargs) + + +Registrar = MachRegistrar() diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py new file mode 100644 index 000000000..9115211e0 --- /dev/null +++ b/python/mach/mach/terminal.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This file contains code for interacting with terminals. + +All the terminal interaction code is consolidated so the complexity can be in +one place, away from code that is commonly looked at. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import sys + + +class LoggingHandler(logging.Handler): + """Custom logging handler that works with terminal window dressing. + + This is alternative terminal logging handler which contains smarts for + emitting terminal control characters properly. Currently, it has generic + support for "footer" elements at the bottom of the screen. Functionality + can be added when needed. + """ + def __init__(self): + logging.Handler.__init__(self) + + self.fh = sys.stdout + self.footer = None + + def flush(self): + self.acquire() + + try: + self.fh.flush() + finally: + self.release() + + def emit(self, record): + msg = self.format(record) + + if self.footer: + self.footer.clear() + + self.fh.write(msg) + self.fh.write('\n') + + if self.footer: + self.footer.draw() + + # If we don't flush, the footer may not get drawn. + self.flush() + + +class TerminalFooter(object): + """Represents something drawn on the bottom of a terminal.""" + def __init__(self, terminal): + self.t = terminal + self.fh = sys.stdout + + def _clear_lines(self, n): + for i in xrange(n): + self.fh.write(self.t.move_x(0)) + self.fh.write(self.t.clear_eol()) + self.fh.write(self.t.move_up()) + + self.fh.write(self.t.move_down()) + self.fh.write(self.t.move_x(0)) + + def clear(self): + raise Exception('clear() must be implemented.') + + def draw(self): + raise Exception('draw() must be implemented.') + diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/mach/mach/test/__init__.py diff --git a/python/mach/mach/test/common.py b/python/mach/mach/test/common.py new file mode 100644 index 000000000..f68ff5095 --- /dev/null +++ b/python/mach/mach/test/common.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from StringIO import StringIO +import os +import unittest + +from mach.main import Mach +from mach.base import CommandContext + +here = os.path.abspath(os.path.dirname(__file__)) + +class TestBase(unittest.TestCase): + provider_dir = os.path.join(here, 'providers') + + def get_mach(self, provider_file=None, entry_point=None, context_handler=None): + m = Mach(os.getcwd()) + m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10) + m.populate_context_handler = context_handler + + if provider_file: + m.load_commands_from_file(os.path.join(self.provider_dir, provider_file)) + + if entry_point: + m.load_commands_from_entry_point(entry_point) + + return m + + def _run_mach(self, argv, *args, **kwargs): + m = self.get_mach(*args, **kwargs) + + stdout = StringIO() + stderr = StringIO() + stdout.encoding = 'UTF-8' + stderr.encoding = 'UTF-8' + + try: + result = m.run(argv, stdout=stdout, stderr=stderr) + except SystemExit: + result = None + + return (result, stdout.getvalue(), stderr.getvalue()) diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/mach/mach/test/providers/__init__.py diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py new file mode 100644 index 000000000..f2e64e6d6 --- /dev/null +++ b/python/mach/mach/test/providers/basic.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + + +@CommandProvider +class ConditionsProvider(object): + @Command('cmd_foo', category='testing') + def run_foo(self): + pass + + @Command('cmd_bar', category='testing') + @CommandArgument('--baz', action="store_true", + help='Run with baz') + def run_bar(self, baz=None): + pass diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py new file mode 100644 index 000000000..a95429752 --- /dev/null +++ b/python/mach/mach/test/providers/conditions.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) + +def is_foo(cls): + """Foo must be true""" + return cls.foo + +def is_bar(cls): + """Bar must be true""" + return cls.bar + +@CommandProvider +class ConditionsProvider(object): + foo = True + bar = False + + @Command('cmd_foo', category='testing', conditions=[is_foo]) + def run_foo(self): + pass + + @Command('cmd_bar', category='testing', conditions=[is_bar]) + def run_bar(self): + pass + + @Command('cmd_foobar', category='testing', conditions=[is_foo, is_bar]) + def run_foobar(self): + pass + +@CommandProvider +class ConditionsContextProvider(object): + def __init__(self, context): + self.foo = context.foo + self.bar = context.bar + + @Command('cmd_foo_ctx', category='testing', conditions=[is_foo]) + def run_foo(self): + pass + + @Command('cmd_bar_ctx', category='testing', conditions=[is_bar]) + def run_bar(self): + pass + + @Command('cmd_foobar_ctx', category='testing', conditions=[is_foo, is_bar]) + def run_foobar(self): + pass diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py new file mode 100644 index 000000000..22284d4dc --- /dev/null +++ b/python/mach/mach/test/providers/conditions_invalid.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) + +@CommandProvider +class ConditionsProvider(object): + @Command('cmd_foo', category='testing', conditions=["invalid"]) + def run_foo(self): + pass diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py new file mode 100644 index 000000000..06bee01ee --- /dev/null +++ b/python/mach/mach/test/providers/throw.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import time + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from mach.test.providers import throw2 + + +@CommandProvider +class TestCommandProvider(object): + @Command('throw', category='testing') + @CommandArgument('--message', '-m', default='General Error') + def throw(self, message): + raise Exception(message) + + @Command('throw_deep', category='testing') + @CommandArgument('--message', '-m', default='General Error') + def throw_deep(self, message): + throw2.throw_deep(message) + diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py new file mode 100644 index 000000000..af0a23fcf --- /dev/null +++ b/python/mach/mach/test/providers/throw2.py @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file exists to trigger the differences in mach error reporting between +# exceptions that occur in mach command modules themselves and in the things +# they call. + +def throw_deep(message): + return throw_real(message) + +def throw_real(message): + raise Exception(message) diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py new file mode 100644 index 000000000..20080687e --- /dev/null +++ b/python/mach/mach/test/test_conditions.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os + +from mach.base import MachError +from mach.main import Mach +from mach.registrar import Registrar +from mach.test.common import TestBase + +from mozunit import main + + +def _populate_context(context, key=None): + if key is None: + return + if key == 'foo': + return True + if key == 'bar': + return False + raise AttributeError(key) + +class TestConditions(TestBase): + """Tests for conditionally filtering commands.""" + + def _run_mach(self, args, context_handler=None): + return TestBase._run_mach(self, args, 'conditions.py', + context_handler=context_handler) + + + def test_conditions_pass(self): + """Test that a command which passes its conditions is runnable.""" + + self.assertEquals((0, '', ''), self._run_mach(['cmd_foo'])) + self.assertEquals((0, '', ''), self._run_mach(['cmd_foo_ctx'], _populate_context)) + + def test_invalid_context_message(self): + """Test that commands which do not pass all their conditions + print the proper failure message.""" + + def is_bar(): + """Bar must be true""" + fail_conditions = [is_bar] + + for name in ('cmd_bar', 'cmd_foobar'): + result, stdout, stderr = self._run_mach([name]) + self.assertEquals(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) + + for name in ('cmd_bar_ctx', 'cmd_foobar_ctx'): + result, stdout, stderr = self._run_mach([name], _populate_context) + self.assertEquals(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) + + def test_invalid_type(self): + """Test that a condition which is not callable raises an exception.""" + + m = Mach(os.getcwd()) + m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10) + self.assertRaises(MachError, m.load_commands_from_file, + os.path.join(self.provider_dir, 'conditions_invalid.py')) + + def test_help_message(self): + """Test that commands that are not runnable do not show up in help.""" + + result, stdout, stderr = self._run_mach(['help'], _populate_context) + self.assertIn('cmd_foo', stdout) + self.assertNotIn('cmd_bar', stdout) + self.assertNotIn('cmd_foobar', stdout) + self.assertIn('cmd_foo_ctx', stdout) + self.assertNotIn('cmd_bar_ctx', stdout) + self.assertNotIn('cmd_foobar_ctx', stdout) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py new file mode 100644 index 000000000..d48dff67b --- /dev/null +++ b/python/mach/mach/test/test_config.py @@ -0,0 +1,297 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +import sys +import unittest + +from mozfile.mozfile import NamedTemporaryFile + +from mach.config import ( + BooleanType, + ConfigException, + ConfigSettings, + IntegerType, + PathType, + PositiveIntegerType, + StringType, +) +from mach.decorators import SettingsProvider + +from mozunit import main + + +if sys.version_info[0] == 3: + str_type = str +else: + str_type = basestring + +CONFIG1 = r""" +[foo] + +bar = bar_value +baz = /baz/foo.c +""" + +CONFIG2 = r""" +[foo] + +bar = value2 +""" + +@SettingsProvider +class Provider1(object): + config_settings = [ + ('foo.bar', StringType), + ('foo.baz', PathType), + ] + + +@SettingsProvider +class ProviderDuplicate(object): + config_settings = [ + ('dupesect.foo', StringType), + ('dupesect.foo', StringType), + ] + + +@SettingsProvider +class Provider2(object): + config_settings = [ + ('a.string', StringType), + ('a.boolean', BooleanType), + ('a.pos_int', PositiveIntegerType), + ('a.int', IntegerType), + ('a.path', PathType), + ] + + +@SettingsProvider +class Provider3(object): + @classmethod + def config_settings(cls): + return [ + ('a.string', 'string'), + ('a.boolean', 'boolean'), + ('a.pos_int', 'pos_int'), + ('a.int', 'int'), + ('a.path', 'path'), + ] + + +@SettingsProvider +class Provider4(object): + config_settings = [ + ('foo.abc', StringType, 'a', {'choices': set('abc')}), + ('foo.xyz', StringType, 'w', {'choices': set('xyz')}), + ] + + +@SettingsProvider +class Provider5(object): + config_settings = [ + ('foo.*', 'string'), + ('foo.bar', 'string'), + ] + + +class TestConfigSettings(unittest.TestCase): + def test_empty(self): + s = ConfigSettings() + + self.assertEqual(len(s), 0) + self.assertNotIn('foo', s) + + def test_duplicate_option(self): + s = ConfigSettings() + + with self.assertRaises(ConfigException): + s.register_provider(ProviderDuplicate) + + def test_simple(self): + s = ConfigSettings() + s.register_provider(Provider1) + + self.assertEqual(len(s), 1) + self.assertIn('foo', s) + + foo = s['foo'] + foo = s.foo + + self.assertEqual(len(foo), 0) + self.assertEqual(len(foo._settings), 2) + + self.assertIn('bar', foo._settings) + self.assertIn('baz', foo._settings) + + self.assertNotIn('bar', foo) + foo['bar'] = 'value1' + self.assertIn('bar', foo) + + self.assertEqual(foo['bar'], 'value1') + self.assertEqual(foo.bar, 'value1') + + def test_assignment_validation(self): + s = ConfigSettings() + s.register_provider(Provider2) + + a = s.a + + # Assigning an undeclared setting raises. + with self.assertRaises(AttributeError): + a.undefined = True + + with self.assertRaises(KeyError): + a['undefined'] = True + + # Basic type validation. + a.string = 'foo' + a.string = 'foo' + + with self.assertRaises(TypeError): + a.string = False + + a.boolean = True + a.boolean = False + + with self.assertRaises(TypeError): + a.boolean = 'foo' + + a.pos_int = 5 + a.pos_int = 0 + + with self.assertRaises(ValueError): + a.pos_int = -1 + + with self.assertRaises(TypeError): + a.pos_int = 'foo' + + a.int = 5 + a.int = 0 + a.int = -5 + + with self.assertRaises(TypeError): + a.int = 1.24 + + with self.assertRaises(TypeError): + a.int = 'foo' + + a.path = '/home/gps' + a.path = 'foo.c' + a.path = 'foo/bar' + a.path = './foo' + + def retrieval_type_helper(self, provider): + s = ConfigSettings() + s.register_provider(provider) + + a = s.a + + a.string = 'foo' + a.boolean = True + a.pos_int = 12 + a.int = -4 + a.path = './foo/bar' + + self.assertIsInstance(a.string, str_type) + self.assertIsInstance(a.boolean, bool) + self.assertIsInstance(a.pos_int, int) + self.assertIsInstance(a.int, int) + self.assertIsInstance(a.path, str_type) + + def test_retrieval_type(self): + self.retrieval_type_helper(Provider2) + self.retrieval_type_helper(Provider3) + + def test_choices_validation(self): + s = ConfigSettings() + s.register_provider(Provider4) + + foo = s.foo + foo.abc + with self.assertRaises(ValueError): + foo.xyz + + with self.assertRaises(ValueError): + foo.abc = 'e' + + foo.abc = 'b' + foo.xyz = 'y' + + def test_wildcard_options(self): + s = ConfigSettings() + s.register_provider(Provider5) + + foo = s.foo + + self.assertIn('*', foo._settings) + self.assertNotIn('*', foo) + + foo.baz = 'value1' + foo.bar = 'value2' + + self.assertIn('baz', foo) + self.assertEqual(foo.baz, 'value1') + + self.assertIn('bar', foo) + self.assertEqual(foo.bar, 'value2') + + def test_file_reading_single(self): + temp = NamedTemporaryFile(mode='wt') + temp.write(CONFIG1) + temp.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_file(temp.name) + + self.assertEqual(s.foo.bar, 'bar_value') + + def test_file_reading_multiple(self): + """Loading multiple files has proper overwrite behavior.""" + temp1 = NamedTemporaryFile(mode='wt') + temp1.write(CONFIG1) + temp1.flush() + + temp2 = NamedTemporaryFile(mode='wt') + temp2.write(CONFIG2) + temp2.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_files([temp1.name, temp2.name]) + + self.assertEqual(s.foo.bar, 'value2') + + def test_file_reading_missing(self): + """Missing files should silently be ignored.""" + + s = ConfigSettings() + + s.load_file('/tmp/foo.ini') + + def test_file_writing(self): + s = ConfigSettings() + s.register_provider(Provider2) + + s.a.string = 'foo' + s.a.boolean = False + + temp = NamedTemporaryFile('wt') + s.write(temp) + temp.flush() + + s2 = ConfigSettings() + s2.register_provider(Provider2) + + s2.load_file(temp.name) + + self.assertEqual(s.a.string, s2.a.string) + self.assertEqual(s.a.boolean, s2.a.boolean) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_dispatcher.py b/python/mach/mach/test/test_dispatcher.py new file mode 100644 index 000000000..3d689a4a2 --- /dev/null +++ b/python/mach/mach/test/test_dispatcher.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +from cStringIO import StringIO + +from mach.base import CommandContext +from mach.registrar import Registrar +from mach.test.common import TestBase + +from mozunit import main + +here = os.path.abspath(os.path.dirname(__file__)) + + +class TestDispatcher(TestBase): + """Tests dispatch related code""" + + def get_parser(self, config=None): + mach = self.get_mach('basic.py') + + for provider in Registrar.settings_providers: + mach.settings.register_provider(provider) + + if config: + if isinstance(config, basestring): + config = StringIO(config) + mach.settings.load_fps([config]) + + context = CommandContext(settings=mach.settings) + return mach.get_argument_parser(context) + + def test_command_aliases(self): + config = """ +[alias] +foo = cmd_foo +bar = cmd_bar +baz = cmd_bar --baz +cmd_bar = cmd_bar --baz +""" + parser = self.get_parser(config=config) + + args = parser.parse_args(['foo']) + self.assertEquals(args.command, 'cmd_foo') + + def assert_bar_baz(argv): + args = parser.parse_args(argv) + self.assertEquals(args.command, 'cmd_bar') + self.assertTrue(args.command_args.baz) + + # The following should all result in |cmd_bar --baz| + assert_bar_baz(['bar', '--baz']) + assert_bar_baz(['baz']) + assert_bar_baz(['cmd_bar']) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py new file mode 100644 index 000000000..7aea91e5e --- /dev/null +++ b/python/mach/mach/test/test_entry_point.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +import imp +import os +import sys + +from mach.base import MachError +from mach.test.common import TestBase +from mock import patch + +from mozunit import main + + +here = os.path.abspath(os.path.dirname(__file__)) + + +class Entry(): + """Stub replacement for pkg_resources.EntryPoint""" + def __init__(self, providers): + self.providers = providers + + def load(self): + def _providers(): + return self.providers + return _providers + + +class TestEntryPoints(TestBase): + """Test integrating with setuptools entry points""" + provider_dir = os.path.join(here, 'providers') + + def _run_mach(self): + return TestBase._run_mach(self, ['help'], entry_point='mach.providers') + + @patch('pkg_resources.iter_entry_points') + def test_load_entry_point_from_directory(self, mock): + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if b'mach.commands' not in sys.modules: + mod = imp.new_module(b'mach.commands') + sys.modules[b'mach.commands'] = mod + + mock.return_value = [Entry([self.provider_dir])] + # Mach error raised due to conditions_invalid.py + with self.assertRaises(MachError): + self._run_mach() + + @patch('pkg_resources.iter_entry_points') + def test_load_entry_point_from_file(self, mock): + mock.return_value = [Entry([os.path.join(self.provider_dir, 'basic.py')])] + + result, stdout, stderr = self._run_mach() + self.assertIsNone(result) + self.assertIn('cmd_foo', stdout) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py new file mode 100644 index 000000000..25553f96b --- /dev/null +++ b/python/mach/mach/test/test_error_output.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.main import ( + COMMAND_ERROR, + MODULE_ERROR +) +from mach.test.common import TestBase + +from mozunit import main + + +class TestErrorOutput(TestBase): + + def _run_mach(self, args): + return TestBase._run_mach(self, args, 'throw.py') + + def test_command_error(self): + result, stdout, stderr = self._run_mach(['throw', '--message', + 'Command Error']) + + self.assertEqual(result, 1) + + self.assertIn(COMMAND_ERROR, stdout) + + def test_invoked_error(self): + result, stdout, stderr = self._run_mach(['throw_deep', '--message', + 'Deep stack']) + + self.assertEqual(result, 1) + + self.assertIn(MODULE_ERROR, stdout) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py new file mode 100644 index 000000000..05592845e --- /dev/null +++ b/python/mach/mach/test/test_logger.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging +import time +import unittest + +from mach.logging import StructuredHumanFormatter + +from mozunit import main + + +class DummyLogger(logging.Logger): + def __init__(self, cb): + logging.Logger.__init__(self, 'test') + + self._cb = cb + + def handle(self, record): + self._cb(record) + + +class TestStructuredHumanFormatter(unittest.TestCase): + def test_non_ascii_logging(self): + # Ensures the formatter doesn't choke when non-ASCII characters are + # present in printed parameters. + formatter = StructuredHumanFormatter(time.time()) + + def on_record(record): + result = formatter.format(record) + relevant = result[9:] + + self.assertEqual(relevant, 'Test: s\xe9curit\xe9') + + logger = DummyLogger(on_record) + + value = 's\xe9curit\xe9' + + logger.log(logging.INFO, 'Test: {utf}', + extra={'action': 'action', 'params': {'utf': value}}) + + +if __name__ == '__main__': + main() diff --git a/python/mach/setup.py b/python/mach/setup.py new file mode 100644 index 000000000..50065546e --- /dev/null +++ b/python/mach/setup.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +try: + from setuptools import setup +except: + from distutils.core import setup + + +VERSION = '0.6' + +README = open('README.rst').read() + +setup( + name='mach', + description='Generic command line command dispatching framework.', + long_description=README, + license='MPL 2.0', + author='Gregory Szorc', + author_email='gregory.szorc@gmail.com', + url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach', + packages=['mach'], + version=VERSION, + classifiers=[ + 'Environment :: Console', + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', + 'Natural Language :: English', + ], + install_requires=[ + 'blessings', + 'mozfile', + 'mozprocess', + ], + tests_require=['mock'], +) + |