summaryrefslogtreecommitdiffstats
path: root/python/mozlint
diff options
context:
space:
mode:
Diffstat (limited to 'python/mozlint')
-rw-r--r--python/mozlint/mozlint/__init__.py7
-rw-r--r--python/mozlint/mozlint/cli.py115
-rw-r--r--python/mozlint/mozlint/errors.py25
-rw-r--r--python/mozlint/mozlint/formatters/__init__.py25
-rw-r--r--python/mozlint/mozlint/formatters/stylish.py122
-rw-r--r--python/mozlint/mozlint/formatters/treeherder.py31
-rw-r--r--python/mozlint/mozlint/parser.py85
-rw-r--r--python/mozlint/mozlint/pathutils.py156
-rw-r--r--python/mozlint/mozlint/result.py88
-rw-r--r--python/mozlint/mozlint/roller.py154
-rw-r--r--python/mozlint/mozlint/types.py142
-rw-r--r--python/mozlint/mozlint/vcs.py62
-rw-r--r--python/mozlint/setup.py26
-rw-r--r--python/mozlint/test/__init__.py0
-rw-r--r--python/mozlint/test/conftest.py42
-rw-r--r--python/mozlint/test/files/foobar.js2
-rw-r--r--python/mozlint/test/files/foobar.py2
-rw-r--r--python/mozlint/test/files/no_foobar.js2
-rw-r--r--python/mozlint/test/linters/badreturncode.lint21
-rw-r--r--python/mozlint/test/linters/explicit_path.lint13
-rw-r--r--python/mozlint/test/linters/external.lint30
-rw-r--r--python/mozlint/test/linters/invalid_exclude.lint10
-rw-r--r--python/mozlint/test/linters/invalid_extension.lnt9
-rw-r--r--python/mozlint/test/linters/invalid_include.lint10
-rw-r--r--python/mozlint/test/linters/invalid_type.lint9
-rw-r--r--python/mozlint/test/linters/missing_attrs.lint7
-rw-r--r--python/mozlint/test/linters/missing_definition.lint4
-rw-r--r--python/mozlint/test/linters/raises.lint19
-rw-r--r--python/mozlint/test/linters/regex.lint15
-rw-r--r--python/mozlint/test/linters/string.lint15
-rw-r--r--python/mozlint/test/linters/structured.lint28
-rw-r--r--python/mozlint/test/test_formatters.py90
-rw-r--r--python/mozlint/test/test_parser.py55
-rw-r--r--python/mozlint/test/test_roller.py82
-rw-r--r--python/mozlint/test/test_types.py50
35 files changed, 1553 insertions, 0 deletions
diff --git a/python/mozlint/mozlint/__init__.py b/python/mozlint/mozlint/__init__.py
new file mode 100644
index 000000000..18eaf5112
--- /dev/null
+++ b/python/mozlint/mozlint/__init__.py
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# flake8: noqa
+
+from .roller import LintRoller
+from .result import ResultContainer
diff --git a/python/mozlint/mozlint/cli.py b/python/mozlint/mozlint/cli.py
new file mode 100644
index 000000000..84c1b6aa4
--- /dev/null
+++ b/python/mozlint/mozlint/cli.py
@@ -0,0 +1,115 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import sys
+from argparse import ArgumentParser, REMAINDER
+
+
+SEARCH_PATHS = []
+
+
+class MozlintParser(ArgumentParser):
+ arguments = [
+ [['paths'],
+ {'nargs': '*',
+ 'default': None,
+ 'help': "Paths to file or directories to lint, like "
+ "'browser/components/loop' or 'mobile/android'. "
+ "Defaults to the current directory if not given.",
+ }],
+ [['-l', '--linter'],
+ {'dest': 'linters',
+ 'default': [],
+ 'action': 'append',
+ 'help': "Linters to run, e.g 'eslint'. By default all linters "
+ "are run for all the appropriate files.",
+ }],
+ [['-f', '--format'],
+ {'dest': 'fmt',
+ 'default': 'stylish',
+ 'help': "Formatter to use. Defaults to 'stylish'.",
+ }],
+ [['-n', '--no-filter'],
+ {'dest': 'use_filters',
+ 'default': True,
+ 'action': 'store_false',
+ 'help': "Ignore all filtering. This is useful for quickly "
+ "testing a directory that otherwise wouldn't be run, "
+ "without needing to modify the config file.",
+ }],
+ [['-r', '--rev'],
+ {'default': None,
+ 'help': "Lint files touched by the given revision(s). Works with "
+ "mercurial or git."
+ }],
+ [['-w', '--workdir'],
+ {'default': False,
+ 'action': 'store_true',
+ 'help': "Lint files touched by changes in the working directory "
+ "(i.e haven't been committed yet). Works with mercurial or git.",
+ }],
+ [['extra_args'],
+ {'nargs': REMAINDER,
+ 'help': "Extra arguments that will be forwarded to the underlying linter.",
+ }],
+ ]
+
+ def __init__(self, **kwargs):
+ ArgumentParser.__init__(self, usage=self.__doc__, **kwargs)
+
+ for cli, args in self.arguments:
+ self.add_argument(*cli, **args)
+
+ def parse_known_args(self, *args, **kwargs):
+ # This is here so the eslint mach command doesn't lose 'extra_args'
+ # when using mach's dispatch functionality.
+ args, extra = ArgumentParser.parse_known_args(self, *args, **kwargs)
+ args.extra_args = extra
+ return args, extra
+
+
+def find_linters(linters=None):
+ lints = []
+ for search_path in SEARCH_PATHS:
+ if not os.path.isdir(search_path):
+ continue
+
+ files = os.listdir(search_path)
+ for f in files:
+ name, ext = os.path.splitext(f)
+ if ext != '.lint':
+ continue
+
+ if linters and name not in linters:
+ continue
+
+ lints.append(os.path.join(search_path, f))
+ return lints
+
+
+def run(paths, linters, fmt, rev, workdir, **lintargs):
+ from mozlint import LintRoller, formatters
+
+ lint = LintRoller(**lintargs)
+ lint.read(find_linters(linters))
+
+ # run all linters
+ results = lint.roll(paths, rev=rev, workdir=workdir)
+
+ formatter = formatters.get(fmt)
+
+ # Explicitly utf-8 encode the output as some of the formatters make
+ # use of unicode characters. This will prevent a UnicodeEncodeError
+ # on environments where utf-8 isn't the default
+ print(formatter(results).encode('utf-8', 'replace'))
+ return lint.return_code
+
+
+if __name__ == '__main__':
+ parser = MozlintParser()
+ args = vars(parser.parse_args())
+ sys.exit(run(**args))
diff --git a/python/mozlint/mozlint/errors.py b/python/mozlint/mozlint/errors.py
new file mode 100644
index 000000000..a899a1974
--- /dev/null
+++ b/python/mozlint/mozlint/errors.py
@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+
+class LintException(Exception):
+ pass
+
+
+class LinterNotFound(LintException):
+ def __init__(self, path):
+ LintException.__init__(self, "Could not find lint file '{}'".format(path))
+
+
+class LinterParseError(LintException):
+ def __init__(self, path, message):
+ LintException.__init__(self, "{}: {}".format(os.path.basename(path), message))
+
+
+class LintersNotConfigured(LintException):
+ def __init__(self):
+ LintException.__init__(self, "No linters registered! Use `LintRoller.read` "
+ "to register a linter.")
diff --git a/python/mozlint/mozlint/formatters/__init__.py b/python/mozlint/mozlint/formatters/__init__.py
new file mode 100644
index 000000000..33aca0446
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/__init__.py
@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from ..result import ResultEncoder
+from .stylish import StylishFormatter
+from .treeherder import TreeherderFormatter
+
+
+class JSONFormatter(object):
+ def __call__(self, results):
+ return json.dumps(results, cls=ResultEncoder)
+
+
+all_formatters = {
+ 'json': JSONFormatter,
+ 'stylish': StylishFormatter,
+ 'treeherder': TreeherderFormatter,
+}
+
+
+def get(name, **fmtargs):
+ return all_formatters[name](**fmtargs)
diff --git a/python/mozlint/mozlint/formatters/stylish.py b/python/mozlint/mozlint/formatters/stylish.py
new file mode 100644
index 000000000..62ddfbeb6
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/stylish.py
@@ -0,0 +1,122 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from ..result import ResultContainer
+
+try:
+ import blessings
+except ImportError:
+ blessings = None
+
+
+class NullTerminal(object):
+ """Replacement for `blessings.Terminal()` that does no formatting."""
+ class NullCallableString(unicode):
+ """A dummy callable Unicode stolen from blessings"""
+ def __new__(cls):
+ new = unicode.__new__(cls, u'')
+ return new
+
+ def __call__(self, *args):
+ if len(args) != 1 or isinstance(args[0], int):
+ return u''
+ return args[0]
+
+ def __getattr__(self, attr):
+ return self.NullCallableString()
+
+
+class StylishFormatter(object):
+ """Formatter based on the eslint default."""
+
+ # Colors later on in the list are fallbacks in case the terminal
+ # doesn't support colors earlier in the list.
+ # See http://www.calmar.ws/vim/256-xterm-24bit-rgb-color-chart.html
+ _colors = {
+ 'grey': [247, 8, 7],
+ 'red': [1],
+ 'yellow': [3],
+ 'brightred': [9, 1],
+ 'brightyellow': [11, 3],
+ }
+ fmt = " {c1}{lineno}{column} {c2}{level}{normal} {message} {c1}{rule}({linter}){normal}"
+ fmt_summary = "{t.bold}{c}\u2716 {problem} ({error}, {warning}){t.normal}"
+
+ def __init__(self, disable_colors=None):
+ if disable_colors or not blessings:
+ self.term = NullTerminal()
+ else:
+ self.term = blessings.Terminal()
+ self.num_colors = self.term.number_of_colors
+
+ def color(self, color):
+ for num in self._colors[color]:
+ if num < self.num_colors:
+ return self.term.color(num)
+ return ''
+
+ def _reset_max(self):
+ self.max_lineno = 0
+ self.max_column = 0
+ self.max_level = 0
+ self.max_message = 0
+
+ def _update_max(self, err):
+ """Calculates the longest length of each token for spacing."""
+ self.max_lineno = max(self.max_lineno, len(str(err.lineno)))
+ if err.column:
+ self.max_column = max(self.max_column, len(str(err.column)))
+ self.max_level = max(self.max_level, len(str(err.level)))
+ self.max_message = max(self.max_message, len(err.message))
+
+ def _pluralize(self, s, num):
+ if num != 1:
+ s += 's'
+ return str(num) + ' ' + s
+
+ def __call__(self, result):
+ message = []
+
+ num_errors = 0
+ num_warnings = 0
+ for path, errors in sorted(result.iteritems()):
+ self._reset_max()
+
+ message.append(self.term.underline(path))
+ # Do a first pass to calculate required padding
+ for err in errors:
+ assert isinstance(err, ResultContainer)
+ self._update_max(err)
+ if err.level == 'error':
+ num_errors += 1
+ else:
+ num_warnings += 1
+
+ for err in errors:
+ message.append(self.fmt.format(
+ normal=self.term.normal,
+ c1=self.color('grey'),
+ c2=self.color('red') if err.level == 'error' else self.color('yellow'),
+ lineno=str(err.lineno).rjust(self.max_lineno),
+ column=(":" + str(err.column).ljust(self.max_column)) if err.column else "",
+ level=err.level.ljust(self.max_level),
+ message=err.message.ljust(self.max_message),
+ rule='{} '.format(err.rule) if err.rule else '',
+ linter=err.linter.lower(),
+ ))
+
+ message.append('') # newline
+
+ # Print a summary
+ message.append(self.fmt_summary.format(
+ t=self.term,
+ c=self.color('brightred') if num_errors else self.color('brightyellow'),
+ problem=self._pluralize('problem', num_errors + num_warnings),
+ error=self._pluralize('error', num_errors),
+ warning=self._pluralize('warning', num_warnings),
+ ))
+
+ return '\n'.join(message)
diff --git a/python/mozlint/mozlint/formatters/treeherder.py b/python/mozlint/mozlint/formatters/treeherder.py
new file mode 100644
index 000000000..7c27011cf
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/treeherder.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from ..result import ResultContainer
+
+
+class TreeherderFormatter(object):
+ """Formatter for treeherder friendly output.
+
+ This formatter looks ugly, but prints output such that
+ treeherder is able to highlight the errors and warnings.
+ This is a stop-gap until bug 1276486 is fixed.
+ """
+ fmt = "TEST-UNEXPECTED-{level} | {path}:{lineno}{column} | {message} ({rule})"
+
+ def __call__(self, result):
+ message = []
+ for path, errors in sorted(result.iteritems()):
+ for err in errors:
+ assert isinstance(err, ResultContainer)
+
+ d = {s: getattr(err, s) for s in err.__slots__}
+ d["column"] = ":%s" % d["column"] if d["column"] else ""
+ d['level'] = d['level'].upper()
+ d['rule'] = d['rule'] or d['linter']
+ message.append(self.fmt.format(**d))
+
+ return "\n".join(message)
diff --git a/python/mozlint/mozlint/parser.py b/python/mozlint/mozlint/parser.py
new file mode 100644
index 000000000..f350d0de7
--- /dev/null
+++ b/python/mozlint/mozlint/parser.py
@@ -0,0 +1,85 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import imp
+import os
+import sys
+import uuid
+
+from .types import supported_types
+from .errors import LinterNotFound, LinterParseError
+
+
+class Parser(object):
+ """Reads and validates `.lint` files."""
+ required_attributes = (
+ 'name',
+ 'description',
+ 'type',
+ 'payload',
+ )
+
+ def __call__(self, path):
+ return self.parse(path)
+
+ def _load_linter(self, path):
+ # Ensure parent module is present otherwise we'll (likely) get
+ # an error due to unknown parent.
+ parent_module = 'mozlint.linters'
+ if parent_module not in sys.modules:
+ mod = imp.new_module(parent_module)
+ sys.modules[parent_module] = mod
+
+ write_bytecode = sys.dont_write_bytecode
+ sys.dont_write_bytecode = True
+
+ module_name = '{}.{}'.format(parent_module, uuid.uuid1().get_hex())
+ imp.load_source(module_name, path)
+
+ sys.dont_write_bytecode = write_bytecode
+
+ mod = sys.modules[module_name]
+
+ if not hasattr(mod, 'LINTER'):
+ raise LinterParseError(path, "No LINTER definition found!")
+
+ definition = mod.LINTER
+ definition['path'] = path
+ return definition
+
+ def _validate(self, linter):
+ missing_attrs = []
+ for attr in self.required_attributes:
+ if attr not in linter:
+ missing_attrs.append(attr)
+
+ if missing_attrs:
+ raise LinterParseError(linter['path'], "Missing required attribute(s): "
+ "{}".format(','.join(missing_attrs)))
+
+ if linter['type'] not in supported_types:
+ raise LinterParseError(linter['path'], "Invalid type '{}'".format(linter['type']))
+
+ for attr in ('include', 'exclude'):
+ if attr in linter and (not isinstance(linter[attr], list) or
+ not all(isinstance(a, basestring) for a in linter[attr])):
+ raise LinterParseError(linter['path'], "The {} directive must be a "
+ "list of strings!".format(attr))
+
+ def parse(self, path):
+ """Read a linter and return its LINTER definition.
+
+ :param path: Path to the linter.
+ :returns: Linter definition (dict)
+ :raises: LinterNotFound, LinterParseError
+ """
+ if not os.path.isfile(path):
+ raise LinterNotFound(path)
+
+ if not path.endswith('.lint'):
+ raise LinterParseError(path, "Invalid filename, linters must end with '.lint'!")
+
+ linter = self._load_linter(path)
+ self._validate(linter)
+ return linter
diff --git a/python/mozlint/mozlint/pathutils.py b/python/mozlint/mozlint/pathutils.py
new file mode 100644
index 000000000..532904dca
--- /dev/null
+++ b/python/mozlint/mozlint/pathutils.py
@@ -0,0 +1,156 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+from mozpack import path as mozpath
+from mozpack.files import FileFinder
+
+
+class FilterPath(object):
+ """Helper class to make comparing and matching file paths easier."""
+ def __init__(self, path, exclude=None):
+ self.path = os.path.normpath(path)
+ self._finder = None
+ self.exclude = exclude
+
+ @property
+ def finder(self):
+ if self._finder:
+ return self._finder
+ self._finder = FileFinder(
+ self.path, find_executables=False, ignore=self.exclude)
+ return self._finder
+
+ @property
+ def ext(self):
+ return os.path.splitext(self.path)[1]
+
+ @property
+ def exists(self):
+ return os.path.exists(self.path)
+
+ @property
+ def isfile(self):
+ return os.path.isfile(self.path)
+
+ @property
+ def isdir(self):
+ return os.path.isdir(self.path)
+
+ def join(self, *args):
+ return FilterPath(os.path.join(self, *args))
+
+ def match(self, patterns):
+ return any(mozpath.match(self.path, pattern.path) for pattern in patterns)
+
+ def contains(self, other):
+ """Return True if other is a subdirectory of self or equals self."""
+ if isinstance(other, FilterPath):
+ other = other.path
+ a = os.path.abspath(self.path)
+ b = os.path.normpath(os.path.abspath(other))
+
+ if b.startswith(a):
+ return True
+ return False
+
+ def __repr__(self):
+ return repr(self.path)
+
+
+def filterpaths(paths, linter, **lintargs):
+ """Filters a list of paths.
+
+ Given a list of paths, and a linter definition plus extra
+ arguments, return the set of paths that should be linted.
+
+ :param paths: A starting list of paths to possibly lint.
+ :param linter: A linter definition.
+ :param lintargs: Extra arguments passed to the linter.
+ :returns: A list of file paths to lint.
+ """
+ include = linter.get('include', [])
+ exclude = lintargs.get('exclude', [])
+ exclude.extend(linter.get('exclude', []))
+ root = lintargs['root']
+
+ if not lintargs.get('use_filters', True) or (not include and not exclude):
+ return paths
+
+ def normalize(path):
+ if not os.path.isabs(path):
+ path = os.path.join(root, path)
+ return FilterPath(path)
+
+ include = map(normalize, include)
+ exclude = map(normalize, exclude)
+
+ # Paths with and without globs will be handled separately,
+ # pull them apart now.
+ includepaths = [p for p in include if p.exists]
+ excludepaths = [p for p in exclude if p.exists]
+
+ includeglobs = [p for p in include if not p.exists]
+ excludeglobs = [p for p in exclude if not p.exists]
+
+ extensions = linter.get('extensions')
+ keep = set()
+ discard = set()
+ for path in map(FilterPath, paths):
+ # Exclude bad file extensions
+ if extensions and path.isfile and path.ext not in extensions:
+ continue
+
+ if path.match(excludeglobs):
+ continue
+
+ # First handle include/exclude directives
+ # that exist (i.e don't have globs)
+ for inc in includepaths:
+ # Only excludes that are subdirectories of the include
+ # path matter.
+ excs = [e for e in excludepaths if inc.contains(e)]
+
+ if path.contains(inc):
+ # If specified path is an ancestor of include path,
+ # then lint the include path.
+ keep.add(inc)
+
+ # We can't apply these exclude paths without explicitly
+ # including every sibling file. Rather than do that,
+ # just return them and hope the underlying linter will
+ # deal with them.
+ discard.update(excs)
+
+ elif inc.contains(path):
+ # If the include path is an ancestor of the specified
+ # path, then add the specified path only if there are
+ # no exclude paths in-between them.
+ if not any(e.contains(path) for e in excs):
+ keep.add(path)
+
+ # Next handle include/exclude directives that
+ # contain globs.
+ if path.isfile:
+ # If the specified path is a file it must be both
+ # matched by an include directive and not matched
+ # by an exclude directive.
+ if not path.match(includeglobs):
+ continue
+
+ keep.add(path)
+ elif path.isdir:
+ # If the specified path is a directory, use a
+ # FileFinder to resolve all relevant globs.
+ path.exclude = [e.path for e in excludeglobs]
+ for pattern in includeglobs:
+ for p, f in path.finder.find(pattern.path):
+ keep.add(path.join(p))
+
+ # Only pass paths we couldn't exclude here to the underlying linter
+ lintargs['exclude'] = [f.path for f in discard]
+ return [f.path for f in keep]
diff --git a/python/mozlint/mozlint/result.py b/python/mozlint/mozlint/result.py
new file mode 100644
index 000000000..0c56f1d76
--- /dev/null
+++ b/python/mozlint/mozlint/result.py
@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from json import dumps, JSONEncoder
+
+
+class ResultContainer(object):
+ """Represents a single lint error and its related metadata.
+
+ :param linter: name of the linter that flagged this error
+ :param path: path to the file containing the error
+ :param message: text describing the error
+ :param lineno: line number that contains the error
+ :param column: column containing the error
+ :param level: severity of the error, either 'warning' or 'error' (default 'error')
+ :param hint: suggestion for fixing the error (optional)
+ :param source: source code context of the error (optional)
+ :param rule: name of the rule that was violated (optional)
+ :param lineoffset: denotes an error spans multiple lines, of the form
+ (<lineno offset>, <num lines>) (optional)
+ """
+
+ __slots__ = (
+ 'linter',
+ 'path',
+ 'message',
+ 'lineno',
+ 'column',
+ 'hint',
+ 'source',
+ 'level',
+ 'rule',
+ 'lineoffset',
+ )
+
+ def __init__(self, linter, path, message, lineno, column=None, hint=None,
+ source=None, level=None, rule=None, lineoffset=None):
+ self.path = path
+ self.message = message
+ self.lineno = lineno
+ self.column = column
+ self.hint = hint
+ self.source = source
+ self.level = level or 'error'
+ self.linter = linter
+ self.rule = rule
+ self.lineoffset = lineoffset
+
+ def __repr__(self):
+ s = dumps(self, cls=ResultEncoder, indent=2)
+ return "ResultContainer({})".format(s)
+
+
+class ResultEncoder(JSONEncoder):
+ """Class for encoding :class:`~result.ResultContainer`s to json.
+
+ Usage:
+
+ json.dumps(results, cls=ResultEncoder)
+ """
+ def default(self, o):
+ if isinstance(o, ResultContainer):
+ return {a: getattr(o, a) for a in o.__slots__}
+ return JSONEncoder.default(self, o)
+
+
+def from_linter(lintobj, **kwargs):
+ """Create a :class:`~result.ResultContainer` from a LINTER definition.
+
+ Convenience method that pulls defaults from a LINTER
+ definition and forwards them.
+
+ :param lintobj: LINTER obj as defined in a .lint file
+ :param kwargs: same as :class:`~result.ResultContainer`
+ :returns: :class:`~result.ResultContainer` object
+ """
+ attrs = {}
+ for attr in ResultContainer.__slots__:
+ attrs[attr] = kwargs.get(attr, lintobj.get(attr))
+
+ if not attrs['linter']:
+ attrs['linter'] = lintobj.get('name')
+
+ if not attrs['message']:
+ attrs['message'] = lintobj.get('description')
+
+ return ResultContainer(**attrs)
diff --git a/python/mozlint/mozlint/roller.py b/python/mozlint/mozlint/roller.py
new file mode 100644
index 000000000..2d1608dd8
--- /dev/null
+++ b/python/mozlint/mozlint/roller.py
@@ -0,0 +1,154 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import signal
+import sys
+import traceback
+from collections import defaultdict
+from Queue import Empty
+from multiprocessing import (
+ Manager,
+ Pool,
+ cpu_count,
+)
+
+from .errors import LintersNotConfigured
+from .types import supported_types
+from .parser import Parser
+from .vcs import VCSFiles
+
+
+def _run_linters(queue, paths, **lintargs):
+ parse = Parser()
+ results = defaultdict(list)
+ return_code = 0
+
+ while True:
+ try:
+ # The astute reader may wonder what is preventing the worker from
+ # grabbing the next linter from the queue after a SIGINT. Because
+ # this is a Manager.Queue(), it is itself in a child process which
+ # also received SIGINT. By the time the worker gets back here, the
+ # Queue is dead and IOError is raised.
+ linter_path = queue.get(False)
+ except (Empty, IOError):
+ return results, return_code
+
+ # Ideally we would pass the entire LINTER definition as an argument
+ # to the worker instead of re-parsing it. But passing a function from
+ # a dynamically created module (with imp) does not seem to be possible
+ # with multiprocessing on Windows.
+ linter = parse(linter_path)
+ func = supported_types[linter['type']]
+ res = func(paths, linter, **lintargs) or []
+
+ if not isinstance(res, (list, tuple)):
+ if res:
+ return_code = 1
+ continue
+
+ for r in res:
+ results[r.path].append(r)
+
+
+def _run_worker(*args, **lintargs):
+ try:
+ return _run_linters(*args, **lintargs)
+ except Exception:
+ # multiprocessing seems to munge worker exceptions, print
+ # it here so it isn't lost.
+ traceback.print_exc()
+ raise
+ finally:
+ sys.stdout.flush()
+
+
+class LintRoller(object):
+ """Registers and runs linters.
+
+ :param root: Path to which relative paths will be joined. If
+ unspecified, root will either be determined from
+ version control or cwd.
+ :param lintargs: Arguments to pass to the underlying linter(s).
+ """
+
+ def __init__(self, root=None, **lintargs):
+ self.parse = Parser()
+ self.vcs = VCSFiles()
+
+ self.linters = []
+ self.lintargs = lintargs
+ self.lintargs['root'] = root or self.vcs.root or os.getcwd()
+
+ self.return_code = None
+
+ def read(self, paths):
+ """Parse one or more linters and add them to the registry.
+
+ :param paths: A path or iterable of paths to linter definitions.
+ """
+ if isinstance(paths, basestring):
+ paths = (paths,)
+
+ for path in paths:
+ self.linters.append(self.parse(path))
+
+ def roll(self, paths=None, rev=None, workdir=None, num_procs=None):
+ """Run all of the registered linters against the specified file paths.
+
+ :param paths: An iterable of files and/or directories to lint.
+ :param rev: Lint all files touched by the specified revision.
+ :param workdir: Lint all files touched in the working directory.
+ :param num_procs: The number of processes to use. Default: cpu count
+ :return: A dictionary with file names as the key, and a list of
+ :class:`~result.ResultContainer`s as the value.
+ """
+ paths = paths or []
+ if isinstance(paths, basestring):
+ paths = [paths]
+
+ if not self.linters:
+ raise LintersNotConfigured
+
+ # Calculate files from VCS
+ if rev:
+ paths.extend(self.vcs.by_rev(rev))
+ if workdir:
+ paths.extend(self.vcs.by_workdir())
+ paths = paths or ['.']
+ paths = map(os.path.abspath, paths)
+
+ # Set up multiprocessing
+ m = Manager()
+ queue = m.Queue()
+
+ for linter in self.linters:
+ queue.put(linter['path'])
+
+ num_procs = num_procs or cpu_count()
+ num_procs = min(num_procs, len(self.linters))
+ pool = Pool(num_procs)
+
+ all_results = defaultdict(list)
+ workers = []
+ for i in range(num_procs):
+ workers.append(
+ pool.apply_async(_run_worker, args=(queue, paths), kwds=self.lintargs))
+ pool.close()
+
+ # ignore SIGINT in parent so we can still get partial results
+ # from child processes. These should shutdown quickly anyway.
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ self.return_code = 0
+ for worker in workers:
+ # parent process blocks on worker.get()
+ results, return_code = worker.get()
+ if results or return_code:
+ self.return_code = 1
+ for k, v in results.iteritems():
+ all_results[k].extend(v)
+ return all_results
diff --git a/python/mozlint/mozlint/types.py b/python/mozlint/mozlint/types.py
new file mode 100644
index 000000000..2f49ae2bf
--- /dev/null
+++ b/python/mozlint/mozlint/types.py
@@ -0,0 +1,142 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import re
+import sys
+from abc import ABCMeta, abstractmethod
+
+from mozlog import get_default_logger, commandline, structuredlog
+from mozlog.reader import LogHandler
+
+from . import result
+from .pathutils import filterpaths
+
+
+class BaseType(object):
+ """Abstract base class for all types of linters."""
+ __metaclass__ = ABCMeta
+ batch = False
+
+ def __call__(self, paths, linter, **lintargs):
+ """Run `linter` against `paths` with `lintargs`.
+
+ :param paths: Paths to lint. Can be a file or directory.
+ :param linter: Linter definition paths are being linted against.
+ :param lintargs: External arguments to the linter not defined in
+ the definition, but passed in by a consumer.
+ :returns: A list of :class:`~result.ResultContainer` objects.
+ """
+ paths = filterpaths(paths, linter, **lintargs)
+ if not paths:
+ print("{}: no files to lint in specified paths".format(linter['name']))
+ return
+
+ if self.batch:
+ return self._lint(paths, linter, **lintargs)
+
+ errors = []
+ try:
+ for p in paths:
+ result = self._lint(p, linter, **lintargs)
+ if result:
+ errors.extend(result)
+ except KeyboardInterrupt:
+ pass
+ return errors
+
+ @abstractmethod
+ def _lint(self, path):
+ pass
+
+
+class LineType(BaseType):
+ """Abstract base class for linter types that check each line individually.
+
+ Subclasses of this linter type will read each file and check the provided
+ payload against each line one by one.
+ """
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def condition(payload, line):
+ pass
+
+ def _lint(self, path, linter, **lintargs):
+ payload = linter['payload']
+
+ with open(path, 'r') as fh:
+ lines = fh.readlines()
+
+ errors = []
+ for i, line in enumerate(lines):
+ if self.condition(payload, line):
+ errors.append(result.from_linter(linter, path=path, lineno=i+1))
+
+ return errors
+
+
+class StringType(LineType):
+ """Linter type that checks whether a substring is found."""
+
+ def condition(self, payload, line):
+ return payload in line
+
+
+class RegexType(LineType):
+ """Linter type that checks whether a regex match is found."""
+
+ def condition(self, payload, line):
+ return re.search(payload, line)
+
+
+class ExternalType(BaseType):
+ """Linter type that runs an external function.
+
+ The function is responsible for properly formatting the results
+ into a list of :class:`~result.ResultContainer` objects.
+ """
+ batch = True
+
+ def _lint(self, files, linter, **lintargs):
+ payload = linter['payload']
+ return payload(files, **lintargs)
+
+
+class LintHandler(LogHandler):
+ def __init__(self, linter):
+ self.linter = linter
+ self.results = []
+
+ def lint(self, data):
+ self.results.append(result.from_linter(self.linter, **data))
+
+
+class StructuredLogType(BaseType):
+ batch = True
+
+ def _lint(self, files, linter, **lintargs):
+ payload = linter["payload"]
+ handler = LintHandler(linter)
+ logger = linter.get("logger")
+ if logger is None:
+ logger = get_default_logger()
+ if logger is None:
+ logger = structuredlog.StructuredLogger(linter["name"])
+ commandline.setup_logging(logger, {}, {"mach": sys.stdout})
+ logger.add_handler(handler)
+ try:
+ payload(files, logger, **lintargs)
+ except KeyboardInterrupt:
+ pass
+ return handler.results
+
+supported_types = {
+ 'string': StringType(),
+ 'regex': RegexType(),
+ 'external': ExternalType(),
+ 'structured_log': StructuredLogType()
+}
+"""Mapping of type string to an associated instance."""
diff --git a/python/mozlint/mozlint/vcs.py b/python/mozlint/mozlint/vcs.py
new file mode 100644
index 000000000..6a118f2e6
--- /dev/null
+++ b/python/mozlint/mozlint/vcs.py
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+
+
+class VCSFiles(object):
+ def __init__(self):
+ self._root = None
+ self._vcs = None
+
+ @property
+ def root(self):
+ if self._root:
+ return self._root
+
+ # First check if we're in an hg repo, if not try git
+ commands = (
+ ['hg', 'root'],
+ ['git', 'rev-parse', '--show-toplevel'],
+ )
+
+ for cmd in commands:
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ output = proc.communicate()[0].strip()
+
+ if proc.returncode == 0:
+ self._vcs = cmd[0]
+ self._root = output
+ return self._root
+
+ @property
+ def vcs(self):
+ return self._vcs or (self.root and self._vcs)
+
+ @property
+ def is_hg(self):
+ return self.vcs == 'hg'
+
+ @property
+ def is_git(self):
+ return self.vcs == 'git'
+
+ def _run(self, cmd):
+ files = subprocess.check_output(cmd).split()
+ return [os.path.join(self.root, f) for f in files]
+
+ def by_rev(self, rev):
+ if self.is_hg:
+ return self._run(['hg', 'log', '--template', '{files % "\\n{file}"}', '-r', rev])
+ elif self.is_git:
+ return self._run(['git', 'diff', '--name-only', rev])
+ return []
+
+ def by_workdir(self):
+ if self.is_hg:
+ return self._run(['hg', 'status', '-amn'])
+ elif self.is_git:
+ return self._run(['git', 'diff', '--name-only'])
+ return []
diff --git a/python/mozlint/setup.py b/python/mozlint/setup.py
new file mode 100644
index 000000000..62d25c38b
--- /dev/null
+++ b/python/mozlint/setup.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+VERSION = 0.1
+DEPS = ["mozlog>=3.4"]
+
+setup(
+ name='mozlint',
+ description='Framework for registering and running micro lints',
+ license='MPL 2.0',
+ author='Andrew Halberstadt',
+ author_email='ahalberstadt@mozilla.com',
+ url='',
+ packages=['mozlint'],
+ version=VERSION,
+ classifiers=[
+ 'Environment :: Console',
+ 'Development Status :: 3 - Alpha',
+ 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+ 'Natural Language :: English',
+ ],
+ install_requires=DEPS,
+)
diff --git a/python/mozlint/test/__init__.py b/python/mozlint/test/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/python/mozlint/test/__init__.py
diff --git a/python/mozlint/test/conftest.py b/python/mozlint/test/conftest.py
new file mode 100644
index 000000000..e171798b0
--- /dev/null
+++ b/python/mozlint/test/conftest.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import pytest
+
+from mozlint import LintRoller
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture
+def lint(request):
+ lintargs = getattr(request.module, 'lintargs', {})
+ return LintRoller(root=here, **lintargs)
+
+
+@pytest.fixture(scope='session')
+def filedir():
+ return os.path.join(here, 'files')
+
+
+@pytest.fixture(scope='module')
+def files(filedir, request):
+ suffix_filter = getattr(request.module, 'files', [''])
+ return [os.path.join(filedir, p) for p in os.listdir(filedir)
+ if any(p.endswith(suffix) for suffix in suffix_filter)]
+
+
+@pytest.fixture(scope='session')
+def lintdir():
+ return os.path.join(here, 'linters')
+
+
+@pytest.fixture(scope='module')
+def linters(lintdir, request):
+ suffix_filter = getattr(request.module, 'linters', ['.lint'])
+ return [os.path.join(lintdir, p) for p in os.listdir(lintdir)
+ if any(p.endswith(suffix) for suffix in suffix_filter)]
diff --git a/python/mozlint/test/files/foobar.js b/python/mozlint/test/files/foobar.js
new file mode 100644
index 000000000..d9754d0a2
--- /dev/null
+++ b/python/mozlint/test/files/foobar.js
@@ -0,0 +1,2 @@
+// Oh no.. we called this variable foobar, bad!
+var foobar = "a string";
diff --git a/python/mozlint/test/files/foobar.py b/python/mozlint/test/files/foobar.py
new file mode 100644
index 000000000..e1677b3fd
--- /dev/null
+++ b/python/mozlint/test/files/foobar.py
@@ -0,0 +1,2 @@
+# Oh no.. we called this variable foobar, bad!
+foobar = "a string"
diff --git a/python/mozlint/test/files/no_foobar.js b/python/mozlint/test/files/no_foobar.js
new file mode 100644
index 000000000..6b95d646c
--- /dev/null
+++ b/python/mozlint/test/files/no_foobar.js
@@ -0,0 +1,2 @@
+// What a relief
+var properlyNamed = "a string";
diff --git a/python/mozlint/test/linters/badreturncode.lint b/python/mozlint/test/linters/badreturncode.lint
new file mode 100644
index 000000000..398d51a55
--- /dev/null
+++ b/python/mozlint/test/linters/badreturncode.lint
@@ -0,0 +1,21 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def lint(files, **lintargs):
+ return 1
+
+
+LINTER = {
+ 'name': "BadReturnCodeLinter",
+ 'description': "Returns an error code no matter what",
+ 'include': [
+ 'files',
+ ],
+ 'type': 'external',
+ 'extensions': ['.js', '.jsm'],
+ 'payload': lint,
+}
diff --git a/python/mozlint/test/linters/explicit_path.lint b/python/mozlint/test/linters/explicit_path.lint
new file mode 100644
index 000000000..8c1a88a1f
--- /dev/null
+++ b/python/mozlint/test/linters/explicit_path.lint
@@ -0,0 +1,13 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "ExplicitPathLinter",
+ 'description': "Only lint a specific file name",
+ 'rule': 'no-foobar',
+ 'include': [
+ 'no_foobar.js',
+ ],
+ 'type': 'string',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/external.lint b/python/mozlint/test/linters/external.lint
new file mode 100644
index 000000000..dcae419db
--- /dev/null
+++ b/python/mozlint/test/linters/external.lint
@@ -0,0 +1,30 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozlint import result
+
+
+def lint(files, **lintargs):
+ results = []
+ for path in files:
+ with open(path, 'r') as fh:
+ for i, line in enumerate(fh.readlines()):
+ if 'foobar' in line:
+ results.append(result.from_linter(
+ LINTER, path=path, lineno=i+1, column=1, rule="no-foobar"))
+ return results
+
+
+LINTER = {
+ 'name': "ExternalLinter",
+ 'description': "It's bad to have the string foobar in js files.",
+ 'include': [
+ 'files',
+ ],
+ 'type': 'external',
+ 'extensions': ['.js', '.jsm'],
+ 'payload': lint,
+}
diff --git a/python/mozlint/test/linters/invalid_exclude.lint b/python/mozlint/test/linters/invalid_exclude.lint
new file mode 100644
index 000000000..be6d0045c
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_exclude.lint
@@ -0,0 +1,10 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "BadExcludeLinter",
+ 'description': "Has an invalid exclude directive.",
+ 'exclude': [0, 1], # should be a list of strings
+ 'type': 'string',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/invalid_extension.lnt b/python/mozlint/test/linters/invalid_extension.lnt
new file mode 100644
index 000000000..3cb8153a0
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_extension.lnt
@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "BadExtensionLinter",
+ 'description': "Has an invalid file extension.",
+ 'type': 'string',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/invalid_include.lint b/python/mozlint/test/linters/invalid_include.lint
new file mode 100644
index 000000000..343d5e195
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_include.lint
@@ -0,0 +1,10 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "BadIncludeLinter",
+ 'description': "Has an invalid include directive.",
+ 'include': 'should be a list',
+ 'type': 'string',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/invalid_type.lint b/python/mozlint/test/linters/invalid_type.lint
new file mode 100644
index 000000000..9e5926c5a
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_type.lint
@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "BadTypeLinter",
+ 'description': "Has an invalid type.",
+ 'type': 'invalid',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/missing_attrs.lint b/python/mozlint/test/linters/missing_attrs.lint
new file mode 100644
index 000000000..380512b64
--- /dev/null
+++ b/python/mozlint/test/linters/missing_attrs.lint
@@ -0,0 +1,7 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "MissingAttrsLinter",
+ 'description': "Missing type and payload",
+}
diff --git a/python/mozlint/test/linters/missing_definition.lint b/python/mozlint/test/linters/missing_definition.lint
new file mode 100644
index 000000000..a84b305d2
--- /dev/null
+++ b/python/mozlint/test/linters/missing_definition.lint
@@ -0,0 +1,4 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+# No LINTER variable
diff --git a/python/mozlint/test/linters/raises.lint b/python/mozlint/test/linters/raises.lint
new file mode 100644
index 000000000..f17e18733
--- /dev/null
+++ b/python/mozlint/test/linters/raises.lint
@@ -0,0 +1,19 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozlint.errors import LintException
+
+
+def lint(files, **lintargs):
+ raise LintException("Oh no something bad happened!")
+
+
+LINTER = {
+ 'name': "RaisesLinter",
+ 'description': "Raises an exception",
+ 'type': 'external',
+ 'payload': lint,
+}
diff --git a/python/mozlint/test/linters/regex.lint b/python/mozlint/test/linters/regex.lint
new file mode 100644
index 000000000..439cadf36
--- /dev/null
+++ b/python/mozlint/test/linters/regex.lint
@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "RegexLinter",
+ 'description': "Make sure the string 'foobar' never appears "
+ "in a js variable file because it is bad.",
+ 'rule': 'no-foobar',
+ 'include': [
+ '**/*.js',
+ '**/*.jsm',
+ ],
+ 'type': 'regex',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/string.lint b/python/mozlint/test/linters/string.lint
new file mode 100644
index 000000000..46bf0e8b8
--- /dev/null
+++ b/python/mozlint/test/linters/string.lint
@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+
+LINTER = {
+ 'name': "StringLinter",
+ 'description': "Make sure the string 'foobar' never appears "
+ "in browser js files because it is bad.",
+ 'rule': 'no-foobar',
+ 'include': [
+ '**/*.js',
+ '**/*.jsm',
+ ],
+ 'type': 'string',
+ 'payload': 'foobar',
+}
diff --git a/python/mozlint/test/linters/structured.lint b/python/mozlint/test/linters/structured.lint
new file mode 100644
index 000000000..e8be8d7b3
--- /dev/null
+++ b/python/mozlint/test/linters/structured.lint
@@ -0,0 +1,28 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def lint(files, logger, **kwargs):
+ for path in files:
+ with open(path, 'r') as fh:
+ for i, line in enumerate(fh.readlines()):
+ if 'foobar' in line:
+ logger.lint_error(path=path,
+ lineno=i+1,
+ column=1,
+ rule="no-foobar")
+
+
+LINTER = {
+ 'name': "StructuredLinter",
+ 'description': "It's bad to have the string foobar in js files.",
+ 'include': [
+ 'files',
+ ],
+ 'type': 'structured_log',
+ 'extensions': ['.js', '.jsm'],
+ 'payload': lint,
+}
diff --git a/python/mozlint/test/test_formatters.py b/python/mozlint/test/test_formatters.py
new file mode 100644
index 000000000..b9e6512b2
--- /dev/null
+++ b/python/mozlint/test/test_formatters.py
@@ -0,0 +1,90 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import json
+import sys
+from collections import defaultdict
+
+import pytest
+
+from mozlint import ResultContainer
+from mozlint import formatters
+
+
+@pytest.fixture
+def results(scope='module'):
+ containers = (
+ ResultContainer(
+ linter='foo',
+ path='a/b/c.txt',
+ message="oh no foo",
+ lineno=1,
+ ),
+ ResultContainer(
+ linter='bar',
+ path='d/e/f.txt',
+ message="oh no bar",
+ hint="try baz instead",
+ level='warning',
+ lineno=4,
+ column=2,
+ rule="bar-not-allowed",
+ ),
+ ResultContainer(
+ linter='baz',
+ path='a/b/c.txt',
+ message="oh no baz",
+ lineno=4,
+ source="if baz:",
+ ),
+ )
+ results = defaultdict(list)
+ for c in containers:
+ results[c.path].append(c)
+ return results
+
+
+def test_stylish_formatter(results):
+ expected = """
+a/b/c.txt
+ 1 error oh no foo (foo)
+ 4 error oh no baz (baz)
+
+d/e/f.txt
+ 4:2 warning oh no bar bar-not-allowed (bar)
+
+\u2716 3 problems (2 errors, 1 warning)
+""".strip()
+
+ fmt = formatters.get('stylish', disable_colors=True)
+ assert expected == fmt(results)
+
+
+def test_treeherder_formatter(results):
+ expected = """
+TEST-UNEXPECTED-ERROR | a/b/c.txt:1 | oh no foo (foo)
+TEST-UNEXPECTED-ERROR | a/b/c.txt:4 | oh no baz (baz)
+TEST-UNEXPECTED-WARNING | d/e/f.txt:4:2 | oh no bar (bar-not-allowed)
+""".strip()
+
+ fmt = formatters.get('treeherder')
+ assert expected == fmt(results)
+
+
+def test_json_formatter(results):
+ fmt = formatters.get('json')
+ formatted = json.loads(fmt(results))
+
+ assert set(formatted.keys()) == set(results.keys())
+
+ slots = ResultContainer.__slots__
+ for errors in formatted.values():
+ for err in errors:
+ assert all(s in err for s in slots)
+
+
+if __name__ == '__main__':
+ sys.exit(pytest.main(['--verbose', __file__]))
diff --git a/python/mozlint/test/test_parser.py b/python/mozlint/test/test_parser.py
new file mode 100644
index 000000000..e18e7a5a9
--- /dev/null
+++ b/python/mozlint/test/test_parser.py
@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+import pytest
+
+from mozlint.parser import Parser
+from mozlint.errors import (
+ LinterNotFound,
+ LinterParseError,
+)
+
+
+@pytest.fixture(scope='module')
+def parse(lintdir):
+ parser = Parser()
+
+ def _parse(name):
+ path = os.path.join(lintdir, name)
+ return parser(path)
+ return _parse
+
+
+def test_parse_valid_linter(parse):
+ lintobj = parse('string.lint')
+ assert isinstance(lintobj, dict)
+ assert 'name' in lintobj
+ assert 'description' in lintobj
+ assert 'type' in lintobj
+ assert 'payload' in lintobj
+
+
+@pytest.mark.parametrize('linter', [
+ 'invalid_type.lint',
+ 'invalid_extension.lnt',
+ 'invalid_include.lint',
+ 'invalid_exclude.lint',
+ 'missing_attrs.lint',
+ 'missing_definition.lint',
+])
+def test_parse_invalid_linter(parse, linter):
+ with pytest.raises(LinterParseError):
+ parse(linter)
+
+
+def test_parse_non_existent_linter(parse):
+ with pytest.raises(LinterNotFound):
+ parse('missing_file.lint')
+
+
+if __name__ == '__main__':
+ sys.exit(pytest.main(['--verbose', __file__]))
diff --git a/python/mozlint/test/test_roller.py b/python/mozlint/test/test_roller.py
new file mode 100644
index 000000000..b4b82c346
--- /dev/null
+++ b/python/mozlint/test/test_roller.py
@@ -0,0 +1,82 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+import pytest
+
+from mozlint import ResultContainer
+from mozlint.errors import LintersNotConfigured, LintException
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+linters = ('string.lint', 'regex.lint', 'external.lint')
+
+
+def test_roll_no_linters_configured(lint, files):
+ with pytest.raises(LintersNotConfigured):
+ lint.roll(files)
+
+
+def test_roll_successful(lint, linters, files):
+ lint.read(linters)
+
+ result = lint.roll(files)
+ assert len(result) == 1
+ assert lint.return_code == 1
+
+ path = result.keys()[0]
+ assert os.path.basename(path) == 'foobar.js'
+
+ errors = result[path]
+ assert isinstance(errors, list)
+ assert len(errors) == 6
+
+ container = errors[0]
+ assert isinstance(container, ResultContainer)
+ assert container.rule == 'no-foobar'
+
+
+def test_roll_catch_exception(lint, lintdir, files):
+ lint.read(os.path.join(lintdir, 'raises.lint'))
+
+ # suppress printed traceback from test output
+ old_stderr = sys.stderr
+ sys.stderr = open(os.devnull, 'w')
+ with pytest.raises(LintException):
+ lint.roll(files)
+ sys.stderr = old_stderr
+
+
+def test_roll_with_excluded_path(lint, linters, files):
+ lint.lintargs.update({'exclude': ['**/foobar.js']})
+
+ lint.read(linters)
+ result = lint.roll(files)
+
+ assert len(result) == 0
+ assert lint.return_code == 0
+
+
+def test_roll_with_invalid_extension(lint, lintdir, filedir):
+ lint.read(os.path.join(lintdir, 'external.lint'))
+ result = lint.roll(os.path.join(filedir, 'foobar.py'))
+ assert len(result) == 0
+ assert lint.return_code == 0
+
+
+def test_roll_with_failure_code(lint, lintdir, files):
+ lint.read(os.path.join(lintdir, 'badreturncode.lint'))
+
+ assert lint.return_code is None
+ result = lint.roll(files)
+ assert len(result) == 0
+ assert lint.return_code == 1
+
+
+if __name__ == '__main__':
+ sys.exit(pytest.main(['--verbose', __file__]))
diff --git a/python/mozlint/test/test_types.py b/python/mozlint/test/test_types.py
new file mode 100644
index 000000000..ee0ea9b63
--- /dev/null
+++ b/python/mozlint/test/test_types.py
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+import pytest
+
+from mozlint.result import ResultContainer
+
+
+@pytest.fixture
+def path(filedir):
+ def _path(name):
+ return os.path.join(filedir, name)
+ return _path
+
+
+@pytest.fixture(params=['string.lint', 'regex.lint', 'external.lint', 'structured.lint'])
+def linter(lintdir, request):
+ return os.path.join(lintdir, request.param)
+
+
+def test_linter_types(lint, linter, files, path):
+ lint.read(linter)
+ result = lint.roll(files)
+ assert isinstance(result, dict)
+ assert path('foobar.js') in result
+ assert path('no_foobar.js') not in result
+
+ result = result[path('foobar.js')][0]
+ assert isinstance(result, ResultContainer)
+
+ name = os.path.basename(linter).split('.')[0]
+ assert result.linter.lower().startswith(name)
+
+
+def test_no_filter(lint, lintdir, files):
+ lint.read(os.path.join(lintdir, 'explicit_path.lint'))
+ result = lint.roll(files)
+ assert len(result) == 0
+
+ lint.lintargs['use_filters'] = False
+ result = lint.roll(files)
+ assert len(result) == 2
+
+
+if __name__ == '__main__':
+ sys.exit(pytest.main(['--verbose', __file__]))