path: root/python/compare-locales/compare_locales/compare.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

'Mozilla l10n compare locales tool'

import codecs
import os
import os.path
import shutil
import re
from difflib import SequenceMatcher
from collections import defaultdict

try:
    from json import dumps
except:
    from simplejson import dumps

from compare_locales import parser
from compare_locales import paths
from compare_locales.checks import getChecker


class Tree(object):
    def __init__(self, valuetype):
        self.branches = dict()
        self.valuetype = valuetype
        self.value = None

    def __getitem__(self, leaf):
        parts = []
        if isinstance(leaf, paths.File):
            parts = [p for p in [leaf.locale, leaf.module] if p] + \
                leaf.file.split('/')
        else:
            parts = leaf.split('/')
        return self.__get(parts)

    def __get(self, parts):
        common = None
        old = None
        new = tuple(parts)
        t = self
        for k, v in self.branches.iteritems():
            for i, part in enumerate(zip(k, parts)):
                if part[0] != part[1]:
                    i -= 1
                    break
            if i < 0:
                continue
            i += 1
            common = tuple(k[:i])
            old = tuple(k[i:])
            new = tuple(parts[i:])
            break
        if old:
            self.branches.pop(k)
            t = Tree(self.valuetype)
            t.branches[old] = v
            self.branches[common] = t
        elif common:
            t = self.branches[common]
        if new:
            if common:
                return t.__get(new)
            t2 = t
            t = Tree(self.valuetype)
            t2.branches[new] = t
        if t.value is None:
            t.value = t.valuetype()
        return t.value

    indent = '  '

    def getContent(self, depth=0):
        '''
        Returns iterator of (depth, flag, key_or_value) tuples.
        If flag is 'value', key_or_value is a value object, otherwise
        (flag is 'key') it's a key string.
        '''
        keys = self.branches.keys()
        keys.sort()
        if self.value is not None:
            yield (depth, 'value', self.value)
        for key in keys:
            yield (depth, 'key', key)
            for child in self.branches[key].getContent(depth + 1):
                yield child

    def toJSON(self):
        '''
        Returns this Tree as a JSON-able tree of hashes.
        Only the values need to take care that they're JSON-able.
        '''
        json = {}
        keys = self.branches.keys()
        keys.sort()
        if self.value is not None:
            json['value'] = self.value
        children = [('/'.join(key), self.branches[key].toJSON())
                    for key in keys]
        if children:
            json['children'] = children
        return json

    def getStrRows(self):
        def tostr(t):
            if t[1] == 'key':
                return self.indent * t[0] + '/'.join(t[2])
            return self.indent * (t[0] + 1) + str(t[2])

        return map(tostr, self.getContent())

    def __str__(self):
        return '\n'.join(self.getStrRows())


class AddRemove(SequenceMatcher):
    def __init__(self):
        SequenceMatcher.__init__(self, None, None, None)

    def set_left(self, left):
        if not isinstance(left, list):
            left = [l for l in left]
        self.set_seq1(left)

    def set_right(self, right):
        if not isinstance(right, list):
            right = [l for l in right]
        self.set_seq2(right)

    def __iter__(self):
        for tag, i1, i2, j1, j2 in self.get_opcodes():
            if tag == 'equal':
                for pair in zip(self.a[i1:i2], self.b[j1:j2]):
                    yield ('equal', pair)
            elif tag == 'delete':
                for item in self.a[i1:i2]:
                    yield ('delete', item)
            elif tag == 'insert':
                for item in self.b[j1:j2]:
                    yield ('add', item)
            else:
                # tag == 'replace'
                for item in self.a[i1:i2]:
                    yield ('delete', item)
                for item in self.b[j1:j2]:
                    yield ('add', item)


class DirectoryCompare(SequenceMatcher):
    def __init__(self, reference):
        SequenceMatcher.__init__(self, None, [i for i in reference],
                                 [])
        self.watcher = None

    def setWatcher(self, watcher):
        self.watcher = watcher

    def compareWith(self, other):
        if not self.watcher:
            return
        self.set_seq2([i for i in other])
        for tag, i1, i2, j1, j2 in self.get_opcodes():
            if tag == 'equal':
                for i, j in zip(xrange(i1, i2), xrange(j1, j2)):
                    self.watcher.compare(self.a[i], self.b[j])
            elif tag == 'delete':
                for i in xrange(i1, i2):
                    self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
            elif tag == 'insert':
                for j in xrange(j1, j2):
                    self.watcher.remove(self.b[j])
            else:
                for j in xrange(j1, j2):
                    self.watcher.remove(self.b[j])
                for i in xrange(i1, i2):
                    self.watcher.add(self.a[i], other.cloneFile(self.a[i]))


class Observer(object):
    stat_cats = ['missing', 'obsolete', 'missingInFiles', 'report',
                 'changed', 'unchanged', 'keys']

    def __init__(self):
        class intdict(defaultdict):
            def __init__(self):
                defaultdict.__init__(self, int)

        self.summary = defaultdict(intdict)
        self.details = Tree(dict)
        self.filter = None

    # support pickling
    def __getstate__(self):
        return dict(summary=self.getSummary(), details=self.details)

    def __setstate__(self, state):
        class intdict(defaultdict):
            def __init__(self):
                defaultdict.__init__(self, int)

        self.summary = defaultdict(intdict)
        if 'summary' in state:
            for loc, stats in state['summary'].iteritems():
                self.summary[loc].update(stats)
        self.details = state['details']
        self.filter = None

    def getSummary(self):
        plaindict = {}
        for k, v in self.summary.iteritems():
            plaindict[k] = dict(v)
        return plaindict

    def toJSON(self):
        return dict(summary=self.getSummary(), details=self.details.toJSON())

    def notify(self, category, file, data):
        rv = "error"
        if category in self.stat_cats:
            # these get called post reporting just for stats
            # return "error" to forward them to other other_observers
            self.summary[file.locale][category] += data
            # keep track of how many strings are in a missing file
            # we got the {'missingFile': 'error'} from the first pass
            if category == 'missingInFiles':
                self.details[file]['strings'] = data
            return "error"
        if category in ['missingFile', 'obsoleteFile']:
            if self.filter is not None:
                rv = self.filter(file)
            if rv != "ignore":
                self.details[file][category] = rv
            return rv
        if category in ['missingEntity', 'obsoleteEntity']:
            if self.filter is not None:
                rv = self.filter(file, data)
            if rv == "ignore":
                return rv
            v = self.details[file]
            try:
                v[category].append(data)
            except KeyError:
                v[category] = [data]
            return rv
        if category == 'error':
            try:
                self.details[file][category].append(data)
            except KeyError:
                self.details[file][category] = [data]
            self.summary[file.locale]['errors'] += 1
        elif category == 'warning':
            try:
                self.details[file][category].append(data)
            except KeyError:
                self.details[file][category] = [data]
            self.summary[file.locale]['warnings'] += 1
        return rv

    def toExhibit(self):
        items = []
        for locale in sorted(self.summary.iterkeys()):
            summary = self.summary[locale]
            if locale is not None:
                item = {'id': 'xxx/' + locale,
                        'label': locale,
                        'locale': locale}
            else:
                item = {'id': 'xxx',
                        'label': 'xxx',
                        'locale': 'xxx'}
            item['type'] = 'Build'
            total = sum([summary[k]
                         for k in ('changed', 'unchanged', 'report', 'missing',
                                   'missingInFiles')
                         if k in summary])
            rate = (('changed' in summary and summary['changed'] * 100) or
                    0) / total
            item.update((k, summary.get(k, 0))
                        for k in ('changed', 'unchanged'))
            item.update((k, summary[k])
                        for k in ('report', 'errors', 'warnings')
                        if k in summary)
            item['missing'] = summary.get('missing', 0) + \
                summary.get('missingInFiles', 0)
            item['completion'] = rate
            item['total'] = total
            result = 'success'
            if item.get('warnings', 0):
                result = 'warning'
            if item.get('errors', 0) or item.get('missing', 0):
                result = 'failure'
            item['result'] = result
            items.append(item)
        data = {
            "properties": dict.fromkeys(
                ("completion", "errors", "warnings", "missing", "report",
                 "unchanged", "changed", "obsolete"),
                {"valueType": "number"}),
            "types": {
                "Build": {"pluralLabel": "Builds"}
            }}
        data['items'] = items
        return dumps(data, indent=2)

    def serialize(self, type="text"):
        if type == "exhibit":
            return self.toExhibit()
        if type == "json":
            return dumps(self.toJSON())

        def tostr(t):
            if t[1] == 'key':
                return '  ' * t[0] + '/'.join(t[2])
            o = []
            indent = '  ' * (t[0] + 1)
            if 'error' in t[2]:
                o += [indent + 'ERROR: ' + e for e in t[2]['error']]
            if 'warning' in t[2]:
                o += [indent + 'WARNING: ' + e for e in t[2]['warning']]
            if 'missingEntity' in t[2] or 'obsoleteEntity' in t[2]:
                missingEntities = ('missingEntity' in t[2] and
                                   t[2]['missingEntity']) or []
                obsoleteEntities = ('obsoleteEntity' in t[2] and
                                    t[2]['obsoleteEntity']) or []
                entities = missingEntities + obsoleteEntities
                entities.sort()
                for entity in entities:
                    op = '+'
                    if entity in obsoleteEntities:
                        op = '-'
                    o.append(indent + op + entity)
            elif 'missingFile' in t[2]:
                o.append(indent + '// add and localize this file')
            elif 'obsoleteFile' in t[2]:
                o.append(indent + '// remove this file')
            return '\n'.join(o)

        out = []
        for locale, summary in sorted(self.summary.iteritems()):
            if locale is not None:
                out.append(locale + ':')
            out += [k + ': ' + str(v) for k, v in sorted(summary.iteritems())]
            total = sum([summary[k]
                         for k in ['changed', 'unchanged', 'report', 'missing',
                                   'missingInFiles']
                         if k in summary])
            rate = 0
            if total:
                rate = (('changed' in summary and summary['changed'] * 100) or
                        0) / total
            out.append('%d%% of entries changed' % rate)
        return '\n'.join(map(tostr, self.details.getContent()) + out)

    def __str__(self):
        return 'observer'


class ContentComparer:
    keyRE = re.compile('[kK]ey')
    nl = re.compile('\n', re.M)

    def __init__(self):
        '''Create a ContentComparer.
        observer is usually a instance of Observer. The return values
        of the notify method are used to control the handling of missing
        entities.
        '''
        self.reference = dict()
        self.observer = Observer()
        self.other_observers = []
        self.merge_stage = None

    def add_observer(self, obs):
        '''Add a non-filtering observer.
        Results from the notify calls are ignored.
        '''
        self.other_observers.append(obs)

    def set_merge_stage(self, merge_stage):
        self.merge_stage = merge_stage

    def merge(self, ref_entities, ref_map, ref_file, l10n_file, missing,
              skips, p):
        outfile = os.path.join(self.merge_stage, l10n_file.module,
                               l10n_file.file)
        outdir = os.path.dirname(outfile)
        if not os.path.isdir(outdir):
            os.makedirs(outdir)
        if not p.canMerge:
            shutil.copyfile(ref_file.fullpath, outfile)
            print "copied reference to " + outfile
            return
        if skips:
            # skips come in ordered by key name, we need them in file order
            skips.sort(key=lambda s: s.span[0])
        trailing = (['\n'] +
                    [ref_entities[ref_map[key]].all for key in missing] +
                    [ref_entities[ref_map[skip.key]].all for skip in skips
                     if not isinstance(skip, parser.Junk)])
        if skips:
            # we need to skip a few errornous blocks in the input, copy by hand
            f = codecs.open(outfile, 'wb', p.encoding)
            offset = 0
            for skip in skips:
                chunk = skip.span
                f.write(p.contents[offset:chunk[0]])
                offset = chunk[1]
            f.write(p.contents[offset:])
        else:
            shutil.copyfile(l10n_file.fullpath, outfile)
            f = codecs.open(outfile, 'ab', p.encoding)
        print "adding to " + outfile

        def ensureNewline(s):
            if not s.endswith('\n'):
                return s + '\n'
            return s

        f.write(''.join(map(ensureNewline, trailing)))
        f.close()

    def notify(self, category, file, data):
        """Check observer for the found data, and if it's
        not to ignore, notify other_observers.
        """
        rv = self.observer.notify(category, file, data)
        if rv == 'ignore':
            return rv
        for obs in self.other_observers:
            # non-filtering other_observers, ignore results
            obs.notify(category, file, data)
        return rv

    def remove(self, obsolete):
        self.notify('obsoleteFile', obsolete, None)
        pass

    def compare(self, ref_file, l10n):
        try:
            p = parser.getParser(ref_file.file)
        except UserWarning:
            # no comparison, XXX report?
            return
        if ref_file not in self.reference:
            # we didn't parse this before
            try:
                p.readContents(ref_file.getContents())
            except Exception, e:
                self.notify('error', ref_file, str(e))
                return
            self.reference[ref_file] = p.parse()
        ref = self.reference[ref_file]
        ref_list = ref[1].keys()
        ref_list.sort()
        try:
            p.readContents(l10n.getContents())
            l10n_entities, l10n_map = p.parse()
        except Exception, e:
            self.notify('error', l10n, str(e))
            return
        lines = []

        def _getLine(offset):
            if not lines:
                lines.append(0)
                for m in self.nl.finditer(p.contents):
                    lines.append(m.end())
            for i in xrange(len(lines), 0, -1):
                if offset >= lines[i - 1]:
                    return (i, offset - lines[i - 1])
            return (1, offset)

        l10n_list = l10n_map.keys()
        l10n_list.sort()
        ar = AddRemove()
        ar.set_left(ref_list)
        ar.set_right(l10n_list)
        report = missing = obsolete = changed = unchanged = keys = 0
        missings = []
        skips = []
        checker = getChecker(l10n, reference=ref[0])
        for action, item_or_pair in ar:
            if action == 'delete':
                # missing entity
                _rv = self.notify('missingEntity', l10n, item_or_pair)
                if _rv == "ignore":
                    continue
                if _rv == "error":
                    # only add to missing entities for l10n-merge on error,
                    # not report
                    missings.append(item_or_pair)
                    missing += 1
                else:
                    # just report
                    report += 1
            elif action == 'add':
                # obsolete entity or junk
                if isinstance(l10n_entities[l10n_map[item_or_pair]],
                              parser.Junk):
                    junk = l10n_entities[l10n_map[item_or_pair]]
                    params = (junk.val,) + junk.span
                    self.notify('error', l10n,
                                'Unparsed content "%s" at %d-%d' % params)
                    if self.merge_stage is not None:
                        skips.append(junk)
                elif self.notify('obsoleteEntity', l10n,
                                 item_or_pair) != 'ignore':
                    obsolete += 1
            else:
                # entity found in both ref and l10n, check for changed
                entity = item_or_pair[0]
                refent = ref[0][ref[1][entity]]
                l10nent = l10n_entities[l10n_map[entity]]
                if self.keyRE.search(entity):
                    keys += 1
                else:
                    if refent.val == l10nent.val:
                        self.doUnchanged(l10nent)
                        unchanged += 1
                    else:
                        self.doChanged(ref_file, refent, l10nent)
                        changed += 1
                        # run checks:
                if checker:
                    for tp, pos, msg, cat in checker.check(refent, l10nent):
                        # compute real src position, if first line,
                        # col needs adjustment
                        _l, _offset = _getLine(l10nent.val_span[0])
                        if isinstance(pos, tuple):
                            # line, column
                            if pos[0] == 1:
                                col = pos[1] + _offset
                            else:
                                col = pos[1]
                            _l += pos[0] - 1
                        else:
                            _l, col = _getLine(l10nent.val_span[0] + pos)
                            # skip error entities when merging
                        if tp == 'error' and self.merge_stage is not None:
                            skips.append(l10nent)
                        self.notify(tp, l10n,
                                    u"%s at line %d, column %d for %s" %
                                    (msg, _l, col, refent.key))
                pass
        if missing:
            self.notify('missing', l10n, missing)
        if self.merge_stage is not None and (missings or skips):
            self.merge(ref[0], ref[1], ref_file, l10n, missings, skips, p)
        if report:
            self.notify('report', l10n, report)
        if obsolete:
            self.notify('obsolete', l10n, obsolete)
        if changed:
            self.notify('changed', l10n, changed)
        if unchanged:
            self.notify('unchanged', l10n, unchanged)
        if keys:
            self.notify('keys', l10n, keys)
        pass

    def add(self, orig, missing):
        if self.notify('missingFile', missing, None) == "ignore":
            # filter said that we don't need this file, don't count it
            return
        f = orig
        try:
            p = parser.getParser(f.file)
        except UserWarning:
            return
        try:
            p.readContents(f.getContents())
            entities, map = p.parse()
        except Exception, e:
            self.notify('error', f, str(e))
            return
        self.notify('missingInFiles', missing, len(map))

    def doUnchanged(self, entity):
        # overload this if needed
        pass

    def doChanged(self, file, ref_entity, l10n_entity):
        # overload this if needed
        pass


def compareApp(app, other_observer=None, merge_stage=None, clobber=False):
    '''Compare locales set in app.

    Optional arguments are:
    - other_observer. A object implementing
        notify(category, _file, data)
      The return values of that callback are ignored.
    - merge_stage. A directory to be used for staging the output of
      l10n-merge.
    - clobber. Clobber the module subdirectories of the merge dir as we go.
      Use wisely, as it might cause data loss.
    '''
    comparer = ContentComparer()
    if other_observer is not None:
        comparer.add_observer(other_observer)
    comparer.observer.filter = app.filter
    for module, reference, locales in app:
        dir_comp = DirectoryCompare(reference)
        dir_comp.setWatcher(comparer)
        for _, localization in locales:
            if merge_stage is not None:
                locale_merge = merge_stage.format(ab_CD=localization.locale)
                comparer.set_merge_stage(locale_merge)
                if clobber:
                    # if clobber, remove the stage for the module if it exists
                    clobberdir = os.path.join(locale_merge, module)
                    if os.path.exists(clobberdir):
                        shutil.rmtree(clobberdir)
                        print "clobbered " + clobberdir
            dir_comp.compareWith(localization)
    return comparer.observer


def compareDirs(reference, locale, other_observer=None, merge_stage=None):
    '''Compare reference and locale dir.

    Optional arguments are:
    - other_observer. A object implementing
        notify(category, _file, data)
      The return values of that callback are ignored.
    '''
    comparer = ContentComparer()
    if other_observer is not None:
        comparer.add_observer(other_observer)
    comparer.set_merge_stage(merge_stage)
    dir_comp = DirectoryCompare(paths.EnumerateDir(reference))
    dir_comp.setWatcher(comparer)
    dir_comp.compareWith(paths.EnumerateDir(locale))
    return comparer.observer