# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
'Mozilla l10n compare locales tool'
import codecs
import os
import os.path
import shutil
import re
from difflib import SequenceMatcher
from collections import defaultdict
try:
from json import dumps
except:
from simplejson import dumps
from compare_locales import parser
from compare_locales import paths
from compare_locales.checks import getChecker
class Tree(object):
def __init__(self, valuetype):
self.branches = dict()
self.valuetype = valuetype
self.value = None
def __getitem__(self, leaf):
parts = []
if isinstance(leaf, paths.File):
parts = [p for p in [leaf.locale, leaf.module] if p] + \
leaf.file.split('/')
else:
parts = leaf.split('/')
return self.__get(parts)
def __get(self, parts):
common = None
old = None
new = tuple(parts)
t = self
for k, v in self.branches.iteritems():
for i, part in enumerate(zip(k, parts)):
if part[0] != part[1]:
i -= 1
break
if i < 0:
continue
i += 1
common = tuple(k[:i])
old = tuple(k[i:])
new = tuple(parts[i:])
break
if old:
self.branches.pop(k)
t = Tree(self.valuetype)
t.branches[old] = v
self.branches[common] = t
elif common:
t = self.branches[common]
if new:
if common:
return t.__get(new)
t2 = t
t = Tree(self.valuetype)
t2.branches[new] = t
if t.value is None:
t.value = t.valuetype()
return t.value
indent = ' '
def getContent(self, depth=0):
'''
Returns iterator of (depth, flag, key_or_value) tuples.
If flag is 'value', key_or_value is a value object, otherwise
(flag is 'key') it's a key string.
'''
keys = self.branches.keys()
keys.sort()
if self.value is not None:
yield (depth, 'value', self.value)
for key in keys:
yield (depth, 'key', key)
for child in self.branches[key].getContent(depth + 1):
yield child
def toJSON(self):
'''
Returns this Tree as a JSON-able tree of hashes.
Only the values need to take care that they're JSON-able.
'''
json = {}
keys = self.branches.keys()
keys.sort()
if self.value is not None:
json['value'] = self.value
children = [('/'.join(key), self.branches[key].toJSON())
for key in keys]
if children:
json['children'] = children
return json
def getStrRows(self):
def tostr(t):
if t[1] == 'key':
return self.indent * t[0] + '/'.join(t[2])
return self.indent * (t[0] + 1) + str(t[2])
return map(tostr, self.getContent())
def __str__(self):
return '\n'.join(self.getStrRows())
class AddRemove(SequenceMatcher):
def __init__(self):
SequenceMatcher.__init__(self, None, None, None)
def set_left(self, left):
if not isinstance(left, list):
left = [l for l in left]
self.set_seq1(left)
def set_right(self, right):
if not isinstance(right, list):
right = [l for l in right]
self.set_seq2(right)
def __iter__(self):
for tag, i1, i2, j1, j2 in self.get_opcodes():
if tag == 'equal':
for pair in zip(self.a[i1:i2], self.b[j1:j2]):
yield ('equal', pair)
elif tag == 'delete':
for item in self.a[i1:i2]:
yield ('delete', item)
elif tag == 'insert':
for item in self.b[j1:j2]:
yield ('add', item)
else:
# tag == 'replace'
for item in self.a[i1:i2]:
yield ('delete', item)
for item in self.b[j1:j2]:
yield ('add', item)
class DirectoryCompare(SequenceMatcher):
def __init__(self, reference):
SequenceMatcher.__init__(self, None, [i for i in reference],
[])
self.watcher = None
def setWatcher(self, watcher):
self.watcher = watcher
def compareWith(self, other):
if not self.watcher:
return
self.set_seq2([i for i in other])
for tag, i1, i2, j1, j2 in self.get_opcodes():
if tag == 'equal':
for i, j in zip(xrange(i1, i2), xrange(j1, j2)):
self.watcher.compare(self.a[i], self.b[j])
elif tag == 'delete':
for i in xrange(i1, i2):
self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
elif tag == 'insert':
for j in xrange(j1, j2):
self.watcher.remove(self.b[j])
else:
for j in xrange(j1, j2):
self.watcher.remove(self.b[j])
for i in xrange(i1, i2):
self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
class Observer(object):
stat_cats = ['missing', 'obsolete', 'missingInFiles', 'report',
'changed', 'unchanged', 'keys']
def __init__(self):
class intdict(defaultdict):
def __init__(self):
defaultdict.__init__(self, int)
self.summary = defaultdict(intdict)
self.details = Tree(dict)
self.filter = None
# support pickling
def __getstate__(self):
return dict(summary=self.getSummary(), details=self.details)
def __setstate__(self, state):
class intdict(defaultdict):
def __init__(self):
defaultdict.__init__(self, int)
self.summary = defaultdict(intdict)
if 'summary' in state:
for loc, stats in state['summary'].iteritems():
self.summary[loc].update(stats)
self.details = state['details']
self.filter = None
def getSummary(self):
plaindict = {}
for k, v in self.summary.iteritems():
plaindict[k] = dict(v)
return plaindict
def toJSON(self):
return dict(summary=self.getSummary(), details=self.details.toJSON())
def notify(self, category, file, data):
rv = "error"
if category in self.stat_cats:
# these get called post reporting just for stats
# return "error" to forward them to other other_observers
self.summary[file.locale][category] += data
# keep track of how many strings are in a missing file
# we got the {'missingFile': 'error'} from the first pass
if category == 'missingInFiles':
self.details[file]['strings'] = data
return "error"
if category in ['missingFile', 'obsoleteFile']:
if self.filter is not None:
rv = self.filter(file)
if rv != "ignore":
self.details[file][category] = rv
return rv
if category in ['missingEntity', 'obsoleteEntity']:
if self.filter is not None:
rv = self.filter(file, data)
if rv == "ignore":
return rv
v = self.details[file]
try:
v[category].append(data)
except KeyError:
v[category] = [data]
return rv
if category == 'error':
try:
self.details[file][category].append(data)
except KeyError:
self.details[file][category] = [data]
self.summary[file.locale]['errors'] += 1
elif category == 'warning':
try:
self.details[file][category].append(data)
except KeyError:
self.details[file][category] = [data]
self.summary[file.locale]['warnings'] += 1
return rv
def toExhibit(self):
items = []
for locale in sorted(self.summary.iterkeys()):
summary = self.summary[locale]
if locale is not None:
item = {'id': 'xxx/' + locale,
'label': locale,
'locale': locale}
else:
item = {'id': 'xxx',
'label': 'xxx',
'locale': 'xxx'}
item['type'] = 'Build'
total = sum([summary[k]
for k in ('changed', 'unchanged', 'report', 'missing',
'missingInFiles')
if k in summary])
rate = (('changed' in summary and summary['changed'] * 100) or
0) / total
item.update((k, summary.get(k, 0))
for k in ('changed', 'unchanged'))
item.update((k, summary[k])
for k in ('report', 'errors', 'warnings')
if k in summary)
item['missing'] = summary.get('missing', 0) + \
summary.get('missingInFiles', 0)
item['completion'] = rate
item['total'] = total
result = 'success'
if item.get('warnings', 0):
result = 'warning'
if item.get('errors', 0) or item.get('missing', 0):
result = 'failure'
item['result'] = result
items.append(item)
data = {
"properties": dict.fromkeys(
("completion", "errors", "warnings", "missing", "report",
"unchanged", "changed", "obsolete"),
{"valueType": "number"}),
"types": {
"Build": {"pluralLabel": "Builds"}
}}
data['items'] = items
return dumps(data, indent=2)
def serialize(self, type="text"):
if type == "exhibit":
return self.toExhibit()
if type == "json":
return dumps(self.toJSON())
def tostr(t):
if t[1] == 'key':
return ' ' * t[0] + '/'.join(t[2])
o = []
indent = ' ' * (t[0] + 1)
if 'error' in t[2]:
o += [indent + 'ERROR: ' + e for e in t[2]['error']]
if 'warning' in t[2]:
o += [indent + 'WARNING: ' + e for e in t[2]['warning']]
if 'missingEntity' in t[2] or 'obsoleteEntity' in t[2]:
missingEntities = ('missingEntity' in t[2] and
t[2]['missingEntity']) or []
obsoleteEntities = ('obsoleteEntity' in t[2] and
t[2]['obsoleteEntity']) or []
entities = missingEntities + obsoleteEntities
entities.sort()
for entity in entities:
op = '+'
if entity in obsoleteEntities:
op = '-'
o.append(indent + op + entity)
elif 'missingFile' in t[2]:
o.append(indent + '// add and localize this file')
elif 'obsoleteFile' in t[2]:
o.append(indent + '// remove this file')
return '\n'.join(o)
out = []
for locale, summary in sorted(self.summary.iteritems()):
if locale is not None:
out.append(locale + ':')
out += [k + ': ' + str(v) for k, v in sorted(summary.iteritems())]
total = sum([summary[k]
for k in ['changed', 'unchanged', 'report', 'missing',
'missingInFiles']
if k in summary])
rate = 0
if total:
rate = (('changed' in summary and summary['changed'] * 100) or
0) / total
out.append('%d%% of entries changed' % rate)
return '\n'.join(map(tostr, self.details.getContent()) + out)
def __str__(self):
return 'observer'
class ContentComparer:
keyRE = re.compile('[kK]ey')
nl = re.compile('\n', re.M)
def __init__(self):
'''Create a ContentComparer.
observer is usually a instance of Observer. The return values
of the notify method are used to control the handling of missing
entities.
'''
self.reference = dict()
self.observer = Observer()
self.other_observers = []
self.merge_stage = None
def add_observer(self, obs):
'''Add a non-filtering observer.
Results from the notify calls are ignored.
'''
self.other_observers.append(obs)
def set_merge_stage(self, merge_stage):
self.merge_stage = merge_stage
def merge(self, ref_entities, ref_map, ref_file, l10n_file, missing,
skips, p):
outfile = os.path.join(self.merge_stage, l10n_file.module,
l10n_file.file)
outdir = os.path.dirname(outfile)
if not os.path.isdir(outdir):
os.makedirs(outdir)
if not p.canMerge:
shutil.copyfile(ref_file.fullpath, outfile)
print "copied reference to " + outfile
return
if skips:
# skips come in ordered by key name, we need them in file order
skips.sort(key=lambda s: s.span[0])
trailing = (['\n'] +
[ref_entities[ref_map[key]].all for key in missing] +
[ref_entities[ref_map[skip.key]].all for skip in skips
if not isinstance(skip, parser.Junk)])
if skips:
# we need to skip a few errornous blocks in the input, copy by hand
f = codecs.open(outfile, 'wb', p.encoding)
offset = 0
for skip in skips:
chunk = skip.span
f.write(p.contents[offset:chunk[0]])
offset = chunk[1]
f.write(p.contents[offset:])
else:
shutil.copyfile(l10n_file.fullpath, outfile)
f = codecs.open(outfile, 'ab', p.encoding)
print "adding to " + outfile
def ensureNewline(s):
if not s.endswith('\n'):
return s + '\n'
return s
f.write(''.join(map(ensureNewline, trailing)))
f.close()
def notify(self, category, file, data):
"""Check observer for the found data, and if it's
not to ignore, notify other_observers.
"""
rv = self.observer.notify(category, file, data)
if rv == 'ignore':
return rv
for obs in self.other_observers:
# non-filtering other_observers, ignore results
obs.notify(category, file, data)
return rv
def remove(self, obsolete):
self.notify('obsoleteFile', obsolete, None)
pass
def compare(self, ref_file, l10n):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
return
if ref_file not in self.reference:
# we didn't parse this before
try:
p.readContents(ref_file.getContents())
except Exception, e:
self.notify('error', ref_file, str(e))
return
self.reference[ref_file] = p.parse()
ref = self.reference[ref_file]
ref_list = ref[1].keys()
ref_list.sort()
try:
p.readContents(l10n.getContents())
l10n_entities, l10n_map = p.parse()
except Exception, e:
self.notify('error', l10n, str(e))
return
lines = []
def _getLine(offset):
if not lines:
lines.append(0)
for m in self.nl.finditer(p.contents):
lines.append(m.end())
for i in xrange(len(lines), 0, -1):
if offset >= lines[i - 1]:
return (i, offset - lines[i - 1])
return (1, offset)
l10n_list = l10n_map.keys()
l10n_list.sort()
ar = AddRemove()
ar.set_left(ref_list)
ar.set_right(l10n_list)
report = missing = obsolete = changed = unchanged = keys = 0
missings = []
skips = []
checker = getChecker(l10n, reference=ref[0])
for action, item_or_pair in ar:
if action == 'delete':
# missing entity
_rv = self.notify('missingEntity', l10n, item_or_pair)
if _rv == "ignore":
continue
if _rv == "error":
# only add to missing entities for l10n-merge on error,
# not report
missings.append(item_or_pair)
missing += 1
else:
# just report
report += 1
elif action == 'add':
# obsolete entity or junk
if isinstance(l10n_entities[l10n_map[item_or_pair]],
parser.Junk):
junk = l10n_entities[l10n_map[item_or_pair]]
params = (junk.val,) + junk.span
self.notify('error', l10n,
'Unparsed content "%s" at %d-%d' % params)
if self.merge_stage is not None:
skips.append(junk)
elif self.notify('obsoleteEntity', l10n,
item_or_pair) != 'ignore':
obsolete += 1
else:
# entity found in both ref and l10n, check for changed
entity = item_or_pair[0]
refent = ref[0][ref[1][entity]]
l10nent = l10n_entities[l10n_map[entity]]
if self.keyRE.search(entity):
keys += 1
else:
if refent.val == l10nent.val:
self.doUnchanged(l10nent)
unchanged += 1
else:
self.doChanged(ref_file, refent, l10nent)
changed += 1
# run checks:
if checker:
for tp, pos, msg, cat in checker.check(refent, l10nent):
# compute real src position, if first line,
# col needs adjustment
_l, _offset = _getLine(l10nent.val_span[0])
if isinstance(pos, tuple):
# line, column
if pos[0] == 1:
col = pos[1] + _offset
else:
col = pos[1]
_l += pos[0] - 1
else:
_l, col = _getLine(l10nent.val_span[0] + pos)
# skip error entities when merging
if tp == 'error' and self.merge_stage is not None:
skips.append(l10nent)
self.notify(tp, l10n,
u"%s at line %d, column %d for %s" %
(msg, _l, col, refent.key))
pass
if missing:
self.notify('missing', l10n, missing)
if self.merge_stage is not None and (missings or skips):
self.merge(ref[0], ref[1], ref_file, l10n, missings, skips, p)
if report:
self.notify('report', l10n, report)
if obsolete:
self.notify('obsolete', l10n, obsolete)
if changed:
self.notify('changed', l10n, changed)
if unchanged:
self.notify('unchanged', l10n, unchanged)
if keys:
self.notify('keys', l10n, keys)
pass
def add(self, orig, missing):
if self.notify('missingFile', missing, None) == "ignore":
# filter said that we don't need this file, don't count it
return
f = orig
try:
p = parser.getParser(f.file)
except UserWarning:
return
try:
p.readContents(f.getContents())
entities, map = p.parse()
except Exception, e:
self.notify('error', f, str(e))
return
self.notify('missingInFiles', missing, len(map))
def doUnchanged(self, entity):
# overload this if needed
pass
def doChanged(self, file, ref_entity, l10n_entity):
# overload this if needed
pass
def compareApp(app, other_observer=None, merge_stage=None, clobber=False):
'''Compare locales set in app.
Optional arguments are:
- other_observer. A object implementing
notify(category, _file, data)
The return values of that callback are ignored.
- merge_stage. A directory to be used for staging the output of
l10n-merge.
- clobber. Clobber the module subdirectories of the merge dir as we go.
Use wisely, as it might cause data loss.
'''
comparer = ContentComparer()
if other_observer is not None:
comparer.add_observer(other_observer)
comparer.observer.filter = app.filter
for module, reference, locales in app:
dir_comp = DirectoryCompare(reference)
dir_comp.setWatcher(comparer)
for _, localization in locales:
if merge_stage is not None:
locale_merge = merge_stage.format(ab_CD=localization.locale)
comparer.set_merge_stage(locale_merge)
if clobber:
# if clobber, remove the stage for the module if it exists
clobberdir = os.path.join(locale_merge, module)
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print "clobbered " + clobberdir
dir_comp.compareWith(localization)
return comparer.observer
def compareDirs(reference, locale, other_observer=None, merge_stage=None):
'''Compare reference and locale dir.
Optional arguments are:
- other_observer. A object implementing
notify(category, _file, data)
The return values of that callback are ignored.
'''
comparer = ContentComparer()
if other_observer is not None:
comparer.add_observer(other_observer)
comparer.set_merge_stage(merge_stage)
dir_comp = DirectoryCompare(paths.EnumerateDir(reference))
dir_comp.setWatcher(comparer)
dir_comp.compareWith(paths.EnumerateDir(locale))
return comparer.observer