1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
'''gaia-style web apps support
This variant supports manifest.webapp localization as well as
.properties files with a naming scheme of locales/foo.*.properties.
'''
from collections import defaultdict
import json
import os
import os.path
import re
from compare_locales.paths import File, EnumerateDir
from compare_locales.compare import AddRemove, ContentComparer
class WebAppCompare(object):
'''For a given directory, analyze
/manifest.webapp
/locales/*.*.properties
Deduce the present locale codes.
'''
ignore_dirs = EnumerateDir.ignore_dirs
reference_locale = 'en-US'
def __init__(self, basedir):
'''Constructor
:param basedir: Directory of the web app to inspect
'''
self.basedir = basedir
self.manifest = Manifest(basedir, self.reference_locale)
self.files = FileComparison(basedir, self.reference_locale)
self.watcher = None
def compare(self, locales):
'''Compare the manifest.webapp and the locales/*.*.properties
'''
if not locales:
locales = self.locales()
self.manifest.compare(locales)
self.files.compare(locales)
def setWatcher(self, watcher):
self.watcher = watcher
self.manifest.watcher = watcher
self.files.watcher = watcher
def locales(self):
'''Inspect files on disk to find present languages.
:rtype: List of locales, sorted, including reference.
'''
locales = set(self.manifest.strings.keys())
locales.update(self.files.locales())
locales = list(sorted(locales))
return locales
class Manifest(object):
'''Class that helps with parsing and inspection of manifest.webapp.
'''
def __init__(self, basedir, reference_locale):
self.file = File(os.path.join(basedir, 'manifest.webapp'),
'manifest.webapp')
self.reference_locale = reference_locale
self._strings = None
self.watcher = None
@property
def strings(self):
if self._strings is None:
self._strings = self.load_and_parse()
return self._strings
def load_and_parse(self):
try:
manifest = json.load(open(self.file.fullpath))
except (ValueError, IOError), e:
if self.watcher:
self.watcher.notify('error', self.file, str(e))
return False
return self.extract_manifest_strings(manifest)
def extract_manifest_strings(self, manifest_fragment):
'''Extract localizable strings from a manifest dict.
This method is recursive, and returns a two-level dict,
first level being locale codes, second level being generated
key and localized value. Keys are generated by concatenating
each level in the json with a ".".
'''
rv = defaultdict(dict)
localizable = manifest_fragment.pop('locales', {})
if localizable:
for locale, keyvalue in localizable.iteritems():
for key, value in keyvalue.iteritems():
key = '.'.join(['locales', 'AB_CD', key])
rv[locale][key] = value
for key, sub_manifest in manifest_fragment.iteritems():
if not isinstance(sub_manifest, dict):
continue
subdict = self.extract_manifest_strings(sub_manifest)
if subdict:
for locale, keyvalue in subdict:
rv[locale].update((key + '.' + subkey, value)
for subkey, value
in keyvalue.iteritems())
return rv
def compare(self, locales):
strings = self.strings
if not strings:
return
# create a copy so that we can mock around with it
strings = strings.copy()
reference = strings.pop(self.reference_locale)
for locale in locales:
if locale == self.reference_locale:
continue
self.compare_strings(reference,
strings.get(locale, {}),
locale)
def compare_strings(self, reference, l10n, locale):
add_remove = AddRemove()
add_remove.set_left(sorted(reference.keys()))
add_remove.set_right(sorted(l10n.keys()))
missing = obsolete = changed = unchanged = 0
for op, item_or_pair in add_remove:
if op == 'equal':
if reference[item_or_pair[0]] == l10n[item_or_pair[1]]:
unchanged += 1
else:
changed += 1
else:
key = item_or_pair.replace('.AB_CD.',
'.%s.' % locale)
if op == 'add':
# obsolete entry
obsolete += 1
self.watcher.notify('obsoleteEntity', self.file, key)
else:
# missing entry
missing += 1
self.watcher.notify('missingEntity', self.file, key)
class FileComparison(object):
'''Compare the locales/*.*.properties files inside a webapp.
'''
prop = re.compile('(?P<base>.*)\\.'
'(?P<locale>[a-zA-Z]+(?:-[a-zA-Z]+)*)'
'\\.properties$')
def __init__(self, basedir, reference_locale):
self.basedir = basedir
self.reference_locale = reference_locale
self.watcher = None
self._reference = self._files = None
def locales(self):
'''Get the locales present in the webapp
'''
self.files()
locales = self._files.keys()
locales.sort()
return locales
def compare(self, locales):
self.files()
for locale in locales:
l10n = self._files[locale]
filecmp = AddRemove()
filecmp.set_left(sorted(self._reference.keys()))
filecmp.set_right(sorted(l10n.keys()))
for op, item_or_pair in filecmp:
if op == 'equal':
self.watcher.compare(self._reference[item_or_pair[0]],
l10n[item_or_pair[1]])
elif op == 'add':
# obsolete file
self.watcher.remove(l10n[item_or_pair])
else:
# missing file
_path = '.'.join([item_or_pair, locale, 'properties'])
missingFile = File(
os.path.join(self.basedir, 'locales', _path),
'locales/' + _path)
self.watcher.add(self._reference[item_or_pair],
missingFile)
def files(self):
'''Read the list of locales from disk.
'''
if self._reference:
return
self._reference = {}
self._files = defaultdict(dict)
path_list = self._listdir()
for path in path_list:
match = self.prop.match(path)
if match is None:
continue
locale = match.group('locale')
if locale == self.reference_locale:
target = self._reference
else:
target = self._files[locale]
fullpath = os.path.join(self.basedir, 'locales', path)
target[match.group('base')] = File(fullpath, 'locales/' + path)
def _listdir(self):
'Monkey-patch this for testing.'
return os.listdir(os.path.join(self.basedir, 'locales'))
def compare_web_app(basedir, locales, other_observer=None):
'''Compare gaia-style web app.
Optional arguments are:
- other_observer. A object implementing
notify(category, _file, data)
The return values of that callback are ignored.
'''
comparer = ContentComparer()
if other_observer is not None:
comparer.add_observer(other_observer)
webapp_comp = WebAppCompare(basedir)
webapp_comp.setWatcher(comparer)
webapp_comp.compare(locales)
return comparer.observer
|
