# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os.path
import os
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from urlparse import urlparse, urljoin
from urllib import pathname2url, url2pathname
from urllib2 import urlopen
from collections import defaultdict
from compare_locales import util
class L10nConfigParser(object):
'''Helper class to gather application information from ini files.
This class is working on synchronous open to read files or web data.
Subclass this and overwrite loadConfigs and addChild if you need async.
'''
def __init__(self, inipath, **kwargs):
"""Constructor for L10nConfigParsers
inipath -- l10n.ini path
Optional keyword arguments are fowarded to the inner ConfigParser as
defaults.
"""
if os.path.isabs(inipath):
self.inipath = 'file:%s' % pathname2url(inipath)
else:
pwdurl = 'file:%s/' % pathname2url(os.getcwd())
self.inipath = urljoin(pwdurl, inipath)
# l10n.ini files can import other l10n.ini files, store the
# corresponding L10nConfigParsers
self.children = []
# we really only care about the l10n directories described in l10n.ini
self.dirs = []
# optional defaults to be passed to the inner ConfigParser (unused?)
self.defaults = kwargs
def getDepth(self, cp):
'''Get the depth for the comparison from the parsed l10n.ini.
Overloadable to get the source depth for fennec and friends.
'''
try:
depth = cp.get('general', 'depth')
except:
depth = '.'
return depth
def getFilters(self):
'''Get the test functions from this ConfigParser and all children.
Only works with synchronous loads, used by compare-locales, which
is local anyway.
'''
filterurl = urljoin(self.inipath, 'filter.py')
try:
l = {}
execfile(url2pathname(urlparse(filterurl).path), {}, l)
if 'test' in l and callable(l['test']):
filters = [l['test']]
else:
filters = []
except:
filters = []
for c in self.children:
filters += c.getFilters()
return filters
def loadConfigs(self):
"""Entry point to load the l10n.ini file this Parser refers to.
This implementation uses synchronous loads, subclasses might overload
this behaviour. If you do, make sure to pass a file-like object
to onLoadConfig.
"""
self.onLoadConfig(urlopen(self.inipath))
def onLoadConfig(self, inifile):
"""Parse a file-like object for the loaded l10n.ini file."""
cp = ConfigParser(self.defaults)
cp.readfp(inifile)
depth = self.getDepth(cp)
self.baseurl = urljoin(self.inipath, depth)
# create child loaders for any other l10n.ini files to be included
try:
for title, path in cp.items('includes'):
# skip default items
if title in self.defaults:
continue
# add child config parser
self.addChild(title, path, cp)
except NoSectionError:
pass
# try to load the "dirs" defined in the "compare" section
try:
self.dirs.extend(cp.get('compare', 'dirs').split())
except (NoOptionError, NoSectionError):
pass
# try getting a top level compare dir, as used for fennec
try:
self.tld = cp.get('compare', 'tld')
# remove tld from comparison dirs
if self.tld in self.dirs:
self.dirs.remove(self.tld)
except (NoOptionError, NoSectionError):
self.tld = None
# try to set "all_path" and "all_url"
try:
self.all_path = cp.get('general', 'all')
self.all_url = urljoin(self.baseurl, self.all_path)
except (NoOptionError, NoSectionError):
self.all_path = None
self.all_url = None
return cp
def addChild(self, title, path, orig_cp):
"""Create a child L10nConfigParser and load it.
title -- indicates the module's name
path -- indicates the path to the module's l10n.ini file
orig_cp -- the configuration parser of this l10n.ini
"""
cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults)
cp.loadConfigs()
self.children.append(cp)
def getTLDPathsTuple(self, basepath):
"""Given the basepath, return the path fragments to be used for
self.tld. For build runs, this is (basepath, self.tld), for
source runs, just (basepath,).
@see overwritten method in SourceTreeConfigParser.
"""
return (basepath, self.tld)
def dirsIter(self):
"""Iterate over all dirs and our base path for this l10n.ini"""
url = urlparse(self.baseurl)
basepath = url2pathname(url.path)
if self.tld is not None:
yield self.tld, self.getTLDPathsTuple(basepath)
for dir in self.dirs:
yield dir, (basepath, dir)
def directories(self):
"""Iterate over all dirs and base paths for this l10n.ini as well
as the included ones.
"""
for t in self.dirsIter():
yield t
for child in self.children:
for t in child.directories():
yield t
def allLocales(self):
"""Return a list of all the locales of this project"""
return util.parseLocales(urlopen(self.all_url).read())
class SourceTreeConfigParser(L10nConfigParser):
'''Subclassing L10nConfigParser to work with just the repos
checked out next to each other instead of intermingled like
we do for real builds.
'''
def __init__(self, inipath, basepath):
'''Add additional arguments basepath.
basepath is used to resolve local paths via branchnames.
'''
L10nConfigParser.__init__(self, inipath)
self.basepath = basepath
self.tld = None
def getDepth(self, cp):
'''Get the depth for the comparison from the parsed l10n.ini.
Overloaded to get the source depth for fennec and friends.
'''
try:
depth = cp.get('general', 'source-depth')
except:
try:
depth = cp.get('general', 'depth')
except:
depth = '.'
return depth
def addChild(self, title, path, orig_cp):
# check if there's a section with details for this include
# we might have to check a different repo, or even VCS
# for example, projects like "mail" indicate in
# an "include_" section where to find the l10n.ini for "toolkit"
details = 'include_' + title
if orig_cp.has_section(details):
branch = orig_cp.get(details, 'mozilla')
inipath = orig_cp.get(details, 'l10n.ini')
path = self.basepath + '/' + branch + '/' + inipath
else:
path = urljoin(self.baseurl, path)
cp = SourceTreeConfigParser(path, self.basepath, **self.defaults)
cp.loadConfigs()
self.children.append(cp)
def getTLDPathsTuple(self, basepath):
"""Overwrite L10nConfigParser's getTLDPathsTuple to just return
the basepath.
"""
return (basepath, )
class File(object):
def __init__(self, fullpath, file, module=None, locale=None):
self.fullpath = fullpath
self.file = file
self.module = module
self.locale = locale
pass
def getContents(self):
# open with universal line ending support and read
return open(self.fullpath, 'rU').read()
def __hash__(self):
f = self.file
if self.module:
f = self.module + '/' + f
return hash(f)
def __str__(self):
return self.fullpath
def __cmp__(self, other):
if not isinstance(other, File):
raise NotImplementedError
rv = cmp(self.module, other.module)
if rv != 0:
return rv
return cmp(self.file, other.file)
class EnumerateDir(object):
ignore_dirs = ['CVS', '.svn', '.hg', '.git']
def __init__(self, basepath, module='', locale=None, ignore_subdirs=[]):
self.basepath = basepath
self.module = module
self.locale = locale
self.ignore_subdirs = ignore_subdirs
pass
def cloneFile(self, other):
'''
Return a File object that this enumerator would return, if it had it.
'''
return File(os.path.join(self.basepath, other.file), other.file,
self.module, self.locale)
def __iter__(self):
# our local dirs are given as a tuple of path segments, starting off
# with an empty sequence for the basepath.
dirs = [()]
while dirs:
dir = dirs.pop(0)
fulldir = os.path.join(self.basepath, *dir)
try:
entries = os.listdir(fulldir)
except OSError:
# we probably just started off in a non-existing dir, ignore
continue
entries.sort()
for entry in entries:
leaf = os.path.join(fulldir, entry)
if os.path.isdir(leaf):
if entry not in self.ignore_dirs and \
leaf not in [os.path.join(self.basepath, d)
for d in self.ignore_subdirs]:
dirs.append(dir + (entry,))
continue
yield File(leaf, '/'.join(dir + (entry,)),
self.module, self.locale)
class LocalesWrap(object):
def __init__(self, base, module, locales, ignore_subdirs=[]):
self.base = base
self.module = module
self.locales = locales
self.ignore_subdirs = ignore_subdirs
def __iter__(self):
for locale in self.locales:
path = os.path.join(self.base, locale, self.module)
yield (locale, EnumerateDir(path, self.module, locale,
self.ignore_subdirs))
class EnumerateApp(object):
reference = 'en-US'
def __init__(self, inipath, l10nbase, locales=None):
self.setupConfigParser(inipath)
self.modules = defaultdict(dict)
self.l10nbase = os.path.abspath(l10nbase)
self.filters = []
drive, tail = os.path.splitdrive(inipath)
self.addFilters(*self.config.getFilters())
self.locales = locales or self.config.allLocales()
self.locales.sort()
def setupConfigParser(self, inipath):
self.config = L10nConfigParser(inipath)
self.config.loadConfigs()
def addFilters(self, *args):
self.filters += args
value_map = {None: None, 'error': 0, 'ignore': 1, 'report': 2}
def filter(self, l10n_file, entity=None):
'''Go through all added filters, and,
- map "error" -> 0, "ignore" -> 1, "report" -> 2
- if filter.test returns a bool, map that to
False -> "ignore" (1), True -> "error" (0)
- take the max of all reported
'''
rv = 0
for f in reversed(self.filters):
try:
_r = f(l10n_file.module, l10n_file.file, entity)
except:
# XXX error handling
continue
if isinstance(_r, bool):
_r = [1, 0][_r]
else:
# map string return value to int, default to 'error',
# None is None
_r = self.value_map.get(_r, 0)
if _r is not None:
rv = max(rv, _r)
return ['error', 'ignore', 'report'][rv]
def __iter__(self):
'''
Iterate over all modules, return en-US directory enumerator, and an
iterator over all locales in each iteration. Per locale, the locale
code and an directory enumerator will be given.
'''
dirmap = dict(self.config.directories())
mods = dirmap.keys()
mods.sort()
for mod in mods:
if self.reference == 'en-US':
base = os.path.join(*(dirmap[mod] + ('locales', 'en-US')))
else:
base = os.path.join(self.l10nbase, self.reference, mod)
yield (mod, EnumerateDir(base, mod, self.reference),
LocalesWrap(self.l10nbase, mod, self.locales,
[m[len(mod)+1:] for m in mods if m.startswith(mod+'/')]))
class EnumerateSourceTreeApp(EnumerateApp):
'''Subclass EnumerateApp to work on side-by-side checked out
repos, and to no pay attention to how the source would actually
be checked out for building.
It's supporting applications like Fennec, too, which have
'locales/en-US/...' in their root dir, but claim to be 'mobile'.
'''
def __init__(self, inipath, basepath, l10nbase, locales=None):
self.basepath = basepath
EnumerateApp.__init__(self, inipath, l10nbase, locales)
def setupConfigParser(self, inipath):
self.config = SourceTreeConfigParser(inipath, self.basepath)
self.config.loadConfigs()
def get_base_path(mod, loc):
'statics for path patterns and conversion'
__l10n = 'l10n/%(loc)s/%(mod)s'
__en_US = 'mozilla/%(mod)s/locales/en-US'
if loc == 'en-US':
return __en_US % {'mod': mod}
return __l10n % {'mod': mod, 'loc': loc}
def get_path(mod, loc, leaf):
return get_base_path(mod, loc) + '/' + leaf