# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os.path import os from ConfigParser import ConfigParser, NoSectionError, NoOptionError from urlparse import urlparse, urljoin from urllib import pathname2url, url2pathname from urllib2 import urlopen from collections import defaultdict from compare_locales import util class L10nConfigParser(object): '''Helper class to gather application information from ini files. This class is working on synchronous open to read files or web data. Subclass this and overwrite loadConfigs and addChild if you need async. ''' def __init__(self, inipath, **kwargs): """Constructor for L10nConfigParsers inipath -- l10n.ini path Optional keyword arguments are fowarded to the inner ConfigParser as defaults. """ if os.path.isabs(inipath): self.inipath = 'file:%s' % pathname2url(inipath) else: pwdurl = 'file:%s/' % pathname2url(os.getcwd()) self.inipath = urljoin(pwdurl, inipath) # l10n.ini files can import other l10n.ini files, store the # corresponding L10nConfigParsers self.children = [] # we really only care about the l10n directories described in l10n.ini self.dirs = [] # optional defaults to be passed to the inner ConfigParser (unused?) self.defaults = kwargs def getDepth(self, cp): '''Get the depth for the comparison from the parsed l10n.ini. Overloadable to get the source depth for fennec and friends. ''' try: depth = cp.get('general', 'depth') except: depth = '.' return depth def getFilters(self): '''Get the test functions from this ConfigParser and all children. Only works with synchronous loads, used by compare-locales, which is local anyway. ''' filterurl = urljoin(self.inipath, 'filter.py') try: l = {} execfile(url2pathname(urlparse(filterurl).path), {}, l) if 'test' in l and callable(l['test']): filters = [l['test']] else: filters = [] except: filters = [] for c in self.children: filters += c.getFilters() return filters def loadConfigs(self): """Entry point to load the l10n.ini file this Parser refers to. This implementation uses synchronous loads, subclasses might overload this behaviour. If you do, make sure to pass a file-like object to onLoadConfig. """ self.onLoadConfig(urlopen(self.inipath)) def onLoadConfig(self, inifile): """Parse a file-like object for the loaded l10n.ini file.""" cp = ConfigParser(self.defaults) cp.readfp(inifile) depth = self.getDepth(cp) self.baseurl = urljoin(self.inipath, depth) # create child loaders for any other l10n.ini files to be included try: for title, path in cp.items('includes'): # skip default items if title in self.defaults: continue # add child config parser self.addChild(title, path, cp) except NoSectionError: pass # try to load the "dirs" defined in the "compare" section try: self.dirs.extend(cp.get('compare', 'dirs').split()) except (NoOptionError, NoSectionError): pass # try getting a top level compare dir, as used for fennec try: self.tld = cp.get('compare', 'tld') # remove tld from comparison dirs if self.tld in self.dirs: self.dirs.remove(self.tld) except (NoOptionError, NoSectionError): self.tld = None # try to set "all_path" and "all_url" try: self.all_path = cp.get('general', 'all') self.all_url = urljoin(self.baseurl, self.all_path) except (NoOptionError, NoSectionError): self.all_path = None self.all_url = None return cp def addChild(self, title, path, orig_cp): """Create a child L10nConfigParser and load it. title -- indicates the module's name path -- indicates the path to the module's l10n.ini file orig_cp -- the configuration parser of this l10n.ini """ cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults) cp.loadConfigs() self.children.append(cp) def getTLDPathsTuple(self, basepath): """Given the basepath, return the path fragments to be used for self.tld. For build runs, this is (basepath, self.tld), for source runs, just (basepath,). @see overwritten method in SourceTreeConfigParser. """ return (basepath, self.tld) def dirsIter(self): """Iterate over all dirs and our base path for this l10n.ini""" url = urlparse(self.baseurl) basepath = url2pathname(url.path) if self.tld is not None: yield self.tld, self.getTLDPathsTuple(basepath) for dir in self.dirs: yield dir, (basepath, dir) def directories(self): """Iterate over all dirs and base paths for this l10n.ini as well as the included ones. """ for t in self.dirsIter(): yield t for child in self.children: for t in child.directories(): yield t def allLocales(self): """Return a list of all the locales of this project""" return util.parseLocales(urlopen(self.all_url).read()) class SourceTreeConfigParser(L10nConfigParser): '''Subclassing L10nConfigParser to work with just the repos checked out next to each other instead of intermingled like we do for real builds. ''' def __init__(self, inipath, basepath): '''Add additional arguments basepath. basepath is used to resolve local paths via branchnames. ''' L10nConfigParser.__init__(self, inipath) self.basepath = basepath self.tld = None def getDepth(self, cp): '''Get the depth for the comparison from the parsed l10n.ini. Overloaded to get the source depth for fennec and friends. ''' try: depth = cp.get('general', 'source-depth') except: try: depth = cp.get('general', 'depth') except: depth = '.' return depth def addChild(self, title, path, orig_cp): # check if there's a section with details for this include # we might have to check a different repo, or even VCS # for example, projects like "mail" indicate in # an "include_" section where to find the l10n.ini for "toolkit" details = 'include_' + title if orig_cp.has_section(details): branch = orig_cp.get(details, 'mozilla') inipath = orig_cp.get(details, 'l10n.ini') path = self.basepath + '/' + branch + '/' + inipath else: path = urljoin(self.baseurl, path) cp = SourceTreeConfigParser(path, self.basepath, **self.defaults) cp.loadConfigs() self.children.append(cp) def getTLDPathsTuple(self, basepath): """Overwrite L10nConfigParser's getTLDPathsTuple to just return the basepath. """ return (basepath, ) class File(object): def __init__(self, fullpath, file, module=None, locale=None): self.fullpath = fullpath self.file = file self.module = module self.locale = locale pass def getContents(self): # open with universal line ending support and read return open(self.fullpath, 'rU').read() def __hash__(self): f = self.file if self.module: f = self.module + '/' + f return hash(f) def __str__(self): return self.fullpath def __cmp__(self, other): if not isinstance(other, File): raise NotImplementedError rv = cmp(self.module, other.module) if rv != 0: return rv return cmp(self.file, other.file) class EnumerateDir(object): ignore_dirs = ['CVS', '.svn', '.hg', '.git'] def __init__(self, basepath, module='', locale=None, ignore_subdirs=[]): self.basepath = basepath self.module = module self.locale = locale self.ignore_subdirs = ignore_subdirs pass def cloneFile(self, other): ''' Return a File object that this enumerator would return, if it had it. ''' return File(os.path.join(self.basepath, other.file), other.file, self.module, self.locale) def __iter__(self): # our local dirs are given as a tuple of path segments, starting off # with an empty sequence for the basepath. dirs = [()] while dirs: dir = dirs.pop(0) fulldir = os.path.join(self.basepath, *dir) try: entries = os.listdir(fulldir) except OSError: # we probably just started off in a non-existing dir, ignore continue entries.sort() for entry in entries: leaf = os.path.join(fulldir, entry) if os.path.isdir(leaf): if entry not in self.ignore_dirs and \ leaf not in [os.path.join(self.basepath, d) for d in self.ignore_subdirs]: dirs.append(dir + (entry,)) continue yield File(leaf, '/'.join(dir + (entry,)), self.module, self.locale) class LocalesWrap(object): def __init__(self, base, module, locales, ignore_subdirs=[]): self.base = base self.module = module self.locales = locales self.ignore_subdirs = ignore_subdirs def __iter__(self): for locale in self.locales: path = os.path.join(self.base, locale, self.module) yield (locale, EnumerateDir(path, self.module, locale, self.ignore_subdirs)) class EnumerateApp(object): reference = 'en-US' def __init__(self, inipath, l10nbase, locales=None): self.setupConfigParser(inipath) self.modules = defaultdict(dict) self.l10nbase = os.path.abspath(l10nbase) self.filters = [] drive, tail = os.path.splitdrive(inipath) self.addFilters(*self.config.getFilters()) self.locales = locales or self.config.allLocales() self.locales.sort() def setupConfigParser(self, inipath): self.config = L10nConfigParser(inipath) self.config.loadConfigs() def addFilters(self, *args): self.filters += args value_map = {None: None, 'error': 0, 'ignore': 1, 'report': 2} def filter(self, l10n_file, entity=None): '''Go through all added filters, and, - map "error" -> 0, "ignore" -> 1, "report" -> 2 - if filter.test returns a bool, map that to False -> "ignore" (1), True -> "error" (0) - take the max of all reported ''' rv = 0 for f in reversed(self.filters): try: _r = f(l10n_file.module, l10n_file.file, entity) except: # XXX error handling continue if isinstance(_r, bool): _r = [1, 0][_r] else: # map string return value to int, default to 'error', # None is None _r = self.value_map.get(_r, 0) if _r is not None: rv = max(rv, _r) return ['error', 'ignore', 'report'][rv] def __iter__(self): ''' Iterate over all modules, return en-US directory enumerator, and an iterator over all locales in each iteration. Per locale, the locale code and an directory enumerator will be given. ''' dirmap = dict(self.config.directories()) mods = dirmap.keys() mods.sort() for mod in mods: if self.reference == 'en-US': base = os.path.join(*(dirmap[mod] + ('locales', 'en-US'))) else: base = os.path.join(self.l10nbase, self.reference, mod) yield (mod, EnumerateDir(base, mod, self.reference), LocalesWrap(self.l10nbase, mod, self.locales, [m[len(mod)+1:] for m in mods if m.startswith(mod+'/')])) class EnumerateSourceTreeApp(EnumerateApp): '''Subclass EnumerateApp to work on side-by-side checked out repos, and to no pay attention to how the source would actually be checked out for building. It's supporting applications like Fennec, too, which have 'locales/en-US/...' in their root dir, but claim to be 'mobile'. ''' def __init__(self, inipath, basepath, l10nbase, locales=None): self.basepath = basepath EnumerateApp.__init__(self, inipath, l10nbase, locales) def setupConfigParser(self, inipath): self.config = SourceTreeConfigParser(inipath, self.basepath) self.config.loadConfigs() def get_base_path(mod, loc): 'statics for path patterns and conversion' __l10n = 'l10n/%(loc)s/%(mod)s' __en_US = 'mozilla/%(mod)s/locales/en-US' if loc == 'en-US': return __en_US % {'mod': mod} return __l10n % {'mod': mod, 'loc': loc} def get_path(mod, loc, leaf): return get_base_path(mod, loc) + '/' + leaf