summaryrefslogtreecommitdiffstats
path: root/python/compare-locales/compare_locales/paths.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /python/compare-locales/compare_locales/paths.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'python/compare-locales/compare_locales/paths.py')
-rw-r--r--python/compare-locales/compare_locales/paths.py398
1 files changed, 398 insertions, 0 deletions
diff --git a/python/compare-locales/compare_locales/paths.py b/python/compare-locales/compare_locales/paths.py
new file mode 100644
index 000000000..f72b3a2e7
--- /dev/null
+++ b/python/compare-locales/compare_locales/paths.py
@@ -0,0 +1,398 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os.path
+import os
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+from urlparse import urlparse, urljoin
+from urllib import pathname2url, url2pathname
+from urllib2 import urlopen
+from collections import defaultdict
+from compare_locales import util
+
+
+class L10nConfigParser(object):
+ '''Helper class to gather application information from ini files.
+
+ This class is working on synchronous open to read files or web data.
+ Subclass this and overwrite loadConfigs and addChild if you need async.
+ '''
+ def __init__(self, inipath, **kwargs):
+ """Constructor for L10nConfigParsers
+
+ inipath -- l10n.ini path
+ Optional keyword arguments are fowarded to the inner ConfigParser as
+ defaults.
+ """
+ if os.path.isabs(inipath):
+ self.inipath = 'file:%s' % pathname2url(inipath)
+ else:
+ pwdurl = 'file:%s/' % pathname2url(os.getcwd())
+ self.inipath = urljoin(pwdurl, inipath)
+ # l10n.ini files can import other l10n.ini files, store the
+ # corresponding L10nConfigParsers
+ self.children = []
+ # we really only care about the l10n directories described in l10n.ini
+ self.dirs = []
+ # optional defaults to be passed to the inner ConfigParser (unused?)
+ self.defaults = kwargs
+
+ def getDepth(self, cp):
+ '''Get the depth for the comparison from the parsed l10n.ini.
+
+ Overloadable to get the source depth for fennec and friends.
+ '''
+ try:
+ depth = cp.get('general', 'depth')
+ except:
+ depth = '.'
+ return depth
+
+ def getFilters(self):
+ '''Get the test functions from this ConfigParser and all children.
+
+ Only works with synchronous loads, used by compare-locales, which
+ is local anyway.
+ '''
+ filterurl = urljoin(self.inipath, 'filter.py')
+ try:
+ l = {}
+ execfile(url2pathname(urlparse(filterurl).path), {}, l)
+ if 'test' in l and callable(l['test']):
+ filters = [l['test']]
+ else:
+ filters = []
+ except:
+ filters = []
+
+ for c in self.children:
+ filters += c.getFilters()
+
+ return filters
+
+ def loadConfigs(self):
+ """Entry point to load the l10n.ini file this Parser refers to.
+
+ This implementation uses synchronous loads, subclasses might overload
+ this behaviour. If you do, make sure to pass a file-like object
+ to onLoadConfig.
+ """
+ self.onLoadConfig(urlopen(self.inipath))
+
+ def onLoadConfig(self, inifile):
+ """Parse a file-like object for the loaded l10n.ini file."""
+ cp = ConfigParser(self.defaults)
+ cp.readfp(inifile)
+ depth = self.getDepth(cp)
+ self.baseurl = urljoin(self.inipath, depth)
+ # create child loaders for any other l10n.ini files to be included
+ try:
+ for title, path in cp.items('includes'):
+ # skip default items
+ if title in self.defaults:
+ continue
+ # add child config parser
+ self.addChild(title, path, cp)
+ except NoSectionError:
+ pass
+ # try to load the "dirs" defined in the "compare" section
+ try:
+ self.dirs.extend(cp.get('compare', 'dirs').split())
+ except (NoOptionError, NoSectionError):
+ pass
+ # try getting a top level compare dir, as used for fennec
+ try:
+ self.tld = cp.get('compare', 'tld')
+ # remove tld from comparison dirs
+ if self.tld in self.dirs:
+ self.dirs.remove(self.tld)
+ except (NoOptionError, NoSectionError):
+ self.tld = None
+ # try to set "all_path" and "all_url"
+ try:
+ self.all_path = cp.get('general', 'all')
+ self.all_url = urljoin(self.baseurl, self.all_path)
+ except (NoOptionError, NoSectionError):
+ self.all_path = None
+ self.all_url = None
+ return cp
+
+ def addChild(self, title, path, orig_cp):
+ """Create a child L10nConfigParser and load it.
+
+ title -- indicates the module's name
+ path -- indicates the path to the module's l10n.ini file
+ orig_cp -- the configuration parser of this l10n.ini
+ """
+ cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults)
+ cp.loadConfigs()
+ self.children.append(cp)
+
+ def getTLDPathsTuple(self, basepath):
+ """Given the basepath, return the path fragments to be used for
+ self.tld. For build runs, this is (basepath, self.tld), for
+ source runs, just (basepath,).
+
+ @see overwritten method in SourceTreeConfigParser.
+ """
+ return (basepath, self.tld)
+
+ def dirsIter(self):
+ """Iterate over all dirs and our base path for this l10n.ini"""
+ url = urlparse(self.baseurl)
+ basepath = url2pathname(url.path)
+ if self.tld is not None:
+ yield self.tld, self.getTLDPathsTuple(basepath)
+ for dir in self.dirs:
+ yield dir, (basepath, dir)
+
+ def directories(self):
+ """Iterate over all dirs and base paths for this l10n.ini as well
+ as the included ones.
+ """
+ for t in self.dirsIter():
+ yield t
+ for child in self.children:
+ for t in child.directories():
+ yield t
+
+ def allLocales(self):
+ """Return a list of all the locales of this project"""
+ return util.parseLocales(urlopen(self.all_url).read())
+
+
+class SourceTreeConfigParser(L10nConfigParser):
+ '''Subclassing L10nConfigParser to work with just the repos
+ checked out next to each other instead of intermingled like
+ we do for real builds.
+ '''
+
+ def __init__(self, inipath, basepath):
+ '''Add additional arguments basepath.
+
+ basepath is used to resolve local paths via branchnames.
+ '''
+ L10nConfigParser.__init__(self, inipath)
+ self.basepath = basepath
+ self.tld = None
+
+ def getDepth(self, cp):
+ '''Get the depth for the comparison from the parsed l10n.ini.
+
+ Overloaded to get the source depth for fennec and friends.
+ '''
+ try:
+ depth = cp.get('general', 'source-depth')
+ except:
+ try:
+ depth = cp.get('general', 'depth')
+ except:
+ depth = '.'
+ return depth
+
+ def addChild(self, title, path, orig_cp):
+ # check if there's a section with details for this include
+ # we might have to check a different repo, or even VCS
+ # for example, projects like "mail" indicate in
+ # an "include_" section where to find the l10n.ini for "toolkit"
+ details = 'include_' + title
+ if orig_cp.has_section(details):
+ branch = orig_cp.get(details, 'mozilla')
+ inipath = orig_cp.get(details, 'l10n.ini')
+ path = self.basepath + '/' + branch + '/' + inipath
+ else:
+ path = urljoin(self.baseurl, path)
+ cp = SourceTreeConfigParser(path, self.basepath, **self.defaults)
+ cp.loadConfigs()
+ self.children.append(cp)
+
+ def getTLDPathsTuple(self, basepath):
+ """Overwrite L10nConfigParser's getTLDPathsTuple to just return
+ the basepath.
+ """
+ return (basepath, )
+
+
+class File(object):
+
+ def __init__(self, fullpath, file, module=None, locale=None):
+ self.fullpath = fullpath
+ self.file = file
+ self.module = module
+ self.locale = locale
+ pass
+
+ def getContents(self):
+ # open with universal line ending support and read
+ return open(self.fullpath, 'rU').read()
+
+ def __hash__(self):
+ f = self.file
+ if self.module:
+ f = self.module + '/' + f
+ return hash(f)
+
+ def __str__(self):
+ return self.fullpath
+
+ def __cmp__(self, other):
+ if not isinstance(other, File):
+ raise NotImplementedError
+ rv = cmp(self.module, other.module)
+ if rv != 0:
+ return rv
+ return cmp(self.file, other.file)
+
+
+class EnumerateDir(object):
+ ignore_dirs = ['CVS', '.svn', '.hg', '.git']
+
+ def __init__(self, basepath, module='', locale=None, ignore_subdirs=[]):
+ self.basepath = basepath
+ self.module = module
+ self.locale = locale
+ self.ignore_subdirs = ignore_subdirs
+ pass
+
+ def cloneFile(self, other):
+ '''
+ Return a File object that this enumerator would return, if it had it.
+ '''
+ return File(os.path.join(self.basepath, other.file), other.file,
+ self.module, self.locale)
+
+ def __iter__(self):
+ # our local dirs are given as a tuple of path segments, starting off
+ # with an empty sequence for the basepath.
+ dirs = [()]
+ while dirs:
+ dir = dirs.pop(0)
+ fulldir = os.path.join(self.basepath, *dir)
+ try:
+ entries = os.listdir(fulldir)
+ except OSError:
+ # we probably just started off in a non-existing dir, ignore
+ continue
+ entries.sort()
+ for entry in entries:
+ leaf = os.path.join(fulldir, entry)
+ if os.path.isdir(leaf):
+ if entry not in self.ignore_dirs and \
+ leaf not in [os.path.join(self.basepath, d)
+ for d in self.ignore_subdirs]:
+ dirs.append(dir + (entry,))
+ continue
+ yield File(leaf, '/'.join(dir + (entry,)),
+ self.module, self.locale)
+
+
+class LocalesWrap(object):
+
+ def __init__(self, base, module, locales, ignore_subdirs=[]):
+ self.base = base
+ self.module = module
+ self.locales = locales
+ self.ignore_subdirs = ignore_subdirs
+
+ def __iter__(self):
+ for locale in self.locales:
+ path = os.path.join(self.base, locale, self.module)
+ yield (locale, EnumerateDir(path, self.module, locale,
+ self.ignore_subdirs))
+
+
+class EnumerateApp(object):
+ reference = 'en-US'
+
+ def __init__(self, inipath, l10nbase, locales=None):
+ self.setupConfigParser(inipath)
+ self.modules = defaultdict(dict)
+ self.l10nbase = os.path.abspath(l10nbase)
+ self.filters = []
+ drive, tail = os.path.splitdrive(inipath)
+ self.addFilters(*self.config.getFilters())
+ self.locales = locales or self.config.allLocales()
+ self.locales.sort()
+
+ def setupConfigParser(self, inipath):
+ self.config = L10nConfigParser(inipath)
+ self.config.loadConfigs()
+
+ def addFilters(self, *args):
+ self.filters += args
+
+ value_map = {None: None, 'error': 0, 'ignore': 1, 'report': 2}
+
+ def filter(self, l10n_file, entity=None):
+ '''Go through all added filters, and,
+ - map "error" -> 0, "ignore" -> 1, "report" -> 2
+ - if filter.test returns a bool, map that to
+ False -> "ignore" (1), True -> "error" (0)
+ - take the max of all reported
+ '''
+ rv = 0
+ for f in reversed(self.filters):
+ try:
+ _r = f(l10n_file.module, l10n_file.file, entity)
+ except:
+ # XXX error handling
+ continue
+ if isinstance(_r, bool):
+ _r = [1, 0][_r]
+ else:
+ # map string return value to int, default to 'error',
+ # None is None
+ _r = self.value_map.get(_r, 0)
+ if _r is not None:
+ rv = max(rv, _r)
+ return ['error', 'ignore', 'report'][rv]
+
+ def __iter__(self):
+ '''
+ Iterate over all modules, return en-US directory enumerator, and an
+ iterator over all locales in each iteration. Per locale, the locale
+ code and an directory enumerator will be given.
+ '''
+ dirmap = dict(self.config.directories())
+ mods = dirmap.keys()
+ mods.sort()
+ for mod in mods:
+ if self.reference == 'en-US':
+ base = os.path.join(*(dirmap[mod] + ('locales', 'en-US')))
+ else:
+ base = os.path.join(self.l10nbase, self.reference, mod)
+ yield (mod, EnumerateDir(base, mod, self.reference),
+ LocalesWrap(self.l10nbase, mod, self.locales,
+ [m[len(mod)+1:] for m in mods if m.startswith(mod+'/')]))
+
+
+class EnumerateSourceTreeApp(EnumerateApp):
+ '''Subclass EnumerateApp to work on side-by-side checked out
+ repos, and to no pay attention to how the source would actually
+ be checked out for building.
+
+ It's supporting applications like Fennec, too, which have
+ 'locales/en-US/...' in their root dir, but claim to be 'mobile'.
+ '''
+
+ def __init__(self, inipath, basepath, l10nbase, locales=None):
+ self.basepath = basepath
+ EnumerateApp.__init__(self, inipath, l10nbase, locales)
+
+ def setupConfigParser(self, inipath):
+ self.config = SourceTreeConfigParser(inipath, self.basepath)
+ self.config.loadConfigs()
+
+
+def get_base_path(mod, loc):
+ 'statics for path patterns and conversion'
+ __l10n = 'l10n/%(loc)s/%(mod)s'
+ __en_US = 'mozilla/%(mod)s/locales/en-US'
+ if loc == 'en-US':
+ return __en_US % {'mod': mod}
+ return __l10n % {'mod': mod, 'loc': loc}
+
+
+def get_path(mod, loc, leaf):
+ return get_base_path(mod, loc) + '/' + leaf