summaryrefslogtreecommitdiffstats
path: root/python/mozbuild/mozpack/packager
diff options
context:
space:
mode:
Diffstat (limited to 'python/mozbuild/mozpack/packager')
-rw-r--r--python/mozbuild/mozpack/packager/__init__.py408
-rw-r--r--python/mozbuild/mozpack/packager/formats.py324
-rw-r--r--python/mozbuild/mozpack/packager/l10n.py259
-rw-r--r--python/mozbuild/mozpack/packager/unpack.py202
4 files changed, 1193 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/packager/__init__.py b/python/mozbuild/mozpack/packager/__init__.py
new file mode 100644
index 000000000..4c98ec3d3
--- /dev/null
+++ b/python/mozbuild/mozpack/packager/__init__.py
@@ -0,0 +1,408 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from mozbuild.preprocessor import Preprocessor
+import re
+import os
+from mozpack.errors import errors
+from mozpack.chrome.manifest import (
+ Manifest,
+ ManifestBinaryComponent,
+ ManifestChrome,
+ ManifestInterfaces,
+ is_manifest,
+ parse_manifest,
+)
+import mozpack.path as mozpath
+from collections import deque
+
+
+class Component(object):
+ '''
+ Class that represents a component in a package manifest.
+ '''
+ def __init__(self, name, destdir=''):
+ if name.find(' ') > 0:
+ errors.fatal('Malformed manifest: space in component name "%s"'
+ % component)
+ self._name = name
+ self._destdir = destdir
+
+ def __repr__(self):
+ s = self.name
+ if self.destdir:
+ s += ' destdir="%s"' % self.destdir
+ return s
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def destdir(self):
+ return self._destdir
+
+ @staticmethod
+ def _triples(lst):
+ '''
+ Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
+ '''
+ return zip(*[iter(lst)] * 3)
+
+ KEY_VALUE_RE = re.compile(r'''
+ \s* # optional whitespace.
+ ([a-zA-Z0-9_]+) # key.
+ \s*=\s* # optional space around =.
+ "([^"]*)" # value without surrounding quotes.
+ (?:\s+|$)
+ ''', re.VERBOSE)
+
+ @staticmethod
+ def _split_options(string):
+ '''
+ Split 'key1="value1" key2="value2"' into
+ {'key1':'value1', 'key2':'value2'}.
+
+ Returned keys and values are all strings.
+
+ Throws ValueError if the input is malformed.
+ '''
+ options = {}
+ splits = Component.KEY_VALUE_RE.split(string)
+ if len(splits) % 3 != 1:
+ # This should never happen -- we expect to always split
+ # into ['', ('key', 'val', '')*].
+ raise ValueError("Bad input")
+ if splits[0]:
+ raise ValueError('Unrecognized input ' + splits[0])
+ for key, val, no_match in Component._triples(splits[1:]):
+ if no_match:
+ raise ValueError('Unrecognized input ' + no_match)
+ options[key] = val
+ return options
+
+ @staticmethod
+ def _split_component_and_options(string):
+ '''
+ Split 'name key1="value1" key2="value2"' into
+ ('name', {'key1':'value1', 'key2':'value2'}).
+
+ Returned name, keys and values are all strings.
+
+ Raises ValueError if the input is malformed.
+ '''
+ splits = string.strip().split(None, 1)
+ if not splits:
+ raise ValueError('No component found')
+ component = splits[0].strip()
+ if not component:
+ raise ValueError('No component found')
+ if not re.match('[a-zA-Z0-9_\-]+$', component):
+ raise ValueError('Bad component name ' + component)
+ options = Component._split_options(splits[1]) if len(splits) > 1 else {}
+ return component, options
+
+ @staticmethod
+ def from_string(string):
+ '''
+ Create a component from a string.
+ '''
+ try:
+ name, options = Component._split_component_and_options(string)
+ except ValueError as e:
+ errors.fatal('Malformed manifest: %s' % e)
+ return
+ destdir = options.pop('destdir', '')
+ if options:
+ errors.fatal('Malformed manifest: options %s not recognized'
+ % options.keys())
+ return Component(name, destdir=destdir)
+
+
+class PackageManifestParser(object):
+ '''
+ Class for parsing of a package manifest, after preprocessing.
+
+ A package manifest is a list of file paths, with some syntaxic sugar:
+ [] designates a toplevel component. Example: [xpcom]
+ - in front of a file specifies it to be removed
+ * wildcard support
+ ** expands to all files and zero or more directories
+ ; file comment
+
+ The parser takes input from the preprocessor line by line, and pushes
+ parsed information to a sink object.
+
+ The add and remove methods of the sink object are called with the
+ current Component instance and a path.
+ '''
+ def __init__(self, sink):
+ '''
+ Initialize the package manifest parser with the given sink.
+ '''
+ self._component = Component('')
+ self._sink = sink
+
+ def handle_line(self, str):
+ '''
+ Handle a line of input and push the parsed information to the sink
+ object.
+ '''
+ # Remove comments.
+ str = str.strip()
+ if not str or str.startswith(';'):
+ return
+ if str.startswith('[') and str.endswith(']'):
+ self._component = Component.from_string(str[1:-1])
+ elif str.startswith('-'):
+ str = str[1:]
+ self._sink.remove(self._component, str)
+ elif ',' in str:
+ errors.fatal('Incompatible syntax')
+ else:
+ self._sink.add(self._component, str)
+
+
+class PreprocessorOutputWrapper(object):
+ '''
+ File-like helper to handle the preprocessor output and send it to a parser.
+ The parser's handle_line method is called in the relevant errors.context.
+ '''
+ def __init__(self, preprocessor, parser):
+ self._parser = parser
+ self._pp = preprocessor
+
+ def write(self, str):
+ file = os.path.normpath(os.path.abspath(self._pp.context['FILE']))
+ with errors.context(file, self._pp.context['LINE']):
+ self._parser.handle_line(str)
+
+
+def preprocess(input, parser, defines={}):
+ '''
+ Preprocess the file-like input with the given defines, and send the
+ preprocessed output line by line to the given parser.
+ '''
+ pp = Preprocessor()
+ pp.context.update(defines)
+ pp.do_filter('substitution')
+ pp.out = PreprocessorOutputWrapper(pp, parser)
+ pp.do_include(input)
+
+
+def preprocess_manifest(sink, manifest, defines={}):
+ '''
+ Preprocess the given file-like manifest with the given defines, and push
+ the parsed information to a sink. See PackageManifestParser documentation
+ for more details on the sink.
+ '''
+ preprocess(manifest, PackageManifestParser(sink), defines)
+
+
+class CallDeque(deque):
+ '''
+ Queue of function calls to make.
+ '''
+ def append(self, function, *args):
+ deque.append(self, (errors.get_context(), function, args))
+
+ def execute(self):
+ while True:
+ try:
+ context, function, args = self.popleft()
+ except IndexError:
+ return
+ if context:
+ with errors.context(context[0], context[1]):
+ function(*args)
+ else:
+ function(*args)
+
+
+class SimplePackager(object):
+ '''
+ Helper used to translate and buffer instructions from the
+ SimpleManifestSink to a formatter. Formatters expect some information to be
+ given first that the simple manifest contents can't guarantee before the
+ end of the input.
+ '''
+ def __init__(self, formatter):
+ self.formatter = formatter
+ # Queue for formatter.add_interfaces()/add_manifest() calls.
+ self._queue = CallDeque()
+ # Queue for formatter.add_manifest() calls for ManifestChrome.
+ self._chrome_queue = CallDeque()
+ # Queue for formatter.add() calls.
+ self._file_queue = CallDeque()
+ # All paths containing addons. (key is path, value is whether it
+ # should be packed or unpacked)
+ self._addons = {}
+ # All manifest paths imported.
+ self._manifests = set()
+ # All manifest paths included from some other manifest.
+ self._included_manifests = {}
+ self._closed = False
+
+ # Parsing RDF is complex, and would require an external library to do
+ # properly. Just go with some hackish but probably sufficient regexp
+ UNPACK_ADDON_RE = re.compile(r'''(?:
+ <em:unpack>true</em:unpack>
+ |em:unpack=(?P<quote>["']?)true(?P=quote)
+ )''', re.VERBOSE)
+
+ def add(self, path, file):
+ '''
+ Add the given BaseFile instance with the given path.
+ '''
+ assert not self._closed
+ if is_manifest(path):
+ self._add_manifest_file(path, file)
+ elif path.endswith('.xpt'):
+ self._queue.append(self.formatter.add_interfaces, path, file)
+ else:
+ self._file_queue.append(self.formatter.add, path, file)
+ if mozpath.basename(path) == 'install.rdf':
+ addon = True
+ install_rdf = file.open().read()
+ if self.UNPACK_ADDON_RE.search(install_rdf):
+ addon = 'unpacked'
+ self._addons[mozpath.dirname(path)] = addon
+
+ def _add_manifest_file(self, path, file):
+ '''
+ Add the given BaseFile with manifest file contents with the given path.
+ '''
+ self._manifests.add(path)
+ base = ''
+ if hasattr(file, 'path'):
+ # Find the directory the given path is relative to.
+ b = mozpath.normsep(file.path)
+ if b.endswith('/' + path) or b == path:
+ base = os.path.normpath(b[:-len(path)])
+ for e in parse_manifest(base, path, file.open()):
+ # ManifestResources need to be given after ManifestChrome, so just
+ # put all ManifestChrome in a separate queue to make them first.
+ if isinstance(e, ManifestChrome):
+ # e.move(e.base) just returns a clone of the entry.
+ self._chrome_queue.append(self.formatter.add_manifest,
+ e.move(e.base))
+ elif not isinstance(e, (Manifest, ManifestInterfaces)):
+ self._queue.append(self.formatter.add_manifest, e.move(e.base))
+ # If a binary component is added to an addon, prevent the addon
+ # from being packed.
+ if isinstance(e, ManifestBinaryComponent):
+ addon = mozpath.basedir(e.base, self._addons)
+ if addon:
+ self._addons[addon] = 'unpacked'
+ if isinstance(e, Manifest):
+ if e.flags:
+ errors.fatal('Flags are not supported on ' +
+ '"manifest" entries')
+ self._included_manifests[e.path] = path
+
+ def get_bases(self, addons=True):
+ '''
+ Return all paths under which root manifests have been found. Root
+ manifests are manifests that are included in no other manifest.
+ `addons` indicates whether to include addon bases as well.
+ '''
+ all_bases = set(mozpath.dirname(m)
+ for m in self._manifests
+ - set(self._included_manifests))
+ if not addons:
+ all_bases -= set(self._addons)
+ else:
+ # If for some reason some detected addon doesn't have a
+ # non-included manifest.
+ all_bases |= set(self._addons)
+ return all_bases
+
+ def close(self):
+ '''
+ Push all instructions to the formatter.
+ '''
+ self._closed = True
+
+ bases = self.get_bases()
+ broken_bases = sorted(
+ m for m, includer in self._included_manifests.iteritems()
+ if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
+ for m in broken_bases:
+ errors.fatal('"%s" is included from "%s", which is outside "%s"' %
+ (m, self._included_manifests[m],
+ mozpath.basedir(m, bases)))
+ for base in sorted(bases):
+ self.formatter.add_base(base, self._addons.get(base, False))
+ self._chrome_queue.execute()
+ self._queue.execute()
+ self._file_queue.execute()
+
+
+class SimpleManifestSink(object):
+ '''
+ Parser sink for "simple" package manifests. Simple package manifests use
+ the format described in the PackageManifestParser documentation, but don't
+ support file removals, and require manifests, interfaces and chrome data to
+ be explicitely listed.
+ Entries starting with bin/ are searched under bin/ in the FileFinder, but
+ are packaged without the bin/ prefix.
+ '''
+ def __init__(self, finder, formatter):
+ '''
+ Initialize the SimpleManifestSink. The given FileFinder is used to
+ get files matching the patterns given in the manifest. The given
+ formatter does the packaging job.
+ '''
+ self._finder = finder
+ self.packager = SimplePackager(formatter)
+ self._closed = False
+ self._manifests = set()
+
+ @staticmethod
+ def normalize_path(path):
+ '''
+ Remove any bin/ prefix.
+ '''
+ if mozpath.basedir(path, ['bin']) == 'bin':
+ return mozpath.relpath(path, 'bin')
+ return path
+
+ def add(self, component, pattern):
+ '''
+ Add files with the given pattern in the given component.
+ '''
+ assert not self._closed
+ added = False
+ for p, f in self._finder.find(pattern):
+ added = True
+ if is_manifest(p):
+ self._manifests.add(p)
+ dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p))
+ self.packager.add(dest, f)
+ if not added:
+ errors.error('Missing file(s): %s' % pattern)
+
+ def remove(self, component, pattern):
+ '''
+ Remove files with the given pattern in the given component.
+ '''
+ assert not self._closed
+ errors.fatal('Removal is unsupported')
+
+ def close(self, auto_root_manifest=True):
+ '''
+ Add possibly missing bits and push all instructions to the formatter.
+ '''
+ if auto_root_manifest:
+ # Simple package manifests don't contain the root manifests, so
+ # find and add them.
+ paths = [mozpath.dirname(m) for m in self._manifests]
+ path = mozpath.dirname(mozpath.commonprefix(paths))
+ for p, f in self._finder.find(mozpath.join(path,
+ 'chrome.manifest')):
+ if not p in self._manifests:
+ self.packager.add(SimpleManifestSink.normalize_path(p), f)
+ self.packager.close()
diff --git a/python/mozbuild/mozpack/packager/formats.py b/python/mozbuild/mozpack/packager/formats.py
new file mode 100644
index 000000000..c4adabab0
--- /dev/null
+++ b/python/mozbuild/mozpack/packager/formats.py
@@ -0,0 +1,324 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from mozpack.chrome.manifest import (
+ Manifest,
+ ManifestInterfaces,
+ ManifestChrome,
+ ManifestBinaryComponent,
+ ManifestResource,
+)
+from urlparse import urlparse
+import mozpack.path as mozpath
+from mozpack.files import (
+ ManifestFile,
+ XPTFile,
+)
+from mozpack.copier import (
+ FileRegistry,
+ FileRegistrySubtree,
+ Jarrer,
+)
+
+STARTUP_CACHE_PATHS = [
+ 'jsloader',
+ 'jssubloader',
+]
+
+'''
+Formatters are classes receiving packaging instructions and creating the
+appropriate package layout.
+
+There are three distinct formatters, each handling one of the different chrome
+formats:
+ - flat: essentially, copies files from the source with the same file system
+ layout. Manifests entries are grouped in a single manifest per directory,
+ as well as XPT interfaces.
+ - jar: chrome content is packaged in jar files.
+ - omni: chrome content, modules, non-binary components, and many other
+ elements are packaged in an omnijar file for each base directory.
+
+The base interface provides the following methods:
+ - add_base(path [, addon])
+ Register a base directory for an application or GRE, or an addon.
+ Base directories usually contain a root manifest (manifests not
+ included in any other manifest) named chrome.manifest.
+ The optional addon argument tells whether the base directory
+ is that of a packed addon (True), unpacked addon ('unpacked') or
+ otherwise (False).
+ - add(path, content)
+ Add the given content (BaseFile instance) at the given virtual path
+ - add_interfaces(path, content)
+ Add the given content (BaseFile instance) and link it to other
+ interfaces in the parent directory of the given virtual path.
+ - add_manifest(entry)
+ Add a ManifestEntry.
+ - contains(path)
+ Returns whether the given virtual path is known of the formatter.
+
+The virtual paths mentioned above are paths as they would be with a flat
+chrome.
+
+Formatters all take a FileCopier instance they will fill with the packaged
+data.
+'''
+
+
+class PiecemealFormatter(object):
+ '''
+ Generic formatter that dispatches across different sub-formatters
+ according to paths.
+ '''
+ def __init__(self, copier):
+ assert isinstance(copier, (FileRegistry, FileRegistrySubtree))
+ self.copier = copier
+ self._sub_formatter = {}
+ self._frozen_bases = False
+
+ def add_base(self, base, addon=False):
+ # Only allow to add a base directory before calls to _get_base()
+ assert not self._frozen_bases
+ assert base not in self._sub_formatter
+ self._add_base(base, addon)
+
+ def _get_base(self, path):
+ '''
+ Return the deepest base directory containing the given path.
+ '''
+ self._frozen_bases = True
+ base = mozpath.basedir(path, self._sub_formatter.keys())
+ relpath = mozpath.relpath(path, base) if base else path
+ return base, relpath
+
+ def add(self, path, content):
+ base, relpath = self._get_base(path)
+ if base is None:
+ return self.copier.add(relpath, content)
+ return self._sub_formatter[base].add(relpath, content)
+
+ def add_manifest(self, entry):
+ base, relpath = self._get_base(entry.base)
+ assert base is not None
+ return self._sub_formatter[base].add_manifest(entry.move(relpath))
+
+ def add_interfaces(self, path, content):
+ base, relpath = self._get_base(path)
+ assert base is not None
+ return self._sub_formatter[base].add_interfaces(relpath, content)
+
+ def contains(self, path):
+ assert '*' not in path
+ base, relpath = self._get_base(path)
+ if base is None:
+ return self.copier.contains(relpath)
+ return self._sub_formatter[base].contains(relpath)
+
+
+class FlatFormatter(PiecemealFormatter):
+ '''
+ Formatter for the flat package format.
+ '''
+ def _add_base(self, base, addon=False):
+ self._sub_formatter[base] = FlatSubFormatter(
+ FileRegistrySubtree(base, self.copier))
+
+
+class FlatSubFormatter(object):
+ '''
+ Sub-formatter for the flat package format.
+ '''
+ def __init__(self, copier):
+ assert isinstance(copier, (FileRegistry, FileRegistrySubtree))
+ self.copier = copier
+
+ def add(self, path, content):
+ self.copier.add(path, content)
+
+ def add_manifest(self, entry):
+ # Store manifest entries in a single manifest per directory, named
+ # after their parent directory, except for root manifests, all named
+ # chrome.manifest.
+ if entry.base:
+ name = mozpath.basename(entry.base)
+ else:
+ name = 'chrome'
+ path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
+ if not self.copier.contains(path):
+ # Add a reference to the manifest file in the parent manifest, if
+ # the manifest file is not a root manifest.
+ if entry.base:
+ parent = mozpath.dirname(entry.base)
+ relbase = mozpath.basename(entry.base)
+ relpath = mozpath.join(relbase,
+ mozpath.basename(path))
+ self.add_manifest(Manifest(parent, relpath))
+ self.copier.add(path, ManifestFile(entry.base))
+ self.copier[path].add(entry)
+
+ def add_interfaces(self, path, content):
+ # Interfaces in the same directory are all linked together in an
+ # interfaces.xpt file.
+ interfaces_path = mozpath.join(mozpath.dirname(path),
+ 'interfaces.xpt')
+ if not self.copier.contains(interfaces_path):
+ self.add_manifest(ManifestInterfaces(mozpath.dirname(path),
+ 'interfaces.xpt'))
+ self.copier.add(interfaces_path, XPTFile())
+ self.copier[interfaces_path].add(content)
+
+ def contains(self, path):
+ assert '*' not in path
+ return self.copier.contains(path)
+
+
+class JarFormatter(PiecemealFormatter):
+ '''
+ Formatter for the jar package format. Assumes manifest entries related to
+ chrome are registered before the chrome data files are added. Also assumes
+ manifest entries for resources are registered after chrome manifest
+ entries.
+ '''
+ def __init__(self, copier, compress=True, optimize=True):
+ PiecemealFormatter.__init__(self, copier)
+ self._compress=compress
+ self._optimize=optimize
+
+ def _add_base(self, base, addon=False):
+ if addon is True:
+ jarrer = Jarrer(self._compress, self._optimize)
+ self.copier.add(base + '.xpi', jarrer)
+ self._sub_formatter[base] = FlatSubFormatter(jarrer)
+ else:
+ self._sub_formatter[base] = JarSubFormatter(
+ FileRegistrySubtree(base, self.copier),
+ self._compress, self._optimize)
+
+
+class JarSubFormatter(PiecemealFormatter):
+ '''
+ Sub-formatter for the jar package format. It is a PiecemealFormatter that
+ dispatches between further sub-formatter for each of the jar files it
+ dispatches the chrome data to, and a FlatSubFormatter for the non-chrome
+ files.
+ '''
+ def __init__(self, copier, compress=True, optimize=True):
+ PiecemealFormatter.__init__(self, copier)
+ self._frozen_chrome = False
+ self._compress = compress
+ self._optimize = optimize
+ self._sub_formatter[''] = FlatSubFormatter(copier)
+
+ def _jarize(self, entry, relpath):
+ '''
+ Transform a manifest entry in one pointing to chrome data in a jar.
+ Return the corresponding chrome path and the new entry.
+ '''
+ base = entry.base
+ basepath = mozpath.split(relpath)[0]
+ chromepath = mozpath.join(base, basepath)
+ entry = entry.rebase(chromepath) \
+ .move(mozpath.join(base, 'jar:%s.jar!' % basepath)) \
+ .rebase(base)
+ return chromepath, entry
+
+ def add_manifest(self, entry):
+ if isinstance(entry, ManifestChrome) and \
+ not urlparse(entry.relpath).scheme:
+ chromepath, entry = self._jarize(entry, entry.relpath)
+ assert not self._frozen_chrome
+ if chromepath not in self._sub_formatter:
+ jarrer = Jarrer(self._compress, self._optimize)
+ self.copier.add(chromepath + '.jar', jarrer)
+ self._sub_formatter[chromepath] = FlatSubFormatter(jarrer)
+ elif isinstance(entry, ManifestResource) and \
+ not urlparse(entry.target).scheme:
+ chromepath, new_entry = self._jarize(entry, entry.target)
+ if chromepath in self._sub_formatter:
+ entry = new_entry
+ PiecemealFormatter.add_manifest(self, entry)
+
+
+class OmniJarFormatter(JarFormatter):
+ '''
+ Formatter for the omnijar package format.
+ '''
+ def __init__(self, copier, omnijar_name, compress=True, optimize=True,
+ non_resources=()):
+ JarFormatter.__init__(self, copier, compress, optimize)
+ self._omnijar_name = omnijar_name
+ self._non_resources = non_resources
+
+ def _add_base(self, base, addon=False):
+ if addon:
+ JarFormatter._add_base(self, base, addon)
+ else:
+ # Initialize a chrome.manifest next to the omnijar file so that
+ # there's always a chrome.manifest file, even an empty one.
+ path = mozpath.normpath(mozpath.join(base, 'chrome.manifest'))
+ if not self.copier.contains(path):
+ self.copier.add(path, ManifestFile(''))
+ self._sub_formatter[base] = OmniJarSubFormatter(
+ FileRegistrySubtree(base, self.copier), self._omnijar_name,
+ self._compress, self._optimize, self._non_resources)
+
+
+class OmniJarSubFormatter(PiecemealFormatter):
+ '''
+ Sub-formatter for the omnijar package format. It is a PiecemealFormatter
+ that dispatches between a FlatSubFormatter for the resources data and
+ another FlatSubFormatter for the other files.
+ '''
+ def __init__(self, copier, omnijar_name, compress=True, optimize=True,
+ non_resources=()):
+ PiecemealFormatter.__init__(self, copier)
+ self._omnijar_name = omnijar_name
+ self._compress = compress
+ self._optimize = optimize
+ self._non_resources = non_resources
+ self._sub_formatter[''] = FlatSubFormatter(copier)
+ jarrer = Jarrer(self._compress, self._optimize)
+ self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer)
+
+ def _get_base(self, path):
+ base = self._omnijar_name if self.is_resource(path) else ''
+ # Only add the omnijar file if something ends up in it.
+ if base and not self.copier.contains(base):
+ self.copier.add(base, self._sub_formatter[base].copier)
+ return base, path
+
+ def add_manifest(self, entry):
+ base = ''
+ if not isinstance(entry, ManifestBinaryComponent):
+ base = self._omnijar_name
+ formatter = self._sub_formatter[base]
+ return formatter.add_manifest(entry)
+
+ def is_resource(self, path):
+ '''
+ Return whether the given path corresponds to a resource to be put in an
+ omnijar archive.
+ '''
+ if any(mozpath.match(path, p.replace('*', '**'))
+ for p in self._non_resources):
+ return False
+ path = mozpath.split(path)
+ if path[0] == 'chrome':
+ return len(path) == 1 or path[1] != 'icons'
+ if path[0] == 'components':
+ return path[-1].endswith(('.js', '.xpt'))
+ if path[0] == 'res':
+ return len(path) == 1 or \
+ (path[1] != 'cursors' and path[1] != 'MainMenu.nib')
+ if path[0] == 'defaults':
+ return len(path) != 3 or \
+ not (path[2] == 'channel-prefs.js' and
+ path[1] in ['pref', 'preferences'])
+ return path[0] in [
+ 'modules',
+ 'greprefs.js',
+ 'hyphenation',
+ 'update.locale',
+ ] or path[0] in STARTUP_CACHE_PATHS
diff --git a/python/mozbuild/mozpack/packager/l10n.py b/python/mozbuild/mozpack/packager/l10n.py
new file mode 100644
index 000000000..758064f59
--- /dev/null
+++ b/python/mozbuild/mozpack/packager/l10n.py
@@ -0,0 +1,259 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+'''
+Replace localized parts of a packaged directory with data from a langpack
+directory.
+'''
+
+import os
+import mozpack.path as mozpath
+from mozpack.packager.formats import (
+ FlatFormatter,
+ JarFormatter,
+ OmniJarFormatter,
+)
+from mozpack.packager import (
+ Component,
+ SimplePackager,
+ SimpleManifestSink,
+)
+from mozpack.files import (
+ ComposedFinder,
+ ManifestFile,
+)
+from mozpack.copier import (
+ FileCopier,
+ Jarrer,
+)
+from mozpack.chrome.manifest import (
+ ManifestLocale,
+ ManifestEntryWithRelPath,
+ is_manifest,
+ ManifestChrome,
+ Manifest,
+)
+from mozpack.errors import errors
+from mozpack.packager.unpack import UnpackFinder
+from createprecomplete import generate_precomplete
+
+
+class LocaleManifestFinder(object):
+ def __init__(self, finder):
+ entries = self.entries = []
+ bases = self.bases = []
+
+ class MockFormatter(object):
+ def add_interfaces(self, path, content):
+ pass
+
+ def add(self, path, content):
+ pass
+
+ def add_manifest(self, entry):
+ if entry.localized:
+ entries.append(entry)
+
+ def add_base(self, base, addon=False):
+ bases.append(base)
+
+ # SimplePackager rejects "manifest foo.manifest" entries with
+ # additional flags (such as "manifest foo.manifest application=bar").
+ # Those type of entries are used by language packs to work as addons,
+ # but are not necessary for the purpose of l10n repacking. So we wrap
+ # the finder in order to remove those entries.
+ class WrapFinder(object):
+ def __init__(self, finder):
+ self._finder = finder
+
+ def find(self, pattern):
+ for p, f in self._finder.find(pattern):
+ if isinstance(f, ManifestFile):
+ unwanted = [
+ e for e in f._entries
+ if isinstance(e, Manifest) and e.flags
+ ]
+ if unwanted:
+ f = ManifestFile(
+ f._base,
+ [e for e in f._entries if e not in unwanted])
+ yield p, f
+
+ sink = SimpleManifestSink(WrapFinder(finder), MockFormatter())
+ sink.add(Component(''), '*')
+ sink.close(False)
+
+ # Find unique locales used in these manifest entries.
+ self.locales = list(set(e.id for e in self.entries
+ if isinstance(e, ManifestLocale)))
+
+
+def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
+ app = LocaleManifestFinder(app_finder)
+ l10n = LocaleManifestFinder(l10n_finder)
+
+ # The code further below assumes there's only one locale replaced with
+ # another one.
+ if len(app.locales) > 1:
+ errors.fatal("Multiple app locales aren't supported: " +
+ ",".join(app.locales))
+ if len(l10n.locales) > 1:
+ errors.fatal("Multiple l10n locales aren't supported: " +
+ ",".join(l10n.locales))
+ locale = app.locales[0]
+ l10n_locale = l10n.locales[0]
+
+ # For each base directory, store what path a locale chrome package name
+ # corresponds to.
+ # e.g., for the following entry under app/chrome:
+ # locale foo en-US path/to/files
+ # keep track that the locale path for foo in app is
+ # app/chrome/path/to/files.
+ l10n_paths = {}
+ for e in l10n.entries:
+ if isinstance(e, ManifestChrome):
+ base = mozpath.basedir(e.path, app.bases)
+ l10n_paths.setdefault(base, {})
+ l10n_paths[base][e.name] = e.path
+
+ # For chrome and non chrome files or directories, store what langpack path
+ # corresponds to a package path.
+ paths = {}
+ for e in app.entries:
+ if isinstance(e, ManifestEntryWithRelPath):
+ base = mozpath.basedir(e.path, app.bases)
+ if base not in l10n_paths:
+ errors.fatal("Locale doesn't contain %s/" % base)
+ # Allow errors to accumulate
+ continue
+ if e.name not in l10n_paths[base]:
+ errors.fatal("Locale doesn't have a manifest entry for '%s'" %
+ e.name)
+ # Allow errors to accumulate
+ continue
+ paths[e.path] = l10n_paths[base][e.name]
+
+ for pattern in non_chrome:
+ for base in app.bases:
+ path = mozpath.join(base, pattern)
+ left = set(p for p, f in app_finder.find(path))
+ right = set(p for p, f in l10n_finder.find(path))
+ for p in right:
+ paths[p] = p
+ for p in left - right:
+ paths[p] = None
+
+ # Create a new package, with non localized bits coming from the original
+ # package, and localized bits coming from the langpack.
+ packager = SimplePackager(formatter)
+ for p, f in app_finder:
+ if is_manifest(p):
+ # Remove localized manifest entries.
+ for e in [e for e in f if e.localized]:
+ f.remove(e)
+ # If the path is one that needs a locale replacement, use the
+ # corresponding file from the langpack.
+ path = None
+ if p in paths:
+ path = paths[p]
+ if not path:
+ continue
+ else:
+ base = mozpath.basedir(p, paths.keys())
+ if base:
+ subpath = mozpath.relpath(p, base)
+ path = mozpath.normpath(mozpath.join(paths[base],
+ subpath))
+ if path:
+ files = [f for p, f in l10n_finder.find(path)]
+ if not len(files):
+ if base not in non_chrome:
+ finderBase = ""
+ if hasattr(l10n_finder, 'base'):
+ finderBase = l10n_finder.base
+ errors.error("Missing file: %s" %
+ os.path.join(finderBase, path))
+ else:
+ packager.add(path, files[0])
+ else:
+ packager.add(p, f)
+
+ # Add localized manifest entries from the langpack.
+ l10n_manifests = []
+ for base in set(e.base for e in l10n.entries):
+ m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
+ path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
+ l10n_manifests.append((path, m))
+ bases = packager.get_bases()
+ for path, m in l10n_manifests:
+ base = mozpath.basedir(path, bases)
+ packager.add(path, m)
+ # Add a "manifest $path" entry in the top manifest under that base.
+ m = ManifestFile(base)
+ m.add(Manifest(base, mozpath.relpath(path, base)))
+ packager.add(mozpath.join(base, 'chrome.manifest'), m)
+
+ packager.close()
+
+ # Add any remaining non chrome files.
+ for pattern in non_chrome:
+ for base in bases:
+ for p, f in l10n_finder.find(mozpath.join(base, pattern)):
+ if not formatter.contains(p):
+ formatter.add(p, f)
+
+ # Transplant jar preloading information.
+ for path, log in app_finder.jarlogs.iteritems():
+ assert isinstance(copier[path], Jarrer)
+ copier[path].preload([l.replace(locale, l10n_locale) for l in log])
+
+
+def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
+ '''
+ Replace localized data from the `source` directory with localized data
+ from `l10n` and `extra_l10n`.
+
+ The `source` argument points to a directory containing a packaged
+ application (in omnijar, jar or flat form).
+ The `l10n` argument points to a directory containing the main localized
+ data (usually in the form of a language pack addon) to use to replace
+ in the packaged application.
+ The `extra_l10n` argument contains a dict associating relative paths in
+ the source to separate directories containing localized data for them.
+ This can be used to point at different language pack addons for different
+ parts of the package application.
+ The `non_resources` argument gives a list of relative paths in the source
+ that should not be added in an omnijar in case the packaged application
+ is in that format.
+ The `non_chrome` argument gives a list of file/directory patterns for
+ localized files that are not listed in a chrome.manifest.
+ '''
+ app_finder = UnpackFinder(source)
+ l10n_finder = UnpackFinder(l10n)
+ if extra_l10n:
+ finders = {
+ '': l10n_finder,
+ }
+ for base, path in extra_l10n.iteritems():
+ finders[base] = UnpackFinder(path)
+ l10n_finder = ComposedFinder(finders)
+ copier = FileCopier()
+ if app_finder.kind == 'flat':
+ formatter = FlatFormatter(copier)
+ elif app_finder.kind == 'jar':
+ formatter = JarFormatter(copier,
+ optimize=app_finder.optimizedjars,
+ compress=app_finder.compressed)
+ elif app_finder.kind == 'omni':
+ formatter = OmniJarFormatter(copier, app_finder.omnijar,
+ optimize=app_finder.optimizedjars,
+ compress=app_finder.compressed,
+ non_resources=non_resources)
+
+ with errors.accumulate():
+ _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
+ copier.copy(source, skip_if_older=False)
+ generate_precomplete(source)
diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py
new file mode 100644
index 000000000..fa2b474e7
--- /dev/null
+++ b/python/mozbuild/mozpack/packager/unpack.py
@@ -0,0 +1,202 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import mozpack.path as mozpath
+from mozpack.files import (
+ BaseFinder,
+ FileFinder,
+ DeflatedFile,
+ ManifestFile,
+)
+from mozpack.chrome.manifest import (
+ parse_manifest,
+ ManifestEntryWithRelPath,
+ ManifestResource,
+ is_manifest,
+)
+from mozpack.mozjar import JarReader
+from mozpack.copier import (
+ FileRegistry,
+ FileCopier,
+)
+from mozpack.packager import SimplePackager
+from mozpack.packager.formats import (
+ FlatFormatter,
+ STARTUP_CACHE_PATHS,
+)
+from urlparse import urlparse
+
+
+class UnpackFinder(BaseFinder):
+ '''
+ Special Finder object that treats the source package directory as if it
+ were in the flat chrome format, whatever chrome format it actually is in.
+
+ This means that for example, paths like chrome/browser/content/... match
+ files under jar:chrome/browser.jar!/content/... in case of jar chrome
+ format.
+
+ The only argument to the constructor is a Finder instance or a path.
+ The UnpackFinder is populated with files from this Finder instance,
+ or with files from a FileFinder using the given path as its root.
+ '''
+ def __init__(self, source):
+ if isinstance(source, BaseFinder):
+ self._finder = source
+ else:
+ self._finder = FileFinder(source)
+ self.base = self._finder.base
+ self.files = FileRegistry()
+ self.kind = 'flat'
+ self.omnijar = None
+ self.jarlogs = {}
+ self.optimizedjars = False
+ self.compressed = True
+
+ jars = set()
+
+ for p, f in self._finder.find('*'):
+ # Skip the precomplete file, which is generated at packaging time.
+ if p == 'precomplete':
+ continue
+ base = mozpath.dirname(p)
+ # If the file is a zip/jar that is not a .xpi, and contains a
+ # chrome.manifest, it is an omnijar. All the files it contains
+ # go in the directory containing the omnijar. Manifests are merged
+ # if there is a corresponding manifest in the directory.
+ if not p.endswith('.xpi') and self._maybe_zip(f) and \
+ (mozpath.basename(p) == self.omnijar or
+ not self.omnijar):
+ jar = self._open_jar(p, f)
+ if 'chrome.manifest' in jar:
+ self.kind = 'omni'
+ self.omnijar = mozpath.basename(p)
+ self._fill_with_jar(base, jar)
+ continue
+ # If the file is a manifest, scan its entries for some referencing
+ # jar: urls. If there are some, the files contained in the jar they
+ # point to, go under a directory named after the jar.
+ if is_manifest(p):
+ m = self.files[p] if self.files.contains(p) \
+ else ManifestFile(base)
+ for e in parse_manifest(self.base, p, f.open()):
+ m.add(self._handle_manifest_entry(e, jars))
+ if self.files.contains(p):
+ continue
+ f = m
+ # If the file is a packed addon, unpack it under a directory named
+ # after the xpi.
+ if p.endswith('.xpi') and self._maybe_zip(f):
+ self._fill_with_jar(p[:-4], self._open_jar(p, f))
+ continue
+ if not p in jars:
+ self.files.add(p, f)
+
+ def _fill_with_jar(self, base, jar):
+ for j in jar:
+ path = mozpath.join(base, j.filename)
+ if is_manifest(j.filename):
+ m = self.files[path] if self.files.contains(path) \
+ else ManifestFile(mozpath.dirname(path))
+ for e in parse_manifest(None, path, j):
+ m.add(e)
+ if not self.files.contains(path):
+ self.files.add(path, m)
+ continue
+ else:
+ self.files.add(path, DeflatedFile(j))
+
+ def _handle_manifest_entry(self, entry, jars):
+ jarpath = None
+ if isinstance(entry, ManifestEntryWithRelPath) and \
+ urlparse(entry.relpath).scheme == 'jar':
+ jarpath, entry = self._unjarize(entry, entry.relpath)
+ elif isinstance(entry, ManifestResource) and \
+ urlparse(entry.target).scheme == 'jar':
+ jarpath, entry = self._unjarize(entry, entry.target)
+ if jarpath:
+ # Don't defer unpacking the jar file. If we already saw
+ # it, take (and remove) it from the registry. If we
+ # haven't, try to find it now.
+ if self.files.contains(jarpath):
+ jar = self.files[jarpath]
+ self.files.remove(jarpath)
+ else:
+ jar = [f for p, f in self._finder.find(jarpath)]
+ assert len(jar) == 1
+ jar = jar[0]
+ if not jarpath in jars:
+ base = mozpath.splitext(jarpath)[0]
+ for j in self._open_jar(jarpath, jar):
+ self.files.add(mozpath.join(base,
+ j.filename),
+ DeflatedFile(j))
+ jars.add(jarpath)
+ self.kind = 'jar'
+ return entry
+
+ def _open_jar(self, path, file):
+ '''
+ Return a JarReader for the given BaseFile instance, keeping a log of
+ the preloaded entries it has.
+ '''
+ jar = JarReader(fileobj=file.open())
+ if jar.is_optimized:
+ self.optimizedjars = True
+ if not any(f.compressed for f in jar):
+ self.compressed = False
+ if jar.last_preloaded:
+ jarlog = jar.entries.keys()
+ self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
+ return jar
+
+ def find(self, path):
+ for p in self.files.match(path):
+ yield p, self.files[p]
+
+ def _maybe_zip(self, file):
+ '''
+ Return whether the given BaseFile looks like a ZIP/Jar.
+ '''
+ header = file.open().read(8)
+ return len(header) == 8 and (header[0:2] == 'PK' or
+ header[4:6] == 'PK')
+
+ def _unjarize(self, entry, relpath):
+ '''
+ Transform a manifest entry pointing to chrome data in a jar in one
+ pointing to the corresponding unpacked path. Return the jar path and
+ the new entry.
+ '''
+ base = entry.base
+ jar, relpath = urlparse(relpath).path.split('!', 1)
+ entry = entry.rebase(mozpath.join(base, 'jar:%s!' % jar)) \
+ .move(mozpath.join(base, mozpath.splitext(jar)[0])) \
+ .rebase(base)
+ return mozpath.join(base, jar), entry
+
+
+def unpack_to_registry(source, registry):
+ '''
+ Transform a jar chrome or omnijar packaged directory into a flat package.
+
+ The given registry is filled with the flat package.
+ '''
+ finder = UnpackFinder(source)
+ packager = SimplePackager(FlatFormatter(registry))
+ for p, f in finder.find('*'):
+ if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
+ packager.add(p, f)
+ packager.close()
+
+
+def unpack(source):
+ '''
+ Transform a jar chrome or omnijar packaged directory into a flat package.
+ '''
+ copier = FileCopier()
+ unpack_to_registry(source, copier)
+ copier.copy(source, skip_if_older=False)