diff options
Diffstat (limited to 'python/mozbuild/mozpack/packager')
| -rw-r--r-- | python/mozbuild/mozpack/packager/__init__.py | 408 | ||||
| -rw-r--r-- | python/mozbuild/mozpack/packager/formats.py | 324 | ||||
| -rw-r--r-- | python/mozbuild/mozpack/packager/l10n.py | 259 | ||||
| -rw-r--r-- | python/mozbuild/mozpack/packager/unpack.py | 202 |
4 files changed, 1193 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/packager/__init__.py b/python/mozbuild/mozpack/packager/__init__.py new file mode 100644 index 000000000..4c98ec3d3 --- /dev/null +++ b/python/mozbuild/mozpack/packager/__init__.py @@ -0,0 +1,408 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from mozbuild.preprocessor import Preprocessor +import re +import os +from mozpack.errors import errors +from mozpack.chrome.manifest import ( + Manifest, + ManifestBinaryComponent, + ManifestChrome, + ManifestInterfaces, + is_manifest, + parse_manifest, +) +import mozpack.path as mozpath +from collections import deque + + +class Component(object): + ''' + Class that represents a component in a package manifest. + ''' + def __init__(self, name, destdir=''): + if name.find(' ') > 0: + errors.fatal('Malformed manifest: space in component name "%s"' + % component) + self._name = name + self._destdir = destdir + + def __repr__(self): + s = self.name + if self.destdir: + s += ' destdir="%s"' % self.destdir + return s + + @property + def name(self): + return self._name + + @property + def destdir(self): + return self._destdir + + @staticmethod + def _triples(lst): + ''' + Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. + ''' + return zip(*[iter(lst)] * 3) + + KEY_VALUE_RE = re.compile(r''' + \s* # optional whitespace. + ([a-zA-Z0-9_]+) # key. + \s*=\s* # optional space around =. + "([^"]*)" # value without surrounding quotes. + (?:\s+|$) + ''', re.VERBOSE) + + @staticmethod + def _split_options(string): + ''' + Split 'key1="value1" key2="value2"' into + {'key1':'value1', 'key2':'value2'}. + + Returned keys and values are all strings. + + Throws ValueError if the input is malformed. + ''' + options = {} + splits = Component.KEY_VALUE_RE.split(string) + if len(splits) % 3 != 1: + # This should never happen -- we expect to always split + # into ['', ('key', 'val', '')*]. + raise ValueError("Bad input") + if splits[0]: + raise ValueError('Unrecognized input ' + splits[0]) + for key, val, no_match in Component._triples(splits[1:]): + if no_match: + raise ValueError('Unrecognized input ' + no_match) + options[key] = val + return options + + @staticmethod + def _split_component_and_options(string): + ''' + Split 'name key1="value1" key2="value2"' into + ('name', {'key1':'value1', 'key2':'value2'}). + + Returned name, keys and values are all strings. + + Raises ValueError if the input is malformed. + ''' + splits = string.strip().split(None, 1) + if not splits: + raise ValueError('No component found') + component = splits[0].strip() + if not component: + raise ValueError('No component found') + if not re.match('[a-zA-Z0-9_\-]+$', component): + raise ValueError('Bad component name ' + component) + options = Component._split_options(splits[1]) if len(splits) > 1 else {} + return component, options + + @staticmethod + def from_string(string): + ''' + Create a component from a string. + ''' + try: + name, options = Component._split_component_and_options(string) + except ValueError as e: + errors.fatal('Malformed manifest: %s' % e) + return + destdir = options.pop('destdir', '') + if options: + errors.fatal('Malformed manifest: options %s not recognized' + % options.keys()) + return Component(name, destdir=destdir) + + +class PackageManifestParser(object): + ''' + Class for parsing of a package manifest, after preprocessing. + + A package manifest is a list of file paths, with some syntaxic sugar: + [] designates a toplevel component. Example: [xpcom] + - in front of a file specifies it to be removed + * wildcard support + ** expands to all files and zero or more directories + ; file comment + + The parser takes input from the preprocessor line by line, and pushes + parsed information to a sink object. + + The add and remove methods of the sink object are called with the + current Component instance and a path. + ''' + def __init__(self, sink): + ''' + Initialize the package manifest parser with the given sink. + ''' + self._component = Component('') + self._sink = sink + + def handle_line(self, str): + ''' + Handle a line of input and push the parsed information to the sink + object. + ''' + # Remove comments. + str = str.strip() + if not str or str.startswith(';'): + return + if str.startswith('[') and str.endswith(']'): + self._component = Component.from_string(str[1:-1]) + elif str.startswith('-'): + str = str[1:] + self._sink.remove(self._component, str) + elif ',' in str: + errors.fatal('Incompatible syntax') + else: + self._sink.add(self._component, str) + + +class PreprocessorOutputWrapper(object): + ''' + File-like helper to handle the preprocessor output and send it to a parser. + The parser's handle_line method is called in the relevant errors.context. + ''' + def __init__(self, preprocessor, parser): + self._parser = parser + self._pp = preprocessor + + def write(self, str): + file = os.path.normpath(os.path.abspath(self._pp.context['FILE'])) + with errors.context(file, self._pp.context['LINE']): + self._parser.handle_line(str) + + +def preprocess(input, parser, defines={}): + ''' + Preprocess the file-like input with the given defines, and send the + preprocessed output line by line to the given parser. + ''' + pp = Preprocessor() + pp.context.update(defines) + pp.do_filter('substitution') + pp.out = PreprocessorOutputWrapper(pp, parser) + pp.do_include(input) + + +def preprocess_manifest(sink, manifest, defines={}): + ''' + Preprocess the given file-like manifest with the given defines, and push + the parsed information to a sink. See PackageManifestParser documentation + for more details on the sink. + ''' + preprocess(manifest, PackageManifestParser(sink), defines) + + +class CallDeque(deque): + ''' + Queue of function calls to make. + ''' + def append(self, function, *args): + deque.append(self, (errors.get_context(), function, args)) + + def execute(self): + while True: + try: + context, function, args = self.popleft() + except IndexError: + return + if context: + with errors.context(context[0], context[1]): + function(*args) + else: + function(*args) + + +class SimplePackager(object): + ''' + Helper used to translate and buffer instructions from the + SimpleManifestSink to a formatter. Formatters expect some information to be + given first that the simple manifest contents can't guarantee before the + end of the input. + ''' + def __init__(self, formatter): + self.formatter = formatter + # Queue for formatter.add_interfaces()/add_manifest() calls. + self._queue = CallDeque() + # Queue for formatter.add_manifest() calls for ManifestChrome. + self._chrome_queue = CallDeque() + # Queue for formatter.add() calls. + self._file_queue = CallDeque() + # All paths containing addons. (key is path, value is whether it + # should be packed or unpacked) + self._addons = {} + # All manifest paths imported. + self._manifests = set() + # All manifest paths included from some other manifest. + self._included_manifests = {} + self._closed = False + + # Parsing RDF is complex, and would require an external library to do + # properly. Just go with some hackish but probably sufficient regexp + UNPACK_ADDON_RE = re.compile(r'''(?: + <em:unpack>true</em:unpack> + |em:unpack=(?P<quote>["']?)true(?P=quote) + )''', re.VERBOSE) + + def add(self, path, file): + ''' + Add the given BaseFile instance with the given path. + ''' + assert not self._closed + if is_manifest(path): + self._add_manifest_file(path, file) + elif path.endswith('.xpt'): + self._queue.append(self.formatter.add_interfaces, path, file) + else: + self._file_queue.append(self.formatter.add, path, file) + if mozpath.basename(path) == 'install.rdf': + addon = True + install_rdf = file.open().read() + if self.UNPACK_ADDON_RE.search(install_rdf): + addon = 'unpacked' + self._addons[mozpath.dirname(path)] = addon + + def _add_manifest_file(self, path, file): + ''' + Add the given BaseFile with manifest file contents with the given path. + ''' + self._manifests.add(path) + base = '' + if hasattr(file, 'path'): + # Find the directory the given path is relative to. + b = mozpath.normsep(file.path) + if b.endswith('/' + path) or b == path: + base = os.path.normpath(b[:-len(path)]) + for e in parse_manifest(base, path, file.open()): + # ManifestResources need to be given after ManifestChrome, so just + # put all ManifestChrome in a separate queue to make them first. + if isinstance(e, ManifestChrome): + # e.move(e.base) just returns a clone of the entry. + self._chrome_queue.append(self.formatter.add_manifest, + e.move(e.base)) + elif not isinstance(e, (Manifest, ManifestInterfaces)): + self._queue.append(self.formatter.add_manifest, e.move(e.base)) + # If a binary component is added to an addon, prevent the addon + # from being packed. + if isinstance(e, ManifestBinaryComponent): + addon = mozpath.basedir(e.base, self._addons) + if addon: + self._addons[addon] = 'unpacked' + if isinstance(e, Manifest): + if e.flags: + errors.fatal('Flags are not supported on ' + + '"manifest" entries') + self._included_manifests[e.path] = path + + def get_bases(self, addons=True): + ''' + Return all paths under which root manifests have been found. Root + manifests are manifests that are included in no other manifest. + `addons` indicates whether to include addon bases as well. + ''' + all_bases = set(mozpath.dirname(m) + for m in self._manifests + - set(self._included_manifests)) + if not addons: + all_bases -= set(self._addons) + else: + # If for some reason some detected addon doesn't have a + # non-included manifest. + all_bases |= set(self._addons) + return all_bases + + def close(self): + ''' + Push all instructions to the formatter. + ''' + self._closed = True + + bases = self.get_bases() + broken_bases = sorted( + m for m, includer in self._included_manifests.iteritems() + if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases)) + for m in broken_bases: + errors.fatal('"%s" is included from "%s", which is outside "%s"' % + (m, self._included_manifests[m], + mozpath.basedir(m, bases))) + for base in sorted(bases): + self.formatter.add_base(base, self._addons.get(base, False)) + self._chrome_queue.execute() + self._queue.execute() + self._file_queue.execute() + + +class SimpleManifestSink(object): + ''' + Parser sink for "simple" package manifests. Simple package manifests use + the format described in the PackageManifestParser documentation, but don't + support file removals, and require manifests, interfaces and chrome data to + be explicitely listed. + Entries starting with bin/ are searched under bin/ in the FileFinder, but + are packaged without the bin/ prefix. + ''' + def __init__(self, finder, formatter): + ''' + Initialize the SimpleManifestSink. The given FileFinder is used to + get files matching the patterns given in the manifest. The given + formatter does the packaging job. + ''' + self._finder = finder + self.packager = SimplePackager(formatter) + self._closed = False + self._manifests = set() + + @staticmethod + def normalize_path(path): + ''' + Remove any bin/ prefix. + ''' + if mozpath.basedir(path, ['bin']) == 'bin': + return mozpath.relpath(path, 'bin') + return path + + def add(self, component, pattern): + ''' + Add files with the given pattern in the given component. + ''' + assert not self._closed + added = False + for p, f in self._finder.find(pattern): + added = True + if is_manifest(p): + self._manifests.add(p) + dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) + self.packager.add(dest, f) + if not added: + errors.error('Missing file(s): %s' % pattern) + + def remove(self, component, pattern): + ''' + Remove files with the given pattern in the given component. + ''' + assert not self._closed + errors.fatal('Removal is unsupported') + + def close(self, auto_root_manifest=True): + ''' + Add possibly missing bits and push all instructions to the formatter. + ''' + if auto_root_manifest: + # Simple package manifests don't contain the root manifests, so + # find and add them. + paths = [mozpath.dirname(m) for m in self._manifests] + path = mozpath.dirname(mozpath.commonprefix(paths)) + for p, f in self._finder.find(mozpath.join(path, + 'chrome.manifest')): + if not p in self._manifests: + self.packager.add(SimpleManifestSink.normalize_path(p), f) + self.packager.close() diff --git a/python/mozbuild/mozpack/packager/formats.py b/python/mozbuild/mozpack/packager/formats.py new file mode 100644 index 000000000..c4adabab0 --- /dev/null +++ b/python/mozbuild/mozpack/packager/formats.py @@ -0,0 +1,324 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from mozpack.chrome.manifest import ( + Manifest, + ManifestInterfaces, + ManifestChrome, + ManifestBinaryComponent, + ManifestResource, +) +from urlparse import urlparse +import mozpack.path as mozpath +from mozpack.files import ( + ManifestFile, + XPTFile, +) +from mozpack.copier import ( + FileRegistry, + FileRegistrySubtree, + Jarrer, +) + +STARTUP_CACHE_PATHS = [ + 'jsloader', + 'jssubloader', +] + +''' +Formatters are classes receiving packaging instructions and creating the +appropriate package layout. + +There are three distinct formatters, each handling one of the different chrome +formats: + - flat: essentially, copies files from the source with the same file system + layout. Manifests entries are grouped in a single manifest per directory, + as well as XPT interfaces. + - jar: chrome content is packaged in jar files. + - omni: chrome content, modules, non-binary components, and many other + elements are packaged in an omnijar file for each base directory. + +The base interface provides the following methods: + - add_base(path [, addon]) + Register a base directory for an application or GRE, or an addon. + Base directories usually contain a root manifest (manifests not + included in any other manifest) named chrome.manifest. + The optional addon argument tells whether the base directory + is that of a packed addon (True), unpacked addon ('unpacked') or + otherwise (False). + - add(path, content) + Add the given content (BaseFile instance) at the given virtual path + - add_interfaces(path, content) + Add the given content (BaseFile instance) and link it to other + interfaces in the parent directory of the given virtual path. + - add_manifest(entry) + Add a ManifestEntry. + - contains(path) + Returns whether the given virtual path is known of the formatter. + +The virtual paths mentioned above are paths as they would be with a flat +chrome. + +Formatters all take a FileCopier instance they will fill with the packaged +data. +''' + + +class PiecemealFormatter(object): + ''' + Generic formatter that dispatches across different sub-formatters + according to paths. + ''' + def __init__(self, copier): + assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) + self.copier = copier + self._sub_formatter = {} + self._frozen_bases = False + + def add_base(self, base, addon=False): + # Only allow to add a base directory before calls to _get_base() + assert not self._frozen_bases + assert base not in self._sub_formatter + self._add_base(base, addon) + + def _get_base(self, path): + ''' + Return the deepest base directory containing the given path. + ''' + self._frozen_bases = True + base = mozpath.basedir(path, self._sub_formatter.keys()) + relpath = mozpath.relpath(path, base) if base else path + return base, relpath + + def add(self, path, content): + base, relpath = self._get_base(path) + if base is None: + return self.copier.add(relpath, content) + return self._sub_formatter[base].add(relpath, content) + + def add_manifest(self, entry): + base, relpath = self._get_base(entry.base) + assert base is not None + return self._sub_formatter[base].add_manifest(entry.move(relpath)) + + def add_interfaces(self, path, content): + base, relpath = self._get_base(path) + assert base is not None + return self._sub_formatter[base].add_interfaces(relpath, content) + + def contains(self, path): + assert '*' not in path + base, relpath = self._get_base(path) + if base is None: + return self.copier.contains(relpath) + return self._sub_formatter[base].contains(relpath) + + +class FlatFormatter(PiecemealFormatter): + ''' + Formatter for the flat package format. + ''' + def _add_base(self, base, addon=False): + self._sub_formatter[base] = FlatSubFormatter( + FileRegistrySubtree(base, self.copier)) + + +class FlatSubFormatter(object): + ''' + Sub-formatter for the flat package format. + ''' + def __init__(self, copier): + assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) + self.copier = copier + + def add(self, path, content): + self.copier.add(path, content) + + def add_manifest(self, entry): + # Store manifest entries in a single manifest per directory, named + # after their parent directory, except for root manifests, all named + # chrome.manifest. + if entry.base: + name = mozpath.basename(entry.base) + else: + name = 'chrome' + path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name)) + if not self.copier.contains(path): + # Add a reference to the manifest file in the parent manifest, if + # the manifest file is not a root manifest. + if entry.base: + parent = mozpath.dirname(entry.base) + relbase = mozpath.basename(entry.base) + relpath = mozpath.join(relbase, + mozpath.basename(path)) + self.add_manifest(Manifest(parent, relpath)) + self.copier.add(path, ManifestFile(entry.base)) + self.copier[path].add(entry) + + def add_interfaces(self, path, content): + # Interfaces in the same directory are all linked together in an + # interfaces.xpt file. + interfaces_path = mozpath.join(mozpath.dirname(path), + 'interfaces.xpt') + if not self.copier.contains(interfaces_path): + self.add_manifest(ManifestInterfaces(mozpath.dirname(path), + 'interfaces.xpt')) + self.copier.add(interfaces_path, XPTFile()) + self.copier[interfaces_path].add(content) + + def contains(self, path): + assert '*' not in path + return self.copier.contains(path) + + +class JarFormatter(PiecemealFormatter): + ''' + Formatter for the jar package format. Assumes manifest entries related to + chrome are registered before the chrome data files are added. Also assumes + manifest entries for resources are registered after chrome manifest + entries. + ''' + def __init__(self, copier, compress=True, optimize=True): + PiecemealFormatter.__init__(self, copier) + self._compress=compress + self._optimize=optimize + + def _add_base(self, base, addon=False): + if addon is True: + jarrer = Jarrer(self._compress, self._optimize) + self.copier.add(base + '.xpi', jarrer) + self._sub_formatter[base] = FlatSubFormatter(jarrer) + else: + self._sub_formatter[base] = JarSubFormatter( + FileRegistrySubtree(base, self.copier), + self._compress, self._optimize) + + +class JarSubFormatter(PiecemealFormatter): + ''' + Sub-formatter for the jar package format. It is a PiecemealFormatter that + dispatches between further sub-formatter for each of the jar files it + dispatches the chrome data to, and a FlatSubFormatter for the non-chrome + files. + ''' + def __init__(self, copier, compress=True, optimize=True): + PiecemealFormatter.__init__(self, copier) + self._frozen_chrome = False + self._compress = compress + self._optimize = optimize + self._sub_formatter[''] = FlatSubFormatter(copier) + + def _jarize(self, entry, relpath): + ''' + Transform a manifest entry in one pointing to chrome data in a jar. + Return the corresponding chrome path and the new entry. + ''' + base = entry.base + basepath = mozpath.split(relpath)[0] + chromepath = mozpath.join(base, basepath) + entry = entry.rebase(chromepath) \ + .move(mozpath.join(base, 'jar:%s.jar!' % basepath)) \ + .rebase(base) + return chromepath, entry + + def add_manifest(self, entry): + if isinstance(entry, ManifestChrome) and \ + not urlparse(entry.relpath).scheme: + chromepath, entry = self._jarize(entry, entry.relpath) + assert not self._frozen_chrome + if chromepath not in self._sub_formatter: + jarrer = Jarrer(self._compress, self._optimize) + self.copier.add(chromepath + '.jar', jarrer) + self._sub_formatter[chromepath] = FlatSubFormatter(jarrer) + elif isinstance(entry, ManifestResource) and \ + not urlparse(entry.target).scheme: + chromepath, new_entry = self._jarize(entry, entry.target) + if chromepath in self._sub_formatter: + entry = new_entry + PiecemealFormatter.add_manifest(self, entry) + + +class OmniJarFormatter(JarFormatter): + ''' + Formatter for the omnijar package format. + ''' + def __init__(self, copier, omnijar_name, compress=True, optimize=True, + non_resources=()): + JarFormatter.__init__(self, copier, compress, optimize) + self._omnijar_name = omnijar_name + self._non_resources = non_resources + + def _add_base(self, base, addon=False): + if addon: + JarFormatter._add_base(self, base, addon) + else: + # Initialize a chrome.manifest next to the omnijar file so that + # there's always a chrome.manifest file, even an empty one. + path = mozpath.normpath(mozpath.join(base, 'chrome.manifest')) + if not self.copier.contains(path): + self.copier.add(path, ManifestFile('')) + self._sub_formatter[base] = OmniJarSubFormatter( + FileRegistrySubtree(base, self.copier), self._omnijar_name, + self._compress, self._optimize, self._non_resources) + + +class OmniJarSubFormatter(PiecemealFormatter): + ''' + Sub-formatter for the omnijar package format. It is a PiecemealFormatter + that dispatches between a FlatSubFormatter for the resources data and + another FlatSubFormatter for the other files. + ''' + def __init__(self, copier, omnijar_name, compress=True, optimize=True, + non_resources=()): + PiecemealFormatter.__init__(self, copier) + self._omnijar_name = omnijar_name + self._compress = compress + self._optimize = optimize + self._non_resources = non_resources + self._sub_formatter[''] = FlatSubFormatter(copier) + jarrer = Jarrer(self._compress, self._optimize) + self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer) + + def _get_base(self, path): + base = self._omnijar_name if self.is_resource(path) else '' + # Only add the omnijar file if something ends up in it. + if base and not self.copier.contains(base): + self.copier.add(base, self._sub_formatter[base].copier) + return base, path + + def add_manifest(self, entry): + base = '' + if not isinstance(entry, ManifestBinaryComponent): + base = self._omnijar_name + formatter = self._sub_formatter[base] + return formatter.add_manifest(entry) + + def is_resource(self, path): + ''' + Return whether the given path corresponds to a resource to be put in an + omnijar archive. + ''' + if any(mozpath.match(path, p.replace('*', '**')) + for p in self._non_resources): + return False + path = mozpath.split(path) + if path[0] == 'chrome': + return len(path) == 1 or path[1] != 'icons' + if path[0] == 'components': + return path[-1].endswith(('.js', '.xpt')) + if path[0] == 'res': + return len(path) == 1 or \ + (path[1] != 'cursors' and path[1] != 'MainMenu.nib') + if path[0] == 'defaults': + return len(path) != 3 or \ + not (path[2] == 'channel-prefs.js' and + path[1] in ['pref', 'preferences']) + return path[0] in [ + 'modules', + 'greprefs.js', + 'hyphenation', + 'update.locale', + ] or path[0] in STARTUP_CACHE_PATHS diff --git a/python/mozbuild/mozpack/packager/l10n.py b/python/mozbuild/mozpack/packager/l10n.py new file mode 100644 index 000000000..758064f59 --- /dev/null +++ b/python/mozbuild/mozpack/packager/l10n.py @@ -0,0 +1,259 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +''' +Replace localized parts of a packaged directory with data from a langpack +directory. +''' + +import os +import mozpack.path as mozpath +from mozpack.packager.formats import ( + FlatFormatter, + JarFormatter, + OmniJarFormatter, +) +from mozpack.packager import ( + Component, + SimplePackager, + SimpleManifestSink, +) +from mozpack.files import ( + ComposedFinder, + ManifestFile, +) +from mozpack.copier import ( + FileCopier, + Jarrer, +) +from mozpack.chrome.manifest import ( + ManifestLocale, + ManifestEntryWithRelPath, + is_manifest, + ManifestChrome, + Manifest, +) +from mozpack.errors import errors +from mozpack.packager.unpack import UnpackFinder +from createprecomplete import generate_precomplete + + +class LocaleManifestFinder(object): + def __init__(self, finder): + entries = self.entries = [] + bases = self.bases = [] + + class MockFormatter(object): + def add_interfaces(self, path, content): + pass + + def add(self, path, content): + pass + + def add_manifest(self, entry): + if entry.localized: + entries.append(entry) + + def add_base(self, base, addon=False): + bases.append(base) + + # SimplePackager rejects "manifest foo.manifest" entries with + # additional flags (such as "manifest foo.manifest application=bar"). + # Those type of entries are used by language packs to work as addons, + # but are not necessary for the purpose of l10n repacking. So we wrap + # the finder in order to remove those entries. + class WrapFinder(object): + def __init__(self, finder): + self._finder = finder + + def find(self, pattern): + for p, f in self._finder.find(pattern): + if isinstance(f, ManifestFile): + unwanted = [ + e for e in f._entries + if isinstance(e, Manifest) and e.flags + ] + if unwanted: + f = ManifestFile( + f._base, + [e for e in f._entries if e not in unwanted]) + yield p, f + + sink = SimpleManifestSink(WrapFinder(finder), MockFormatter()) + sink.add(Component(''), '*') + sink.close(False) + + # Find unique locales used in these manifest entries. + self.locales = list(set(e.id for e in self.entries + if isinstance(e, ManifestLocale))) + + +def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): + app = LocaleManifestFinder(app_finder) + l10n = LocaleManifestFinder(l10n_finder) + + # The code further below assumes there's only one locale replaced with + # another one. + if len(app.locales) > 1: + errors.fatal("Multiple app locales aren't supported: " + + ",".join(app.locales)) + if len(l10n.locales) > 1: + errors.fatal("Multiple l10n locales aren't supported: " + + ",".join(l10n.locales)) + locale = app.locales[0] + l10n_locale = l10n.locales[0] + + # For each base directory, store what path a locale chrome package name + # corresponds to. + # e.g., for the following entry under app/chrome: + # locale foo en-US path/to/files + # keep track that the locale path for foo in app is + # app/chrome/path/to/files. + l10n_paths = {} + for e in l10n.entries: + if isinstance(e, ManifestChrome): + base = mozpath.basedir(e.path, app.bases) + l10n_paths.setdefault(base, {}) + l10n_paths[base][e.name] = e.path + + # For chrome and non chrome files or directories, store what langpack path + # corresponds to a package path. + paths = {} + for e in app.entries: + if isinstance(e, ManifestEntryWithRelPath): + base = mozpath.basedir(e.path, app.bases) + if base not in l10n_paths: + errors.fatal("Locale doesn't contain %s/" % base) + # Allow errors to accumulate + continue + if e.name not in l10n_paths[base]: + errors.fatal("Locale doesn't have a manifest entry for '%s'" % + e.name) + # Allow errors to accumulate + continue + paths[e.path] = l10n_paths[base][e.name] + + for pattern in non_chrome: + for base in app.bases: + path = mozpath.join(base, pattern) + left = set(p for p, f in app_finder.find(path)) + right = set(p for p, f in l10n_finder.find(path)) + for p in right: + paths[p] = p + for p in left - right: + paths[p] = None + + # Create a new package, with non localized bits coming from the original + # package, and localized bits coming from the langpack. + packager = SimplePackager(formatter) + for p, f in app_finder: + if is_manifest(p): + # Remove localized manifest entries. + for e in [e for e in f if e.localized]: + f.remove(e) + # If the path is one that needs a locale replacement, use the + # corresponding file from the langpack. + path = None + if p in paths: + path = paths[p] + if not path: + continue + else: + base = mozpath.basedir(p, paths.keys()) + if base: + subpath = mozpath.relpath(p, base) + path = mozpath.normpath(mozpath.join(paths[base], + subpath)) + if path: + files = [f for p, f in l10n_finder.find(path)] + if not len(files): + if base not in non_chrome: + finderBase = "" + if hasattr(l10n_finder, 'base'): + finderBase = l10n_finder.base + errors.error("Missing file: %s" % + os.path.join(finderBase, path)) + else: + packager.add(path, files[0]) + else: + packager.add(p, f) + + # Add localized manifest entries from the langpack. + l10n_manifests = [] + for base in set(e.base for e in l10n.entries): + m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) + path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) + l10n_manifests.append((path, m)) + bases = packager.get_bases() + for path, m in l10n_manifests: + base = mozpath.basedir(path, bases) + packager.add(path, m) + # Add a "manifest $path" entry in the top manifest under that base. + m = ManifestFile(base) + m.add(Manifest(base, mozpath.relpath(path, base))) + packager.add(mozpath.join(base, 'chrome.manifest'), m) + + packager.close() + + # Add any remaining non chrome files. + for pattern in non_chrome: + for base in bases: + for p, f in l10n_finder.find(mozpath.join(base, pattern)): + if not formatter.contains(p): + formatter.add(p, f) + + # Transplant jar preloading information. + for path, log in app_finder.jarlogs.iteritems(): + assert isinstance(copier[path], Jarrer) + copier[path].preload([l.replace(locale, l10n_locale) for l in log]) + + +def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()): + ''' + Replace localized data from the `source` directory with localized data + from `l10n` and `extra_l10n`. + + The `source` argument points to a directory containing a packaged + application (in omnijar, jar or flat form). + The `l10n` argument points to a directory containing the main localized + data (usually in the form of a language pack addon) to use to replace + in the packaged application. + The `extra_l10n` argument contains a dict associating relative paths in + the source to separate directories containing localized data for them. + This can be used to point at different language pack addons for different + parts of the package application. + The `non_resources` argument gives a list of relative paths in the source + that should not be added in an omnijar in case the packaged application + is in that format. + The `non_chrome` argument gives a list of file/directory patterns for + localized files that are not listed in a chrome.manifest. + ''' + app_finder = UnpackFinder(source) + l10n_finder = UnpackFinder(l10n) + if extra_l10n: + finders = { + '': l10n_finder, + } + for base, path in extra_l10n.iteritems(): + finders[base] = UnpackFinder(path) + l10n_finder = ComposedFinder(finders) + copier = FileCopier() + if app_finder.kind == 'flat': + formatter = FlatFormatter(copier) + elif app_finder.kind == 'jar': + formatter = JarFormatter(copier, + optimize=app_finder.optimizedjars, + compress=app_finder.compressed) + elif app_finder.kind == 'omni': + formatter = OmniJarFormatter(copier, app_finder.omnijar, + optimize=app_finder.optimizedjars, + compress=app_finder.compressed, + non_resources=non_resources) + + with errors.accumulate(): + _repack(app_finder, l10n_finder, copier, formatter, non_chrome) + copier.copy(source, skip_if_older=False) + generate_precomplete(source) diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py new file mode 100644 index 000000000..fa2b474e7 --- /dev/null +++ b/python/mozbuild/mozpack/packager/unpack.py @@ -0,0 +1,202 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import mozpack.path as mozpath +from mozpack.files import ( + BaseFinder, + FileFinder, + DeflatedFile, + ManifestFile, +) +from mozpack.chrome.manifest import ( + parse_manifest, + ManifestEntryWithRelPath, + ManifestResource, + is_manifest, +) +from mozpack.mozjar import JarReader +from mozpack.copier import ( + FileRegistry, + FileCopier, +) +from mozpack.packager import SimplePackager +from mozpack.packager.formats import ( + FlatFormatter, + STARTUP_CACHE_PATHS, +) +from urlparse import urlparse + + +class UnpackFinder(BaseFinder): + ''' + Special Finder object that treats the source package directory as if it + were in the flat chrome format, whatever chrome format it actually is in. + + This means that for example, paths like chrome/browser/content/... match + files under jar:chrome/browser.jar!/content/... in case of jar chrome + format. + + The only argument to the constructor is a Finder instance or a path. + The UnpackFinder is populated with files from this Finder instance, + or with files from a FileFinder using the given path as its root. + ''' + def __init__(self, source): + if isinstance(source, BaseFinder): + self._finder = source + else: + self._finder = FileFinder(source) + self.base = self._finder.base + self.files = FileRegistry() + self.kind = 'flat' + self.omnijar = None + self.jarlogs = {} + self.optimizedjars = False + self.compressed = True + + jars = set() + + for p, f in self._finder.find('*'): + # Skip the precomplete file, which is generated at packaging time. + if p == 'precomplete': + continue + base = mozpath.dirname(p) + # If the file is a zip/jar that is not a .xpi, and contains a + # chrome.manifest, it is an omnijar. All the files it contains + # go in the directory containing the omnijar. Manifests are merged + # if there is a corresponding manifest in the directory. + if not p.endswith('.xpi') and self._maybe_zip(f) and \ + (mozpath.basename(p) == self.omnijar or + not self.omnijar): + jar = self._open_jar(p, f) + if 'chrome.manifest' in jar: + self.kind = 'omni' + self.omnijar = mozpath.basename(p) + self._fill_with_jar(base, jar) + continue + # If the file is a manifest, scan its entries for some referencing + # jar: urls. If there are some, the files contained in the jar they + # point to, go under a directory named after the jar. + if is_manifest(p): + m = self.files[p] if self.files.contains(p) \ + else ManifestFile(base) + for e in parse_manifest(self.base, p, f.open()): + m.add(self._handle_manifest_entry(e, jars)) + if self.files.contains(p): + continue + f = m + # If the file is a packed addon, unpack it under a directory named + # after the xpi. + if p.endswith('.xpi') and self._maybe_zip(f): + self._fill_with_jar(p[:-4], self._open_jar(p, f)) + continue + if not p in jars: + self.files.add(p, f) + + def _fill_with_jar(self, base, jar): + for j in jar: + path = mozpath.join(base, j.filename) + if is_manifest(j.filename): + m = self.files[path] if self.files.contains(path) \ + else ManifestFile(mozpath.dirname(path)) + for e in parse_manifest(None, path, j): + m.add(e) + if not self.files.contains(path): + self.files.add(path, m) + continue + else: + self.files.add(path, DeflatedFile(j)) + + def _handle_manifest_entry(self, entry, jars): + jarpath = None + if isinstance(entry, ManifestEntryWithRelPath) and \ + urlparse(entry.relpath).scheme == 'jar': + jarpath, entry = self._unjarize(entry, entry.relpath) + elif isinstance(entry, ManifestResource) and \ + urlparse(entry.target).scheme == 'jar': + jarpath, entry = self._unjarize(entry, entry.target) + if jarpath: + # Don't defer unpacking the jar file. If we already saw + # it, take (and remove) it from the registry. If we + # haven't, try to find it now. + if self.files.contains(jarpath): + jar = self.files[jarpath] + self.files.remove(jarpath) + else: + jar = [f for p, f in self._finder.find(jarpath)] + assert len(jar) == 1 + jar = jar[0] + if not jarpath in jars: + base = mozpath.splitext(jarpath)[0] + for j in self._open_jar(jarpath, jar): + self.files.add(mozpath.join(base, + j.filename), + DeflatedFile(j)) + jars.add(jarpath) + self.kind = 'jar' + return entry + + def _open_jar(self, path, file): + ''' + Return a JarReader for the given BaseFile instance, keeping a log of + the preloaded entries it has. + ''' + jar = JarReader(fileobj=file.open()) + if jar.is_optimized: + self.optimizedjars = True + if not any(f.compressed for f in jar): + self.compressed = False + if jar.last_preloaded: + jarlog = jar.entries.keys() + self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1] + return jar + + def find(self, path): + for p in self.files.match(path): + yield p, self.files[p] + + def _maybe_zip(self, file): + ''' + Return whether the given BaseFile looks like a ZIP/Jar. + ''' + header = file.open().read(8) + return len(header) == 8 and (header[0:2] == 'PK' or + header[4:6] == 'PK') + + def _unjarize(self, entry, relpath): + ''' + Transform a manifest entry pointing to chrome data in a jar in one + pointing to the corresponding unpacked path. Return the jar path and + the new entry. + ''' + base = entry.base + jar, relpath = urlparse(relpath).path.split('!', 1) + entry = entry.rebase(mozpath.join(base, 'jar:%s!' % jar)) \ + .move(mozpath.join(base, mozpath.splitext(jar)[0])) \ + .rebase(base) + return mozpath.join(base, jar), entry + + +def unpack_to_registry(source, registry): + ''' + Transform a jar chrome or omnijar packaged directory into a flat package. + + The given registry is filled with the flat package. + ''' + finder = UnpackFinder(source) + packager = SimplePackager(FlatFormatter(registry)) + for p, f in finder.find('*'): + if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS: + packager.add(p, f) + packager.close() + + +def unpack(source): + ''' + Transform a jar chrome or omnijar packaged directory into a flat package. + ''' + copier = FileCopier() + unpack_to_registry(source, copier) + copier.copy(source, skip_if_older=False) |