diff options
Diffstat (limited to 'python/mozbuild/mozpack/packager/__init__.py')
| -rw-r--r-- | python/mozbuild/mozpack/packager/__init__.py | 408 |
1 files changed, 408 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/packager/__init__.py b/python/mozbuild/mozpack/packager/__init__.py new file mode 100644 index 000000000..4c98ec3d3 --- /dev/null +++ b/python/mozbuild/mozpack/packager/__init__.py @@ -0,0 +1,408 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from mozbuild.preprocessor import Preprocessor +import re +import os +from mozpack.errors import errors +from mozpack.chrome.manifest import ( + Manifest, + ManifestBinaryComponent, + ManifestChrome, + ManifestInterfaces, + is_manifest, + parse_manifest, +) +import mozpack.path as mozpath +from collections import deque + + +class Component(object): + ''' + Class that represents a component in a package manifest. + ''' + def __init__(self, name, destdir=''): + if name.find(' ') > 0: + errors.fatal('Malformed manifest: space in component name "%s"' + % component) + self._name = name + self._destdir = destdir + + def __repr__(self): + s = self.name + if self.destdir: + s += ' destdir="%s"' % self.destdir + return s + + @property + def name(self): + return self._name + + @property + def destdir(self): + return self._destdir + + @staticmethod + def _triples(lst): + ''' + Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. + ''' + return zip(*[iter(lst)] * 3) + + KEY_VALUE_RE = re.compile(r''' + \s* # optional whitespace. + ([a-zA-Z0-9_]+) # key. + \s*=\s* # optional space around =. + "([^"]*)" # value without surrounding quotes. + (?:\s+|$) + ''', re.VERBOSE) + + @staticmethod + def _split_options(string): + ''' + Split 'key1="value1" key2="value2"' into + {'key1':'value1', 'key2':'value2'}. + + Returned keys and values are all strings. + + Throws ValueError if the input is malformed. + ''' + options = {} + splits = Component.KEY_VALUE_RE.split(string) + if len(splits) % 3 != 1: + # This should never happen -- we expect to always split + # into ['', ('key', 'val', '')*]. + raise ValueError("Bad input") + if splits[0]: + raise ValueError('Unrecognized input ' + splits[0]) + for key, val, no_match in Component._triples(splits[1:]): + if no_match: + raise ValueError('Unrecognized input ' + no_match) + options[key] = val + return options + + @staticmethod + def _split_component_and_options(string): + ''' + Split 'name key1="value1" key2="value2"' into + ('name', {'key1':'value1', 'key2':'value2'}). + + Returned name, keys and values are all strings. + + Raises ValueError if the input is malformed. + ''' + splits = string.strip().split(None, 1) + if not splits: + raise ValueError('No component found') + component = splits[0].strip() + if not component: + raise ValueError('No component found') + if not re.match('[a-zA-Z0-9_\-]+$', component): + raise ValueError('Bad component name ' + component) + options = Component._split_options(splits[1]) if len(splits) > 1 else {} + return component, options + + @staticmethod + def from_string(string): + ''' + Create a component from a string. + ''' + try: + name, options = Component._split_component_and_options(string) + except ValueError as e: + errors.fatal('Malformed manifest: %s' % e) + return + destdir = options.pop('destdir', '') + if options: + errors.fatal('Malformed manifest: options %s not recognized' + % options.keys()) + return Component(name, destdir=destdir) + + +class PackageManifestParser(object): + ''' + Class for parsing of a package manifest, after preprocessing. + + A package manifest is a list of file paths, with some syntaxic sugar: + [] designates a toplevel component. Example: [xpcom] + - in front of a file specifies it to be removed + * wildcard support + ** expands to all files and zero or more directories + ; file comment + + The parser takes input from the preprocessor line by line, and pushes + parsed information to a sink object. + + The add and remove methods of the sink object are called with the + current Component instance and a path. + ''' + def __init__(self, sink): + ''' + Initialize the package manifest parser with the given sink. + ''' + self._component = Component('') + self._sink = sink + + def handle_line(self, str): + ''' + Handle a line of input and push the parsed information to the sink + object. + ''' + # Remove comments. + str = str.strip() + if not str or str.startswith(';'): + return + if str.startswith('[') and str.endswith(']'): + self._component = Component.from_string(str[1:-1]) + elif str.startswith('-'): + str = str[1:] + self._sink.remove(self._component, str) + elif ',' in str: + errors.fatal('Incompatible syntax') + else: + self._sink.add(self._component, str) + + +class PreprocessorOutputWrapper(object): + ''' + File-like helper to handle the preprocessor output and send it to a parser. + The parser's handle_line method is called in the relevant errors.context. + ''' + def __init__(self, preprocessor, parser): + self._parser = parser + self._pp = preprocessor + + def write(self, str): + file = os.path.normpath(os.path.abspath(self._pp.context['FILE'])) + with errors.context(file, self._pp.context['LINE']): + self._parser.handle_line(str) + + +def preprocess(input, parser, defines={}): + ''' + Preprocess the file-like input with the given defines, and send the + preprocessed output line by line to the given parser. + ''' + pp = Preprocessor() + pp.context.update(defines) + pp.do_filter('substitution') + pp.out = PreprocessorOutputWrapper(pp, parser) + pp.do_include(input) + + +def preprocess_manifest(sink, manifest, defines={}): + ''' + Preprocess the given file-like manifest with the given defines, and push + the parsed information to a sink. See PackageManifestParser documentation + for more details on the sink. + ''' + preprocess(manifest, PackageManifestParser(sink), defines) + + +class CallDeque(deque): + ''' + Queue of function calls to make. + ''' + def append(self, function, *args): + deque.append(self, (errors.get_context(), function, args)) + + def execute(self): + while True: + try: + context, function, args = self.popleft() + except IndexError: + return + if context: + with errors.context(context[0], context[1]): + function(*args) + else: + function(*args) + + +class SimplePackager(object): + ''' + Helper used to translate and buffer instructions from the + SimpleManifestSink to a formatter. Formatters expect some information to be + given first that the simple manifest contents can't guarantee before the + end of the input. + ''' + def __init__(self, formatter): + self.formatter = formatter + # Queue for formatter.add_interfaces()/add_manifest() calls. + self._queue = CallDeque() + # Queue for formatter.add_manifest() calls for ManifestChrome. + self._chrome_queue = CallDeque() + # Queue for formatter.add() calls. + self._file_queue = CallDeque() + # All paths containing addons. (key is path, value is whether it + # should be packed or unpacked) + self._addons = {} + # All manifest paths imported. + self._manifests = set() + # All manifest paths included from some other manifest. + self._included_manifests = {} + self._closed = False + + # Parsing RDF is complex, and would require an external library to do + # properly. Just go with some hackish but probably sufficient regexp + UNPACK_ADDON_RE = re.compile(r'''(?: + <em:unpack>true</em:unpack> + |em:unpack=(?P<quote>["']?)true(?P=quote) + )''', re.VERBOSE) + + def add(self, path, file): + ''' + Add the given BaseFile instance with the given path. + ''' + assert not self._closed + if is_manifest(path): + self._add_manifest_file(path, file) + elif path.endswith('.xpt'): + self._queue.append(self.formatter.add_interfaces, path, file) + else: + self._file_queue.append(self.formatter.add, path, file) + if mozpath.basename(path) == 'install.rdf': + addon = True + install_rdf = file.open().read() + if self.UNPACK_ADDON_RE.search(install_rdf): + addon = 'unpacked' + self._addons[mozpath.dirname(path)] = addon + + def _add_manifest_file(self, path, file): + ''' + Add the given BaseFile with manifest file contents with the given path. + ''' + self._manifests.add(path) + base = '' + if hasattr(file, 'path'): + # Find the directory the given path is relative to. + b = mozpath.normsep(file.path) + if b.endswith('/' + path) or b == path: + base = os.path.normpath(b[:-len(path)]) + for e in parse_manifest(base, path, file.open()): + # ManifestResources need to be given after ManifestChrome, so just + # put all ManifestChrome in a separate queue to make them first. + if isinstance(e, ManifestChrome): + # e.move(e.base) just returns a clone of the entry. + self._chrome_queue.append(self.formatter.add_manifest, + e.move(e.base)) + elif not isinstance(e, (Manifest, ManifestInterfaces)): + self._queue.append(self.formatter.add_manifest, e.move(e.base)) + # If a binary component is added to an addon, prevent the addon + # from being packed. + if isinstance(e, ManifestBinaryComponent): + addon = mozpath.basedir(e.base, self._addons) + if addon: + self._addons[addon] = 'unpacked' + if isinstance(e, Manifest): + if e.flags: + errors.fatal('Flags are not supported on ' + + '"manifest" entries') + self._included_manifests[e.path] = path + + def get_bases(self, addons=True): + ''' + Return all paths under which root manifests have been found. Root + manifests are manifests that are included in no other manifest. + `addons` indicates whether to include addon bases as well. + ''' + all_bases = set(mozpath.dirname(m) + for m in self._manifests + - set(self._included_manifests)) + if not addons: + all_bases -= set(self._addons) + else: + # If for some reason some detected addon doesn't have a + # non-included manifest. + all_bases |= set(self._addons) + return all_bases + + def close(self): + ''' + Push all instructions to the formatter. + ''' + self._closed = True + + bases = self.get_bases() + broken_bases = sorted( + m for m, includer in self._included_manifests.iteritems() + if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases)) + for m in broken_bases: + errors.fatal('"%s" is included from "%s", which is outside "%s"' % + (m, self._included_manifests[m], + mozpath.basedir(m, bases))) + for base in sorted(bases): + self.formatter.add_base(base, self._addons.get(base, False)) + self._chrome_queue.execute() + self._queue.execute() + self._file_queue.execute() + + +class SimpleManifestSink(object): + ''' + Parser sink for "simple" package manifests. Simple package manifests use + the format described in the PackageManifestParser documentation, but don't + support file removals, and require manifests, interfaces and chrome data to + be explicitely listed. + Entries starting with bin/ are searched under bin/ in the FileFinder, but + are packaged without the bin/ prefix. + ''' + def __init__(self, finder, formatter): + ''' + Initialize the SimpleManifestSink. The given FileFinder is used to + get files matching the patterns given in the manifest. The given + formatter does the packaging job. + ''' + self._finder = finder + self.packager = SimplePackager(formatter) + self._closed = False + self._manifests = set() + + @staticmethod + def normalize_path(path): + ''' + Remove any bin/ prefix. + ''' + if mozpath.basedir(path, ['bin']) == 'bin': + return mozpath.relpath(path, 'bin') + return path + + def add(self, component, pattern): + ''' + Add files with the given pattern in the given component. + ''' + assert not self._closed + added = False + for p, f in self._finder.find(pattern): + added = True + if is_manifest(p): + self._manifests.add(p) + dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) + self.packager.add(dest, f) + if not added: + errors.error('Missing file(s): %s' % pattern) + + def remove(self, component, pattern): + ''' + Remove files with the given pattern in the given component. + ''' + assert not self._closed + errors.fatal('Removal is unsupported') + + def close(self, auto_root_manifest=True): + ''' + Add possibly missing bits and push all instructions to the formatter. + ''' + if auto_root_manifest: + # Simple package manifests don't contain the root manifests, so + # find and add them. + paths = [mozpath.dirname(m) for m in self._manifests] + path = mozpath.dirname(mozpath.commonprefix(paths)) + for p, f in self._finder.find(mozpath.join(path, + 'chrome.manifest')): + if not p in self._manifests: + self.packager.add(SimpleManifestSink.normalize_path(p), f) + self.packager.close() |