summaryrefslogtreecommitdiffstats
path: root/toolkit/mozapps/installer/packager.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /toolkit/mozapps/installer/packager.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'toolkit/mozapps/installer/packager.py')
-rw-r--r--toolkit/mozapps/installer/packager.py415
1 files changed, 415 insertions, 0 deletions
diff --git a/toolkit/mozapps/installer/packager.py b/toolkit/mozapps/installer/packager.py
new file mode 100644
index 000000000..f2dc3fac6
--- /dev/null
+++ b/toolkit/mozapps/installer/packager.py
@@ -0,0 +1,415 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozpack.packager.formats import (
+ FlatFormatter,
+ JarFormatter,
+ OmniJarFormatter,
+)
+from mozpack.packager import (
+ preprocess_manifest,
+ preprocess,
+ Component,
+ SimpleManifestSink,
+)
+from mozpack.files import (
+ GeneratedFile,
+ FileFinder,
+ File,
+)
+from mozpack.copier import (
+ FileCopier,
+ Jarrer,
+)
+from mozpack.errors import errors
+from mozpack.unify import UnifiedBuildFinder
+import mozpack.path as mozpath
+import buildconfig
+from argparse import ArgumentParser
+import os
+from StringIO import StringIO
+import subprocess
+import platform
+import mozinfo
+
+# List of libraries to shlibsign.
+SIGN_LIBS = [
+ 'softokn3',
+ 'nssdbm3',
+ 'freebl3',
+ 'freeblpriv3',
+ 'freebl_32fpu_3',
+ 'freebl_32int_3',
+ 'freebl_32int64_3',
+ 'freebl_64fpu_3',
+ 'freebl_64int_3',
+]
+
+
+class ToolLauncher(object):
+ '''
+ Helper to execute tools like xpcshell with the appropriate environment.
+ launcher = ToolLauncher()
+ launcher.tooldir = '/path/to/tools'
+ launcher.launch(['xpcshell', '-e', 'foo.js'])
+ '''
+ def __init__(self):
+ self.tooldir = None
+
+ def launch(self, cmd, extra_linker_path=None, extra_env={}):
+ '''
+ Launch the given command, passed as a list. The first item in the
+ command list is the program name, without a path and without a suffix.
+ These are determined from the tooldir member and the BIN_SUFFIX value.
+ An extra_linker_path may be passed to give an additional directory
+ to add to the search paths for the dynamic linker.
+ An extra_env dict may be passed to give additional environment
+ variables to export when running the command.
+ '''
+ assert self.tooldir
+ cmd[0] = os.path.join(self.tooldir, 'bin',
+ cmd[0] + buildconfig.substs['BIN_SUFFIX'])
+ if not extra_linker_path:
+ extra_linker_path = os.path.join(self.tooldir, 'bin')
+ env = dict(os.environ)
+ for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
+ if p in env:
+ env[p] = extra_linker_path + ':' + env[p]
+ else:
+ env[p] = extra_linker_path
+ for e in extra_env:
+ env[e] = extra_env[e]
+
+ # For VC12+, make sure we can find the right bitness of pgort1x0.dll
+ if not buildconfig.substs.get('HAVE_64BIT_BUILD'):
+ for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
+ if e not in env:
+ continue
+
+ vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
+ if os.path.exists(vcdir):
+ env['PATH'] = '%s;%s' % (vcdir, env['PATH'])
+ break
+
+ # Work around a bug in Python 2.7.2 and lower where unicode types in
+ # environment variables aren't handled by subprocess.
+ for k, v in env.items():
+ if isinstance(v, unicode):
+ env[k] = v.encode('utf-8')
+
+ print >>errors.out, 'Executing', ' '.join(cmd)
+ errors.out.flush()
+ return subprocess.call(cmd, env=env)
+
+ def can_launch(self):
+ return self.tooldir is not None
+
+launcher = ToolLauncher()
+
+
+class LibSignFile(File):
+ '''
+ File class for shlibsign signatures.
+ '''
+ def copy(self, dest, skip_if_older=True):
+ assert isinstance(dest, basestring)
+ # os.path.getmtime returns a result in seconds with precision up to the
+ # microsecond. But microsecond is too precise because shutil.copystat
+ # only copies milliseconds, and seconds is not enough precision.
+ if os.path.exists(dest) and skip_if_older and \
+ int(os.path.getmtime(self.path) * 1000) <= \
+ int(os.path.getmtime(dest) * 1000):
+ return False
+ if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
+ errors.fatal('Error while signing %s' % self.path)
+
+
+def precompile_cache(registry, source_path, gre_path, app_path):
+ '''
+ Create startup cache for the given application directory, using the
+ given GRE path.
+ - registry is a FileRegistry-like instance where to add the startup cache.
+ - source_path is the base path of the package.
+ - gre_path is the GRE path, relative to source_path.
+ - app_path is the application path, relative to source_path.
+ Startup cache for all resources under resource://app/ are generated,
+ except when gre_path == app_path, in which case it's under
+ resource://gre/.
+ '''
+ from tempfile import mkstemp
+ source_path = os.path.abspath(source_path)
+ if app_path != gre_path:
+ resource = 'app'
+ else:
+ resource = 'gre'
+ app_path = os.path.join(source_path, app_path)
+ gre_path = os.path.join(source_path, gre_path)
+
+ fd, cache = mkstemp('.zip')
+ os.close(fd)
+ os.remove(cache)
+
+ try:
+ extra_env = {'MOZ_STARTUP_CACHE': cache}
+ if buildconfig.substs.get('MOZ_TSAN'):
+ extra_env['TSAN_OPTIONS'] = 'report_bugs=0'
+ if buildconfig.substs.get('MOZ_ASAN'):
+ extra_env['ASAN_OPTIONS'] = 'detect_leaks=0'
+ if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
+ '-f', os.path.join(os.path.dirname(__file__),
+ 'precompile_cache.js'),
+ '-e', 'precompile_startupcache("resource://%s/");'
+ % resource],
+ extra_linker_path=gre_path,
+ extra_env=extra_env):
+ errors.fatal('Error while running startup cache precompilation')
+ return
+ from mozpack.mozjar import JarReader
+ jar = JarReader(cache)
+ resource = '/resource/%s/' % resource
+ for f in jar:
+ if resource in f.filename:
+ path = f.filename[f.filename.index(resource) + len(resource):]
+ if registry.contains(path):
+ registry.add(f.filename, GeneratedFile(f.read()))
+ jar.close()
+ finally:
+ if os.path.exists(cache):
+ os.remove(cache)
+
+
+class RemovedFiles(GeneratedFile):
+ '''
+ File class for removed-files. Is used as a preprocessor parser.
+ '''
+ def __init__(self, copier):
+ self.copier = copier
+ GeneratedFile.__init__(self, '')
+
+ def handle_line(self, str):
+ f = str.strip()
+ if not f:
+ return
+ if self.copier.contains(f):
+ errors.error('Removal of packaged file(s): %s' % f)
+ self.content += f + '\n'
+
+
+def split_define(define):
+ '''
+ Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
+ 1. Numeric VALs are returned as ints.
+ '''
+ if '=' in define:
+ name, value = define.split('=', 1)
+ try:
+ value = int(value)
+ except ValueError:
+ pass
+ return (name, value)
+ return (define, 1)
+
+
+class NoPkgFilesRemover(object):
+ '''
+ Formatter wrapper to handle NO_PKG_FILES.
+ '''
+ def __init__(self, formatter, has_manifest):
+ assert 'NO_PKG_FILES' in os.environ
+ self._formatter = formatter
+ self._files = os.environ['NO_PKG_FILES'].split()
+ if has_manifest:
+ self._error = errors.error
+ self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
+ else:
+ self._error = errors.warn
+ self._msg = 'Skipping %s'
+
+ def add_base(self, base, *args):
+ self._formatter.add_base(base, *args)
+
+ def add(self, path, content):
+ if not any(mozpath.match(path, spec) for spec in self._files):
+ self._formatter.add(path, content)
+ else:
+ self._error(self._msg % path)
+
+ def add_manifest(self, entry):
+ self._formatter.add_manifest(entry)
+
+ def add_interfaces(self, path, content):
+ self._formatter.add_interfaces(path, content)
+
+ def contains(self, path):
+ return self._formatter.contains(path)
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument('-D', dest='defines', action='append',
+ metavar="VAR[=VAL]", help='Define a variable')
+ parser.add_argument('--format', default='omni',
+ help='Choose the chrome format for packaging ' +
+ '(omni, jar or flat ; default: %(default)s)')
+ parser.add_argument('--removals', default=None,
+ help='removed-files source file')
+ parser.add_argument('--ignore-errors', action='store_true', default=False,
+ help='Transform errors into warnings.')
+ parser.add_argument('--minify', action='store_true', default=False,
+ help='Make some files more compact while packaging')
+ parser.add_argument('--minify-js', action='store_true',
+ help='Minify JavaScript files while packaging.')
+ parser.add_argument('--js-binary',
+ help='Path to js binary. This is used to verify '
+ 'minified JavaScript. If this is not defined, '
+ 'minification verification will not be performed.')
+ parser.add_argument('--jarlog', default='', help='File containing jar ' +
+ 'access logs')
+ parser.add_argument('--optimizejars', action='store_true', default=False,
+ help='Enable jar optimizations')
+ parser.add_argument('--unify', default='',
+ help='Base directory of another build to unify with')
+ parser.add_argument('--disable-compression', action='store_false',
+ dest='compress', default=True,
+ help='Disable jar compression')
+ parser.add_argument('manifest', default=None, nargs='?',
+ help='Manifest file name')
+ parser.add_argument('source', help='Source directory')
+ parser.add_argument('destination', help='Destination directory')
+ parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
+ default=[],
+ help='Extra files not to be considered as resources')
+ args = parser.parse_args()
+
+ defines = dict(buildconfig.defines)
+ if args.ignore_errors:
+ errors.ignore_errors()
+
+ if args.defines:
+ for name, value in [split_define(d) for d in args.defines]:
+ defines[name] = value
+
+ copier = FileCopier()
+ if args.format == 'flat':
+ formatter = FlatFormatter(copier)
+ elif args.format == 'jar':
+ formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
+ elif args.format == 'omni':
+ formatter = OmniJarFormatter(copier,
+ buildconfig.substs['OMNIJAR_NAME'],
+ compress=args.compress,
+ optimize=args.optimizejars,
+ non_resources=args.non_resource)
+ else:
+ errors.fatal('Unknown format: %s' % args.format)
+
+ # Adjust defines according to the requested format.
+ if isinstance(formatter, OmniJarFormatter):
+ defines['MOZ_OMNIJAR'] = 1
+ elif 'MOZ_OMNIJAR' in defines:
+ del defines['MOZ_OMNIJAR']
+
+ respath = ''
+ if 'RESPATH' in defines:
+ respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
+ while respath.startswith('/'):
+ respath = respath[1:]
+
+ if args.unify:
+ def is_native(path):
+ path = os.path.abspath(path)
+ return platform.machine() in mozpath.split(path)
+
+ # Invert args.unify and args.source if args.unify points to the
+ # native architecture.
+ args.source, args.unify = sorted([args.source, args.unify],
+ key=is_native, reverse=True)
+ if is_native(args.source) and not buildconfig.substs['CROSS_COMPILE']:
+ launcher.tooldir = args.source
+ elif not buildconfig.substs['CROSS_COMPILE']:
+ launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
+
+ with errors.accumulate():
+ finder_args = dict(
+ minify=args.minify,
+ minify_js=args.minify_js,
+ )
+ if args.js_binary:
+ finder_args['minify_js_verify_command'] = [
+ args.js_binary,
+ os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ 'js-compare-ast.js')
+ ]
+ if args.unify:
+ finder = UnifiedBuildFinder(FileFinder(args.source),
+ FileFinder(args.unify),
+ **finder_args)
+ else:
+ finder = FileFinder(args.source, **finder_args)
+ if 'NO_PKG_FILES' in os.environ:
+ sinkformatter = NoPkgFilesRemover(formatter,
+ args.manifest is not None)
+ else:
+ sinkformatter = formatter
+ sink = SimpleManifestSink(finder, sinkformatter)
+ if args.manifest:
+ preprocess_manifest(sink, args.manifest, defines)
+ else:
+ sink.add(Component(''), 'bin/*')
+ sink.close(args.manifest is not None)
+
+ if args.removals:
+ removals_in = StringIO(open(args.removals).read())
+ removals_in.name = args.removals
+ removals = RemovedFiles(copier)
+ preprocess(removals_in, removals, defines)
+ copier.add(mozpath.join(respath, 'removed-files'), removals)
+
+ # shlibsign libraries
+ if launcher.can_launch():
+ if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
+ for lib in SIGN_LIBS:
+ libbase = mozpath.join(respath, '%s%s') \
+ % (buildconfig.substs['DLL_PREFIX'], lib)
+ libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
+ if copier.contains(libname):
+ copier.add(libbase + '.chk',
+ LibSignFile(os.path.join(args.destination,
+ libname)))
+
+ # Setup preloading
+ if args.jarlog and os.path.exists(args.jarlog):
+ from mozpack.mozjar import JarLog
+ log = JarLog(args.jarlog)
+ for p, f in copier:
+ if not isinstance(f, Jarrer):
+ continue
+ key = JarLog.canonicalize(os.path.join(args.destination, p))
+ if key in log:
+ f.preload(log[key])
+
+ # Fill startup cache
+ if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
+ and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
+ gre_path = None
+ def get_bases():
+ for b in sink.packager.get_bases(addons=False):
+ for p in (mozpath.join('bin', b), b):
+ if os.path.exists(os.path.join(args.source, p)):
+ yield p
+ break
+ for base in sorted(get_bases()):
+ if not gre_path:
+ gre_path = base
+ omnijar_path = mozpath.join(sink.normalize_path(base),
+ buildconfig.substs['OMNIJAR_NAME'])
+ if formatter.contains(omnijar_path):
+ precompile_cache(formatter.copier[omnijar_path],
+ args.source, gre_path, base)
+
+ copier.copy(args.destination)
+
+
+if __name__ == '__main__':
+ main()