path: root/toolkit/mozapps/installer/packager.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from mozpack.packager.formats import (
    FlatFormatter,
    JarFormatter,
    OmniJarFormatter,
)
from mozpack.packager import (
    preprocess_manifest,
    preprocess,
    Component,
    SimpleManifestSink,
)
from mozpack.files import (
    GeneratedFile,
    FileFinder,
    File,
)
from mozpack.copier import (
    FileCopier,
    Jarrer,
)
from mozpack.errors import errors
from mozpack.mozjar import JAR_BROTLI
from mozpack.unify import UnifiedBuildFinder
import mozpack.path as mozpath
import buildconfig
from argparse import ArgumentParser
import os
from StringIO import StringIO
import subprocess
import platform
import mozinfo

# List of libraries to shlibsign.
SIGN_LIBS = [
    'softokn3',
    'nssdbm3',
    'freebl3',
    'freeblpriv3',
    'freebl_32fpu_3',
    'freebl_32int_3',
    'freebl_32int64_3',
    'freebl_64fpu_3',
    'freebl_64int_3',
]


class ToolLauncher(object):
    '''
    Helper to execute tools like xpcshell with the appropriate environment.
        launcher = ToolLauncher()
        launcher.tooldir = '/path/to/tools'
        launcher.launch(['xpcshell', '-e', 'foo.js'])
    '''
    def __init__(self):
        self.tooldir = None

    def launch(self, cmd, extra_linker_path=None, extra_env={}):
        '''
        Launch the given command, passed as a list. The first item in the
        command list is the program name, without a path and without a suffix.
        These are determined from the tooldir member and the BIN_SUFFIX value.
        An extra_linker_path may be passed to give an additional directory
        to add to the search paths for the dynamic linker.
        An extra_env dict may be passed to give additional environment
        variables to export when running the command.
        '''
        assert self.tooldir
        cmd[0] = os.path.join(self.tooldir, 'bin',
                              cmd[0] + buildconfig.substs['BIN_SUFFIX'])
        if not extra_linker_path:
            extra_linker_path = os.path.join(self.tooldir, 'bin')
        env = dict(os.environ)
        for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
            if p in env:
                env[p] = extra_linker_path + ':' + env[p]
            else:
                env[p] = extra_linker_path
        for e in extra_env:
            env[e] = extra_env[e]

        # For VC12+, make sure we can find the right bitness of pgort1x0.dll
        if not buildconfig.substs.get('HAVE_64BIT_BUILD'):
            for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
                if e not in env:
                    continue

                vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
                if os.path.exists(vcdir):
                    env['PATH'] = '%s;%s' % (vcdir, env['PATH'])
                    break

        # Work around a bug in Python 2.7.2 and lower where unicode types in
        # environment variables aren't handled by subprocess.
        for k, v in env.items():
            if isinstance(v, unicode):
                env[k] = v.encode('utf-8')

        print >>errors.out, 'Executing', ' '.join(cmd)
        errors.out.flush()
        return subprocess.call(cmd, env=env)

    def can_launch(self):
        return self.tooldir is not None

launcher = ToolLauncher()


class LibSignFile(File):
    '''
    File class for shlibsign signatures.
    '''
    def copy(self, dest, skip_if_older=True):
        assert isinstance(dest, basestring)
        # os.path.getmtime returns a result in seconds with precision up to the
        # microsecond. But microsecond is too precise because shutil.copystat
        # only copies milliseconds, and seconds is not enough precision.
        if os.path.exists(dest) and skip_if_older and \
                int(os.path.getmtime(self.path) * 1000) <= \
                int(os.path.getmtime(dest) * 1000):
            return False
        if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
            errors.fatal('Error while signing %s' % self.path)


def precompile_cache(registry, source_path, gre_path, app_path):
    '''
    Create startup cache for the given application directory, using the
    given GRE path.
    - registry is a FileRegistry-like instance where to add the startup cache.
    - source_path is the base path of the package.
    - gre_path is the GRE path, relative to source_path.
    - app_path is the application path, relative to source_path.
    Startup cache for all resources under resource://app/ are generated,
    except when gre_path == app_path, in which case it's under
    resource://gre/.
    '''
    from tempfile import mkstemp
    source_path = os.path.abspath(source_path)
    if app_path != gre_path:
        resource = 'app'
    else:
        resource = 'gre'
    app_path = os.path.join(source_path, app_path)
    gre_path = os.path.join(source_path, gre_path)

    fd, cache = mkstemp('.zip')
    os.close(fd)
    os.remove(cache)

    try:
        extra_env = {'MOZ_STARTUP_CACHE': cache}
        if buildconfig.substs.get('MOZ_TSAN'):
            extra_env['TSAN_OPTIONS'] = 'report_bugs=0'
        if buildconfig.substs.get('MOZ_ASAN'):
            extra_env['ASAN_OPTIONS'] = 'detect_leaks=0'
        if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
                            '-f', os.path.join(os.path.dirname(__file__),
                            'precompile_cache.js'),
                            '-e', 'precompile_startupcache("resource://%s/");'
                                  % resource],
                           extra_linker_path=gre_path,
                           extra_env=extra_env):
            errors.fatal('Error while running startup cache precompilation')
            return
        from mozpack.mozjar import JarReader
        jar = JarReader(cache)
        resource = '/resource/%s/' % resource
        for f in jar:
            if resource in f.filename:
                path = f.filename[f.filename.index(resource) + len(resource):]
                if registry.contains(path):
                    registry.add(f.filename, GeneratedFile(f.read()))
        jar.close()
    finally:
        if os.path.exists(cache):
            os.remove(cache)


class RemovedFiles(GeneratedFile):
    '''
    File class for removed-files. Is used as a preprocessor parser.
    '''
    def __init__(self, copier):
        self.copier = copier
        GeneratedFile.__init__(self, '')

    def handle_line(self, str):
        f = str.strip()
        if not f:
            return
        if self.copier.contains(f):
            errors.error('Removal of packaged file(s): %s' % f)
        self.content += f + '\n'


def split_define(define):
    '''
    Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
    1. Numeric VALs are returned as ints.
    '''
    if '=' in define:
        name, value = define.split('=', 1)
        try:
            value = int(value)
        except ValueError:
            pass
        return (name, value)
    return (define, 1)


class NoPkgFilesRemover(object):
    '''
    Formatter wrapper to handle NO_PKG_FILES.
    '''
    def __init__(self, formatter, has_manifest):
        assert 'NO_PKG_FILES' in os.environ
        self._formatter = formatter
        self._files = os.environ['NO_PKG_FILES'].split()
        if has_manifest:
            self._error = errors.error
            self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
        else:
            self._error = errors.warn
            self._msg = 'Skipping %s'

    def add_base(self, base, *args):
        self._formatter.add_base(base, *args)

    def add(self, path, content):
        if not any(mozpath.match(path, spec) for spec in self._files):
            self._formatter.add(path, content)
        else:
            self._error(self._msg % path)

    def add_manifest(self, entry):
        self._formatter.add_manifest(entry)

    def add_interfaces(self, path, content):
        self._formatter.add_interfaces(path, content)

    def contains(self, path):
        return self._formatter.contains(path)


def main():
    parser = ArgumentParser()
    parser.add_argument('-D', dest='defines', action='append',
                        metavar="VAR[=VAL]", help='Define a variable')
    parser.add_argument('--format', default='omni',
                        help='Choose the chrome format for packaging ' +
                        '(omni, jar or flat ; default: %(default)s)')
    parser.add_argument('--removals', default=None,
                        help='removed-files source file')
    parser.add_argument('--ignore-errors', action='store_true', default=False,
                        help='Transform errors into warnings.')
    parser.add_argument('--minify', action='store_true', default=False,
                        help='Make some files more compact while packaging')
    parser.add_argument('--minify-js', action='store_true',
                        help='Minify JavaScript files while packaging.')
    parser.add_argument('--js-binary',
                        help='Path to js binary. This is used to verify '
                        'minified JavaScript. If this is not defined, '
                        'minification verification will not be performed.')
    parser.add_argument('--jarlog', default='', help='File containing jar ' +
                        'access logs')
    parser.add_argument('--optimizejars', action='store_true', default=False,
                        help='Enable jar optimizations')
    parser.add_argument('--unify', default='',
                        help='Base directory of another build to unify with')
    parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
                        default='deflate',
                        help='Use given jar compression (default: deflate)')
    parser.add_argument('manifest', default=None, nargs='?',
                        help='Manifest file name')
    parser.add_argument('source', help='Source directory')
    parser.add_argument('destination', help='Destination directory')
    parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                        default=[],
                        help='Extra files not to be considered as resources')
    args = parser.parse_args()

    defines = dict(buildconfig.defines)
    if args.ignore_errors:
        errors.ignore_errors()

    if args.defines:
        for name, value in [split_define(d) for d in args.defines]:
            defines[name] = value

    compress = {
        'none': False,
        'deflate': True,
        'brotli': JAR_BROTLI,
    }[args.compress]

    copier = FileCopier()
    if args.format == 'flat':
        formatter = FlatFormatter(copier)
    elif args.format == 'jar':
        formatter = JarFormatter(copier, compress=compress, optimize=args.optimizejars)
    elif args.format == 'omni':
        formatter = OmniJarFormatter(copier,
                                     buildconfig.substs['OMNIJAR_NAME'],
                                     compress=compress,
                                     optimize=args.optimizejars,
                                     non_resources=args.non_resource)
    else:
        errors.fatal('Unknown format: %s' % args.format)

    # Adjust defines according to the requested format.
    if isinstance(formatter, OmniJarFormatter):
        defines['MOZ_OMNIJAR'] = 1
    elif 'MOZ_OMNIJAR' in defines:
        del defines['MOZ_OMNIJAR']

    respath = ''
    if 'RESPATH' in defines:
        respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
    while respath.startswith('/'):
        respath = respath[1:]

    if args.unify:
        def is_native(path):
            path = os.path.abspath(path)
            return platform.machine() in mozpath.split(path)

        # Invert args.unify and args.source if args.unify points to the
        # native architecture.
        args.source, args.unify = sorted([args.source, args.unify],
                                         key=is_native, reverse=True)
        if is_native(args.source) and not buildconfig.substs['CROSS_COMPILE']:
            launcher.tooldir = args.source
    elif not buildconfig.substs['CROSS_COMPILE']:
        launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')

    with errors.accumulate():
        finder_args = dict(
            minify=args.minify,
            minify_js=args.minify_js,
        )
        if args.js_binary:
            finder_args['minify_js_verify_command'] = [
                args.js_binary,
                os.path.join(os.path.abspath(os.path.dirname(__file__)),
                    'js-compare-ast.js')
            ]
        if args.unify:
            finder = UnifiedBuildFinder(FileFinder(args.source),
                                        FileFinder(args.unify),
                                        **finder_args)
        else:
            finder = FileFinder(args.source, **finder_args)
        if 'NO_PKG_FILES' in os.environ:
            sinkformatter = NoPkgFilesRemover(formatter,
                                              args.manifest is not None)
        else:
            sinkformatter = formatter
        sink = SimpleManifestSink(finder, sinkformatter)
        if args.manifest:
            preprocess_manifest(sink, args.manifest, defines)
        else:
            sink.add(Component(''), 'bin/*')
        sink.close(args.manifest is not None)

        if args.removals:
            removals_in = StringIO(open(args.removals).read())
            removals_in.name = args.removals
            removals = RemovedFiles(copier)
            preprocess(removals_in, removals, defines)
            copier.add(mozpath.join(respath, 'removed-files'), removals)

    # shlibsign libraries
    if launcher.can_launch():
        if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
            for lib in SIGN_LIBS:
                libbase = mozpath.join(respath, '%s%s') \
                    % (buildconfig.substs['DLL_PREFIX'], lib)
                libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
                if copier.contains(libname):
                    copier.add(libbase + '.chk',
                               LibSignFile(os.path.join(args.destination,
                                                        libname)))

    # Setup preloading
    if args.jarlog and os.path.exists(args.jarlog):
        from mozpack.mozjar import JarLog
        log = JarLog(args.jarlog)
        for p, f in copier:
            if not isinstance(f, Jarrer):
                continue
            key = JarLog.canonicalize(os.path.join(args.destination, p))
            if key in log:
                f.preload(log[key])

    # Fill startup cache
    if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
      and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1' \
      and buildconfig.substs['MOZ_DISABLE_PRECOMPILED_STARTUPCACHE'] != '1':
        gre_path = None
        def get_bases():
            for b in sink.packager.get_bases(addons=False):
                for p in (mozpath.join('bin', b), b):
                    if os.path.exists(os.path.join(args.source, p)):
                        yield p
                        break
        for base in sorted(get_bases()):
            if not gre_path:
                gre_path = base
            omnijar_path = mozpath.join(sink.normalize_path(base),
                                        buildconfig.substs['OMNIJAR_NAME'])
            if formatter.contains(omnijar_path):
                precompile_cache(formatter.copier[omnijar_path],
                                 args.source, gre_path, base)

    copier.copy(args.destination)


if __name__ == '__main__':
    main()