diff options
Diffstat (limited to 'config/expandlibs_exec.py')
| -rw-r--r-- | config/expandlibs_exec.py | 354 |
1 files changed, 354 insertions, 0 deletions
diff --git a/config/expandlibs_exec.py b/config/expandlibs_exec.py new file mode 100644 index 000000000..c05343022 --- /dev/null +++ b/config/expandlibs_exec.py @@ -0,0 +1,354 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +'''expandlibs-exec.py applies expandlibs rules, and some more (see below) to +a given command line, and executes that command line with the expanded +arguments. + +With the --extract argument (useful for e.g. $(AR)), it extracts object files +from static libraries (or use those listed in library descriptors directly). + +With the --uselist argument (useful for e.g. $(CC)), it replaces all object +files with a list file. This can be used to avoid limitations in the length +of a command line. The kind of list file format used depends on the +EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list) +or 'linkerscript' for GNU ld linker scripts. +See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details. + +With the --symbol-order argument, followed by a file name, it will add the +relevant linker options to change the order in which the linker puts the +symbols appear in the resulting binary. Only works for ELF targets. +''' +from __future__ import with_statement +import sys +import os +from expandlibs import ( + ExpandArgs, + relativize, + isDynamicLib, + isObject, +) +import expandlibs_config as conf +from optparse import OptionParser +import subprocess +import tempfile +import shutil +import subprocess +import re +from mozbuild.makeutil import Makefile + +# The are the insert points for a GNU ld linker script, assuming a more +# or less "standard" default linker script. This is not a dict because +# order is important. +SECTION_INSERT_BEFORE = [ + ('.text', '.fini'), + ('.rodata', '.rodata1'), + ('.data.rel.ro', '.dynamic'), + ('.data', '.data1'), +] + +class ExpandArgsMore(ExpandArgs): + ''' Meant to be used as 'with ExpandArgsMore(args) as ...: ''' + def __enter__(self): + self.tmp = [] + return self + + def __exit__(self, type, value, tb): + '''Automatically remove temporary files''' + for tmp in self.tmp: + if os.path.isdir(tmp): + shutil.rmtree(tmp, True) + else: + os.remove(tmp) + + def extract(self): + self[0:] = self._extract(self) + + def _extract(self, args): + '''When a static library name is found, either extract its contents + in a temporary directory or use the information found in the + corresponding lib descriptor. + ''' + ar_extract = conf.AR_EXTRACT.split() + newlist = [] + + def lookup(base, f): + for root, dirs, files in os.walk(base): + if f in files: + return os.path.join(root, f) + + for arg in args: + if os.path.splitext(arg)[1] == conf.LIB_SUFFIX: + if os.path.exists(arg + conf.LIBS_DESC_SUFFIX): + newlist += self._extract(self._expand_desc(arg)) + continue + elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'): + tmp = tempfile.mkdtemp(dir=os.curdir) + self.tmp.append(tmp) + if conf.AR == 'lib': + out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg]) + files = out.splitlines() + # If lib -list returns a list full of dlls, it's an + # import lib. + if all(isDynamicLib(f) for f in files): + newlist += [arg] + continue + for f in files: + subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp) + else: + subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp) + objs = [] + basedir = os.path.dirname(arg) + for root, dirs, files in os.walk(tmp): + for f in files: + if isObject(f): + # If the file extracted from the library also + # exists in the directory containing the + # library, or one of its subdirectories, use + # that instead. + maybe_obj = lookup(os.path.join(basedir, os.path.relpath(root, tmp)), f) + if maybe_obj: + objs.append(relativize(maybe_obj)) + else: + objs.append(relativize(os.path.join(root, f))) + newlist += sorted(objs) + continue + newlist += [arg] + return newlist + + def makelist(self): + '''Replaces object file names with a temporary list file, using a + list format depending on the EXPAND_LIBS_LIST_STYLE variable + ''' + objs = [o for o in self if isObject(o)] + if not len(objs): return + fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir) + if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript": + content = ['INPUT("%s")\n' % obj for obj in objs] + ref = tmp + elif conf.EXPAND_LIBS_LIST_STYLE == "filelist": + content = ["%s\n" % obj for obj in objs] + ref = "-Wl,-filelist," + tmp + elif conf.EXPAND_LIBS_LIST_STYLE == "list": + content = ["%s\n" % obj for obj in objs] + ref = "@" + tmp + else: + os.close(fd) + os.remove(tmp) + return + self.tmp.append(tmp) + f = os.fdopen(fd, "w") + f.writelines(content) + f.close() + idx = self.index(objs[0]) + newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs] + self[0:] = newlist + + def _getFoldedSections(self): + '''Returns a dict about folded sections. + When section A and B are folded into section C, the dict contains: + { 'A': 'C', + 'B': 'C', + 'C': ['A', 'B'] }''' + if not conf.LD_PRINT_ICF_SECTIONS: + return {} + + proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + (stdout, stderr) = proc.communicate() + result = {} + # gold's --print-icf-sections output looks like the following: + # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o' + # In terms of words, chances are this will change in the future, + # especially considering "into" is misplaced. Splitting on quotes + # seems safer. + for l in stderr.split('\n'): + quoted = l.split("'") + if len(quoted) > 5 and quoted[1] != quoted[5]: + result[quoted[1]] = [quoted[5]] + if quoted[5] in result: + result[quoted[5]].append(quoted[1]) + else: + result[quoted[5]] = [quoted[1]] + return result + + def _getOrderedSections(self, ordered_symbols): + '''Given an ordered list of symbols, returns the corresponding list + of sections following the order.''' + if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']: + raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) + finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX]) + folded = self._getFoldedSections() + sections = set() + ordered_sections = [] + for symbol in ordered_symbols: + symbol_sections = finder.getSections(symbol) + all_symbol_sections = [] + for section in symbol_sections: + if section in folded: + if isinstance(folded[section], str): + section = folded[section] + all_symbol_sections.append(section) + all_symbol_sections.extend(folded[section]) + else: + all_symbol_sections.append(section) + for section in all_symbol_sections: + if not section in sections: + ordered_sections.append(section) + sections.add(section) + return ordered_sections + + def orderSymbols(self, order): + '''Given a file containing a list of symbols, adds the appropriate + argument to make the linker put the symbols in that order.''' + with open(order) as file: + sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()]) + split_sections = {} + linked_sections = [s[0] for s in SECTION_INSERT_BEFORE] + for s in sections: + for linked_section in linked_sections: + if s.startswith(linked_section): + if linked_section in split_sections: + split_sections[linked_section].append(s) + else: + split_sections[linked_section] = [s] + break + content = [] + # Order is important + linked_sections = [s for s in linked_sections if s in split_sections] + + if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file': + option = '-Wl,--section-ordering-file,%s' + content = sections + for linked_section in linked_sections: + content.extend(split_sections[linked_section]) + content.append('%s.*' % linked_section) + content.append(linked_section) + + elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript': + option = '-Wl,-T,%s' + section_insert_before = dict(SECTION_INSERT_BEFORE) + for linked_section in linked_sections: + content.append('SECTIONS {') + content.append(' %s : {' % linked_section) + content.extend(' *(%s)' % s for s in split_sections[linked_section]) + content.append(' }') + content.append('}') + content.append('INSERT BEFORE %s' % section_insert_before[linked_section]) + else: + raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) + + fd, tmp = tempfile.mkstemp(dir=os.curdir) + f = os.fdopen(fd, "w") + f.write('\n'.join(content)+'\n') + f.close() + self.tmp.append(tmp) + self.append(option % tmp) + +class SectionFinder(object): + '''Instances of this class allow to map symbol names to sections in + object files.''' + + def __init__(self, objs): + '''Creates an instance, given a list of object files.''' + if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']: + raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) + self.mapping = {} + for obj in objs: + if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX: + raise Exception('%s is not an object nor a static library' % obj) + for symbol, section in SectionFinder._getSymbols(obj): + sym = SectionFinder._normalize(symbol) + if sym in self.mapping: + if not section in self.mapping[sym]: + self.mapping[sym].append(section) + else: + self.mapping[sym] = [section] + + def getSections(self, symbol): + '''Given a symbol, returns a list of sections containing it or the + corresponding thunks. When the given symbol is a thunk, returns the + list of sections containing its corresponding normal symbol and the + other thunks for that symbol.''' + sym = SectionFinder._normalize(symbol) + if sym in self.mapping: + return self.mapping[sym] + return [] + + @staticmethod + def _normalize(symbol): + '''For normal symbols, return the given symbol. For thunks, return + the corresponding normal symbol.''' + if re.match('^_ZThn[0-9]+_', symbol): + return re.sub('^_ZThn[0-9]+_', '_Z', symbol) + return symbol + + @staticmethod + def _getSymbols(obj): + '''Returns a list of (symbol, section) contained in the given object + file.''' + proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + (stdout, stderr) = proc.communicate() + syms = [] + for line in stdout.splitlines(): + # Each line has the following format: + # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol> + tmp = line.split(' ',1) + # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"] + # We only need to consider cases where "<section>\t<length> <symbol>" is present, + # and where the [FfO] flag is either F (function) or O (object). + if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']: + tmp = tmp[1][8:].split() + # That gives us ["<section>","<length>", "<symbol>"] + syms.append((tmp[-1], tmp[0])) + return syms + +def print_command(out, args): + print >>out, "Executing: " + " ".join(args) + for tmp in [f for f in args.tmp if os.path.isfile(f)]: + print >>out, tmp + ":" + with open(tmp) as file: + print >>out, "".join([" " + l for l in file.readlines()]) + out.flush() + +def main(args, proc_callback=None): + parser = OptionParser() + parser.add_option("--extract", action="store_true", dest="extract", + help="when a library has no descriptor file, extract it first, when possible") + parser.add_option("--uselist", action="store_true", dest="uselist", + help="use a list file for objects when executing a command") + parser.add_option("--verbose", action="store_true", dest="verbose", + help="display executed command and temporary files content") + parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE", + help="use the given list of symbols to order symbols in the resulting binary when using with a linker") + + (options, args) = parser.parse_args(args) + + with ExpandArgsMore(args) as args: + if options.extract: + args.extract() + if options.symbol_order: + args.orderSymbols(options.symbol_order) + if options.uselist: + args.makelist() + + if options.verbose: + print_command(sys.stderr, args) + try: + proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + if proc_callback: + proc_callback(proc) + except Exception, e: + print >>sys.stderr, 'error: Launching', args, ':', e + raise e + (stdout, stderr) = proc.communicate() + if proc.returncode and not options.verbose: + print_command(sys.stderr, args) + sys.stderr.write(stdout) + sys.stderr.flush() + if proc.returncode: + return proc.returncode + return 0 + +if __name__ == '__main__': + exit(main(sys.argv[1:])) |