diff options
Diffstat (limited to 'dom/bindings/mozwebidlcodegen/__init__.py')
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/__init__.py | 583 |
1 files changed, 583 insertions, 0 deletions
diff --git a/dom/bindings/mozwebidlcodegen/__init__.py b/dom/bindings/mozwebidlcodegen/__init__.py new file mode 100644 index 000000000..b6e351e52 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/__init__.py @@ -0,0 +1,583 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module contains code for managing WebIDL files and bindings for +# the build system. + +from __future__ import unicode_literals + +import errno +import hashlib +import json +import logging +import os + +from copy import deepcopy + +from mach.mixin.logging import LoggingMixin + +from mozbuild.base import MozbuildObject +from mozbuild.makeutil import Makefile +from mozbuild.pythonutil import iter_modules_in_path +from mozbuild.util import FileAvoidWrite + +import mozpack.path as mozpath + +# There are various imports in this file in functions to avoid adding +# dependencies to config.status. See bug 949875. + + +class BuildResult(object): + """Represents the result of processing WebIDL files. + + This holds a summary of output file generation during code generation. + """ + + def __init__(self): + # The .webidl files that had their outputs regenerated. + self.inputs = set() + + # The output files that were created. + self.created = set() + + # The output files that changed. + self.updated = set() + + # The output files that didn't change. + self.unchanged = set() + + +class WebIDLCodegenManagerState(dict): + """Holds state for the WebIDL code generation manager. + + State is currently just an extended dict. The internal implementation of + state should be considered a black box to everyone except + WebIDLCodegenManager. But we'll still document it. + + Fields: + + version + The integer version of the format. This is to detect incompatible + changes between state. It should be bumped whenever the format + changes or semantics change. + + webidls + A dictionary holding information about every known WebIDL input. + Keys are the basenames of input WebIDL files. Values are dicts of + metadata. Keys in those dicts are: + + * filename - The full path to the input filename. + * inputs - A set of full paths to other webidl files this webidl + depends on. + * outputs - Set of full output paths that are created/derived from + this file. + * sha1 - The hexidecimal SHA-1 of the input filename from the last + processing time. + + global_inputs + A dictionary defining files that influence all processing. Keys + are full filenames. Values are hexidecimal SHA-1 from the last + processing time. + """ + + VERSION = 1 + + def __init__(self, fh=None): + self['version'] = self.VERSION + self['webidls'] = {} + self['global_depends'] = {} + + if not fh: + return + + state = json.load(fh) + if state['version'] != self.VERSION: + raise Exception('Unknown state version: %s' % state['version']) + + self['version'] = state['version'] + self['global_depends'] = state['global_depends'] + + for k, v in state['webidls'].items(): + self['webidls'][k] = v + + # Sets are converted to lists for serialization because JSON + # doesn't support sets. + self['webidls'][k]['inputs'] = set(v['inputs']) + self['webidls'][k]['outputs'] = set(v['outputs']) + + def dump(self, fh): + """Dump serialized state to a file handle.""" + normalized = deepcopy(self) + + for k, v in self['webidls'].items(): + # Convert sets to lists because JSON doesn't support sets. + normalized['webidls'][k]['outputs'] = sorted(v['outputs']) + normalized['webidls'][k]['inputs'] = sorted(v['inputs']) + + json.dump(normalized, fh, sort_keys=True) + + +class WebIDLCodegenManager(LoggingMixin): + """Manages all code generation around WebIDL. + + To facilitate testing, this object is meant to be generic and reusable. + Paths, etc should be parameters and not hardcoded. + """ + + # Global parser derived declaration files. + GLOBAL_DECLARE_FILES = { + 'GeneratedAtomList.h', + 'GeneratedEventList.h', + 'PrototypeList.h', + 'RegisterBindings.h', + 'RegisterWorkerBindings.h', + 'RegisterWorkerDebuggerBindings.h', + 'RegisterWorkletBindings.h', + 'ResolveSystemBinding.h', + 'UnionConversions.h', + 'UnionTypes.h', + } + + # Global parser derived definition files. + GLOBAL_DEFINE_FILES = { + 'RegisterBindings.cpp', + 'RegisterWorkerBindings.cpp', + 'RegisterWorkerDebuggerBindings.cpp', + 'RegisterWorkletBindings.cpp', + 'ResolveSystemBinding.cpp', + 'UnionTypes.cpp', + 'PrototypeList.cpp', + } + + def __init__(self, config_path, inputs, exported_header_dir, + codegen_dir, state_path, cache_dir=None, make_deps_path=None, + make_deps_target=None): + """Create an instance that manages WebIDLs in the build system. + + config_path refers to a WebIDL config file (e.g. Bindings.conf). + inputs is a 4-tuple describing the input .webidl files and how to + process them. Members are: + (set(.webidl files), set(basenames of exported files), + set(basenames of generated events files), + set(example interface names)) + + exported_header_dir and codegen_dir are directories where generated + files will be written to. + state_path is the path to a file that will receive JSON state from our + actions. + make_deps_path is the path to a make dependency file that we can + optionally write. + make_deps_target is the target that receives the make dependencies. It + must be defined if using make_deps_path. + """ + self.populate_logger() + + input_paths, exported_stems, generated_events_stems, example_interfaces = inputs + + self._config_path = config_path + self._input_paths = set(input_paths) + self._exported_stems = set(exported_stems) + self._generated_events_stems = set(generated_events_stems) + self._generated_events_stems_as_array = generated_events_stems + self._example_interfaces = set(example_interfaces) + self._exported_header_dir = exported_header_dir + self._codegen_dir = codegen_dir + self._state_path = state_path + self._cache_dir = cache_dir + self._make_deps_path = make_deps_path + self._make_deps_target = make_deps_target + + if ((make_deps_path and not make_deps_target) or + (not make_deps_path and make_deps_target)): + raise Exception('Must define both make_deps_path and make_deps_target ' + 'if one is defined.') + + self._parser_results = None + self._config = None + self._state = WebIDLCodegenManagerState() + + if os.path.exists(state_path): + with open(state_path, 'rb') as fh: + try: + self._state = WebIDLCodegenManagerState(fh=fh) + except Exception as e: + self.log(logging.WARN, 'webidl_bad_state', {'msg': str(e)}, + 'Bad WebIDL state: {msg}') + + @property + def config(self): + if not self._config: + self._parse_webidl() + + return self._config + + def generate_build_files(self): + """Generate files required for the build. + + This function is in charge of generating all the .h/.cpp files derived + from input .webidl files. Please note that there are build actions + required to produce .webidl files and these build actions are + explicitly not captured here: this function assumes all .webidl files + are present and up to date. + + This routine is called as part of the build to ensure files that need + to exist are present and up to date. This routine may not be called if + the build dependencies (generated as a result of calling this the first + time) say everything is up to date. + + Because reprocessing outputs for every .webidl on every invocation + is expensive, we only regenerate the minimal set of files on every + invocation. The rules for deciding what needs done are roughly as + follows: + + 1. If any .webidl changes, reparse all .webidl files and regenerate + the global derived files. Only regenerate output files (.h/.cpp) + impacted by the modified .webidl files. + 2. If an non-.webidl dependency (Python files, config file) changes, + assume everything is out of date and regenerate the world. This + is because changes in those could globally impact every output + file. + 3. If an output file is missing, ensure it is present by performing + necessary regeneration. + """ + # Despite #1 above, we assume the build system is smart enough to not + # invoke us if nothing has changed. Therefore, any invocation means + # something has changed. And, if anything has changed, we need to + # parse the WebIDL. + self._parse_webidl() + + result = BuildResult() + + # If we parse, we always update globals - they are cheap and it is + # easier that way. + created, updated, unchanged = self._write_global_derived() + result.created |= created + result.updated |= updated + result.unchanged |= unchanged + + # If any of the extra dependencies changed, regenerate the world. + global_changed, global_hashes = self._global_dependencies_changed() + if global_changed: + # Make a copy because we may modify. + changed_inputs = set(self._input_paths) + else: + changed_inputs = self._compute_changed_inputs() + + self._state['global_depends'] = global_hashes + + # Generate bindings from .webidl files. + for filename in sorted(changed_inputs): + basename = mozpath.basename(filename) + result.inputs.add(filename) + written, deps = self._generate_build_files_for_webidl(filename) + result.created |= written[0] + result.updated |= written[1] + result.unchanged |= written[2] + + self._state['webidls'][basename] = dict( + filename=filename, + outputs=written[0] | written[1] | written[2], + inputs=set(deps), + sha1=self._input_hashes[filename], + ) + + # Process some special interfaces required for testing. + for interface in self._example_interfaces: + written = self.generate_example_files(interface) + result.created |= written[0] + result.updated |= written[1] + result.unchanged |= written[2] + + # Generate a make dependency file. + if self._make_deps_path: + mk = Makefile() + codegen_rule = mk.create_rule([self._make_deps_target]) + codegen_rule.add_dependencies(global_hashes.keys()) + codegen_rule.add_dependencies(self._input_paths) + + with FileAvoidWrite(self._make_deps_path) as fh: + mk.dump(fh) + + self._save_state() + + return result + + def generate_example_files(self, interface): + """Generates example files for a given interface.""" + from Codegen import CGExampleRoot + + root = CGExampleRoot(self.config, interface) + + example_paths = self._example_paths(interface) + for path in example_paths: + print "Generating %s" % path + + return self._maybe_write_codegen(root, *example_paths) + + def _parse_webidl(self): + import WebIDL + from Configuration import Configuration + + self.log(logging.INFO, 'webidl_parse', + {'count': len(self._input_paths)}, + 'Parsing {count} WebIDL files.') + + hashes = {} + parser = WebIDL.Parser(self._cache_dir) + + for path in sorted(self._input_paths): + with open(path, 'rb') as fh: + data = fh.read() + hashes[path] = hashlib.sha1(data).hexdigest() + parser.parse(data, path) + + self._parser_results = parser.finish() + self._config = Configuration(self._config_path, self._parser_results, + self._generated_events_stems_as_array) + self._input_hashes = hashes + + def _write_global_derived(self): + from Codegen import GlobalGenRoots + + things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES] + things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES) + + result = (set(), set(), set()) + + for what, filename in things: + stem = mozpath.splitext(filename)[0] + root = getattr(GlobalGenRoots, stem)(self._config) + + if what == 'declare': + code = root.declare() + output_root = self._exported_header_dir + elif what == 'define': + code = root.define() + output_root = self._codegen_dir + else: + raise Exception('Unknown global gen type: %s' % what) + + output_path = mozpath.join(output_root, filename) + self._maybe_write_file(output_path, code, result) + + return result + + def _compute_changed_inputs(self): + """Compute the set of input files that need to be regenerated.""" + changed_inputs = set() + expected_outputs = self.expected_build_output_files() + + # Look for missing output files. + if any(not os.path.exists(f) for f in expected_outputs): + # FUTURE Bug 940469 Only regenerate minimum set. + changed_inputs |= self._input_paths + + # That's it for examining output files. We /could/ examine SHA-1's of + # output files from a previous run to detect modifications. But that's + # a lot of extra work and most build systems don't do that anyway. + + # Now we move on to the input files. + old_hashes = {v['filename']: v['sha1'] + for v in self._state['webidls'].values()} + + old_filenames = set(old_hashes.keys()) + new_filenames = self._input_paths + + # If an old file has disappeared or a new file has arrived, mark + # it. + changed_inputs |= old_filenames ^ new_filenames + + # For the files in common between runs, compare content. If the file + # has changed, mark it. We don't need to perform mtime comparisons + # because content is a stronger validator. + for filename in old_filenames & new_filenames: + if old_hashes[filename] != self._input_hashes[filename]: + changed_inputs.add(filename) + + # We've now populated the base set of inputs that have changed. + + # Inherit dependencies from previous run. The full set of dependencies + # is associated with each record, so we don't need to perform any fancy + # graph traversal. + for v in self._state['webidls'].values(): + if any(dep for dep in v['inputs'] if dep in changed_inputs): + changed_inputs.add(v['filename']) + + # Only use paths that are known to our current state. + # This filters out files that were deleted or changed type (e.g. from + # static to preprocessed). + return changed_inputs & self._input_paths + + def _binding_info(self, p): + """Compute binding metadata for an input path. + + Returns a tuple of: + + (stem, binding_stem, is_event, output_files) + + output_files is itself a tuple. The first two items are the binding + header and C++ paths, respectively. The 2nd pair are the event header + and C++ paths or None if this isn't an event binding. + """ + basename = mozpath.basename(p) + stem = mozpath.splitext(basename)[0] + binding_stem = '%sBinding' % stem + + if stem in self._exported_stems: + header_dir = self._exported_header_dir + else: + header_dir = self._codegen_dir + + is_event = stem in self._generated_events_stems + + files = ( + mozpath.join(header_dir, '%s.h' % binding_stem), + mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem), + mozpath.join(header_dir, '%s.h' % stem) if is_event else None, + mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None, + ) + + return stem, binding_stem, is_event, header_dir, files + + def _example_paths(self, interface): + return ( + mozpath.join(self._codegen_dir, '%s-example.h' % interface), + mozpath.join(self._codegen_dir, '%s-example.cpp' % interface)) + + def expected_build_output_files(self): + """Obtain the set of files generate_build_files() should write.""" + paths = set() + + # Account for global generation. + for p in self.GLOBAL_DECLARE_FILES: + paths.add(mozpath.join(self._exported_header_dir, p)) + for p in self.GLOBAL_DEFINE_FILES: + paths.add(mozpath.join(self._codegen_dir, p)) + + for p in self._input_paths: + stem, binding_stem, is_event, header_dir, files = self._binding_info(p) + paths |= {f for f in files if f} + + for interface in self._example_interfaces: + for p in self._example_paths(interface): + paths.add(p) + + return paths + + def _generate_build_files_for_webidl(self, filename): + from Codegen import ( + CGBindingRoot, + CGEventRoot, + ) + + self.log(logging.INFO, 'webidl_generate_build_for_input', + {'filename': filename}, + 'Generating WebIDL files derived from {filename}') + + stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) + root = CGBindingRoot(self._config, binding_stem, filename) + + result = self._maybe_write_codegen(root, files[0], files[1]) + + if is_event: + generated_event = CGEventRoot(self._config, stem) + result = self._maybe_write_codegen(generated_event, files[2], + files[3], result) + + return result, root.deps() + + def _global_dependencies_changed(self): + """Determine whether the global dependencies have changed.""" + current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) + + # We need to catch other .py files from /dom/bindings. We assume these + # are in the same directory as the config file. + current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) + + current_files.add(self._config_path) + + current_hashes = {} + for f in current_files: + # This will fail if the file doesn't exist. If a current global + # dependency doesn't exist, something else is wrong. + with open(f, 'rb') as fh: + current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() + + # The set of files has changed. + if current_files ^ set(self._state['global_depends'].keys()): + return True, current_hashes + + # Compare hashes. + for f, sha1 in current_hashes.items(): + if sha1 != self._state['global_depends'][f]: + return True, current_hashes + + return False, current_hashes + + def _save_state(self): + with open(self._state_path, 'wb') as fh: + self._state.dump(fh) + + def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): + assert declare_path and define_path + if not result: + result = (set(), set(), set()) + + self._maybe_write_file(declare_path, obj.declare(), result) + self._maybe_write_file(define_path, obj.define(), result) + + return result + + def _maybe_write_file(self, path, content, result): + fh = FileAvoidWrite(path) + fh.write(content) + existed, updated = fh.close() + + if not existed: + result[0].add(path) + elif updated: + result[1].add(path) + else: + result[2].add(path) + + +def create_build_system_manager(topsrcdir, topobjdir, dist_dir): + """Create a WebIDLCodegenManager for use by the build system.""" + src_dir = os.path.join(topsrcdir, 'dom', 'bindings') + obj_dir = os.path.join(topobjdir, 'dom', 'bindings') + + with open(os.path.join(obj_dir, 'file-lists.json'), 'rb') as fh: + files = json.load(fh) + + inputs = (files['webidls'], files['exported_stems'], + files['generated_events_stems'], files['example_interfaces']) + + cache_dir = os.path.join(obj_dir, '_cache') + try: + os.makedirs(cache_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + return WebIDLCodegenManager( + os.path.join(src_dir, 'Bindings.conf'), + inputs, + os.path.join(dist_dir, 'include', 'mozilla', 'dom'), + obj_dir, + os.path.join(obj_dir, 'codegen.json'), + cache_dir=cache_dir, + # The make rules include a codegen.pp file containing dependencies. + make_deps_path=os.path.join(obj_dir, 'codegen.pp'), + make_deps_target='codegen.pp', + ) + + +class BuildSystemWebIDL(MozbuildObject): + @property + def manager(self): + if not hasattr(self, '_webidl_manager'): + self._webidl_manager = create_build_system_manager( + self.topsrcdir, self.topobjdir, self.distdir) + + return self._webidl_manager |