diff options
Diffstat (limited to 'addon-sdk/source/python-lib/cuddlefish/packaging.py')
| -rw-r--r-- | addon-sdk/source/python-lib/cuddlefish/packaging.py | 463 |
1 files changed, 463 insertions, 0 deletions
diff --git a/addon-sdk/source/python-lib/cuddlefish/packaging.py b/addon-sdk/source/python-lib/cuddlefish/packaging.py new file mode 100644 index 000000000..0c5357e8e --- /dev/null +++ b/addon-sdk/source/python-lib/cuddlefish/packaging.py @@ -0,0 +1,463 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +import re +import copy + +import simplejson as json +from cuddlefish.bunch import Bunch + +MANIFEST_NAME = 'package.json' +DEFAULT_LOADER = 'addon-sdk' + +# Is different from root_dir when running tests +env_root = os.environ.get('CUDDLEFISH_ROOT') + +DEFAULT_PROGRAM_MODULE = 'main' + +DEFAULT_ICON = 'icon.png' +DEFAULT_ICON64 = 'icon64.png' + +METADATA_PROPS = ['name', 'description', 'keywords', 'author', 'version', + 'developers', 'translators', 'contributors', 'license', 'homepage', + 'icon', 'icon64', 'main', 'directories', 'permissions', 'preferences'] + +RESOURCE_HOSTNAME_RE = re.compile(r'^[a-z0-9_\-]+$') + +class Error(Exception): + pass + +class MalformedPackageError(Error): + pass + +class MalformedJsonFileError(Error): + pass + +class DuplicatePackageError(Error): + pass + +class PackageNotFoundError(Error): + def __init__(self, missing_package, reason): + self.missing_package = missing_package + self.reason = reason + def __str__(self): + return "%s (%s)" % (self.missing_package, self.reason) + +class BadChromeMarkerError(Error): + pass + +def validate_resource_hostname(name): + """ + Validates the given hostname for a resource: URI. + + For more information, see: + + https://bugzilla.mozilla.org/show_bug.cgi?id=566812#c13 + + Examples: + + >>> validate_resource_hostname('blarg') + + >>> validate_resource_hostname('bl arg') + Traceback (most recent call last): + ... + ValueError: Error: the name of your package contains an invalid character. + Package names can contain only lower-case letters, numbers, underscores, and dashes. + Current package name: bl arg + + >>> validate_resource_hostname('BLARG') + Traceback (most recent call last): + ... + ValueError: Error: the name of your package contains upper-case letters. + Package names can contain only lower-case letters, numbers, underscores, and dashes. + Current package name: BLARG + + >>> validate_resource_hostname('foo@bar') + Traceback (most recent call last): + ... + ValueError: Error: the name of your package contains an invalid character. + Package names can contain only lower-case letters, numbers, underscores, and dashes. + Current package name: foo@bar + """ + + # See https://bugzilla.mozilla.org/show_bug.cgi?id=568131 for details. + if not name.lower() == name: + raise ValueError("""Error: the name of your package contains upper-case letters. +Package names can contain only lower-case letters, numbers, underscores, and dashes. +Current package name: %s""" % name) + + if not RESOURCE_HOSTNAME_RE.match(name): + raise ValueError("""Error: the name of your package contains an invalid character. +Package names can contain only lower-case letters, numbers, underscores, and dashes. +Current package name: %s""" % name) + +def find_packages_with_module(pkg_cfg, name): + # TODO: Make this support more than just top-level modules. + filename = "%s.js" % name + packages = [] + for cfg in pkg_cfg.packages.itervalues(): + if 'lib' in cfg: + matches = [dirname for dirname in resolve_dirs(cfg, cfg.lib) + if os.path.exists(os.path.join(dirname, filename))] + if matches: + packages.append(cfg.name) + return packages + +def resolve_dirs(pkg_cfg, dirnames): + for dirname in dirnames: + yield resolve_dir(pkg_cfg, dirname) + +def resolve_dir(pkg_cfg, dirname): + return os.path.join(pkg_cfg.root_dir, dirname) + +def validate_permissions(perms): + if (perms.get('cross-domain-content') and + not isinstance(perms.get('cross-domain-content'), list)): + raise ValueError("Error: `cross-domain-content` permissions in \ + package.json file must be an array of strings:\n %s" % perms) + +def get_metadata(pkg_cfg, deps): + metadata = Bunch() + for pkg_name in deps: + cfg = pkg_cfg.packages[pkg_name] + metadata[pkg_name] = Bunch() + for prop in METADATA_PROPS: + if cfg.get(prop): + if prop == 'permissions': + validate_permissions(cfg[prop]) + metadata[pkg_name][prop] = cfg[prop] + return metadata + +def set_section_dir(base_json, name, base_path, dirnames, allow_root=False): + resolved = compute_section_dir(base_json, base_path, dirnames, allow_root) + if resolved: + base_json[name] = os.path.abspath(resolved) + +def compute_section_dir(base_json, base_path, dirnames, allow_root): + # PACKAGE_JSON.lib is highest priority + # then PACKAGE_JSON.directories.lib + # then lib/ (if it exists) + # then . (but only if allow_root=True) + for dirname in dirnames: + if base_json.get(dirname): + return os.path.join(base_path, base_json[dirname]) + if "directories" in base_json: + for dirname in dirnames: + if dirname in base_json.directories: + return os.path.join(base_path, base_json.directories[dirname]) + for dirname in dirnames: + if os.path.isdir(os.path.join(base_path, dirname)): + return os.path.join(base_path, dirname) + if allow_root: + return os.path.abspath(base_path) + return None + +def normalize_string_or_array(base_json, key): + if base_json.get(key): + if isinstance(base_json[key], basestring): + base_json[key] = [base_json[key]] + +def load_json_file(path): + data = open(path, 'r').read() + try: + return Bunch(json.loads(data)) + except ValueError, e: + raise MalformedJsonFileError('%s when reading "%s"' % (str(e), + path)) + +def get_config_in_dir(path): + package_json = os.path.join(path, MANIFEST_NAME) + if not (os.path.exists(package_json) and + os.path.isfile(package_json)): + raise MalformedPackageError('%s not found in "%s"' % (MANIFEST_NAME, + path)) + base_json = load_json_file(package_json) + + if 'name' not in base_json: + base_json.name = os.path.basename(path) + + # later processing steps will expect to see the following keys in the + # base_json that we return: + # + # name: name of the package + # lib: list of directories with .js files + # test: list of directories with test-*.js files + # doc: list of directories with documentation .md files + # data: list of directories with bundled arbitrary data files + # packages: ? + + if (not base_json.get('tests') and + os.path.isdir(os.path.join(path, 'test'))): + base_json['tests'] = 'test' + + set_section_dir(base_json, 'lib', path, ['lib'], True) + set_section_dir(base_json, 'tests', path, ['test', 'tests'], False) + set_section_dir(base_json, 'doc', path, ['doc', 'docs']) + set_section_dir(base_json, 'data', path, ['data']) + set_section_dir(base_json, 'packages', path, ['packages']) + set_section_dir(base_json, 'locale', path, ['locale']) + + if (not base_json.get('icon') and + os.path.isfile(os.path.join(path, DEFAULT_ICON))): + base_json['icon'] = DEFAULT_ICON + + if (not base_json.get('icon64') and + os.path.isfile(os.path.join(path, DEFAULT_ICON64))): + base_json['icon64'] = DEFAULT_ICON64 + + for key in ['lib', 'tests', 'dependencies', 'packages']: + # TODO: lib/tests can be an array?? consider interaction with + # compute_section_dir above + normalize_string_or_array(base_json, key) + + if 'main' not in base_json and 'lib' in base_json: + for dirname in base_json['lib']: + program = os.path.join(path, dirname, + '%s.js' % DEFAULT_PROGRAM_MODULE) + if os.path.exists(program): + base_json['main'] = DEFAULT_PROGRAM_MODULE + break + + base_json.root_dir = path + + if "dependencies" in base_json: + deps = base_json["dependencies"] + deps = [x for x in deps if x not in ["addon-kit", "api-utils"]] + deps.append("addon-sdk") + base_json["dependencies"] = deps + + return base_json + +def _is_same_file(a, b): + if hasattr(os.path, 'samefile'): + return os.path.samefile(a, b) + return a == b + +def build_config(root_dir, target_cfg, packagepath=[]): + dirs_to_scan = [env_root] # root is addon-sdk dir, diff from root_dir in tests + + def add_packages_from_config(pkgconfig): + if 'packages' in pkgconfig: + for package_dir in resolve_dirs(pkgconfig, pkgconfig.packages): + dirs_to_scan.append(package_dir) + + add_packages_from_config(target_cfg) + + packages_dir = os.path.join(root_dir, 'packages') + if os.path.exists(packages_dir) and os.path.isdir(packages_dir): + dirs_to_scan.append(packages_dir) + dirs_to_scan.extend(packagepath) + + packages = Bunch({target_cfg.name: target_cfg}) + + while dirs_to_scan: + packages_dir = dirs_to_scan.pop() + if os.path.exists(os.path.join(packages_dir, "package.json")): + package_paths = [packages_dir] + else: + package_paths = [os.path.join(packages_dir, dirname) + for dirname in os.listdir(packages_dir) + if not dirname.startswith('.')] + package_paths = [dirname for dirname in package_paths + if os.path.isdir(dirname)] + + for path in package_paths: + pkgconfig = get_config_in_dir(path) + if pkgconfig.name in packages: + otherpkg = packages[pkgconfig.name] + if not _is_same_file(otherpkg.root_dir, path): + raise DuplicatePackageError(path, otherpkg.root_dir) + else: + packages[pkgconfig.name] = pkgconfig + add_packages_from_config(pkgconfig) + + return Bunch(packages=packages) + +def get_deps_for_targets(pkg_cfg, targets): + visited = [] + deps_left = [[dep, None] for dep in list(targets)] + + while deps_left: + [dep, required_by] = deps_left.pop() + if dep not in visited: + visited.append(dep) + if dep not in pkg_cfg.packages: + required_reason = ("required by '%s'" % (required_by)) \ + if required_by is not None \ + else "specified as target" + raise PackageNotFoundError(dep, required_reason) + dep_cfg = pkg_cfg.packages[dep] + deps_left.extend([[i, dep] for i in dep_cfg.get('dependencies', [])]) + deps_left.extend([[i, dep] for i in dep_cfg.get('extra_dependencies', [])]) + + return visited + +def generate_build_for_target(pkg_cfg, target, deps, + include_tests=True, + include_dep_tests=False, + is_running_tests=False, + default_loader=DEFAULT_LOADER): + + build = Bunch(# Contains section directories for all packages: + packages=Bunch(), + locale=Bunch() + ) + + def add_section_to_build(cfg, section, is_code=False, + is_data=False): + if section in cfg: + dirnames = cfg[section] + if isinstance(dirnames, basestring): + # This is just for internal consistency within this + # function, it has nothing to do w/ a non-canonical + # configuration dict. + dirnames = [dirnames] + for dirname in resolve_dirs(cfg, dirnames): + # ensure that package name is valid + try: + validate_resource_hostname(cfg.name) + except ValueError, err: + print err + sys.exit(1) + # ensure that this package has an entry + if not cfg.name in build.packages: + build.packages[cfg.name] = Bunch() + # detect duplicated sections + if section in build.packages[cfg.name]: + raise KeyError("package's section already defined", + cfg.name, section) + # Register this section (lib, data, tests) + build.packages[cfg.name][section] = dirname + + def add_locale_to_build(cfg): + # Bug 730776: Ignore locales for addon-kit, that are only for unit tests + if not is_running_tests and cfg.name == "addon-sdk": + return + + path = resolve_dir(cfg, cfg['locale']) + files = os.listdir(path) + for filename in files: + fullpath = os.path.join(path, filename) + if os.path.isfile(fullpath) and filename.endswith('.properties'): + language = filename[:-len('.properties')] + + from property_parser import parse_file, MalformedLocaleFileError + try: + content = parse_file(fullpath) + except MalformedLocaleFileError, msg: + print msg[0] + sys.exit(1) + + # Merge current locales into global locale hashtable. + # Locale files only contains one big JSON object + # that act as an hastable of: + # "keys to translate" => "translated keys" + if language in build.locale: + merge = (build.locale[language].items() + + content.items()) + build.locale[language] = Bunch(merge) + else: + build.locale[language] = content + + def add_dep_to_build(dep): + dep_cfg = pkg_cfg.packages[dep] + add_section_to_build(dep_cfg, "lib", is_code=True) + add_section_to_build(dep_cfg, "data", is_data=True) + if include_tests and include_dep_tests: + add_section_to_build(dep_cfg, "tests", is_code=True) + if 'locale' in dep_cfg: + add_locale_to_build(dep_cfg) + if ("loader" in dep_cfg) and ("loader" not in build): + build.loader = "%s/%s" % (dep, + dep_cfg.loader) + + target_cfg = pkg_cfg.packages[target] + + if include_tests and not include_dep_tests: + add_section_to_build(target_cfg, "tests", is_code=True) + + for dep in deps: + add_dep_to_build(dep) + + if 'loader' not in build: + add_dep_to_build(DEFAULT_LOADER) + + if 'icon' in target_cfg: + build['icon'] = os.path.join(target_cfg.root_dir, target_cfg.icon) + del target_cfg['icon'] + + if 'icon64' in target_cfg: + build['icon64'] = os.path.join(target_cfg.root_dir, target_cfg.icon64) + del target_cfg['icon64'] + + if 'id' in target_cfg: + # NOTE: logic duplicated from buildJID() + jid = target_cfg['id'] + if not ('@' in jid or jid.startswith('{')): + jid += '@jetpack' + build['preferencesBranch'] = jid + + if 'preferences-branch' in target_cfg: + build['preferencesBranch'] = target_cfg['preferences-branch'] + + return build + +def _get_files_in_dir(path): + data = {} + files = os.listdir(path) + for filename in files: + fullpath = os.path.join(path, filename) + if os.path.isdir(fullpath): + data[filename] = _get_files_in_dir(fullpath) + else: + try: + info = os.stat(fullpath) + data[filename] = ("file", dict(size=info.st_size)) + except OSError: + pass + return ("directory", data) + +def build_pkg_index(pkg_cfg): + pkg_cfg = copy.deepcopy(pkg_cfg) + for pkg in pkg_cfg.packages: + root_dir = pkg_cfg.packages[pkg].root_dir + files = _get_files_in_dir(root_dir) + pkg_cfg.packages[pkg].files = files + try: + readme = open(root_dir + '/README.md').read() + pkg_cfg.packages[pkg].readme = readme + except IOError: + pass + del pkg_cfg.packages[pkg].root_dir + return pkg_cfg.packages + +def build_pkg_cfg(root): + pkg_cfg = build_config(root, Bunch(name='dummy')) + del pkg_cfg.packages['dummy'] + return pkg_cfg + +def call_plugins(pkg_cfg, deps): + for dep in deps: + dep_cfg = pkg_cfg.packages[dep] + dirnames = dep_cfg.get('python-lib', []) + for dirname in resolve_dirs(dep_cfg, dirnames): + sys.path.append(dirname) + module_names = dep_cfg.get('python-plugins', []) + for module_name in module_names: + module = __import__(module_name) + module.init(root_dir=dep_cfg.root_dir) + +def call_cmdline_tool(env_root, pkg_name): + pkg_cfg = build_config(env_root, Bunch(name='dummy')) + if pkg_name not in pkg_cfg.packages: + print "This tool requires the '%s' package." % pkg_name + sys.exit(1) + cfg = pkg_cfg.packages[pkg_name] + for dirname in resolve_dirs(cfg, cfg['python-lib']): + sys.path.append(dirname) + module_name = cfg.get('python-cmdline-tool') + module = __import__(module_name) + module.run() |