From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- python/devtools/migrate-l10n/migrate/__init__.py | 0 .../devtools/migrate-l10n/migrate/conf/bug1294186 | 22 ++ .../migrate-l10n/migrate/conf/bug1308500_1309191 | 97 ++++++++ python/devtools/migrate-l10n/migrate/main.py | 261 +++++++++++++++++++++ .../migrate-l10n/migrate/tests/__init__.py | 0 5 files changed, 380 insertions(+) create mode 100644 python/devtools/migrate-l10n/migrate/__init__.py create mode 100644 python/devtools/migrate-l10n/migrate/conf/bug1294186 create mode 100644 python/devtools/migrate-l10n/migrate/conf/bug1308500_1309191 create mode 100644 python/devtools/migrate-l10n/migrate/main.py create mode 100644 python/devtools/migrate-l10n/migrate/tests/__init__.py (limited to 'python/devtools/migrate-l10n/migrate') diff --git a/python/devtools/migrate-l10n/migrate/__init__.py b/python/devtools/migrate-l10n/migrate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/devtools/migrate-l10n/migrate/conf/bug1294186 b/python/devtools/migrate-l10n/migrate/conf/bug1294186 new file mode 100644 index 000000000..0b91b4d58 --- /dev/null +++ b/python/devtools/migrate-l10n/migrate/conf/bug1294186 @@ -0,0 +1,22 @@ +font-inspector.properties:fontinspector.seeAll.tooltip = font-inspector.dtd:showAllFonts +font-inspector.properties:fontinspector.seeAll = font-inspector.dtd:showAllFontsUsed +font-inspector.properties:fontinspector.usedAs = font-inspector.dtd:usedAs +font-inspector.properties:fontinspector.system = font-inspector.dtd:system +font-inspector.properties:fontinspector.remote = font-inspector.dtd:remote +font-inspector.properties:fontinspector.previewText = font-inspector.dtd:previewHint + +inspector.properties:inspector.eyedropper.label = inspector.dtd:inspectorEyeDropper.label +inspector.properties:inspector.breadcrumbs.label = inspector.dtd:inspectorBreadcrumbsGroup + +boxmodel.properties:boxmodel.title = layoutview.dtd:layoutViewTitle +boxmodel.properties:boxmodel.margin = layoutview.dtd:margin.tooltip +boxmodel.properties:boxmodel.padding = layoutview.dtd:padding.tooltip +boxmodel.properties:boxmodel.border = layoutview.dtd:border.tooltip +boxmodel.properties:boxmodel.content = layoutview.dtd:content.tooltip +boxmodel.properties:boxmodel.geometryButton.tooltip = layoutview.dtd:geometry.button.tooltip + +inspector.properties:inspector.browserStyles.label = styleinspector.dtd:browserStylesLabel +inspector.properties:inspector.filterStyles.placeholder = styleinspector.dtd:filterStylesPlaceholder +inspector.properties:inspector.addRule.tooltip = styleinspector.dtd:addRuleButtonTooltip +inspector.properties:inspector.togglePseudo.tooltip = styleinspector.dtd:togglePseudoClassPanel +inspector.properties:inspector.noProperties = styleinspector.dtd:noPropertiesFound diff --git a/python/devtools/migrate-l10n/migrate/conf/bug1308500_1309191 b/python/devtools/migrate-l10n/migrate/conf/bug1308500_1309191 new file mode 100644 index 000000000..177236b33 --- /dev/null +++ b/python/devtools/migrate-l10n/migrate/conf/bug1308500_1309191 @@ -0,0 +1,97 @@ +netmonitor.properties:netmonitor.perfNotice1 = netmonitor.dtd:netmonitorUI.perfNotice1 +netmonitor.properties:netmonitor.perfNotice2 = netmonitor.dtd:netmonitorUI.perfNotice2 +netmonitor.properties:netmonitor.perfNotice3 = netmonitor.dtd:netmonitorUI.perfNotice3 +netmonitor.properties:netmonitor.reloadNotice1 = netmonitor.dtd:netmonitorUI.reloadNotice1 +netmonitor.properties:netmonitor.reloadNotice2 = netmonitor.dtd:netmonitorUI.reloadNotice2 +netmonitor.properties:netmonitor.reloadNotice3 = netmonitor.dtd:netmonitorUI.reloadNotice3 +netmonitor.properties:netmonitor.toolbar.status3 = netmonitor.dtd:netmonitorUI.toolbar.status3 +netmonitor.properties:netmonitor.toolbar.method = netmonitor.dtd:netmonitorUI.toolbar.method +netmonitor.properties:netmonitor.toolbar.file = netmonitor.dtd:netmonitorUI.toolbar.file +netmonitor.properties:netmonitor.toolbar.domain = netmonitor.dtd:netmonitorUI.toolbar.domain +netmonitor.properties:netmonitor.toolbar.cause = netmonitor.dtd:netmonitorUI.toolbar.cause +netmonitor.properties:netmonitor.toolbar.type = netmonitor.dtd:netmonitorUI.toolbar.type +netmonitor.properties:netmonitor.toolbar.transferred = netmonitor.dtd:netmonitorUI.toolbar.transferred +netmonitor.properties:netmonitor.toolbar.size = netmonitor.dtd:netmonitorUI.toolbar.size +netmonitor.properties:netmonitor.toolbar.waterfall = netmonitor.dtd:netmonitorUI.toolbar.waterfall +netmonitor.properties:netmonitor.tab.headers = netmonitor.dtd:netmonitorUI.tab.headers +netmonitor.properties:netmonitor.tab.cookies = netmonitor.dtd:netmonitorUI.tab.cookies +netmonitor.properties:netmonitor.tab.params = netmonitor.dtd:netmonitorUI.tab.params +netmonitor.properties:netmonitor.tab.response = netmonitor.dtd:netmonitorUI.tab.response +netmonitor.properties:netmonitor.tab.timings = netmonitor.dtd:netmonitorUI.tab.timings +netmonitor.properties:netmonitor.tab.preview = netmonitor.dtd:netmonitorUI.tab.preview +netmonitor.properties:netmonitor.tab.security = netmonitor.dtd:netmonitorUI.tab.security +netmonitor.properties:netmonitor.toolbar.filter.all = netmonitor.dtd:netmonitorUI.footer.filterAll +netmonitor.properties:netmonitor.toolbar.filter.html = netmonitor.dtd:netmonitorUI.footer.filterHTML +netmonitor.properties:netmonitor.toolbar.filter.css = netmonitor.dtd:netmonitorUI.footer.filterCSS +netmonitor.properties:netmonitor.toolbar.filter.js = netmonitor.dtd:netmonitorUI.footer.filterJS +netmonitor.properties:netmonitor.toolbar.filter.xhr = netmonitor.dtd:netmonitorUI.footer.filterXHR +netmonitor.properties:netmonitor.toolbar.filter.fonts = netmonitor.dtd:netmonitorUI.footer.filterFonts +netmonitor.properties:netmonitor.toolbar.filter.images = netmonitor.dtd:netmonitorUI.footer.filterImages +netmonitor.properties:netmonitor.toolbar.filter.media = netmonitor.dtd:netmonitorUI.footer.filterMedia +netmonitor.properties:netmonitor.toolbar.filter.flash = netmonitor.dtd:netmonitorUI.footer.filterFlash +netmonitor.properties:netmonitor.toolbar.filter.ws = netmonitor.dtd:netmonitorUI.footer.filterWS +netmonitor.properties:netmonitor.toolbar.filter.other = netmonitor.dtd:netmonitorUI.footer.filterOther +netmonitor.properties:netmonitor.toolbar.filterFreetext.label = netmonitor.dtd:netmonitorUI.footer.filterFreetext.label +netmonitor.properties:netmonitor.toolbar.clear = netmonitor.dtd:netmonitorUI.footer.clear +netmonitor.properties:netmonitor.toolbar.perf = netmonitor.dtd:netmonitorUI.footer.perf +netmonitor.properties:netmonitor.panesButton.tooltip = netmonitor.dtd:netmonitorUI.panesButton.tooltip +netmonitor.properties:netmonitor.summary.url = netmonitor.dtd:netmonitorUI.summary.url +netmonitor.properties:netmonitor.summary.method = netmonitor.dtd:netmonitorUI.summary.method +netmonitor.properties:netmonitor.summary.address = netmonitor.dtd:netmonitorUI.summary.address +netmonitor.properties:netmonitor.summary.status = netmonitor.dtd:netmonitorUI.summary.status +netmonitor.properties:netmonitor.summary.version = netmonitor.dtd:netmonitorUI.summary.version +netmonitor.properties:netmonitor.summary.editAndResend = netmonitor.dtd:netmonitorUI.summary.editAndResend +netmonitor.properties:netmonitor.summary.rawHeaders = netmonitor.dtd:netmonitorUI.summary.rawHeaders +netmonitor.properties:netmonitor.summary.rawHeaders.requestHeaders = netmonitor.dtd:netmonitorUI.summary.rawHeaders.requestHeaders +netmonitor.properties:netmonitor.summary.rawHeaders.responseHeaders = netmonitor.dtd:netmonitorUI.summary.rawHeaders.responseHeaders +netmonitor.properties:netmonitor.summary.size = netmonitor.dtd:netmonitorUI.summary.size +netmonitor.properties:netmonitor.response.name = netmonitor.dtd:netmonitorUI.response.name +netmonitor.properties:netmonitor.response.dimensions = netmonitor.dtd:netmonitorUI.response.dimensions +netmonitor.properties:netmonitor.response.mime = netmonitor.dtd:netmonitorUI.response.mime +netmonitor.properties:netmonitor.timings.blocked = netmonitor.dtd:netmonitorUI.timings.blocked +netmonitor.properties:netmonitor.timings.dns = netmonitor.dtd:netmonitorUI.timings.dns +netmonitor.properties:netmonitor.timings.connect = netmonitor.dtd:netmonitorUI.timings.connect +netmonitor.properties:netmonitor.timings.send = netmonitor.dtd:netmonitorUI.timings.send +netmonitor.properties:netmonitor.timings.wait = netmonitor.dtd:netmonitorUI.timings.wait +netmonitor.properties:netmonitor.timings.receive = netmonitor.dtd:netmonitorUI.timings.receive +netmonitor.properties:netmonitor.security.warning.cipher = netmonitor.dtd:netmonitorUI.security.warning.cipher +netmonitor.properties:netmonitor.security.error = netmonitor.dtd:netmonitorUI.security.error +netmonitor.properties:netmonitor.security.protocolVersion = netmonitor.dtd:netmonitorUI.security.protocolVersion +netmonitor.properties:netmonitor.security.cipherSuite = netmonitor.dtd:netmonitorUI.security.cipherSuite +netmonitor.properties:netmonitor.security.hsts = netmonitor.dtd:netmonitorUI.security.hsts +netmonitor.properties:netmonitor.security.hpkp = netmonitor.dtd:netmonitorUI.security.hpkp +netmonitor.properties:netmonitor.security.connection = netmonitor.dtd:netmonitorUI.security.connection +netmonitor.properties:netmonitor.security.certificate = netmonitor.dtd:netmonitorUI.security.certificate +netmonitor.properties:netmonitor.context.copyUrl = netmonitor.dtd:netmonitorUI.context.copyUrl +netmonitor.properties:netmonitor.context.copyUrl.accesskey = netmonitor.dtd:netmonitorUI.context.copyUrl.accesskey +netmonitor.properties:netmonitor.context.copyUrlParams = netmonitor.dtd:netmonitorUI.context.copyUrlParams +netmonitor.properties:netmonitor.context.copyUrlParams.accesskey = netmonitor.dtd:netmonitorUI.context.copyUrlParams.accesskey +netmonitor.properties:netmonitor.context.copyPostData = netmonitor.dtd:netmonitorUI.context.copyPostData +netmonitor.properties:netmonitor.context.copyPostData.accesskey = netmonitor.dtd:netmonitorUI.context.copyPostData.accesskey +netmonitor.properties:netmonitor.context.copyAsCurl = netmonitor.dtd:netmonitorUI.context.copyAsCurl +netmonitor.properties:netmonitor.context.copyAsCurl.accesskey = netmonitor.dtd:netmonitorUI.context.copyAsCurl.accesskey +netmonitor.properties:netmonitor.context.copyRequestHeaders = netmonitor.dtd:netmonitorUI.context.copyRequestHeaders +netmonitor.properties:netmonitor.context.copyRequestHeaders.accesskey = netmonitor.dtd:netmonitorUI.context.copyRequestHeaders.accesskey +netmonitor.properties:netmonitor.context.copyResponseHeaders = netmonitor.dtd:netmonitorUI.context.copyResponseHeaders +netmonitor.properties:netmonitor.context.copyResponseHeaders.accesskey = netmonitor.dtd:netmonitorUI.context.copyResponseHeaders.accesskey +netmonitor.properties:netmonitor.context.copyResponse = netmonitor.dtd:netmonitorUI.context.copyResponse +netmonitor.properties:netmonitor.context.copyResponse.accesskey = netmonitor.dtd:netmonitorUI.context.copyResponse.accesskey +netmonitor.properties:netmonitor.context.copyImageAsDataUri = netmonitor.dtd:netmonitorUI.context.copyImageAsDataUri +netmonitor.properties:netmonitor.context.copyImageAsDataUri.accesskey = netmonitor.dtd:netmonitorUI.context.copyImageAsDataUri.accesskey +netmonitor.properties:netmonitor.context.copyAllAsHar = netmonitor.dtd:netmonitorUI.context.copyAllAsHar +netmonitor.properties:netmonitor.context.copyAllAsHar.accesskey = netmonitor.dtd:netmonitorUI.context.copyAllAsHar.accesskey +netmonitor.properties:netmonitor.context.saveAllAsHar = netmonitor.dtd:netmonitorUI.context.saveAllAsHar +netmonitor.properties:netmonitor.context.saveAllAsHar.accesskey = netmonitor.dtd:netmonitorUI.context.saveAllAsHar.accesskey +netmonitor.properties:netmonitor.context.editAndResend = netmonitor.dtd:netmonitorUI.summary.editAndResend +netmonitor.properties:netmonitor.context.editAndResend.accesskey = netmonitor.dtd:netmonitorUI.summary.editAndResend.accesskey +netmonitor.properties:netmonitor.context.newTab = netmonitor.dtd:netmonitorUI.context.newTab +netmonitor.properties:netmonitor.context.newTab.accesskey = netmonitor.dtd:netmonitorUI.context.newTab.accesskey +netmonitor.properties:netmonitor.context.perfTools = netmonitor.dtd:netmonitorUI.context.perfTools +netmonitor.properties:netmonitor.context.perfTools.accesskey = netmonitor.dtd:netmonitorUI.context.perfTools.accesskey +netmonitor.properties:netmonitor.custom.newRequest = netmonitor.dtd:netmonitorUI.custom.newRequest +netmonitor.properties:netmonitor.custom.query = netmonitor.dtd:netmonitorUI.custom.query +netmonitor.properties:netmonitor.custom.headers = netmonitor.dtd:netmonitorUI.custom.headers +netmonitor.properties:netmonitor.custom.postData = netmonitor.dtd:netmonitorUI.custom.postData +netmonitor.properties:netmonitor.custom.send = netmonitor.dtd:netmonitorUI.custom.send +netmonitor.properties:netmonitor.custom.cancel = netmonitor.dtd:netmonitorUI.custom.cancel +netmonitor.properties:netmonitor.backButton = netmonitor.dtd:netmonitorUI.backButton diff --git a/python/devtools/migrate-l10n/migrate/main.py b/python/devtools/migrate-l10n/migrate/main.py new file mode 100644 index 000000000..0a1d468a8 --- /dev/null +++ b/python/devtools/migrate-l10n/migrate/main.py @@ -0,0 +1,261 @@ +import argparse +import glob +import HTMLParser +import logging +import os +import re +import sys +import urllib2 + + +# Import compare-locales parser from parent folder. +script_path = os.path.dirname(os.path.realpath(__file__)) +compare_locales_path = os.path.join(script_path, '../../../compare-locales') +sys.path.insert(0, compare_locales_path) +from compare_locales import parser + + +# Configure logging format and level +logging.basicConfig(format=' [%(levelname)s] %(message)s', level=logging.INFO) + + +# License header to use when creating new properties files. +DEFAULT_HEADER = ('# This Source Code Form is subject to the terms of the ' + 'Mozilla Public\n# License, v. 2.0. If a copy of the MPL ' + 'was not distributed with this\n# file, You can obtain ' + 'one at http://mozilla.org/MPL/2.0/.\n') + + +# Base url to retrieve properties files on central, that will be parsed for +# localization notes. +CENTRAL_BASE_URL = ('https://hg.mozilla.org/' + 'mozilla-central/raw-file/tip/' + 'devtools/client/locales/en-US/') + + +# HTML parser to translate HTML entities in dtd files. +HTML_PARSER = HTMLParser.HTMLParser() + +# Cache to store properties files retrieved over the network. +central_prop_cache = {} + +# Cache the parsed entities from the existing DTD files. +dtd_entities_cache = {} + + +# Retrieve the content of the current version of a properties file for the +# provided filename, from devtools/client on mozilla central. Will return an +# empty array if the file can't be retrieved or read. +def get_central_prop_content(prop_filename): + if prop_filename in central_prop_cache: + return central_prop_cache[prop_filename] + + url = CENTRAL_BASE_URL + prop_filename + logging.info('loading localization file from central: {%s}' % url) + + try: + central_prop_cache[prop_filename] = urllib2.urlopen(url).readlines() + except: + logging.error('failed to load properties file from central: {%s}' + % url) + central_prop_cache[prop_filename] = [] + + return central_prop_cache[prop_filename] + + +# Retrieve the current en-US localization notes for the provided prop_name. +def get_localization_note(prop_name, prop_filename): + prop_content = get_central_prop_content(prop_filename) + + comment_buffer = [] + for i, line in enumerate(prop_content): + # Remove line breaks. + line = line.strip('\n').strip('\r') + + if line.startswith('#'): + # Comment line, add to the current comment buffer. + comment_buffer.append(line) + elif re.search('(^|\n)' + re.escape(prop_name) + '\s*=', line): + # Property found, the current comment buffer is the localization + # note. + break; + else: + # No match, not a comment, reinitialize the comment buffer. + comment_buffer = [] + + return '\n'.join(comment_buffer) + + +# Retrieve the parsed DTD entities for a provided path. Results are cached by +# dtd path. +def get_dtd_entities(dtd_path): + if dtd_path in dtd_entities_cache: + return dtd_entities_cache[dtd_path] + + dtd_parser = parser.getParser('.dtd') + dtd_parser.readFile(dtd_path) + dtd_entities_cache[dtd_path] = dtd_parser.parse() + return dtd_entities_cache[dtd_path] + + +# Extract the value of an entity in a dtd file. +def get_translation_from_dtd(dtd_path, entity_name): + entities, map = get_dtd_entities(dtd_path) + if entity_name not in map: + # Bail out if translation is missing. + return + + key = map[entity_name] + entity = entities[key] + translation = HTML_PARSER.unescape(entity.val) + return translation.encode('utf-8') + + +# Extract the header and file wide comments for the provided properties file +# filename. +def get_properties_header(prop_filename): + prop_content = get_central_prop_content(prop_filename) + + # if the file content is empty, return the default license header. + if len(prop_content) == 0: + return DEFAULT_HEADER + + header_buffer = [] + for i, line in enumerate(prop_content): + # remove line breaks. + line = line.strip('\n').strip('\r') + + # regexp matching keys extracted form parser.py. + is_entity_line = re.search('^(\s*)' + '((?:[#!].*?\n\s*)*)' + '([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', line) + is_loc_note = re.search('^(\s*)' + '\#\s*LOCALIZATION NOTE\s*\([^)]+\)', line) + if is_entity_line or is_loc_note: + # header finished, break the loop. + break + else: + # header line, add to the current buffer. + header_buffer.append(line) + + # concatenate the current buffer and return. + return '\n'.join(header_buffer) + + +# Create a new properties file at the provided path. +def create_properties_file(prop_path): + logging.info('creating new *.properties file: {%s}' % prop_path) + + prop_filename = os.path.basename(prop_path) + header = get_properties_header(prop_filename) + + prop_file = open(prop_path, 'w+') + prop_file.write(header) + prop_file.close() + + +# Migrate a single string entry for a dtd to a properties file. +def migrate_string(dtd_path, prop_path, dtd_name, prop_name): + if not os.path.isfile(dtd_path): + logging.error('dtd file can not be found at: {%s}' % dtd_path) + return + + translation = get_translation_from_dtd(dtd_path, dtd_name) + if not translation: + logging.error('translation could not be found for: {%s} in {%s}' + % (dtd_name, dtd_path)) + return + + # Create properties file if missing. + if not os.path.isfile(prop_path): + create_properties_file(prop_path) + + if not os.path.isfile(prop_path): + logging.error('could not create new properties file at: {%s}' + % prop_path) + return + + prop_line = prop_name + '=' + translation + '\n' + + # Skip the string if it already exists in the destination file. + prop_file_content = open(prop_path, 'r').read() + if prop_line in prop_file_content: + logging.warning('string already migrated, skipping: {%s}' % prop_name) + return + + # Skip the string and log an error if an existing entry is found, but with + # a different value. + if re.search('(^|\n)' + re.escape(prop_name) + '\s*=', prop_file_content): + logging.error('existing string found, skipping: {%s}' % prop_name) + return + + prop_filename = os.path.basename(prop_path) + logging.info('migrating {%s} in {%s}' % (prop_name, prop_filename)) + with open(prop_path, 'a') as prop_file: + localization_note = get_localization_note(prop_name, prop_filename) + if len(localization_note): + prop_file.write('\n' + localization_note) + else: + logging.warning('localization notes could not be found for: {%s}' + % prop_name) + prop_file.write('\n' + prop_line) + + +# Apply the migration instructions in the provided configuration file. +def migrate_conf(conf_path, l10n_path): + f = open(conf_path, 'r') + lines = f.readlines() + f.close() + + for i, line in enumerate(lines): + # Remove line breaks. + line = line.strip('\n').strip('\r') + + # Skip invalid lines. + if ' = ' not in line: + continue + + # Expected syntax: ${prop_path}:${prop_name} = ${dtd_path}:${dtd_name}. + prop_info, dtd_info = line.split(' = ') + prop_path, prop_name = prop_info.split(':') + dtd_path, dtd_name = dtd_info.split(':') + + dtd_path = os.path.join(l10n_path, dtd_path) + prop_path = os.path.join(l10n_path, prop_path) + + migrate_string(dtd_path, prop_path, dtd_name, prop_name) + + +def main(): + # Read command line arguments. + arg_parser = argparse.ArgumentParser( + description='Migrate devtools localized strings.') + arg_parser.add_argument('path', type=str, help='path to l10n repository') + arg_parser.add_argument('-c', '--config', type=str, + help='path to configuration file or folder') + args = arg_parser.parse_args() + + # Retrieve path to devtools localization files in l10n repository. + devtools_l10n_path = os.path.join(args.path, 'devtools/client/') + if not os.path.exists(devtools_l10n_path): + logging.error('l10n path is invalid: {%s}' % devtools_l10n_path) + exit() + logging.info('l10n path is valid: {%s}' % devtools_l10n_path) + + # Retrieve configuration files to apply. + if os.path.isdir(args.config): + conf_files = glob.glob(args.config + '*') + elif os.path.isfile(args.config): + conf_files = [args.config] + else: + logging.error('config path is invalid: {%s}' % args.config) + exit() + + # Perform migration for each configuration file. + for conf_file in conf_files: + logging.info('performing migration for config file: {%s}' % conf_file) + migrate_conf(conf_file, devtools_l10n_path) + + +if __name__ == '__main__': + main() diff --git a/python/devtools/migrate-l10n/migrate/tests/__init__.py b/python/devtools/migrate-l10n/migrate/tests/__init__.py new file mode 100644 index 000000000..e69de29bb -- cgit v1.2.3