diff options
Diffstat (limited to 'python/compare-locales/compare_locales/tests')
12 files changed, 1293 insertions, 0 deletions
diff --git a/python/compare-locales/compare_locales/tests/__init__.py b/python/compare-locales/compare_locales/tests/__init__.py new file mode 100644 index 000000000..8808d78f4 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/__init__.py @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +'''Mixins for parser tests. +''' + +from itertools import izip_longest +from pkg_resources import resource_string +import re + +from compare_locales.parser import getParser + + +class ParserTestMixin(): + '''Utility methods used by the parser tests. + ''' + filename = None + + def setUp(self): + '''Create a parser for this test. + ''' + self.parser = getParser(self.filename) + + def tearDown(self): + 'tear down this test' + del self.parser + + def resource(self, name): + testcontent = resource_string(__name__, 'data/' + name) + # fake universal line endings + testcontent = re.sub('\r\n?', lambda m: '\n', testcontent) + return testcontent + + def _test(self, content, refs): + '''Helper to test the parser. + Compares the result of parsing content with the given list + of reference keys and values. + ''' + self.parser.readContents(content) + entities = [entity for entity in self.parser] + for entity, ref in izip_longest(entities, refs): + self.assertTrue(entity, 'excess reference entity') + self.assertTrue(ref, 'excess parsed entity') + self.assertEqual(entity.val, ref[1]) + if ref[0].startswith('_junk'): + self.assertTrue(re.match(ref[0], entity.key)) + else: + self.assertEqual(entity.key, ref[0]) diff --git a/python/compare-locales/compare_locales/tests/data/bug121341.properties b/python/compare-locales/compare_locales/tests/data/bug121341.properties new file mode 100644 index 000000000..b45fc9698 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/data/bug121341.properties @@ -0,0 +1,68 @@ +# simple check +1=abc +# test whitespace trimming in key and value + 2 = xy +# test parsing of escaped values +3 = \u1234\t\r\n\uAB\ +\u1\n +# test multiline properties +4 = this is \ +multiline property +5 = this is \ + another multiline property +# property with DOS EOL
+6 = test\u0036
+# test multiline property with with DOS EOL +7 = yet another multi\
+ line propery
+# trimming should not trim escaped whitespaces +8 = \ttest5\u0020 +# another variant of #8 +9 = \ test6\t +# test UTF-8 encoded property/value +10aሴb = c췯d +# next property should test unicode escaping at the boundary of parsing buffer +# buffer size is expected to be 4096 so add comments to get to this offset +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +############################################################################### +11 = \uABCD diff --git a/python/compare-locales/compare_locales/tests/data/test.properties b/python/compare-locales/compare_locales/tests/data/test.properties new file mode 100644 index 000000000..19cae9702 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/data/test.properties @@ -0,0 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +1=1 + 2=2 +3 =3 + 4 =4 +5=5 +6= 6 +7=7 +8= 8 +# this is a comment +9=this is the first part of a continued line \ + and here is the 2nd part diff --git a/python/compare-locales/compare_locales/tests/data/triple-license.dtd b/python/compare-locales/compare_locales/tests/data/triple-license.dtd new file mode 100644 index 000000000..4a28b17a6 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/data/triple-license.dtd @@ -0,0 +1,38 @@ +<!-- ***** BEGIN LICENSE BLOCK ***** +#if 0 + - Version: MPL 1.1/GPL 2.0/LGPL 2.1 + - + - The contents of this file are subject to the Mozilla Public License Version + - 1.1 (the "License"); you may not use this file except in compliance with + - the License. You may obtain a copy of the License at + - http://www.mozilla.org/MPL/ + - + - Software distributed under the License is distributed on an "AS IS" basis, + - WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + - for the specific language governing rights and limitations under the + - License. + - + - The Original Code is mozilla.org Code. + - + - The Initial Developer of the Original Code is dummy. + - Portions created by the Initial Developer are Copyright (C) 2005 + - the Initial Developer. All Rights Reserved. + - + - Contributor(s): + - + - Alternatively, the contents of this file may be used under the terms of + - either the GNU General Public License Version 2 or later (the "GPL"), or + - the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + - in which case the provisions of the GPL or the LGPL are applicable instead + - of those above. If you wish to allow use of your version of this file only + - under the terms of either the GPL or the LGPL, and not to allow others to + - use your version of this file under the terms of the MPL, indicate your + - decision by deleting the provisions above and replace them with the notice + - and other provisions required by the LGPL or the GPL. If you do not delete + - the provisions above, a recipient may use your version of this file under + - the terms of any one of the MPL, the GPL or the LGPL. + - +#endif + - ***** END LICENSE BLOCK ***** --> + +<!ENTITY foo "value"> diff --git a/python/compare-locales/compare_locales/tests/test_checks.py b/python/compare-locales/compare_locales/tests/test_checks.py new file mode 100644 index 000000000..b995d43f9 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_checks.py @@ -0,0 +1,403 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from compare_locales.checks import getChecker +from compare_locales.parser import getParser, Entity +from compare_locales.paths import File + + +class BaseHelper(unittest.TestCase): + file = None + refContent = None + + def setUp(self): + p = getParser(self.file.file) + p.readContents(self.refContent) + self.refList, self.refMap = p.parse() + + def _test(self, content, refWarnOrErrors, with_ref_file=False): + p = getParser(self.file.file) + p.readContents(content) + l10n = [e for e in p] + assert len(l10n) == 1 + l10n = l10n[0] + if with_ref_file: + kwargs = { + 'reference': self.refList + } + else: + kwargs = {} + checker = getChecker(self.file, **kwargs) + ref = self.refList[self.refMap[l10n.key]] + found = tuple(checker.check(ref, l10n)) + self.assertEqual(found, refWarnOrErrors) + + +class TestProperties(BaseHelper): + file = File('foo.properties', 'foo.properties') + refContent = '''some = value +''' + + def testGood(self): + self._test('''some = localized''', + tuple()) + + def testMissedEscape(self): + self._test(r'''some = \u67ood escape, bad \escape''', + (('warning', 20, r'unknown escape sequence, \e', + 'escape'),)) + + +class TestPlurals(BaseHelper): + file = File('foo.properties', 'foo.properties') + refContent = '''\ +# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. +# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals +# #1 number of files +# example: 111 files - Downloads +downloadsTitleFiles=#1 file - Downloads;#1 files - #2 +''' + + def testGood(self): + self._test('''\ +# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. +# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals +# #1 number of files +# example: 111 files - Downloads +downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 filers +''', + tuple()) + + def testNotUsed(self): + self._test('''\ +# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. +# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals +# #1 number of files +# example: 111 files - Downloads +downloadsTitleFiles=#1 file - Downloads;#1 files - Downloads;#1 filers +''', + (('warning', 0, 'not all variables used in l10n', + 'plural'),)) + + def testNotDefined(self): + self._test('''\ +# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. +# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals +# #1 number of files +# example: 111 files - Downloads +downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 #3 +''', + (('error', 0, 'unreplaced variables in l10n', 'plural'),)) + + +class TestDTDs(BaseHelper): + file = File('foo.dtd', 'foo.dtd') + refContent = '''<!ENTITY foo "This is 'good'"> +<!ENTITY width "10ch"> +<!ENTITY style "width: 20ch; height: 280px;"> +<!ENTITY minStyle "min-height: 50em;"> +<!ENTITY ftd "0"> +<!ENTITY formatPercent "This is 100% correct"> +<!ENTITY some.key "K"> +''' + + def testWarning(self): + self._test('''<!ENTITY foo "This is ¬ good"> +''', + (('warning', (0, 0), 'Referencing unknown entity `not`', + 'xmlparse'),)) + # make sure we only handle translated entity references + self._test(u'''<!ENTITY foo "This is &ƞǿŧ; good"> +'''.encode('utf-8'), + (('warning', (0, 0), u'Referencing unknown entity `ƞǿŧ`', + 'xmlparse'),)) + + def testErrorFirstLine(self): + self._test('''<!ENTITY foo "This is </bad> stuff"> +''', + (('error', (1, 10), 'mismatched tag', 'xmlparse'),)) + + def testErrorSecondLine(self): + self._test('''<!ENTITY foo "This is + </bad> +stuff"> +''', + (('error', (2, 4), 'mismatched tag', 'xmlparse'),)) + + def testKeyErrorSingleAmpersand(self): + self._test('''<!ENTITY some.key "&"> +''', + (('error', (1, 1), 'not well-formed (invalid token)', + 'xmlparse'),)) + + def testXMLEntity(self): + self._test('''<!ENTITY foo "This is "good""> +''', + tuple()) + + def testPercentEntity(self): + self._test('''<!ENTITY formatPercent "Another 100%"> +''', + tuple()) + self._test('''<!ENTITY formatPercent "Bad 100% should fail"> +''', + (('error', (0, 32), 'not well-formed (invalid token)', + 'xmlparse'),)) + + def testNoNumber(self): + self._test('''<!ENTITY ftd "foo">''', + (('warning', 0, 'reference is a number', 'number'),)) + + def testNoLength(self): + self._test('''<!ENTITY width "15miles">''', + (('error', 0, 'reference is a CSS length', 'css'),)) + + def testNoStyle(self): + self._test('''<!ENTITY style "15ch">''', + (('error', 0, 'reference is a CSS spec', 'css'),)) + self._test('''<!ENTITY style "junk">''', + (('error', 0, 'reference is a CSS spec', 'css'),)) + + def testStyleWarnings(self): + self._test('''<!ENTITY style "width:15ch">''', + (('warning', 0, 'height only in reference', 'css'),)) + self._test('''<!ENTITY style "width:15em;height:200px;">''', + (('warning', 0, "units for width don't match (em != ch)", + 'css'),)) + + def testNoWarning(self): + self._test('''<!ENTITY width "12em">''', tuple()) + self._test('''<!ENTITY style "width:12ch;height:200px;">''', tuple()) + self._test('''<!ENTITY ftd "0">''', tuple()) + + +class TestEntitiesInDTDs(BaseHelper): + file = File('foo.dtd', 'foo.dtd') + refContent = '''<!ENTITY short "This is &brandShortName;"> +<!ENTITY shorter "This is &brandShorterName;"> +<!ENTITY ent.start "Using &brandShorterName; start to"> +<!ENTITY ent.end " end"> +''' + + def testOK(self): + self._test('''<!ENTITY ent.start "Mit &brandShorterName;">''', tuple(), + with_ref_file=True) + + def testMismatch(self): + self._test('''<!ENTITY ent.start "Mit &brandShortName;">''', + (('warning', (0, 0), + 'Entity brandShortName referenced, ' + 'but brandShorterName used in context', + 'xmlparse'),), + with_ref_file=True) + + def testAcross(self): + self._test('''<!ENTITY ent.end "Mit &brandShorterName;">''', + tuple(), + with_ref_file=True) + + def testAcrossWithMismatch(self): + '''If we could tell that ent.start and ent.end are one string, + we should warn. Sadly, we can't, so this goes without warning.''' + self._test('''<!ENTITY ent.end "Mit &brandShortName;">''', + tuple(), + with_ref_file=True) + + def testUnknownWithRef(self): + self._test('''<!ENTITY ent.start "Mit &foopy;">''', + (('warning', + (0, 0), + 'Referencing unknown entity `foopy` ' + '(brandShorterName used in context, ' + 'brandShortName known)', + 'xmlparse'),), + with_ref_file=True) + + def testUnknown(self): + self._test('''<!ENTITY ent.end "Mit &foopy;">''', + (('warning', + (0, 0), + 'Referencing unknown entity `foopy`' + ' (brandShortName, brandShorterName known)', + 'xmlparse'),), + with_ref_file=True) + + +class TestAndroid(unittest.TestCase): + """Test Android checker + + Make sure we're hitting our extra rules only if + we're passing in a DTD file in the embedding/android module. + """ + apos_msg = u"Apostrophes in Android DTDs need escaping with \\' or " + \ + u"\\u0027, or use \u2019, or put string in quotes." + quot_msg = u"Quotes in Android DTDs need escaping with \\\" or " + \ + u"\\u0022, or put string in apostrophes." + + def getEntity(self, v): + return Entity(v, lambda s: s, (0, len(v)), (), (0, 0), (), (), + (0, len(v)), ()) + + def getDTDEntity(self, v): + v = v.replace('"', '"') + return Entity('<!ENTITY foo "%s">' % v, + lambda s: s, + (0, len(v) + 16), (), (0, 0), (), (9, 12), + (14, len(v) + 14), ()) + + def test_android_dtd(self): + """Testing the actual android checks. The logic is involved, + so this is a lot of nitty gritty detail tests. + """ + f = File("embedding/android/strings.dtd", "strings.dtd", + "embedding/android") + checker = getChecker(f) + # good string + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # dtd warning + l10n = self.getDTDEntity("plain localized string &ref;") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('warning', (0, 0), + 'Referencing unknown entity `ref`', 'xmlparse'),)) + # no report on stray ampersand or quote, if not completely quoted + for i in xrange(3): + # make sure we're catching unescaped apostrophes, + # try 0..5 backticks + l10n = self.getDTDEntity("\\"*(2*i) + "'") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 2*i, self.apos_msg, 'android'),)) + l10n = self.getDTDEntity("\\"*(2*i + 1) + "'") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # make sure we don't report if apos string is quoted + l10n = self.getDTDEntity('"' + "\\"*(2*i) + "'\"") + tpl = tuple(checker.check(ref, l10n)) + self.assertEqual(tpl, (), + "`%s` shouldn't fail but got %s" + % (l10n.val, str(tpl))) + l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"") + tpl = tuple(checker.check(ref, l10n)) + self.assertEqual(tpl, (), + "`%s` shouldn't fail but got %s" + % (l10n.val, str(tpl))) + # make sure we're catching unescaped quotes, try 0..5 backticks + l10n = self.getDTDEntity("\\"*(2*i) + "\"") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 2*i, self.quot_msg, 'android'),)) + l10n = self.getDTDEntity("\\"*(2*i + 1) + "'") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # make sure we don't report if quote string is single quoted + l10n = self.getDTDEntity("'" + "\\"*(2*i) + "\"'") + tpl = tuple(checker.check(ref, l10n)) + self.assertEqual(tpl, (), + "`%s` shouldn't fail but got %s" % + (l10n.val, str(tpl))) + l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"") + tpl = tuple(checker.check(ref, l10n)) + self.assertEqual(tpl, (), + "`%s` shouldn't fail but got %s" % + (l10n.val, str(tpl))) + # check for mixed quotes and ampersands + l10n = self.getDTDEntity("'\"") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 0, self.apos_msg, 'android'), + ('error', 1, self.quot_msg, 'android'))) + l10n = self.getDTDEntity("''\"'") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 1, self.apos_msg, 'android'),)) + l10n = self.getDTDEntity('"\'""') + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 2, self.quot_msg, 'android'),)) + + # broken unicode escape + l10n = self.getDTDEntity("Some broken \u098 unicode") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 12, 'truncated \\uXXXX escape', + 'android'),)) + # broken unicode escape, try to set the error off + l10n = self.getDTDEntity(u"\u9690"*14+"\u006"+" "+"\u0064") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 14, 'truncated \\uXXXX escape', + 'android'),)) + + def test_android_prop(self): + f = File("embedding/android/strings.properties", "strings.properties", + "embedding/android") + checker = getChecker(f) + # good plain string + ref = self.getEntity("plain string") + l10n = self.getEntity("plain localized string") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # no dtd warning + ref = self.getEntity("plain string") + l10n = self.getEntity("plain localized string &ref;") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # no report on stray ampersand + ref = self.getEntity("plain string") + l10n = self.getEntity("plain localized string with apos: '") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # report on bad printf + ref = self.getEntity("string with %s") + l10n = self.getEntity("string with %S") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('error', 0, 'argument 1 `S` should be `s`', + 'printf'),)) + + def test_non_android_dtd(self): + f = File("browser/strings.dtd", "strings.dtd", "browser") + checker = getChecker(f) + # good string + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # dtd warning + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string &ref;") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('warning', (0, 0), + 'Referencing unknown entity `ref`', 'xmlparse'),)) + # no report on stray ampersand + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string with apos: '") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + + def test_entities_across_dtd(self): + f = File("browser/strings.dtd", "strings.dtd", "browser") + p = getParser(f.file) + p.readContents('<!ENTITY other "some &good.ref;">') + ref = p.parse() + checker = getChecker(f, reference=ref[0]) + # good string + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + # dtd warning + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string &ref;") + self.assertEqual(tuple(checker.check(ref, l10n)), + (('warning', (0, 0), + 'Referencing unknown entity `ref` (good.ref known)', + 'xmlparse'),)) + # no report on stray ampersand + ref = self.getDTDEntity("plain string") + l10n = self.getDTDEntity("plain localized string with &good.ref;") + self.assertEqual(tuple(checker.check(ref, l10n)), + ()) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/compare-locales/compare_locales/tests/test_compare.py b/python/compare-locales/compare_locales/tests/test_compare.py new file mode 100644 index 000000000..51ba7cd8c --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_compare.py @@ -0,0 +1,90 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from compare_locales import compare + + +class TestTree(unittest.TestCase): + '''Test the Tree utility class + + Tree value classes need to be in-place editable + ''' + + def test_empty_dict(self): + tree = compare.Tree(dict) + self.assertEqual(list(tree.getContent()), []) + self.assertDictEqual( + tree.toJSON(), + {} + ) + + def test_disjoint_dict(self): + tree = compare.Tree(dict) + tree['one/entry']['leaf'] = 1 + tree['two/other']['leaf'] = 2 + self.assertEqual( + list(tree.getContent()), + [ + (0, 'key', ('one', 'entry')), + (1, 'value', {'leaf': 1}), + (0, 'key', ('two', 'other')), + (1, 'value', {'leaf': 2}) + ] + ) + self.assertDictEqual( + tree.toJSON(), + { + 'children': [ + ('one/entry', + {'value': {'leaf': 1}} + ), + ('two/other', + {'value': {'leaf': 2}} + ) + ] + } + ) + self.assertMultiLineEqual( + str(tree), + '''\ +one/entry + {'leaf': 1} +two/other + {'leaf': 2}\ +''' + ) + + def test_overlapping_dict(self): + tree = compare.Tree(dict) + tree['one/entry']['leaf'] = 1 + tree['one/other']['leaf'] = 2 + self.assertEqual( + list(tree.getContent()), + [ + (0, 'key', ('one',)), + (1, 'key', ('entry',)), + (2, 'value', {'leaf': 1}), + (1, 'key', ('other',)), + (2, 'value', {'leaf': 2}) + ] + ) + self.assertDictEqual( + tree.toJSON(), + { + 'children': [ + ('one', { + 'children': [ + ('entry', + {'value': {'leaf': 1}} + ), + ('other', + {'value': {'leaf': 2}} + ) + ] + }) + ] + } + ) diff --git a/python/compare-locales/compare_locales/tests/test_dtd.py b/python/compare-locales/compare_locales/tests/test_dtd.py new file mode 100644 index 000000000..87ddcde30 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_dtd.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +'''Tests for the DTD parser. +''' + +import unittest +import re + +from compare_locales.parser import getParser +from compare_locales.tests import ParserTestMixin + + +class TestDTD(ParserTestMixin, unittest.TestCase): + '''Tests for the DTD Parser.''' + filename = 'foo.dtd' + + def test_one_entity(self): + self._test('''<!ENTITY foo.label "stuff">''', + (('foo.label', 'stuff'),)) + + quoteContent = '''<!ENTITY good.one "one"> +<!ENTITY bad.one "bad " quote"> +<!ENTITY good.two "two"> +<!ENTITY bad.two "bad "quoted" word"> +<!ENTITY good.three "three"> +<!ENTITY good.four "good ' quote"> +<!ENTITY good.five "good 'quoted' word"> +''' + quoteRef = ( + ('good.one', 'one'), + ('_junk_\\d_25-56$', '<!ENTITY bad.one "bad " quote">'), + ('good.two', 'two'), + ('_junk_\\d_82-119$', '<!ENTITY bad.two "bad "quoted" word">'), + ('good.three', 'three'), + ('good.four', 'good \' quote'), + ('good.five', 'good \'quoted\' word'),) + + def test_quotes(self): + self._test(self.quoteContent, self.quoteRef) + + def test_apos(self): + qr = re.compile('[\'"]', re.M) + + def quot2apos(s): + return qr.sub(lambda m: m.group(0) == '"' and "'" or '"', s) + + self._test(quot2apos(self.quoteContent), + map(lambda t: (t[0], quot2apos(t[1])), self.quoteRef)) + + def test_parsed_ref(self): + self._test('''<!ENTITY % fooDTD SYSTEM "chrome://brand.dtd"> + %fooDTD; +''', + (('fooDTD', '"chrome://brand.dtd"'),)) + + def test_trailing_comment(self): + self._test('''<!ENTITY first "string"> +<!ENTITY second "string"> +<!-- +<!ENTITY commented "out"> +--> +''', + (('first', 'string'), ('second', 'string'))) + + def test_license_header(self): + p = getParser('foo.dtd') + p.readContents(self.resource('triple-license.dtd')) + for e in p: + self.assertEqual(e.key, 'foo') + self.assertEqual(e.val, 'value') + self.assert_('MPL' in p.header) + p.readContents('''\ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> +<!ENTITY foo "value"> +''') + for e in p: + self.assertEqual(e.key, 'foo') + self.assertEqual(e.val, 'value') + self.assert_('MPL' in p.header) + +if __name__ == '__main__': + unittest.main() diff --git a/python/compare-locales/compare_locales/tests/test_ini.py b/python/compare-locales/compare_locales/tests/test_ini.py new file mode 100644 index 000000000..4c8cc03e1 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_ini.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from compare_locales.tests import ParserTestMixin + + +mpl2 = '''\ +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this file, +; You can obtain one at http://mozilla.org/MPL/2.0/. +''' + + +class TestIniParser(ParserTestMixin, unittest.TestCase): + + filename = 'foo.ini' + + def testSimpleHeader(self): + self._test('''; This file is in the UTF-8 encoding +[Strings] +TitleText=Some Title +''', (('TitleText', 'Some Title'),)) + self.assert_('UTF-8' in self.parser.header) + + def testMPL2_Space_UTF(self): + self._test(mpl2 + ''' +; This file is in the UTF-8 encoding +[Strings] +TitleText=Some Title +''', (('TitleText', 'Some Title'),)) + self.assert_('MPL' in self.parser.header) + + def testMPL2_Space(self): + self._test(mpl2 + ''' +[Strings] +TitleText=Some Title +''', (('TitleText', 'Some Title'),)) + self.assert_('MPL' in self.parser.header) + + def testMPL2_MultiSpace(self): + self._test(mpl2 + '''\ + +; more comments + +[Strings] +TitleText=Some Title +''', (('TitleText', 'Some Title'),)) + self.assert_('MPL' in self.parser.header) + + def testMPL2_JunkBeforeCategory(self): + self._test(mpl2 + '''\ +Junk +[Strings] +TitleText=Some Title +''', (('_junk_\\d+_0-213$', mpl2 + '''\ +Junk +[Strings]'''), ('TitleText', 'Some Title'))) + self.assert_('MPL' not in self.parser.header) + + def test_TrailingComment(self): + self._test(mpl2 + ''' +[Strings] +TitleText=Some Title +;Stray trailing comment +''', (('TitleText', 'Some Title'),)) + self.assert_('MPL' in self.parser.header) + + def test_SpacedTrailingComments(self): + self._test(mpl2 + ''' +[Strings] +TitleText=Some Title + +;Stray trailing comment +;Second stray comment + +''', (('TitleText', 'Some Title'),)) + self.assert_('MPL' in self.parser.header) + + def test_TrailingCommentsAndJunk(self): + self._test(mpl2 + ''' +[Strings] +TitleText=Some Title + +;Stray trailing comment +Junk +;Second stray comment + +''', (('TitleText', 'Some Title'), ('_junk_\\d+_231-284$', '''\ + +;Stray trailing comment +Junk +;Second stray comment + +'''))) + self.assert_('MPL' in self.parser.header) + + def test_JunkInbetweenEntries(self): + self._test(mpl2 + ''' +[Strings] +TitleText=Some Title + +Junk + +Good=other string +''', (('TitleText', 'Some Title'), ('_junk_\\d+_231-236$', '''\ + +Junk'''), ('Good', 'other string'))) + self.assert_('MPL' in self.parser.header) + +if __name__ == '__main__': + unittest.main() diff --git a/python/compare-locales/compare_locales/tests/test_merge.py b/python/compare-locales/compare_locales/tests/test_merge.py new file mode 100644 index 000000000..c006edbb5 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_merge.py @@ -0,0 +1,265 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import os +from tempfile import mkdtemp +import shutil + +from compare_locales.parser import getParser +from compare_locales.paths import File +from compare_locales.compare import ContentComparer + + +class ContentMixin(object): + maxDiff = None # we got big dictionaries to compare + extension = None # OVERLOAD + + def reference(self, content): + self.ref = os.path.join(self.tmp, "en-reference" + self.extension) + open(self.ref, "w").write(content) + + def localized(self, content): + self.l10n = os.path.join(self.tmp, "l10n" + self.extension) + open(self.l10n, "w").write(content) + + +class TestProperties(unittest.TestCase, ContentMixin): + extension = '.properties' + + def setUp(self): + self.tmp = mkdtemp() + os.mkdir(os.path.join(self.tmp, "merge")) + + def tearDown(self): + shutil.rmtree(self.tmp) + del self.tmp + + def testGood(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""foo = fooVal +bar = barVal +eff = effVal""") + self.localized("""foo = lFoo +bar = lBar +eff = lEff +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.properties", ""), + File(self.l10n, "l10n.properties", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'changed': 3 + }}, + 'details': {} + } + ) + self.assert_(not os.path.exists(os.path.join(cc.merge_stage, + 'l10n.properties'))) + + def testMissing(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""foo = fooVal +bar = barVal +eff = effVal""") + self.localized("""bar = lBar +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.properties", ""), + File(self.l10n, "l10n.properties", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'changed': 1, 'missing': 2 + }}, + 'details': { + 'children': [ + ('l10n.properties', + {'value': {'missingEntity': [u'eff', u'foo']}} + ) + ]} + } + ) + mergefile = os.path.join(self.tmp, "merge", "l10n.properties") + self.assertTrue(os.path.isfile(mergefile)) + p = getParser(mergefile) + p.readFile(mergefile) + [m, n] = p.parse() + self.assertEqual(map(lambda e: e.key, m), ["bar", "eff", "foo"]) + + def testError(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""foo = fooVal +bar = %d barVal +eff = effVal""") + self.localized("""bar = %S lBar +eff = leffVal +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.properties", ""), + File(self.l10n, "l10n.properties", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'changed': 2, 'errors': 1, 'missing': 1 + }}, + 'details': { + 'children': [ + ('l10n.properties', + {'value': { + 'error': [u'argument 1 `S` should be `d` ' + u'at line 1, column 6 for bar'], + 'missingEntity': [u'foo']}} + ) + ]} + } + ) + mergefile = os.path.join(self.tmp, "merge", "l10n.properties") + self.assertTrue(os.path.isfile(mergefile)) + p = getParser(mergefile) + p.readFile(mergefile) + [m, n] = p.parse() + self.assertEqual([e.key for e in m], ["eff", "foo", "bar"]) + self.assertEqual(m[n['bar']].val, '%d barVal') + + def testObsolete(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""foo = fooVal +eff = effVal""") + self.localized("""foo = fooVal +other = obsolete +eff = leffVal +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.properties", ""), + File(self.l10n, "l10n.properties", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'changed': 1, 'obsolete': 1, 'unchanged': 1 + }}, + 'details': { + 'children': [ + ('l10n.properties', + {'value': {'obsoleteEntity': [u'other']}})]}, + } + ) + + +class TestDTD(unittest.TestCase, ContentMixin): + extension = '.dtd' + + def setUp(self): + self.tmp = mkdtemp() + os.mkdir(os.path.join(self.tmp, "merge")) + + def tearDown(self): + shutil.rmtree(self.tmp) + del self.tmp + + def testGood(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""<!ENTITY foo 'fooVal'> +<!ENTITY bar 'barVal'> +<!ENTITY eff 'effVal'>""") + self.localized("""<!ENTITY foo 'lFoo'> +<!ENTITY bar 'lBar'> +<!ENTITY eff 'lEff'> +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.dtd", ""), + File(self.l10n, "l10n.dtd", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'changed': 3 + }}, + 'details': {} + } + ) + self.assert_( + not os.path.exists(os.path.join(cc.merge_stage, 'l10n.dtd'))) + + def testMissing(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""<!ENTITY foo 'fooVal'> +<!ENTITY bar 'barVal'> +<!ENTITY eff 'effVal'>""") + self.localized("""<!ENTITY bar 'lBar'> +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.dtd", ""), + File(self.l10n, "l10n.dtd", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'changed': 1, 'missing': 2 + }}, + 'details': { + 'children': [ + ('l10n.dtd', + {'value': {'missingEntity': [u'eff', u'foo']}} + ) + ]} + } + ) + mergefile = os.path.join(self.tmp, "merge", "l10n.dtd") + self.assertTrue(os.path.isfile(mergefile)) + p = getParser(mergefile) + p.readFile(mergefile) + [m, n] = p.parse() + self.assertEqual(map(lambda e: e.key, m), ["bar", "eff", "foo"]) + + def testJunk(self): + self.assertTrue(os.path.isdir(self.tmp)) + self.reference("""<!ENTITY foo 'fooVal'> +<!ENTITY bar 'barVal'> +<!ENTITY eff 'effVal'>""") + self.localized("""<!ENTITY foo 'fooVal'> +<!ENTY bar 'gimmick'> +<!ENTITY eff 'effVal'> +""") + cc = ContentComparer() + cc.set_merge_stage(os.path.join(self.tmp, "merge")) + cc.compare(File(self.ref, "en-reference.dtd", ""), + File(self.l10n, "l10n.dtd", "")) + self.assertDictEqual( + cc.observer.toJSON(), + {'summary': + {None: { + 'errors': 1, 'missing': 1, 'unchanged': 2 + }}, + 'details': { + 'children': [ + ('l10n.dtd', + {'value': { + 'error': [u'Unparsed content "<!ENTY bar ' + u'\'gimmick\'>" at 23-44'], + 'missingEntity': [u'bar']}} + ) + ]} + } + ) + mergefile = os.path.join(self.tmp, "merge", "l10n.dtd") + self.assertTrue(os.path.isfile(mergefile)) + p = getParser(mergefile) + p.readFile(mergefile) + [m, n] = p.parse() + self.assertEqual(map(lambda e: e.key, m), ["foo", "eff", "bar"]) + +if __name__ == '__main__': + unittest.main() diff --git a/python/compare-locales/compare_locales/tests/test_properties.py b/python/compare-locales/compare_locales/tests/test_properties.py new file mode 100644 index 000000000..331a1a57c --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_properties.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from compare_locales.tests import ParserTestMixin + + +class TestPropertiesParser(ParserTestMixin, unittest.TestCase): + + filename = 'foo.properties' + + def testBackslashes(self): + self._test(r'''one_line = This is one line +two_line = This is the first \ +of two lines +one_line_trailing = This line ends in \\ +and has junk +two_lines_triple = This line is one of two and ends in \\\ +and still has another line coming +''', ( + ('one_line', 'This is one line'), + ('two_line', u'This is the first of two lines'), + ('one_line_trailing', u'This line ends in \\'), + ('_junk_\\d+_113-126$', 'and has junk\n'), + ('two_lines_triple', 'This line is one of two and ends in \\' + 'and still has another line coming'))) + + def testProperties(self): + # port of netwerk/test/PropertiesTest.cpp + self.parser.readContents(self.resource('test.properties')) + ref = ['1', '2', '3', '4', '5', '6', '7', '8', + 'this is the first part of a continued line ' + 'and here is the 2nd part'] + i = iter(self.parser) + for r, e in zip(ref, i): + self.assertEqual(e.val, r) + + def test_bug121341(self): + # port of xpcom/tests/unit/test_bug121341.js + self.parser.readContents(self.resource('bug121341.properties')) + ref = ['abc', 'xy', u"\u1234\t\r\n\u00AB\u0001\n", + "this is multiline property", + "this is another multiline property", u"test\u0036", + "yet another multiline propery", u"\ttest5\u0020", " test6\t", + u"c\uCDEFd", u"\uABCD"] + i = iter(self.parser) + for r, e in zip(ref, i): + self.assertEqual(e.val, r) + + def test_comment_in_multi(self): + self._test(r'''bar=one line with a \ +# part that looks like a comment \ +and an end''', (('bar', 'one line with a # part that looks like a comment ' + 'and an end'),)) + + def test_license_header(self): + self._test('''\ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +foo=value +''', (('foo', 'value'),)) + self.assert_('MPL' in self.parser.header) + + def test_escapes(self): + self.parser.readContents(r''' +# unicode escapes +zero = some \unicode +one = \u0 +two = \u41 +three = \u042 +four = \u0043 +five = \u0044a +six = \a +seven = \n\r\t\\ +''') + ref = ['some unicode', chr(0), 'A', 'B', 'C', 'Da', 'a', '\n\r\t\\'] + for r, e in zip(ref, self.parser): + self.assertEqual(e.val, r) + + def test_trailing_comment(self): + self._test('''first = string +second = string + +# +#commented out +''', (('first', 'string'), ('second', 'string'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/compare-locales/compare_locales/tests/test_util.py b/python/compare-locales/compare_locales/tests/test_util.py new file mode 100644 index 000000000..fd2d2c92b --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_util.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from compare_locales import util + + +class ParseLocalesTest(unittest.TestCase): + def test_empty(self): + self.assertEquals(util.parseLocales(''), []) + + def test_all(self): + self.assertEquals(util.parseLocales('''af +de'''), ['af', 'de']) + + def test_shipped(self): + self.assertEquals(util.parseLocales('''af +ja win mac +de'''), ['af', 'de', 'ja']) + + def test_sparse(self): + self.assertEquals(util.parseLocales(''' +af + +de + +'''), ['af', 'de']) diff --git a/python/compare-locales/compare_locales/tests/test_webapps.py b/python/compare-locales/compare_locales/tests/test_webapps.py new file mode 100644 index 000000000..2f1223649 --- /dev/null +++ b/python/compare-locales/compare_locales/tests/test_webapps.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from compare_locales import webapps + + +class TestFileComparison(unittest.TestCase): + + def mock_FileComparison(self, mock_listdir): + class Target(webapps.FileComparison): + def _listdir(self): + return mock_listdir() + return Target('.', 'en-US') + + def test_just_reference(self): + def _listdir(): + return ['my_app.en-US.properties'] + filecomp = self.mock_FileComparison(_listdir) + filecomp.files() + self.assertEqual(filecomp.locales(), []) + self.assertEqual(filecomp._reference.keys(), ['my_app']) + file_ = filecomp._reference['my_app'] + self.assertEqual(file_.file, 'locales/my_app.en-US.properties') + + def test_just_locales(self): + def _listdir(): + return ['my_app.ar.properties', + 'my_app.sr-Latn.properties', + 'my_app.sv-SE.properties', + 'my_app.po_SI.properties'] + filecomp = self.mock_FileComparison(_listdir) + filecomp.files() + self.assertEqual(filecomp.locales(), + ['ar', 'sr-Latn', 'sv-SE']) + self.assertEqual(filecomp._files['ar'].keys(), ['my_app']) + file_ = filecomp._files['ar']['my_app'] + self.assertEqual(file_.file, 'locales/my_app.ar.properties') |