Add Python cmake parser

Unfortunately, upstream has removed the configure+make build system our import script relies on to generate code and moz.build files.

2 files changed, 468 insertions, 0 deletions

diff --git a/media/libaom/cmakeparser.py b/media/libaom/cmakeparser.py
new file mode 100644
index 000000000..9a2ee4b85
--- /dev/null
+++ b/media/libaom/cmakeparser.py
@@ -0,0 +1,278 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from pyparsing import (CharsNotIn, Group, Forward, Literal, Suppress, Word,
+                       QuotedString, ZeroOrMore, alphas, alphanums)
+from string import Template
+import re
+
+# Grammar for CMake
+comment = Literal('#') + ZeroOrMore(CharsNotIn('\n'))
+quoted_argument = QuotedString('\"', '\\', multiline=True)
+unquoted_argument = CharsNotIn('\n ()#\"\\')
+argument = quoted_argument | unquoted_argument | Suppress(comment)
+arguments = Forward()
+arguments << (argument | (Literal('(') + ZeroOrMore(arguments) + Literal(')')))
+identifier = Word(alphas, alphanums+'_')
+command = Group(identifier + Literal('(') + ZeroOrMore(arguments) + Literal(')'))
+file_elements = command | Suppress(comment)
+cmake = ZeroOrMore(file_elements)
+
+
+def extract_arguments(parsed):
+    """Extract the command arguments skipping the parentheses"""
+    return parsed[2:len(parsed) - 1]
+
+
+def match_block(command, parsed, start):
+    """Find the end of block starting with the command"""
+    depth = 0
+    end = start + 1
+    endcommand = 'end' + command
+    while parsed[end][0] != endcommand or depth > 0:
+        if parsed[end][0] == command:
+            depth += 1
+        elif parsed[end][0] == endcommand:
+            depth -= 1
+        end = end + 1
+        if end == len(parsed):
+            print('error: eof when trying to match block statement: %s'
+                  % parsed[start])
+    return end
+
+
+def parse_if(parsed, start):
+    """Parse if/elseif/else/endif into a list of conditions and commands"""
+    depth = 0
+    conditions = []
+    condition = [extract_arguments(parsed[start])]
+    start = start + 1
+    end = start
+
+    while parsed[end][0] != 'endif' or depth > 0:
+        command = parsed[end][0]
+        if command == 'if':
+            depth += 1
+        elif command == 'else' and depth == 0:
+            condition.append(parsed[start:end])
+            conditions.append(condition)
+            start = end + 1
+            condition = [['TRUE']]
+        elif command == 'elseif' and depth == 0:
+            condition.append(parsed[start:end])
+            conditions.append(condition)
+            condition = [extract_arguments(parsed[end])]
+            start = end + 1
+        elif command == 'endif':
+            depth -= 1
+        end = end + 1
+        if end == len(parsed):
+            print('error: eof when trying to match if statement: %s'
+                  % parsed[start])
+    condition.append(parsed[start:end])
+    conditions.append(condition)
+    return end, conditions
+
+
+def substs(variables, values):
+    """Substitute variables into values"""
+    new_values = []
+    for value in values:
+        t = Template(value)
+        new_value = t.safe_substitute(variables)
+
+        # Safe substitute leaves unrecognized variables in place.
+        # We replace them with the empty string.
+        new_values.append(re.sub('\$\{\w+\}', '', new_value))
+    return new_values
+
+
+def evaluate(variables, cache_variables, parsed):
+    """Evaluate a list of parsed commands, returning sources to build"""
+    i = 0
+    sources = []
+    while i < len(parsed):
+        command = parsed[i][0]
+        arguments = substs(variables, extract_arguments(parsed[i]))
+
+        if command == 'foreach':
+            end = match_block(command, parsed, i)
+            for argument in arguments[1:]:
+                # ; is also a valid divider, why have one when you can have two?
+                argument = argument.replace(';', ' ')
+                for value in argument.split():
+                    variables[arguments[0]] = value
+                    cont_eval, new_sources = evaluate(variables, cache_variables,
+                                                      parsed[i+1:end])
+                    sources.extend(new_sources)
+                    if not cont_eval:
+                        return cont_eval, sources
+        elif command == 'function':
+            # for now we just execute functions inline at point of declaration
+            # as this is sufficient to build libaom
+            pass
+        elif command == 'if':
+            i, conditions = parse_if(parsed, i)
+            for condition in conditions:
+                if evaluate_boolean(variables, condition[0]):
+                    cont_eval, new_sources = evaluate(variables,
+                                                      cache_variables,
+                                                      condition[1])
+                    sources.extend(new_sources)
+                    if not cont_eval:
+                        return cont_eval, sources
+                    break
+        elif command == 'include':
+            if arguments:
+                try:
+                    print('including: %s' % arguments[0])
+                    sources.extend(parse(variables, cache_variables, arguments[0]))
+                except IOError:
+                    print('warning: could not include: %s' % arguments[0])
+        elif command == 'list':
+            try:
+                action = arguments[0]
+                variable = arguments[1]
+                values = arguments[2:]
+                if action == 'APPEND':
+                    if not variables.has_key(variable):
+                        variables[variable] = ' '.join(values)
+                    else:
+                        variables[variable] += ' ' + ' '.join(values)
+            except (IndexError, KeyError):
+                pass
+        elif command == 'option':
+            variable = arguments[0]
+            value = arguments[2]
+            # Allow options to be override without changing CMake files
+            if not variables.has_key(variable):
+                variables[variable] = value
+        elif command == 'return':
+            return False, sources
+        elif command == 'set':
+            variable = arguments[0]
+            values = arguments[1:]
+            # CACHE variables are not set if already present
+            try:
+                cache = values.index('CACHE')
+                values = values[0:cache]
+                if not variables.has_key(variable):
+                    variables[variable] = ' '.join(values)
+                cache_variables.append(variable)
+            except ValueError:
+                variables[variable] = ' '.join(values)
+        elif command == 'add_asm_library':
+            try:
+                sources.extend(variables[arguments[1]].split(' '))
+            except (IndexError, KeyError):
+                pass
+        elif command == 'add_intrinsics_object_library':
+            try:
+                sources.extend(variables[arguments[3]].split(' '))
+            except (IndexError, KeyError):
+                pass
+        elif command == 'add_library':
+            for source in arguments[1:]:
+                sources.extend(source.split(' '))
+        elif command == 'target_sources':
+            for source in arguments[1:]:
+                sources.extend(source.split(' '))
+        elif command == 'MOZDEBUG':
+            print('>>>> MOZDEBUG: %s' % ' '.join(arguments))
+        i += 1
+    return True, sources
+
+
+def evaluate_boolean(variables, arguments):
+    """Evaluate a boolean expression"""
+    if not arguments:
+        return False
+
+    argument = arguments[0]
+
+    if argument == 'NOT':
+        return not evaluate_boolean(variables, arguments[1:])
+
+    if argument == '(':
+        i = 0
+        depth = 1
+        while depth > 0 and i < len(arguments):
+            i += 1
+            if arguments[i] == '(':
+                depth += 1
+            if arguments[i] == ')':
+                depth -= 1
+        return evaluate_boolean(variables, arguments[1:i])
+
+    def evaluate_constant(argument):
+        try:
+            as_int = int(argument)
+            if as_int != 0:
+                return True
+            else:
+                return False
+        except ValueError:
+            upper = argument.upper()
+            if upper in ['ON', 'YES', 'TRUE', 'Y']:
+                return True
+            elif upper in ['OFF', 'NO', 'FALSE', 'N', 'IGNORE', '', 'NOTFOUND']:
+                return False
+            elif upper.endswith('-NOTFOUND'):
+                return False
+        return None
+
+    def lookup_variable(argument):
+        # If statements can have old-style variables which are not demarcated
+        # like ${VARIABLE}. Attempt to look up the variable both ways.
+        try:
+            if re.search('\$\{\w+\}', argument):
+                try:
+                    t = Template(argument)
+                    value = t.substitute(variables)
+                    try:
+                        # Attempt an old-style variable lookup with the
+                        # substituted value.
+                        return variables[value]
+                    except KeyError:
+                        return value
+                except ValueError:
+                    # TODO: CMake supports nesting, e.g. ${${foo}}
+                    return None
+            else:
+                return variables[argument]
+        except KeyError:
+            return None
+
+    lhs = lookup_variable(argument)
+    if lhs is None:
+        # variable resolution failed, treat as string
+        lhs = argument
+
+    if len(arguments) > 1:
+        op = arguments[1]
+        if op == 'AND':
+            return evaluate_constant(lhs) and evaluate_boolean(variables, arguments[2:])
+        elif op == 'MATCHES':
+            rhs = lookup_variable(arguments[2])
+            if not rhs:
+                rhs = arguments[2]
+            return not re.match(rhs, lhs) is None
+        elif op == 'OR':
+            return evaluate_constant(lhs) or evaluate_boolean(variables, arguments[2:])
+        elif op == 'STREQUAL':
+            rhs = lookup_variable(arguments[2])
+            if not rhs:
+                rhs = arguments[2]
+            return lhs == rhs
+    else:
+        lhs = evaluate_constant(lhs)
+        if lhs is None:
+            lhs = lookup_variable(argument)
+
+    return lhs
+
+
+def parse(variables, cache_variables, filename):
+    parsed = cmake.parseFile(filename)
+    cont_eval, sources = evaluate(variables, cache_variables, parsed)
+    return sources
diff --git a/media/libaom/test_cmakeparser.py b/media/libaom/test_cmakeparser.py
new file mode 100644
index 000000000..af68c3313
--- /dev/null
+++ b/media/libaom/test_cmakeparser.py
@@ -0,0 +1,190 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from pyparsing import ParseException
+import unittest
+
+import cmakeparser as cp
+
+class TestCMakeParser(unittest.TestCase):
+    def test_arguments(self):
+        self.assertEqual(cp.arguments.parseString('1').asList(), ['1'])
+        self.assertEqual(cp.arguments.parseString('(1 2)').asList(),
+                         ['(', '1', '2', ')'])
+
+    def test_command(self):
+        self.assertEqual(cp.command.parseString('blah()').asList(),
+                         [['blah', '(', ')']])
+        self.assertEqual(cp.command.parseString('blah(1)').asList(),
+                         [['blah', '(', '1', ')']])
+        self.assertEqual(cp.command.parseString('blah(1 (2 3))').asList(),
+                         [['blah', '(', '1', '(', '2', '3', ')', ')']])
+
+    def test_evaluate_boolean(self):
+        self.assertTrue(cp.evaluate_boolean({}, ['TRUE']))
+        self.assertFalse(cp.evaluate_boolean({}, ['NOT', 'TRUE']))
+        self.assertFalse(cp.evaluate_boolean({}, ['TRUE', 'AND', 'FALSE']))
+        self.assertTrue(cp.evaluate_boolean({}, ['ABC', 'MATCHES', '^AB']))
+        self.assertTrue(cp.evaluate_boolean({}, ['TRUE', 'OR', 'FALSE']))
+        self.assertTrue(cp.evaluate_boolean({}, ['ABC', 'STREQUAL', 'ABC']))
+        self.assertTrue(cp.evaluate_boolean({'ABC': '1'}, ['ABC']))
+        self.assertFalse(cp.evaluate_boolean({'ABC': '0'}, ['${ABC}']))
+        self.assertFalse(cp.evaluate_boolean({}, ['ABC']))
+        self.assertFalse(cp.evaluate_boolean({}, ['${ABC}']))
+        self.assertTrue(cp.evaluate_boolean({'YES_ABC': 1, 'VAL': 'ABC'},
+                                            ['YES_${VAL}']))
+        self.assertTrue(cp.evaluate_boolean({'ABC': 'DEF', 'DEF': 1},
+                                            ['${ABC}']))
+        self.assertTrue(cp.evaluate_boolean({}, ['FALSE', 'OR', '(', 'FALSE', 'OR', 'TRUE', ')']))
+        self.assertFalse(cp.evaluate_boolean({}, ['FALSE', 'OR', '(', 'FALSE', 'AND', 'TRUE', ')']))
+
+    def test_foreach(self):
+        s = """
+            set(STUFF A B C D E F)
+            foreach(item ${STUFF})
+                set(YES_${item} 1)
+            endforeach ()
+            """
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cp.evaluate(variables, [], parsed)
+        for k in ['A', 'B', 'C', 'D', 'E', 'F']:
+            self.assertEqual(variables['YES_%s' % k], '1')
+
+        s = """
+            set(STUFF "A;B;C;D;E;F")
+            foreach(item ${STUFF})
+                set(${item} 1)
+            endforeach ()
+            """
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cp.evaluate(variables, [], parsed)
+        for k in ['A', 'B', 'C', 'D', 'E', 'F']:
+            self.assertEqual(variables[k], '1')
+
+        s = """
+            set(STUFF D E F)
+            foreach(item A B C ${STUFF})
+                set(${item} 1)
+            endforeach ()
+            """
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cp.evaluate(variables, [], parsed)
+        for k in ['A', 'B', 'C', 'D', 'E', 'F']:
+            self.assertEqual(variables[k], '1')
+
+    def test_list(self):
+        s = 'list(APPEND TEST 1 1 2 3 5 8 13)'
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cache_variables = []
+        cp.evaluate(variables, cache_variables, parsed)
+        self.assertEqual(variables['TEST'], '1 1 2 3 5 8 13')
+        self.assertEqual(len(cache_variables), 0)
+
+        s = """
+            set(TEST 1)
+            list(APPEND TEST 1 2 3 5 8 13)
+            """
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cache_variables = []
+        cp.evaluate(variables, cache_variables, parsed)
+        self.assertEqual(variables['TEST'], '1 1 2 3 5 8 13')
+        self.assertEqual(len(cache_variables), 0)
+
+    def test_malformed_input(self):
+        self.assertEqual(len(cp.cmake.parseString('func ((A)')), 0)
+        self.assertEqual(len(cp.cmake.parseString('cmd"arg"(arg2)')), 0)
+        self.assertRaises(ParseException, cp.command.parseString, 'func ((A)')
+        self.assertRaises(ParseException, cp.identifier.parseString,
+                          '-asdlf')
+        self.assertEqual(cp.identifier.parseString('asd-lf')[0], 'asd')
+        self.assertRaises(ParseException, cp.quoted_argument.parseString,
+                          'blah"')
+        s = """
+            " blah blah
+            blah
+            """
+        self.assertRaises(ParseException, cp.quoted_argument.parseString,
+                          s)
+        self.assertRaises(ParseException, cp.quoted_argument.parseString,
+                          'asdlf')
+        self.assertRaises(ParseException, cp.unquoted_argument.parseString,
+                          '#asdflkj')
+
+    def test_parse_if(self):
+        s = """
+            if (A)
+                B()
+            elseif (C)
+                D()
+            elseif (E)
+                F()
+            else ()
+                H()
+            endif ()
+            I()
+            """
+        parsed = cp.cmake.parseString(s)
+        self.assertEqual(len(parsed), 10)
+        end, conditions = cp.parse_if(parsed, 0)
+        self.assertEqual(parsed[end][0], 'endif')
+        self.assertEqual(len(conditions), 4)
+        self.assertEqual(conditions[0][0], ['A'])
+        self.assertEqual(conditions[0][1][0], parsed[1])
+        self.assertEqual(conditions[1][0], ['C'])
+        self.assertEqual(conditions[1][1][0], parsed[3])
+        self.assertEqual(conditions[2][0], ['E'])
+        self.assertEqual(conditions[2][1][0], parsed[5])
+        self.assertEqual(conditions[3][0], ['TRUE'])
+        self.assertEqual(conditions[3][1][0], parsed[7])
+
+    def test_return(self):
+        s = """
+            set(TEST 2)
+            if (true)
+                return()
+            endif ()
+            set(TEST 3)
+            """
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cache_variables = []
+        cp.evaluate(variables, cache_variables, parsed)
+        self.assertEqual(variables['TEST'], '2')
+        self.assertEqual(len(cache_variables), 0)
+
+    def test_set(self):
+        s = """set(TEST 2)"""
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cache_variables = []
+        cp.evaluate(variables, cache_variables, parsed)
+        self.assertEqual(variables['TEST'], '2')
+        self.assertEqual(len(cache_variables), 0)
+
+        s = """set(TEST 3 CACHE "documentation")"""
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cache_variables = []
+        cp.evaluate(variables, cache_variables, parsed)
+        self.assertEqual(variables['TEST'], '3')
+        self.assertTrue('TEST' in cache_variables)
+
+        s = """set(TEST A B C D)"""
+        parsed = cp.cmake.parseString(s)
+        variables = {}
+        cp.evaluate(variables, [], parsed)
+        self.assertEqual(variables['TEST'], 'A B C D')
+
+        s = """set(TEST ${TEST} E F G H)"""
+        parsed = cp.cmake.parseString(s)
+        cp.evaluate(variables, [], parsed)
+        self.assertEqual(variables['TEST'], 'A B C D E F G H')
+
+
+if __name__ == '__main__':
+    unittest.main()