diff options
Diffstat (limited to 'build/pymake/pymake/parser.py')
| -rw-r--r-- | build/pymake/pymake/parser.py | 822 |
1 files changed, 822 insertions, 0 deletions
diff --git a/build/pymake/pymake/parser.py b/build/pymake/pymake/parser.py new file mode 100644 index 000000000..4bff53368 --- /dev/null +++ b/build/pymake/pymake/parser.py @@ -0,0 +1,822 @@ +""" +Module for parsing Makefile syntax. + +Makefiles use a line-based parsing system. Continuations and substitutions are handled differently based on the +type of line being parsed: + +Lines with makefile syntax condense continuations to a single space, no matter the actual trailing whitespace +of the first line or the leading whitespace of the continuation. In other situations, trailing whitespace is +relevant. + +Lines with command syntax do not condense continuations: the backslash and newline are part of the command. +(GNU Make is buggy in this regard, at least on mac). + +Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding). +Otherwise, they are parsed as makefile syntax. + +This file parses into the data structures defined in the parserdata module. Those classes are what actually +do the dirty work of "executing" the parsed data into a data.Makefile. + +Four iterator functions are available: +* iterdata +* itermakefilechars +* itercommandchars + +The iterators handle line continuations and comments in different ways, but share a common calling +convention: + +Called with (data, startoffset, tokenlist, finditer) + +yield 4-tuples (flatstr, token, tokenoffset, afteroffset) +flatstr is data, guaranteed to have no tokens (may be '') +token, tokenoffset, afteroffset *may be None*. That means there is more text +coming. +""" + +import logging, re, os, sys +import data, functions, util, parserdata + +_log = logging.getLogger('pymake.parser') + +class SyntaxError(util.MakeError): + pass + +_skipws = re.compile('\S') +class Data(object): + """ + A single virtual "line", which can be multiple source lines joined with + continuations. + """ + + __slots__ = ('s', 'lstart', 'lend', 'loc') + + def __init__(self, s, lstart, lend, loc): + self.s = s + self.lstart = lstart + self.lend = lend + self.loc = loc + + @staticmethod + def fromstring(s, path): + return Data(s, 0, len(s), parserdata.Location(path, 1, 0)) + + def getloc(self, offset): + assert offset >= self.lstart and offset <= self.lend + return self.loc.offset(self.s, self.lstart, offset) + + def skipwhitespace(self, offset): + """ + Return the offset of the first non-whitespace character in data starting at offset, or None if there are + only whitespace characters remaining. + """ + m = _skipws.search(self.s, offset, self.lend) + if m is None: + return self.lend + + return m.start(0) + +_linere = re.compile(r'\\*\n') +def enumeratelines(s, filename): + """ + Enumerate lines in a string as Data objects, joining line + continuations. + """ + + off = 0 + lineno = 1 + curlines = 0 + for m in _linere.finditer(s): + curlines += 1 + start, end = m.span(0) + + if (start - end) % 2 == 0: + # odd number of backslashes is a continuation + continue + + yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0)) + + lineno += curlines + curlines = 0 + off = end + + yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0)) + +_alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes + := | + \+= | + \?= | + :: | + (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character + :(?![\\/]) | # colon followed by anything except a slash (Windows path detection) + [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE) + +def iterdata(d, offset, tokenlist, it): + """ + Iterate over flat data without line continuations, comments, or any special escaped characters. + + Typically used to parse recursively-expanded variables. + """ + + assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!" + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + + if offset == d.lend: + return + + s = d.s + for m in it: + mstart, mend = m.span(0) + token = s[mstart:mend] + if token in tokenlist or (token[0] == '$' and '$' in tokenlist): + yield s[offset:mstart], token, mstart, mend + else: + yield s[offset:mend], None, None, mend + offset = mend + + yield s[offset:d.lend], None, None, None + +# multiple backslashes before a newline are unescaped, halving their total number +_makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*') +def _replacemakecontinuations(m): + start, end = m.span(1) + if start == -1: + return ' ' + return ' '.rjust((end - start) / 2 + 1, '\\') + +def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False): + """ + Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines + are converted to single-space continuations. + """ + + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + + if offset == d.lend: + return + + s = d.s + for m in it: + mstart, mend = m.span(0) + token = s[mstart:mend] + + starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart]) + + if token[-1] == '#' and not ignorecomments: + l = mend - mstart + # multiple backslashes before a hash are unescaped, halving their total number + if l % 2: + # found a comment + yield starttext + token[:(l - 1) / 2], None, None, None + return + else: + yield starttext + token[-l / 2:], None, None, mend + elif token in tokenlist or (token[0] == '$' and '$' in tokenlist): + yield starttext, token, mstart, mend + else: + yield starttext + token, None, None, mend + offset = mend + + yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None + +_findcomment = re.compile(r'\\*\#') +def flattenmakesyntax(d, offset): + """ + A shortcut method for flattening line continuations and comments in makefile syntax without + looking for other tokens. + """ + + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + if offset == d.lend: + return '' + + s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend]) + + elements = [] + offset = 0 + for m in _findcomment.finditer(s): + mstart, mend = m.span(0) + elements.append(s[offset:mstart]) + if (mend - mstart) % 2: + # even number of backslashes... it's a comment + elements.append(''.ljust((mend - mstart - 1) / 2, '\\')) + return ''.join(elements) + + # odd number of backslashes + elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#') + offset = mend + + elements.append(s[offset:]) + return ''.join(elements) + +def itercommandchars(d, offset, tokenlist, it): + """ + Iterate over command syntax. # comment markers are not special, and escaped newlines are included + in the output text. + """ + + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + + if offset == d.lend: + return + + s = d.s + for m in it: + mstart, mend = m.span(0) + token = s[mstart:mend] + starttext = s[offset:mstart].replace('\n\t', '\n') + + if token in tokenlist or (token[0] == '$' and '$' in tokenlist): + yield starttext, token, mstart, mend + else: + yield starttext + token, None, None, mend + offset = mend + + yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None + +_redefines = re.compile('\s*define|\s*endef') +def iterdefinelines(it, startloc): + """ + Process the insides of a define. Most characters are included literally. Escaped newlines are treated + as they would be in makefile syntax. Internal define/endef pairs are ignored. + """ + + results = [] + + definecount = 1 + for d in it: + m = _redefines.match(d.s, d.lstart, d.lend) + if m is not None: + directive = m.group(0).strip() + if directive == 'endef': + definecount -= 1 + if definecount == 0: + return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results)) + else: + definecount += 1 + + results.append(d.s[d.lstart:d.lend]) + + # Falling off the end is an unterminated define! + raise SyntaxError("define without matching endef", startloc) + +def _ensureend(d, offset, msg): + """ + Ensure that only whitespace remains in this data. + """ + + s = flattenmakesyntax(d, offset) + if s != '' and not s.isspace(): + raise SyntaxError(msg, d.getloc(offset)) + +_eqargstokenlist = ('(', "'", '"') + +def ifeq(d, offset): + if offset > d.lend - 1: + raise SyntaxError("No arguments after conditional", d.getloc(offset)) + + # the variety of formats for this directive is rather maddening + token = d.s[offset] + if token not in _eqargstokenlist: + raise SyntaxError("No arguments after conditional", d.getloc(offset)) + + offset += 1 + + if token == '(': + arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars) + if t is None: + raise SyntaxError("Expected two arguments in conditional", d.getloc(d.lend)) + + arg1.rstrip() + + offset = d.skipwhitespace(offset) + arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars) + if t is None: + raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) + + _ensureend(d, offset, "Unexpected text after conditional") + else: + arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars) + if t is None: + raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend)) + + offset = d.skipwhitespace(offset) + if offset == d.lend: + raise SyntaxError("Expected two arguments in conditional", d.getloc(offset)) + + token = d.s[offset] + if token not in '\'"': + raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) + + arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars) + + _ensureend(d, offset, "Unexpected text after conditional") + + return parserdata.EqCondition(arg1, arg2) + +def ifneq(d, offset): + c = ifeq(d, offset) + c.expected = False + return c + +def ifdef(d, offset): + e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) + e.rstrip() + + return parserdata.IfdefCondition(e) + +def ifndef(d, offset): + c = ifdef(d, offset) + c.expected = False + return c + +_conditionkeywords = { + 'ifeq': ifeq, + 'ifneq': ifneq, + 'ifdef': ifdef, + 'ifndef': ifndef + } + +_conditiontokens = tuple(_conditionkeywords.iterkeys()) +_conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens)) + +_directivestokenlist = _conditiontokens + \ + ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport') + +_directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist)) + +_varsettokens = (':=', '+=', '?=', '=') + +def _parsefile(pathname): + fd = open(pathname, "rU") + stmts = parsestring(fd.read(), pathname) + stmts.mtime = os.fstat(fd.fileno()).st_mtime + fd.close() + return stmts + +def _checktime(path, stmts): + mtime = os.path.getmtime(path) + if mtime != stmts.mtime: + _log.debug("Re-parsing makefile '%s': mtimes differ", path) + return False + + return True + +_parsecache = util.MostUsedCache(50, _parsefile, _checktime) + +def parsefile(pathname): + """ + Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing + makefiles that have already been parsed and have not changed. + """ + + pathname = os.path.realpath(pathname) + return _parsecache.get(pathname) + +# colon followed by anything except a slash (Windows path detection) +_depfilesplitter = re.compile(r':(?![\\/])') +# simple variable references +_vars = re.compile('\$\((\w+)\)') + +def parsedepfile(pathname): + """ + Parse a filename listing only depencencies into a parserdata.StatementList. + Simple variable references are allowed in such files. + """ + def continuation_iter(lines): + current_line = [] + for line in lines: + line = line.rstrip() + if line.endswith("\\"): + current_line.append(line.rstrip("\\")) + continue + if not len(line): + continue + current_line.append(line) + yield ''.join(current_line) + current_line = [] + if current_line: + yield ''.join(current_line) + + def get_expansion(s): + if '$' in s: + expansion = data.Expansion() + # for an input like e.g. "foo $(bar) baz", + # _vars.split returns ["foo", "bar", "baz"] + # every other element is a variable name. + for i, element in enumerate(_vars.split(s)): + if i % 2: + expansion.appendfunc(functions.VariableRef(None, + data.StringExpansion(element, None))) + elif element: + expansion.appendstr(element) + + return expansion + + return data.StringExpansion(s, None) + + pathname = os.path.realpath(pathname) + stmts = parserdata.StatementList() + for line in continuation_iter(open(pathname).readlines()): + target, deps = _depfilesplitter.split(line, 1) + stmts.append(parserdata.Rule(get_expansion(target), + get_expansion(deps), False)) + return stmts + +def parsestring(s, filename): + """ + Parse a string containing makefile data into a parserdata.StatementList. + """ + + currule = False + condstack = [parserdata.StatementList()] + + fdlines = enumeratelines(s, filename) + for d in fdlines: + assert len(condstack) > 0 + + offset = d.lstart + + if currule and offset < d.lend and d.s[offset] == '\t': + e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars) + assert token is None + assert offset is None + condstack[-1].append(parserdata.Command(e)) + continue + + # To parse Makefile syntax, we first strip leading whitespace and + # look for initial keywords. If there are no keywords, it's either + # setting a variable or writing a rule. + + offset = d.skipwhitespace(offset) + if offset is None: + continue + + m = _directivesre.match(d.s, offset, d.lend) + if m is not None: + kword = m.group(1) + offset = m.end(0) + + if kword == 'endif': + _ensureend(d, offset, "Unexpected data after 'endif' directive") + if len(condstack) == 1: + raise SyntaxError("unmatched 'endif' directive", + d.getloc(offset)) + + condstack.pop().endloc = d.getloc(offset) + continue + + if kword == 'else': + if len(condstack) == 1: + raise SyntaxError("unmatched 'else' directive", + d.getloc(offset)) + + m = _conditionre.match(d.s, offset, d.lend) + if m is None: + _ensureend(d, offset, "Unexpected data after 'else' directive.") + condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition()) + else: + kword = m.group(1) + if kword not in _conditionkeywords: + raise SyntaxError("Unexpected condition after 'else' directive.", + d.getloc(offset)) + + startoffset = offset + offset = d.skipwhitespace(m.end(1)) + c = _conditionkeywords[kword](d, offset) + condstack[-1].addcondition(d.getloc(startoffset), c) + continue + + if kword in _conditionkeywords: + c = _conditionkeywords[kword](d, offset) + cb = parserdata.ConditionBlock(d.getloc(d.lstart), c) + condstack[-1].append(cb) + condstack.append(cb) + continue + + if kword == 'endef': + raise SyntaxError("endef without matching define", d.getloc(offset)) + + if kword == 'define': + currule = False + vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars) + vname.rstrip() + + startloc = d.getloc(d.lstart) + value = iterdefinelines(fdlines, startloc) + condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None)) + continue + + if kword in ('include', '-include', 'includedeps', '-includedeps'): + if kword.startswith('-'): + required = False + kword = kword[1:] + else: + required = True + + deps = kword == 'includedeps' + + currule = False + incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) + condstack[-1].append(parserdata.Include(incfile, required, deps)) + + continue + + if kword == 'vpath': + currule = False + e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) + condstack[-1].append(parserdata.VPathDirective(e)) + continue + + if kword == 'override': + currule = False + vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) + vname.lstrip() + vname.rstrip() + + if token is None: + raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart)) + + value = flattenmakesyntax(d, offset).lstrip() + + condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE)) + continue + + if kword == 'export': + currule = False + e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) + e.lstrip() + e.rstrip() + + if token is None: + condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=False)) + else: + condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=True)) + + value = flattenmakesyntax(d, offset).lstrip() + condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) + + continue + + if kword == 'unexport': + e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars) + condstack[-1].append(parserdata.UnexportDirective(e)) + continue + + e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars) + if token is None: + e.rstrip() + e.lstrip() + if not e.isempty(): + condstack[-1].append(parserdata.EmptyDirective(e)) + continue + + # if we encountered real makefile syntax, the current rule is over + currule = False + + if token in _varsettokens: + e.lstrip() + e.rstrip() + + value = flattenmakesyntax(d, offset).lstrip() + + condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) + else: + doublecolon = token == '::' + + # `e` is targets or target patterns, which can end up as + # * a rule + # * an implicit rule + # * a static pattern rule + # * a target-specific variable definition + # * a pattern-specific variable definition + # any of the rules may have order-only prerequisites + # delimited by |, and a command delimited by ; + targets = e + + e, token, offset = parsemakesyntax(d, offset, + _varsettokens + (':', '|', ';'), + itermakefilechars) + if token in (None, ';'): + condstack[-1].append(parserdata.Rule(targets, e, doublecolon)) + currule = True + + if token == ';': + offset = d.skipwhitespace(offset) + e, t, offset = parsemakesyntax(d, offset, (), itercommandchars) + condstack[-1].append(parserdata.Command(e)) + + elif token in _varsettokens: + e.lstrip() + e.rstrip() + + value = flattenmakesyntax(d, offset).lstrip() + condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets)) + elif token == '|': + raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset)) + else: + assert token == ':' + # static pattern rule + + pattern = e + + deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars) + + condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon)) + currule = True + + if token == ';': + offset = d.skipwhitespace(offset) + e, token, offset = parsemakesyntax(d, offset, (), itercommandchars) + condstack[-1].append(parserdata.Command(e)) + + if len(condstack) != 1: + raise SyntaxError("Condition never terminated with endif", condstack[-1].loc) + + return condstack[0] + +_PARSESTATE_TOPLEVEL = 0 # at the top level +_PARSESTATE_FUNCTION = 1 # expanding a function call +_PARSESTATE_VARNAME = 2 # expanding a variable expansion. +_PARSESTATE_SUBSTFROM = 3 # expanding a variable expansion substitution "from" value +_PARSESTATE_SUBSTTO = 4 # expanding a variable expansion substitution "to" value +_PARSESTATE_PARENMATCH = 5 # inside nested parentheses/braces that must be matched + +class ParseStackFrame(object): + __slots__ = ('parsestate', 'parent', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom') + + def __init__(self, parsestate, parent, expansion, tokenlist, openbrace, closebrace, function=None, loc=None): + self.parsestate = parsestate + self.parent = parent + self.expansion = expansion + self.tokenlist = tokenlist + self.openbrace = openbrace + self.closebrace = closebrace + self.function = function + self.loc = loc + + def __str__(self): + return "<state=%i expansion=%s tokenlist=%s openbrace=%s closebrace=%s>" % (self.parsestate, self.expansion, self.tokenlist, self.openbrace, self.closebrace) + +_matchingbrace = { + '(': ')', + '{': '}', + } + +def parsemakesyntax(d, offset, stopon, iterfunc): + """ + Given Data, parse it into a data.Expansion. + + @param stopon (sequence) + Indicate characters where toplevel parsing should stop. + + @param iterfunc (generator function) + A function which is used to iterate over d, yielding (char, offset, loc) + @see iterdata + @see itermakefilechars + @see itercommandchars + + @return a tuple (expansion, token, offset). If all the data is consumed, + token and offset will be None + """ + + assert callable(iterfunc) + + stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)), + tokenlist=stopon + ('$',), + openbrace=None, closebrace=None) + + tokeniterator = _alltokens.finditer(d.s, offset, d.lend) + + di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) + while True: # this is not a for loop because `di` changes during the function + assert stacktop is not None + try: + s, token, tokenoffset, offset = di.next() + except StopIteration: + break + + stacktop.expansion.appendstr(s) + if token is None: + continue + + parsestate = stacktop.parsestate + + if token[0] == '$': + if tokenoffset + 1 == d.lend: + # an unterminated $ expands to nothing + break + + loc = d.getloc(tokenoffset) + c = token[1] + if c == '$': + assert len(token) == 2 + stacktop.expansion.appendstr('$') + elif c in ('(', '{'): + closebrace = _matchingbrace[c] + + if len(token) > 2: + fname = token[2:].rstrip() + fn = functions.functionmap[fname](loc) + e = data.Expansion() + if len(fn) + 1 == fn.maxargs: + tokenlist = (c, closebrace, '$') + else: + tokenlist = (',', c, closebrace, '$') + + stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop, + e, tokenlist, function=fn, + openbrace=c, closebrace=closebrace) + else: + e = data.Expansion() + tokenlist = (':', c, closebrace, '$') + stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop, + e, tokenlist, + openbrace=c, closebrace=closebrace, loc=loc) + else: + assert len(token) == 2 + e = data.Expansion.fromstring(c, loc) + stacktop.expansion.appendfunc(functions.VariableRef(loc, e)) + elif token in ('(', '{'): + assert token == stacktop.openbrace + + stacktop.expansion.appendstr(token) + stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop, + stacktop.expansion, + (token, stacktop.closebrace, '$'), + openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset)) + elif parsestate == _PARSESTATE_PARENMATCH: + assert token == stacktop.closebrace + stacktop.expansion.appendstr(token) + stacktop = stacktop.parent + elif parsestate == _PARSESTATE_TOPLEVEL: + assert stacktop.parent is None + return stacktop.expansion.finish(), token, offset + elif parsestate == _PARSESTATE_FUNCTION: + if token == ',': + stacktop.function.append(stacktop.expansion.finish()) + + stacktop.expansion = data.Expansion() + if len(stacktop.function) + 1 == stacktop.function.maxargs: + tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') + stacktop.tokenlist = tokenlist + elif token in (')', '}'): + fn = stacktop.function + fn.append(stacktop.expansion.finish()) + fn.setup() + + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Not reached, _PARSESTATE_FUNCTION" + elif parsestate == _PARSESTATE_VARNAME: + if token == ':': + stacktop.varname = stacktop.expansion + stacktop.parsestate = _PARSESTATE_SUBSTFROM + stacktop.expansion = data.Expansion() + stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$') + elif token in (')', '}'): + fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish()) + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Not reached, _PARSESTATE_VARNAME" + elif parsestate == _PARSESTATE_SUBSTFROM: + if token == '=': + stacktop.substfrom = stacktop.expansion + stacktop.parsestate = _PARSESTATE_SUBSTTO + stacktop.expansion = data.Expansion() + stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') + elif token in (')', '}'): + # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make + # parses it. Issue a warning. Combine the varname and substfrom expansions to + # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME + _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc) + stacktop.varname.appendstr(':') + stacktop.varname.concat(stacktop.expansion) + fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish()) + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Not reached, _PARSESTATE_SUBSTFROM" + elif parsestate == _PARSESTATE_SUBSTTO: + assert token in (')','}'), "Not reached, _PARSESTATE_SUBSTTO" + + fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(), + stacktop.substfrom.finish(), stacktop.expansion.finish()) + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Unexpected parse state %s" % stacktop.parsestate + + if stacktop.parent is not None and iterfunc == itercommandchars: + di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator, + ignorecomments=True) + else: + di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) + + if stacktop.parent is not None: + raise SyntaxError("Unterminated function call", d.getloc(offset)) + + assert stacktop.parsestate == _PARSESTATE_TOPLEVEL + + return stacktop.expansion.finish(), None, None |