summaryrefslogtreecommitdiffstats
path: root/build/pymake
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /build/pymake
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'build/pymake')
-rw-r--r--build/pymake/LICENSE21
-rw-r--r--build/pymake/README64
-rwxr-xr-xbuild/pymake/make.py35
-rwxr-xr-xbuild/pymake/mkformat.py13
-rwxr-xr-xbuild/pymake/mkparse.py12
-rw-r--r--build/pymake/pymake/__init__.py0
-rw-r--r--build/pymake/pymake/builtins.py120
-rw-r--r--build/pymake/pymake/command.py278
-rw-r--r--build/pymake/pymake/data.py1842
-rw-r--r--build/pymake/pymake/functions.py873
-rw-r--r--build/pymake/pymake/globrelative.py68
-rw-r--r--build/pymake/pymake/implicit.py14
-rw-r--r--build/pymake/pymake/parser.py822
-rw-r--r--build/pymake/pymake/parserdata.py1006
-rw-r--r--build/pymake/pymake/process.py556
-rw-r--r--build/pymake/pymake/util.py150
-rw-r--r--build/pymake/pymake/win32process.py28
-rw-r--r--build/pymake/tests/automatic-variables.mk79
-rw-r--r--build/pymake/tests/bad-command-continuation.mk3
-rw-r--r--build/pymake/tests/call.mk12
-rw-r--r--build/pymake/tests/cmd-stripdotslash.mk5
-rw-r--r--build/pymake/tests/cmdgoals.mk9
-rw-r--r--build/pymake/tests/commandmodifiers.mk21
-rw-r--r--build/pymake/tests/comment-parsing.mk29
-rw-r--r--build/pymake/tests/continuations-in-functions.mk6
-rw-r--r--build/pymake/tests/datatests.py237
-rw-r--r--build/pymake/tests/default-goal-set-first.mk7
-rw-r--r--build/pymake/tests/default-goal.mk8
-rw-r--r--build/pymake/tests/default-target.mk14
-rw-r--r--build/pymake/tests/default-target2.mk6
-rw-r--r--build/pymake/tests/define-directive.mk69
-rw-r--r--build/pymake/tests/depfailed.mk4
-rw-r--r--build/pymake/tests/depfailedj.mk10
-rw-r--r--build/pymake/tests/diamond-deps.mk13
-rw-r--r--build/pymake/tests/dotslash-dir.mk8
-rw-r--r--build/pymake/tests/dotslash-parse.mk4
-rw-r--r--build/pymake/tests/dotslash-phony.mk3
-rw-r--r--build/pymake/tests/dotslash.mk9
-rw-r--r--build/pymake/tests/doublecolon-exists.mk16
-rw-r--r--build/pymake/tests/doublecolon-priordeps.mk19
-rw-r--r--build/pymake/tests/doublecolon-remake.mk4
-rw-r--r--build/pymake/tests/dynamic-var.mk18
-rw-r--r--build/pymake/tests/empty-arg.mk2
-rw-r--r--build/pymake/tests/empty-command-semicolon.mk5
-rw-r--r--build/pymake/tests/empty-with-deps.mk4
-rw-r--r--build/pymake/tests/env-var-append.mk7
-rw-r--r--build/pymake/tests/env-var-append2.mk8
-rw-r--r--build/pymake/tests/eof-continuation.mk5
-rw-r--r--build/pymake/tests/escape-chars.mk26
-rw-r--r--build/pymake/tests/escaped-continuation.mk6
-rw-r--r--build/pymake/tests/eval-duringexecute.mk12
-rw-r--r--build/pymake/tests/eval.mk7
-rw-r--r--build/pymake/tests/exit-code.mk5
-rw-r--r--build/pymake/tests/file-functions-symlinks.mk22
-rw-r--r--build/pymake/tests/file-functions.mk19
-rw-r--r--build/pymake/tests/foreach-local-variable.mk8
-rw-r--r--build/pymake/tests/formattingtests.py289
-rw-r--r--build/pymake/tests/func-refs.mk11
-rw-r--r--build/pymake/tests/functions.mk36
-rw-r--r--build/pymake/tests/functiontests.py54
-rw-r--r--build/pymake/tests/if-syntaxerr.mk6
-rw-r--r--build/pymake/tests/ifdefs-nesting.mk13
-rw-r--r--build/pymake/tests/ifdefs.mk127
-rw-r--r--build/pymake/tests/ignore-error.mk13
-rw-r--r--build/pymake/tests/implicit-chain.mk12
-rw-r--r--build/pymake/tests/implicit-dir.mk16
-rw-r--r--build/pymake/tests/implicit-terminal.mk16
-rw-r--r--build/pymake/tests/implicitsubdir.mk12
-rw-r--r--build/pymake/tests/include-dynamic.mk21
-rw-r--r--build/pymake/tests/include-file.inc1
-rw-r--r--build/pymake/tests/include-missing.mk9
-rw-r--r--build/pymake/tests/include-notfound.mk19
-rw-r--r--build/pymake/tests/include-optional-warning.mk4
-rw-r--r--build/pymake/tests/include-regen.mk10
-rw-r--r--build/pymake/tests/include-regen2.mk10
-rw-r--r--build/pymake/tests/include-regen3.mk10
-rw-r--r--build/pymake/tests/include-test.mk8
-rw-r--r--build/pymake/tests/includedeps-norebuild.mk15
-rw-r--r--build/pymake/tests/includedeps-sideeffects.mk10
-rw-r--r--build/pymake/tests/includedeps-stripdotslash.deps1
-rw-r--r--build/pymake/tests/includedeps-stripdotslash.mk8
-rw-r--r--build/pymake/tests/includedeps-variables.deps1
-rw-r--r--build/pymake/tests/includedeps-variables.mk10
-rw-r--r--build/pymake/tests/includedeps.deps1
-rw-r--r--build/pymake/tests/includedeps.mk9
-rw-r--r--build/pymake/tests/info.mk8
-rw-r--r--build/pymake/tests/justprint-native.mk28
-rw-r--r--build/pymake/tests/justprint.mk5
-rw-r--r--build/pymake/tests/keep-going-doublecolon.mk16
-rw-r--r--build/pymake/tests/keep-going-parallel.mk11
-rw-r--r--build/pymake/tests/keep-going.mk14
-rw-r--r--build/pymake/tests/line-continuations.mk24
-rw-r--r--build/pymake/tests/link-search.mk7
-rw-r--r--build/pymake/tests/makeflags.mk7
-rw-r--r--build/pymake/tests/matchany.mk14
-rw-r--r--build/pymake/tests/matchany2.mk13
-rw-r--r--build/pymake/tests/matchany3.mk10
-rw-r--r--build/pymake/tests/mkdir-fail.mk7
-rw-r--r--build/pymake/tests/mkdir.mk27
-rw-r--r--build/pymake/tests/multiple-rules-prerequisite-merge.mk25
-rw-r--r--build/pymake/tests/native-command-delay-load.mk12
-rw-r--r--build/pymake/tests/native-command-raise.mk9
-rw-r--r--build/pymake/tests/native-command-return-fail1.mk8
-rw-r--r--build/pymake/tests/native-command-return-fail2.mk8
-rw-r--r--build/pymake/tests/native-command-return.mk11
-rw-r--r--build/pymake/tests/native-command-shell-glob.mk11
-rw-r--r--build/pymake/tests/native-command-sys-exit-fail1.mk8
-rw-r--r--build/pymake/tests/native-command-sys-exit-fail2.mk8
-rw-r--r--build/pymake/tests/native-command-sys-exit.mk11
-rw-r--r--build/pymake/tests/native-environment.mk11
-rw-r--r--build/pymake/tests/native-pycommandpath-sep.mk21
-rw-r--r--build/pymake/tests/native-pycommandpath.mk15
-rw-r--r--build/pymake/tests/native-simple.mk12
-rw-r--r--build/pymake/tests/native-touch.mk15
-rw-r--r--build/pymake/tests/newlines.mk30
-rw-r--r--build/pymake/tests/no-remake.mk7
-rw-r--r--build/pymake/tests/nosuchfile.mk4
-rw-r--r--build/pymake/tests/notargets.mk5
-rw-r--r--build/pymake/tests/notparallel.mk8
-rw-r--r--build/pymake/tests/oneline-command-continuations.mk5
-rw-r--r--build/pymake/tests/override-propagate.mk37
-rw-r--r--build/pymake/tests/parallel-dep-resolution.mk8
-rw-r--r--build/pymake/tests/parallel-dep-resolution2.mk9
-rw-r--r--build/pymake/tests/parallel-native.mk21
-rw-r--r--build/pymake/tests/parallel-simple.mk27
-rw-r--r--build/pymake/tests/parallel-submake.mk17
-rw-r--r--build/pymake/tests/parallel-toserial.mk31
-rw-r--r--build/pymake/tests/parallel-waiting.mk21
-rw-r--r--build/pymake/tests/parentheses.mk2
-rw-r--r--build/pymake/tests/parsertests.py314
-rw-r--r--build/pymake/tests/path-length.mk9
-rwxr-xr-xbuild/pymake/tests/pathdir/pathtest2
-rw-r--r--build/pymake/tests/pathdir/pathtest.exebin0 -> 45056 bytes
-rw-r--r--build/pymake/tests/pathdir/src/Makefile2
-rw-r--r--build/pymake/tests/pathdir/src/pathtest.cpp6
-rw-r--r--build/pymake/tests/patsubst.mk7
-rw-r--r--build/pymake/tests/phony.mk10
-rw-r--r--build/pymake/tests/pycmd.py38
-rw-r--r--build/pymake/tests/recursive-set.mk7
-rw-r--r--build/pymake/tests/recursive-set2.mk8
-rw-r--r--build/pymake/tests/remake-mtime.mk14
-rw-r--r--build/pymake/tests/rm-fail.mk7
-rw-r--r--build/pymake/tests/rm.mk21
-rw-r--r--build/pymake/tests/runtests.py215
-rw-r--r--build/pymake/tests/serial-dep-resolution.mk5
-rw-r--r--build/pymake/tests/serial-doublecolon-execution.mk18
-rw-r--r--build/pymake/tests/serial-rule-execution.mk5
-rw-r--r--build/pymake/tests/serial-rule-execution2.mk13
-rw-r--r--build/pymake/tests/serial-toparallel.mk5
-rw-r--r--build/pymake/tests/shellfunc.mk7
-rw-r--r--build/pymake/tests/simple-makeflags.mk10
-rw-r--r--build/pymake/tests/sort.mk4
-rw-r--r--build/pymake/tests/specified-target.mk7
-rw-r--r--build/pymake/tests/static-pattern.mk5
-rw-r--r--build/pymake/tests/static-pattern2.mk10
-rw-r--r--build/pymake/tests/subdir/delayload.py1
-rw-r--r--build/pymake/tests/subdir/pymod.py5
-rw-r--r--build/pymake/tests/subdir/testmodule.py3
-rw-r--r--build/pymake/tests/submake-path.makefile211
-rw-r--r--build/pymake/tests/submake-path.mk16
-rw-r--r--build/pymake/tests/submake.makefile224
-rw-r--r--build/pymake/tests/submake.mk16
-rw-r--r--build/pymake/tests/subprocess-path.mk32
-rw-r--r--build/pymake/tests/tab-intro.mk16
-rw-r--r--build/pymake/tests/target-specific.mk30
-rw-r--r--build/pymake/tests/unexport.mk15
-rw-r--r--build/pymake/tests/unexport.submk15
-rw-r--r--build/pymake/tests/unterminated-dollar.mk6
-rw-r--r--build/pymake/tests/var-change-flavor.mk12
-rw-r--r--build/pymake/tests/var-commandline.mk8
-rw-r--r--build/pymake/tests/var-overrides.mk21
-rw-r--r--build/pymake/tests/var-ref.mk19
-rw-r--r--build/pymake/tests/var-set.mk55
-rw-r--r--build/pymake/tests/var-substitutions.mk49
-rw-r--r--build/pymake/tests/vpath-directive-dynamic.mk12
-rw-r--r--build/pymake/tests/vpath-directive.mk31
-rw-r--r--build/pymake/tests/vpath.mk18
-rw-r--r--build/pymake/tests/vpath2.mk18
-rw-r--r--build/pymake/tests/wildcards.mk22
-rw-r--r--build/pymake/tests/windows-paths.mk5
180 files changed, 9220 insertions, 0 deletions
diff --git a/build/pymake/LICENSE b/build/pymake/LICENSE
new file mode 100644
index 000000000..04a7d641d
--- /dev/null
+++ b/build/pymake/LICENSE
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2009 The Mozilla Foundation <http://www.mozilla.org/>
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/build/pymake/README b/build/pymake/README
new file mode 100644
index 000000000..4f0fdfea4
--- /dev/null
+++ b/build/pymake/README
@@ -0,0 +1,64 @@
+INTRODUCTION
+
+make.py (and the pymake modules that support it) are an implementation of the make tool
+which are mostly compatible with makefiles written for GNU make.
+
+PURPOSE
+
+The Mozilla project inspired this tool with several goals:
+
+* Improve build speeds, especially on Windows. This can be done by reducing the total number
+ of processes that are launched, especially MSYS shell processes which are expensive.
+
+* Allow writing some complicated build logic directly in Python instead of in shell.
+
+* Allow computing dependencies for special targets, such as members within ZIP files.
+
+* Enable experiments with build system. By writing a makefile parser, we can experiment
+ with converting in-tree makefiles to another build system, such as SCons, waf, ant, ...insert
+ your favorite build tool here. Or we could experiment along the lines of makepp, keeping
+ our existing makefiles, but change the engine to build a global dependency graph.
+
+KNOWN INCOMPATIBILITIES
+
+* Order-only prerequisites are not yet supported
+
+* Secondary expansion is not yet supported.
+
+* Target-specific variables behave differently than in GNU make: in pymake, the target-specific
+ variable only applies to the specific target that is mentioned, and does not apply recursively
+ to all dependencies which are remade. This is an intentional change: the behavior of GNU make
+ is neither deterministic nor intuitive.
+
+* $(eval) is only supported during the parse phase. Any attempt to recursively expand
+ an $(eval) function during command execution will fail. This is an intentional incompatibility.
+
+* There is a subtle difference in execution order that can cause unexpected changes in the
+ following circumstance:
+** A file `foo.c` exists on the VPATH
+** A rule for `foo.c` exists with a dependency on `tool` and no commands
+** `tool` is remade for some other reason earlier in the file
+ In this case, pymake resets the VPATH of `foo.c`, while GNU make does not. This shouldn't
+ happen in the real world, since a target found on the VPATH without commands is silly. But
+ mozilla/js/src happens to have a rule, which I'm patching.
+
+* pymake does not implement any of the builtin implicit rules or the related variables. Mozilla
+ only cares because pymake doesn't implicitly define $(RM), which I'm also fixing in the Mozilla
+ code.
+
+ISSUES
+
+* Speed is a problem.
+
+FUTURE WORK
+
+* implement a new type of command which is implemented in python. This would allow us
+to replace the current `nsinstall` binary (and execution costs for the shell and binary) with an
+in-process python solution.
+
+AUTHOR
+
+Initial code was written by Benjamin Smedberg <benjamin@smedbergs.us>. For future releases see
+http://benjamin.smedbergs.us/pymake/
+
+See the LICENSE file for license information (MIT license)
diff --git a/build/pymake/make.py b/build/pymake/make.py
new file mode 100755
index 000000000..0857f3f8c
--- /dev/null
+++ b/build/pymake/make.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+"""
+make.py
+
+A drop-in or mostly drop-in replacement for GNU make.
+"""
+
+import sys, os
+import pymake.command, pymake.process
+
+import gc
+
+if __name__ == '__main__':
+ if 'TINDERBOX_OUTPUT' in os.environ:
+ # When building on mozilla build slaves, execute mozmake instead. Until bug
+ # 978211, this is the easiest, albeit hackish, way to do this.
+ import subprocess
+ mozmake = os.path.join(os.path.dirname(__file__), '..', '..',
+ 'mozmake.exe')
+ cmd = [mozmake]
+ cmd.extend(sys.argv[1:])
+ shell = os.environ.get('SHELL')
+ if shell and not shell.lower().endswith('.exe'):
+ cmd += ['SHELL=%s.exe' % shell]
+ sys.exit(subprocess.call(cmd))
+
+ sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+ sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
+
+ gc.disable()
+
+ pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit)
+ pymake.process.ParallelContext.spin()
+ assert False, "Not reached"
diff --git a/build/pymake/mkformat.py b/build/pymake/mkformat.py
new file mode 100755
index 000000000..41dd761b2
--- /dev/null
+++ b/build/pymake/mkformat.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import sys
+import pymake.parser
+
+filename = sys.argv[1]
+source = None
+
+with open(filename, 'rU') as fh:
+ source = fh.read()
+
+statements = pymake.parser.parsestring(source, filename)
+print statements.to_source()
diff --git a/build/pymake/mkparse.py b/build/pymake/mkparse.py
new file mode 100755
index 000000000..253683948
--- /dev/null
+++ b/build/pymake/mkparse.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+import sys
+import pymake.parser
+
+for f in sys.argv[1:]:
+ print "Parsing %s" % f
+ fd = open(f, 'rU')
+ s = fd.read()
+ fd.close()
+ stmts = pymake.parser.parsestring(s, f)
+ print stmts
diff --git a/build/pymake/pymake/__init__.py b/build/pymake/pymake/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/build/pymake/pymake/__init__.py
diff --git a/build/pymake/pymake/builtins.py b/build/pymake/pymake/builtins.py
new file mode 100644
index 000000000..eb6f2e11b
--- /dev/null
+++ b/build/pymake/pymake/builtins.py
@@ -0,0 +1,120 @@
+# Basic commands implemented in Python
+import errno, sys, os, shutil, time
+from getopt import getopt, GetoptError
+
+from process import PythonException
+
+__all__ = ["mkdir", "rm", "sleep", "touch"]
+
+def mkdir(args):
+ """
+ Emulate some of the behavior of mkdir(1).
+ Only supports the -p (--parents) argument.
+ """
+ try:
+ opts, args = getopt(args, "p", ["parents"])
+ except GetoptError, e:
+ raise PythonException, ("mkdir: %s" % e, 1)
+ parents = False
+ for o, a in opts:
+ if o in ('-p', '--parents'):
+ parents = True
+ for f in args:
+ try:
+ if parents:
+ os.makedirs(f)
+ else:
+ os.mkdir(f)
+ except OSError, e:
+ if e.errno == errno.EEXIST and parents:
+ pass
+ else:
+ raise PythonException, ("mkdir: %s" % e, 1)
+
+def rm(args):
+ """
+ Emulate most of the behavior of rm(1).
+ Only supports the -r (--recursive) and -f (--force) arguments.
+ """
+ try:
+ opts, args = getopt(args, "rRf", ["force", "recursive"])
+ except GetoptError, e:
+ raise PythonException, ("rm: %s" % e, 1)
+ force = False
+ recursive = False
+ for o, a in opts:
+ if o in ('-f', '--force'):
+ force = True
+ elif o in ('-r', '-R', '--recursive'):
+ recursive = True
+ for f in args:
+ if os.path.isdir(f):
+ if not recursive:
+ raise PythonException, ("rm: cannot remove '%s': Is a directory" % f, 1)
+ else:
+ shutil.rmtree(f, force)
+ elif os.path.exists(f):
+ try:
+ os.unlink(f)
+ except:
+ if not force:
+ raise PythonException, ("rm: failed to remove '%s': %s" % (f, sys.exc_info()[0]), 1)
+ elif not force:
+ raise PythonException, ("rm: cannot remove '%s': No such file or directory" % f, 1)
+
+def sleep(args):
+ """
+ Emulate the behavior of sleep(1).
+ """
+ total = 0
+ values = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
+ for a in args:
+ multiplier = 1
+ for k, v in values.iteritems():
+ if a.endswith(k):
+ a = a[:-1]
+ multiplier = v
+ break
+ try:
+ f = float(a)
+ total += f * multiplier
+ except ValueError:
+ raise PythonException, ("sleep: invalid time interval '%s'" % a, 1)
+ time.sleep(total)
+
+def touch(args):
+ """
+ Emulate the behavior of touch(1).
+ """
+ try:
+ opts, args = getopt(args, "t:")
+ except GetoptError, e:
+ raise PythonException, ("touch: %s" % e, 1)
+ opts = dict(opts)
+ times = None
+ if '-t' in opts:
+ import re
+ from time import mktime, localtime
+ m = re.match('^(?P<Y>(?:\d\d)?\d\d)?(?P<M>\d\d)(?P<D>\d\d)(?P<h>\d\d)(?P<m>\d\d)(?:\.(?P<s>\d\d))?$', opts['-t'])
+ if not m:
+ raise PythonException, ("touch: invalid date format '%s'" % opts['-t'], 1)
+ def normalized_field(m, f):
+ if f == 'Y':
+ if m.group(f) is None:
+ return localtime()[0]
+ y = int(m.group(f))
+ if y < 69:
+ y += 2000
+ elif y < 100:
+ y += 1900
+ return y
+ if m.group(f) is None:
+ return localtime()[0] if f == 'Y' else 0
+ return int(m.group(f))
+ time = [normalized_field(m, f) for f in ['Y', 'M', 'D', 'h', 'm', 's']] + [0, 0, -1]
+ time = mktime(time)
+ times = (time, time)
+ for f in args:
+ if not os.path.exists(f):
+ open(f, 'a').close()
+ os.utime(f, times)
diff --git a/build/pymake/pymake/command.py b/build/pymake/pymake/command.py
new file mode 100644
index 000000000..cd68e4fdb
--- /dev/null
+++ b/build/pymake/pymake/command.py
@@ -0,0 +1,278 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Makefile execution.
+
+Multiple `makes` can be run within the same process. Each one has an entirely data.Makefile and .Target
+structure, environment, and working directory. Typically they will all share a parallel execution context,
+except when a submake specifies -j1 when the parent make is building in parallel.
+"""
+
+import os, subprocess, sys, logging, time, traceback, re
+from optparse import OptionParser
+import data, parserdata, process, util
+
+# TODO: If this ever goes from relocatable package to system-installed, this may need to be
+# a configured-in path.
+
+makepypath = util.normaljoin(os.path.dirname(__file__), '../make.py')
+
+_simpleopts = re.compile(r'^[a-zA-Z]+(\s|$)')
+def parsemakeflags(env):
+ """
+ Parse MAKEFLAGS from the environment into a sequence of command-line arguments.
+ """
+
+ makeflags = env.get('MAKEFLAGS', '')
+ makeflags = makeflags.strip()
+
+ if makeflags == '':
+ return []
+
+ if _simpleopts.match(makeflags):
+ makeflags = '-' + makeflags
+
+ opts = []
+ curopt = ''
+
+ i = 0
+ while i < len(makeflags):
+ c = makeflags[i]
+ if c.isspace():
+ opts.append(curopt)
+ curopt = ''
+ i += 1
+ while i < len(makeflags) and makeflags[i].isspace():
+ i += 1
+ continue
+
+ if c == '\\':
+ i += 1
+ if i == len(makeflags):
+ raise data.DataError("MAKEFLAGS has trailing backslash")
+ c = makeflags[i]
+
+ curopt += c
+ i += 1
+
+ if curopt != '':
+ opts.append(curopt)
+
+ return opts
+
+def _version(*args):
+ print """pymake: GNU-compatible make program
+Copyright (C) 2009 The Mozilla Foundation <http://www.mozilla.org/>
+This is free software; see the source for copying conditions.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE."""
+
+_log = logging.getLogger('pymake.execution')
+
+class _MakeContext(object):
+ def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb):
+ self.makeflags = makeflags
+ self.makelevel = makelevel
+
+ self.workdir = workdir
+ self.context = context
+ self.env = env
+ self.targets = targets
+ self.options = options
+ self.ostmts = ostmts
+ self.overrides = overrides
+ self.cb = cb
+
+ self.restarts = 0
+
+ self.remakecb(True)
+
+ def remakecb(self, remade, error=None):
+ if error is not None:
+ print error
+ self.context.defer(self.cb, 2)
+ return
+
+ if remade:
+ if self.restarts > 0:
+ _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel)
+
+ self.makefile = data.Makefile(restarts=self.restarts,
+ make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
+ makeflags=self.makeflags,
+ makeoverrides=self.overrides,
+ workdir=self.workdir,
+ context=self.context,
+ env=self.env,
+ makelevel=self.makelevel,
+ targets=self.targets,
+ keepgoing=self.options.keepgoing,
+ silent=self.options.silent,
+ justprint=self.options.justprint)
+
+ self.restarts += 1
+
+ try:
+ self.ostmts.execute(self.makefile)
+ for f in self.options.makefiles:
+ self.makefile.include(f)
+ self.makefile.finishparsing()
+ self.makefile.remakemakefiles(self.remakecb)
+ except util.MakeError, e:
+ print e
+ self.context.defer(self.cb, 2)
+
+ return
+
+ if len(self.targets) == 0:
+ if self.makefile.defaulttarget is None:
+ print "No target specified and no default target found."
+ self.context.defer(self.cb, 2)
+ return
+
+ _log.info("Making default target %s", self.makefile.defaulttarget)
+ self.realtargets = [self.makefile.defaulttarget]
+ self.tstack = ['<default-target>']
+ else:
+ self.realtargets = self.targets
+ self.tstack = ['<command-line>']
+
+ self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, cb=self.makecb)
+
+ def makecb(self, error, didanything):
+ assert error in (True, False)
+
+ if error:
+ self.context.defer(self.cb, 2)
+ return
+
+ if not len(self.realtargets):
+ if self.options.printdir:
+ print "make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir)
+ sys.stdout.flush()
+
+ self.context.defer(self.cb, 0)
+ else:
+ self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, self.makecb)
+
+def main(args, env, cwd, cb):
+ """
+ Start a single makefile execution, given a command line, working directory, and environment.
+
+ @param cb a callback to notify with an exit code when make execution is finished.
+ """
+
+ try:
+ makelevel = int(env.get('MAKELEVEL', '0'))
+
+ op = OptionParser()
+ op.add_option('-f', '--file', '--makefile',
+ action='append',
+ dest='makefiles',
+ default=[])
+ op.add_option('-d',
+ action="store_true",
+ dest="verbose", default=False)
+ op.add_option('-k', '--keep-going',
+ action="store_true",
+ dest="keepgoing", default=False)
+ op.add_option('--debug-log',
+ dest="debuglog", default=None)
+ op.add_option('-C', '--directory',
+ dest="directory", default=None)
+ op.add_option('-v', '--version', action="store_true",
+ dest="printversion", default=False)
+ op.add_option('-j', '--jobs', type="int",
+ dest="jobcount", default=1)
+ op.add_option('-w', '--print-directory', action="store_true",
+ dest="printdir")
+ op.add_option('--no-print-directory', action="store_false",
+ dest="printdir", default=True)
+ op.add_option('-s', '--silent', action="store_true",
+ dest="silent", default=False)
+ op.add_option('-n', '--just-print', '--dry-run', '--recon',
+ action="store_true",
+ dest="justprint", default=False)
+
+ options, arguments1 = op.parse_args(parsemakeflags(env))
+ options, arguments2 = op.parse_args(args, values=options)
+
+ op.destroy()
+
+ arguments = arguments1 + arguments2
+
+ if options.printversion:
+ _version()
+ cb(0)
+ return
+
+ shortflags = []
+ longflags = []
+
+ if options.keepgoing:
+ shortflags.append('k')
+
+ if options.printdir:
+ shortflags.append('w')
+
+ if options.silent:
+ shortflags.append('s')
+ options.printdir = False
+
+ if options.justprint:
+ shortflags.append('n')
+
+ loglevel = logging.WARNING
+ if options.verbose:
+ loglevel = logging.DEBUG
+ shortflags.append('d')
+
+ logkwargs = {}
+ if options.debuglog:
+ logkwargs['filename'] = options.debuglog
+ longflags.append('--debug-log=%s' % options.debuglog)
+
+ if options.directory is None:
+ workdir = cwd
+ else:
+ workdir = util.normaljoin(cwd, options.directory)
+
+ if options.jobcount != 1:
+ longflags.append('-j%i' % (options.jobcount,))
+
+ makeflags = ''.join(shortflags)
+ if len(longflags):
+ makeflags += ' ' + ' '.join(longflags)
+
+ logging.basicConfig(level=loglevel, **logkwargs)
+
+ context = process.getcontext(options.jobcount)
+
+ if options.printdir:
+ print "make.py[%i]: Entering directory '%s'" % (makelevel, workdir)
+ sys.stdout.flush()
+
+ if len(options.makefiles) == 0:
+ if os.path.exists(util.normaljoin(workdir, 'Makefile')):
+ options.makefiles.append('Makefile')
+ else:
+ print "No makefile found"
+ cb(2)
+ return
+
+ ostmts, targets, overrides = parserdata.parsecommandlineargs(arguments)
+
+ _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb)
+ except (util.MakeError), e:
+ print e
+ if options.printdir:
+ print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
+ sys.stdout.flush()
+ cb(2)
+ return
diff --git a/build/pymake/pymake/data.py b/build/pymake/pymake/data.py
new file mode 100644
index 000000000..dcd7e9225
--- /dev/null
+++ b/build/pymake/pymake/data.py
@@ -0,0 +1,1842 @@
+"""
+A representation of makefile data structures.
+"""
+
+import logging, re, os, sys
+import parserdata, parser, functions, process, util, implicit
+from cStringIO import StringIO
+
+if sys.version_info[0] < 3:
+ str_type = basestring
+else:
+ str_type = str
+
+_log = logging.getLogger('pymake.data')
+
+class DataError(util.MakeError):
+ pass
+
+class ResolutionError(DataError):
+ """
+ Raised when dependency resolution fails, either due to recursion or to missing
+ prerequisites.This is separately catchable so that implicit rule search can try things
+ without having to commit.
+ """
+ pass
+
+def withoutdups(it):
+ r = set()
+ for i in it:
+ if not i in r:
+ r.add(i)
+ yield i
+
+def mtimeislater(deptime, targettime):
+ """
+ Is the mtime of the dependency later than the target?
+ """
+
+ if deptime is None:
+ return True
+ if targettime is None:
+ return False
+ # int(1000*x) because of http://bugs.python.org/issue10148
+ return int(1000 * deptime) > int(1000 * targettime)
+
+def getmtime(path):
+ try:
+ s = os.stat(path)
+ return s.st_mtime
+ except OSError:
+ return None
+
+def stripdotslash(s):
+ if s.startswith('./'):
+ st = s[2:]
+ return st if st != '' else '.'
+ return s
+
+def stripdotslashes(sl):
+ for s in sl:
+ yield stripdotslash(s)
+
+def getindent(stack):
+ return ''.ljust(len(stack) - 1)
+
+def _if_else(c, t, f):
+ if c:
+ return t()
+ return f()
+
+
+class BaseExpansion(object):
+ """Base class for expansions.
+
+ A make expansion is the parsed representation of a string, which may
+ contain references to other elements.
+ """
+
+ @property
+ def is_static_string(self):
+ """Returns whether the expansion is composed of static string content.
+
+ This is always True for StringExpansion. It will be True for Expansion
+ only if all elements of that Expansion are static strings.
+ """
+ raise Exception('Must be implemented in child class.')
+
+ def functions(self, descend=False):
+ """Obtain all functions inside this expansion.
+
+ This is a generator for pymake.functions.Function instances.
+
+ By default, this only returns functions existing as the primary
+ elements of this expansion. If `descend` is True, it will descend into
+ child expansions and extract all functions in the tree.
+ """
+ # An empty generator. Yeah, it's weird.
+ for x in []:
+ yield x
+
+ def variable_references(self, descend=False):
+ """Obtain all variable references in this expansion.
+
+ This is a generator for pymake.functionsVariableRef instances.
+
+ To retrieve the names of variables, simply query the `vname` field on
+ the returned instances. Most of the time these will be StringExpansion
+ instances.
+ """
+ for f in self.functions(descend=descend):
+ if not isinstance(f, functions.VariableRef):
+ continue
+
+ yield f
+
+ @property
+ def is_filesystem_dependent(self):
+ """Whether this expansion may query the filesystem for evaluation.
+
+ This effectively asks "is any function in this expansion dependent on
+ the filesystem.
+ """
+ for f in self.functions(descend=True):
+ if f.is_filesystem_dependent:
+ return True
+
+ return False
+
+ @property
+ def is_shell_dependent(self):
+ """Whether this expansion may invoke a shell for evaluation."""
+
+ for f in self.functions(descend=True):
+ if isinstance(f, functions.ShellFunction):
+ return True
+
+ return False
+
+
+class StringExpansion(BaseExpansion):
+ """An Expansion representing a static string.
+
+ This essentially wraps a single str instance.
+ """
+
+ __slots__ = ('loc', 's',)
+ simple = True
+
+ def __init__(self, s, loc):
+ assert isinstance(s, str_type)
+ self.s = s
+ self.loc = loc
+
+ def lstrip(self):
+ self.s = self.s.lstrip()
+
+ def rstrip(self):
+ self.s = self.s.rstrip()
+
+ def isempty(self):
+ return self.s == ''
+
+ def resolve(self, i, j, fd, k=None):
+ fd.write(self.s)
+
+ def resolvestr(self, i, j, k=None):
+ return self.s
+
+ def resolvesplit(self, i, j, k=None):
+ return self.s.split()
+
+ def clone(self):
+ e = Expansion(self.loc)
+ e.appendstr(self.s)
+ return e
+
+ @property
+ def is_static_string(self):
+ return True
+
+ def __len__(self):
+ return 1
+
+ def __getitem__(self, i):
+ assert i == 0
+ return self.s, False
+
+ def __repr__(self):
+ return "Exp<%s>(%r)" % (self.loc, self.s)
+
+ def __eq__(self, other):
+ """We only compare the string contents."""
+ return self.s == other
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def to_source(self, escape_variables=False, escape_comments=False):
+ s = self.s
+
+ if escape_comments:
+ s = s.replace('#', '\\#')
+
+ if escape_variables:
+ return s.replace('$', '$$')
+
+ return s
+
+
+class Expansion(BaseExpansion, list):
+ """A representation of expanded data.
+
+ This is effectively an ordered list of StringExpansion and
+ pymake.function.Function instances. Every item in the collection appears in
+ the same context in a make file.
+ """
+
+ __slots__ = ('loc',)
+ simple = False
+
+ def __init__(self, loc=None):
+ # A list of (element, isfunc) tuples
+ # element is either a string or a function
+ self.loc = loc
+
+ @staticmethod
+ def fromstring(s, path):
+ return StringExpansion(s, parserdata.Location(path, 1, 0))
+
+ def clone(self):
+ e = Expansion()
+ e.extend(self)
+ return e
+
+ def appendstr(self, s):
+ assert isinstance(s, str_type)
+ if s == '':
+ return
+
+ self.append((s, False))
+
+ def appendfunc(self, func):
+ assert isinstance(func, functions.Function)
+ self.append((func, True))
+
+ def concat(self, o):
+ """Concatenate the other expansion on to this one."""
+ if o.simple:
+ self.appendstr(o.s)
+ else:
+ self.extend(o)
+
+ def isempty(self):
+ return (not len(self)) or self[0] == ('', False)
+
+ def lstrip(self):
+ """Strip leading literal whitespace from this expansion."""
+ while True:
+ i, isfunc = self[0]
+ if isfunc:
+ return
+
+ i = i.lstrip()
+ if i != '':
+ self[0] = i, False
+ return
+
+ del self[0]
+
+ def rstrip(self):
+ """Strip trailing literal whitespace from this expansion."""
+ while True:
+ i, isfunc = self[-1]
+ if isfunc:
+ return
+
+ i = i.rstrip()
+ if i != '':
+ self[-1] = i, False
+ return
+
+ del self[-1]
+
+ def finish(self):
+ # Merge any adjacent literal strings:
+ strings = []
+ elements = []
+ for (e, isfunc) in self:
+ if isfunc:
+ if strings:
+ s = ''.join(strings)
+ if s:
+ elements.append((s, False))
+ strings = []
+ elements.append((e, True))
+ else:
+ strings.append(e)
+
+ if not elements:
+ # This can only happen if there were no function elements.
+ return StringExpansion(''.join(strings), self.loc)
+
+ if strings:
+ s = ''.join(strings)
+ if s:
+ elements.append((s, False))
+
+ if len(elements) < len(self):
+ self[:] = elements
+
+ return self
+
+ def resolve(self, makefile, variables, fd, setting=[]):
+ """
+ Resolve this variable into a value, by interpolating the value
+ of other variables.
+
+ @param setting (Variable instance) the variable currently
+ being set, if any. Setting variables must avoid self-referential
+ loops.
+ """
+ assert isinstance(makefile, Makefile)
+ assert isinstance(variables, Variables)
+ assert isinstance(setting, list)
+
+ for e, isfunc in self:
+ if isfunc:
+ e.resolve(makefile, variables, fd, setting)
+ else:
+ assert isinstance(e, str_type)
+ fd.write(e)
+
+ def resolvestr(self, makefile, variables, setting=[]):
+ fd = StringIO()
+ self.resolve(makefile, variables, fd, setting)
+ return fd.getvalue()
+
+ def resolvesplit(self, makefile, variables, setting=[]):
+ return self.resolvestr(makefile, variables, setting).split()
+
+ @property
+ def is_static_string(self):
+ """An Expansion is static if all its components are strings, not
+ functions."""
+ for e, is_func in self:
+ if is_func:
+ return False
+
+ return True
+
+ def functions(self, descend=False):
+ for e, is_func in self:
+ if is_func:
+ yield e
+
+ if descend:
+ for exp in e.expansions(descend=True):
+ for f in exp.functions(descend=True):
+ yield f
+
+ def __repr__(self):
+ return "<Expansion with elements: %r>" % ([e for e, isfunc in self],)
+
+ def to_source(self, escape_variables=False, escape_comments=False):
+ parts = []
+ for e, is_func in self:
+ if is_func:
+ parts.append(e.to_source())
+ continue
+
+ if escape_variables:
+ parts.append(e.replace('$', '$$'))
+ continue
+
+ parts.append(e)
+
+ return ''.join(parts)
+
+ def __eq__(self, other):
+ if not isinstance(other, (Expansion, StringExpansion)):
+ return False
+
+ # Expansions are equivalent if adjacent string literals normalize to
+ # the same value. So, we must normalize before any comparisons are
+ # made.
+ a = self.clone().finish()
+
+ if isinstance(other, StringExpansion):
+ if isinstance(a, StringExpansion):
+ return a == other
+
+ # A normalized Expansion != StringExpansion.
+ return False
+
+ b = other.clone().finish()
+
+ # b could be a StringExpansion now.
+ if isinstance(b, StringExpansion):
+ if isinstance(a, StringExpansion):
+ return a == b
+
+ # Our normalized Expansion != normalized StringExpansion.
+ return False
+
+ if len(a) != len(b):
+ return False
+
+ for i in xrange(len(self)):
+ e1, is_func1 = a[i]
+ e2, is_func2 = b[i]
+
+ if is_func1 != is_func2:
+ return False
+
+ if type(e1) != type(e2):
+ return False
+
+ if e1 != e2:
+ return False
+
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+class Variables(object):
+ """
+ A mapping from variable names to variables. Variables have flavor, source, and value. The value is an
+ expansion object.
+ """
+
+ __slots__ = ('parent', '_map')
+
+ FLAVOR_RECURSIVE = 0
+ FLAVOR_SIMPLE = 1
+ FLAVOR_APPEND = 2
+
+ SOURCE_OVERRIDE = 0
+ SOURCE_COMMANDLINE = 1
+ SOURCE_MAKEFILE = 2
+ SOURCE_ENVIRONMENT = 3
+ SOURCE_AUTOMATIC = 4
+ SOURCE_IMPLICIT = 5
+
+ def __init__(self, parent=None):
+ self._map = {} # vname -> flavor, source, valuestr, valueexp
+ self.parent = parent
+
+ def readfromenvironment(self, env):
+ for k, v in env.iteritems():
+ self.set(k, self.FLAVOR_RECURSIVE, self.SOURCE_ENVIRONMENT, v)
+
+ def get(self, name, expand=True):
+ """
+ Get the value of a named variable. Returns a tuple (flavor, source, value)
+
+ If the variable is not present, returns (None, None, None)
+
+ @param expand If true, the value will be returned as an expansion. If false,
+ it will be returned as an unexpanded string.
+ """
+ flavor, source, valuestr, valueexp = self._map.get(name, (None, None, None, None))
+ if flavor is not None:
+ if expand and flavor != self.FLAVOR_SIMPLE and valueexp is None:
+ d = parser.Data.fromstring(valuestr, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0))
+ valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
+ self._map[name] = flavor, source, valuestr, valueexp
+
+ if flavor == self.FLAVOR_APPEND:
+ if self.parent:
+ pflavor, psource, pvalue = self.parent.get(name, expand)
+ else:
+ pflavor, psource, pvalue = None, None, None
+
+ if pvalue is None:
+ flavor = self.FLAVOR_RECURSIVE
+ # fall through
+ else:
+ if source > psource:
+ # TODO: log a warning?
+ return pflavor, psource, pvalue
+
+ if not expand:
+ return pflavor, psource, pvalue + ' ' + valuestr
+
+ pvalue = pvalue.clone()
+ pvalue.appendstr(' ')
+ pvalue.concat(valueexp)
+
+ return pflavor, psource, pvalue
+
+ if not expand:
+ return flavor, source, valuestr
+
+ if flavor == self.FLAVOR_RECURSIVE:
+ val = valueexp
+ else:
+ val = Expansion.fromstring(valuestr, "Expansion of variable '%s'" % (name,))
+
+ return flavor, source, val
+
+ if self.parent is not None:
+ return self.parent.get(name, expand)
+
+ return (None, None, None)
+
+ def set(self, name, flavor, source, value, force=False):
+ assert flavor in (self.FLAVOR_RECURSIVE, self.FLAVOR_SIMPLE)
+ assert source in (self.SOURCE_OVERRIDE, self.SOURCE_COMMANDLINE, self.SOURCE_MAKEFILE, self.SOURCE_ENVIRONMENT, self.SOURCE_AUTOMATIC, self.SOURCE_IMPLICIT)
+ assert isinstance(value, str_type), "expected str, got %s" % type(value)
+
+ prevflavor, prevsource, prevvalue = self.get(name)
+ if prevsource is not None and source > prevsource and not force:
+ # TODO: give a location for this warning
+ _log.info("not setting variable '%s', set by higher-priority source to value '%s'" % (name, prevvalue))
+ return
+
+ self._map[name] = flavor, source, value, None
+
+ def append(self, name, source, value, variables, makefile):
+ assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC)
+ assert isinstance(value, str_type)
+
+ if name not in self._map:
+ self._map[name] = self.FLAVOR_APPEND, source, value, None
+ return
+
+ prevflavor, prevsource, prevvalue, valueexp = self._map[name]
+ if source > prevsource:
+ # TODO: log a warning?
+ return
+
+ if prevflavor == self.FLAVOR_SIMPLE:
+ d = parser.Data.fromstring(value, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0))
+ valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
+
+ val = valueexp.resolvestr(makefile, variables, [name])
+ self._map[name] = prevflavor, prevsource, prevvalue + ' ' + val, None
+ return
+
+ newvalue = prevvalue + ' ' + value
+ self._map[name] = prevflavor, prevsource, newvalue, None
+
+ def merge(self, other):
+ assert isinstance(other, Variables)
+ for k, flavor, source, value in other:
+ self.set(k, flavor, source, value)
+
+ def __iter__(self):
+ for k, (flavor, source, value, valueexp) in self._map.iteritems():
+ yield k, flavor, source, value
+
+ def __contains__(self, item):
+ return item in self._map
+
+class Pattern(object):
+ """
+ A pattern is a string, possibly with a % substitution character. From the GNU make manual:
+
+ '%' characters in pattern rules can be quoted with precending backslashes ('\'). Backslashes that
+ would otherwise quote '%' charcters can be quoted with more backslashes. Backslashes that
+ quote '%' characters or other backslashes are removed from the pattern before it is compared t
+ file names or has a stem substituted into it. Backslashes that are not in danger of quoting '%'
+ characters go unmolested. For example, the pattern the\%weird\\%pattern\\ has `the%weird\' preceding
+ the operative '%' character, and 'pattern\\' following it. The final two backslashes are left alone
+ because they cannot affect any '%' character.
+
+ This insane behavior probably doesn't matter, but we're compatible just for shits and giggles.
+ """
+
+ __slots__ = ('data')
+
+ def __init__(self, s):
+ r = []
+ i = 0
+ slen = len(s)
+ while i < slen:
+ c = s[i]
+ if c == '\\':
+ nc = s[i + 1]
+ if nc == '%':
+ r.append('%')
+ i += 1
+ elif nc == '\\':
+ r.append('\\')
+ i += 1
+ else:
+ r.append(c)
+ elif c == '%':
+ self.data = (''.join(r), s[i+1:])
+ return
+ else:
+ r.append(c)
+ i += 1
+
+ # This is different than (s,) because \% and \\ have been unescaped. Parsing patterns is
+ # context-sensitive!
+ self.data = (''.join(r),)
+
+ def ismatchany(self):
+ return self.data == ('','')
+
+ def ispattern(self):
+ return len(self.data) == 2
+
+ def __hash__(self):
+ return self.data.__hash__()
+
+ def __eq__(self, o):
+ assert isinstance(o, Pattern)
+ return self.data == o.data
+
+ def gettarget(self):
+ assert not self.ispattern()
+ return self.data[0]
+
+ def hasslash(self):
+ return self.data[0].find('/') != -1 or self.data[1].find('/') != -1
+
+ def match(self, word):
+ """
+ Match this search pattern against a word (string).
+
+ @returns None if the word doesn't match, or the matching stem.
+ If this is a %-less pattern, the stem will always be ''
+ """
+ d = self.data
+ if len(d) == 1:
+ if word == d[0]:
+ return word
+ return None
+
+ d0, d1 = d
+ l1 = len(d0)
+ l2 = len(d1)
+ if len(word) >= l1 + l2 and word.startswith(d0) and word.endswith(d1):
+ if l2 == 0:
+ return word[l1:]
+ return word[l1:-l2]
+
+ return None
+
+ def resolve(self, dir, stem):
+ if self.ispattern():
+ return dir + self.data[0] + stem + self.data[1]
+
+ return self.data[0]
+
+ def subst(self, replacement, word, mustmatch):
+ """
+ Given a word, replace the current pattern with the replacement pattern, a la 'patsubst'
+
+ @param mustmatch If true and this pattern doesn't match the word, throw a DataError. Otherwise
+ return word unchanged.
+ """
+ assert isinstance(replacement, str_type)
+
+ stem = self.match(word)
+ if stem is None:
+ if mustmatch:
+ raise DataError("target '%s' doesn't match pattern" % (word,))
+ return word
+
+ if not self.ispattern():
+ # if we're not a pattern, the replacement is not parsed as a pattern either
+ return replacement
+
+ return Pattern(replacement).resolve('', stem)
+
+ def __repr__(self):
+ return "<Pattern with data %r>" % (self.data,)
+
+ _backre = re.compile(r'[%\\]')
+ def __str__(self):
+ if not self.ispattern():
+ return self._backre.sub(r'\\\1', self.data[0])
+
+ return self._backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1]
+
+class RemakeTargetSerially(object):
+ __slots__ = ('target', 'makefile', 'indent', 'rlist')
+
+ def __init__(self, target, makefile, indent, rlist):
+ self.target = target
+ self.makefile = makefile
+ self.indent = indent
+ self.rlist = rlist
+ self.commandscb(False)
+
+ def resolvecb(self, error, didanything):
+ assert error in (True, False)
+
+ if didanything:
+ self.target.didanything = True
+
+ if error:
+ self.target.error = True
+ self.makefile.error = True
+ if not self.makefile.keepgoing:
+ self.target.notifydone(self.makefile)
+ return
+ else:
+ # don't run the commands!
+ del self.rlist[0]
+ self.commandscb(error=False)
+ else:
+ self.rlist.pop(0).runcommands(self.indent, self.commandscb)
+
+ def commandscb(self, error):
+ assert error in (True, False)
+
+ if error:
+ self.target.error = True
+ self.makefile.error = True
+
+ if self.target.error and not self.makefile.keepgoing:
+ self.target.notifydone(self.makefile)
+ return
+
+ if not len(self.rlist):
+ self.target.notifydone(self.makefile)
+ else:
+ self.rlist[0].resolvedeps(True, self.resolvecb)
+
+class RemakeTargetParallel(object):
+ __slots__ = ('target', 'makefile', 'indent', 'rlist', 'rulesremaining', 'currunning')
+
+ def __init__(self, target, makefile, indent, rlist):
+ self.target = target
+ self.makefile = makefile
+ self.indent = indent
+ self.rlist = rlist
+
+ self.rulesremaining = len(rlist)
+ self.currunning = False
+
+ for r in rlist:
+ makefile.context.defer(self.doresolve, r)
+
+ def doresolve(self, r):
+ if self.makefile.error and not self.makefile.keepgoing:
+ r.error = True
+ self.resolvecb(True, False)
+ else:
+ r.resolvedeps(False, self.resolvecb)
+
+ def resolvecb(self, error, didanything):
+ assert error in (True, False)
+
+ if error:
+ self.target.error = True
+
+ if didanything:
+ self.target.didanything = True
+
+ self.rulesremaining -= 1
+
+ # commandscb takes care of the details if we're currently building
+ # something
+ if self.currunning:
+ return
+
+ self.runnext()
+
+ def runnext(self):
+ assert not self.currunning
+
+ if self.makefile.error and not self.makefile.keepgoing:
+ self.rlist = []
+ else:
+ while len(self.rlist) and self.rlist[0].error:
+ del self.rlist[0]
+
+ if not len(self.rlist):
+ if not self.rulesremaining:
+ self.target.notifydone(self.makefile)
+ return
+
+ if self.rlist[0].depsremaining != 0:
+ return
+
+ self.currunning = True
+ rule = self.rlist.pop(0)
+ self.makefile.context.defer(rule.runcommands, self.indent, self.commandscb)
+
+ def commandscb(self, error):
+ assert error in (True, False)
+ if error:
+ self.target.error = True
+ self.makefile.error = True
+
+ assert self.currunning
+ self.currunning = False
+ self.runnext()
+
+class RemakeRuleContext(object):
+ def __init__(self, target, makefile, rule, deps,
+ targetstack, avoidremakeloop):
+ self.target = target
+ self.makefile = makefile
+ self.rule = rule
+ self.deps = deps
+ self.targetstack = targetstack
+ self.avoidremakeloop = avoidremakeloop
+
+ self.running = False
+ self.error = False
+ self.depsremaining = len(deps) + 1
+ self.remake = False
+
+ def resolvedeps(self, serial, cb):
+ self.resolvecb = cb
+ self.didanything = False
+ if serial:
+ self._resolvedepsserial()
+ else:
+ self._resolvedepsparallel()
+
+ def _weakdepfinishedserial(self, error, didanything):
+ if error:
+ self.remake = True
+ self._depfinishedserial(False, didanything)
+
+ def _depfinishedserial(self, error, didanything):
+ assert error in (True, False)
+
+ if didanything:
+ self.didanything = True
+
+ if error:
+ self.error = True
+ if not self.makefile.keepgoing:
+ self.resolvecb(error=True, didanything=self.didanything)
+ return
+
+ if len(self.resolvelist):
+ dep, weak = self.resolvelist.pop(0)
+ self.makefile.context.defer(dep.make,
+ self.makefile, self.targetstack, weak and self._weakdepfinishedserial or self._depfinishedserial)
+ else:
+ self.resolvecb(error=self.error, didanything=self.didanything)
+
+ def _resolvedepsserial(self):
+ self.resolvelist = list(self.deps)
+ self._depfinishedserial(False, False)
+
+ def _startdepparallel(self, d):
+ dep, weak = d
+ if weak:
+ depfinished = self._weakdepfinishedparallel
+ else:
+ depfinished = self._depfinishedparallel
+ if self.makefile.error:
+ depfinished(True, False)
+ else:
+ dep.make(self.makefile, self.targetstack, depfinished)
+
+ def _weakdepfinishedparallel(self, error, didanything):
+ if error:
+ self.remake = True
+ self._depfinishedparallel(False, didanything)
+
+ def _depfinishedparallel(self, error, didanything):
+ assert error in (True, False)
+
+ if error:
+ print "<%s>: Found error" % self.target.target
+ self.error = True
+ if didanything:
+ self.didanything = True
+
+ self.depsremaining -= 1
+ if self.depsremaining == 0:
+ self.resolvecb(error=self.error, didanything=self.didanything)
+
+ def _resolvedepsparallel(self):
+ self.depsremaining -= 1
+ if self.depsremaining == 0:
+ self.resolvecb(error=self.error, didanything=self.didanything)
+ return
+
+ self.didanything = False
+
+ for d in self.deps:
+ self.makefile.context.defer(self._startdepparallel, d)
+
+ def _commandcb(self, error):
+ assert error in (True, False)
+
+ if error:
+ self.runcb(error=True)
+ return
+
+ if len(self.commands):
+ self.commands.pop(0)(self._commandcb)
+ else:
+ self.runcb(error=False)
+
+ def runcommands(self, indent, cb):
+ assert not self.running
+ self.running = True
+
+ self.runcb = cb
+
+ if self.rule is None or not len(self.rule.commands):
+ if self.target.mtime is None:
+ self.target.beingremade()
+ else:
+ for d, weak in self.deps:
+ if mtimeislater(d.mtime, self.target.mtime):
+ if d.mtime is None:
+ self.target.beingremade()
+ else:
+ _log.info("%sNot remaking %s ubecause it would have no effect, even though %s is newer.", indent, self.target.target, d.target)
+ break
+ cb(error=False)
+ return
+
+ if self.rule.doublecolon:
+ if len(self.deps) == 0:
+ if self.avoidremakeloop:
+ _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc)
+ cb(error=False)
+ return
+
+ remake = self.remake
+ if remake:
+ _log.info("%sRemaking %s using rule at %s: weak dependency was not found.", indent, self.target.target, self.rule.loc)
+ else:
+ if self.target.mtime is None:
+ remake = True
+ _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc)
+
+ if not remake:
+ if self.rule.doublecolon:
+ if len(self.deps) == 0:
+ _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc)
+ remake = True
+
+ if not remake:
+ for d, weak in self.deps:
+ if mtimeislater(d.mtime, self.target.mtime):
+ _log.info("%sRemaking %s using rule at %s because %s is newer.", indent, self.target.target, self.rule.loc, d.target)
+ remake = True
+ break
+
+ if remake:
+ self.target.beingremade()
+ self.target.didanything = True
+ try:
+ self.commands = [c for c in self.rule.getcommands(self.target, self.makefile)]
+ except util.MakeError, e:
+ print e
+ sys.stdout.flush()
+ cb(error=True)
+ return
+
+ self._commandcb(False)
+ else:
+ cb(error=False)
+
+MAKESTATE_NONE = 0
+MAKESTATE_FINISHED = 1
+MAKESTATE_WORKING = 2
+
+class Target(object):
+ """
+ An actual (non-pattern) target.
+
+ It holds target-specific variables and a list of rules. It may also point to a parent
+ PatternTarget, if this target is being created by an implicit rule.
+
+ The rules associated with this target may be Rule instances or, in the case of static pattern
+ rules, PatternRule instances.
+ """
+
+ wasremade = False
+
+ def __init__(self, target, makefile):
+ assert isinstance(target, str_type)
+ self.target = target
+ self.vpathtarget = None
+ self.rules = []
+ self.variables = Variables(makefile.variables)
+ self.explicit = False
+ self._state = MAKESTATE_NONE
+
+ def addrule(self, rule):
+ assert isinstance(rule, (Rule, PatternRuleInstance))
+ if len(self.rules) and rule.doublecolon != self.rules[0].doublecolon:
+ raise DataError("Cannot have single- and double-colon rules for the same target. Prior rule location: %s" % self.rules[0].loc, rule.loc)
+
+ if isinstance(rule, PatternRuleInstance):
+ if len(rule.prule.targetpatterns) != 1:
+ raise DataError("Static pattern rules must only have one target pattern", rule.prule.loc)
+ if rule.prule.targetpatterns[0].match(self.target) is None:
+ raise DataError("Static pattern rule doesn't match target '%s'" % self.target, rule.loc)
+
+ self.rules.append(rule)
+
+ def isdoublecolon(self):
+ return self.rules[0].doublecolon
+
+ def isphony(self, makefile):
+ """Is this a phony target? We don't check for existence of phony targets."""
+ return makefile.gettarget('.PHONY').hasdependency(self.target)
+
+ def hasdependency(self, t):
+ for rule in self.rules:
+ if t in rule.prerequisites:
+ return True
+
+ return False
+
+ def resolveimplicitrule(self, makefile, targetstack, rulestack):
+ """
+ Try to resolve an implicit rule to build this target.
+ """
+ # The steps in the GNU make manual Implicit-Rule-Search.html are very detailed. I hope they can be trusted.
+
+ indent = getindent(targetstack)
+
+ _log.info("%sSearching for implicit rule to make '%s'", indent, self.target)
+
+ dir, s, file = util.strrpartition(self.target, '/')
+ dir = dir + s
+
+ candidates = [] # list of PatternRuleInstance
+
+ hasmatch = util.any((r.hasspecificmatch(file) for r in makefile.implicitrules))
+
+ for r in makefile.implicitrules:
+ if r in rulestack:
+ _log.info("%s %s: Avoiding implicit rule recursion", indent, r.loc)
+ continue
+
+ if not len(r.commands):
+ continue
+
+ for ri in r.matchesfor(dir, file, hasmatch):
+ candidates.append(ri)
+
+ newcandidates = []
+
+ for r in candidates:
+ depfailed = None
+ for p in r.prerequisites:
+ t = makefile.gettarget(p)
+ t.resolvevpath(makefile)
+ if not t.explicit and t.mtime is None:
+ depfailed = p
+ break
+
+ if depfailed is not None:
+ if r.doublecolon:
+ _log.info("%s Terminal rule at %s doesn't match: prerequisite '%s' not mentioned and doesn't exist.", indent, r.loc, depfailed)
+ else:
+ newcandidates.append(r)
+ continue
+
+ _log.info("%sFound implicit rule at %s for target '%s'", indent, r.loc, self.target)
+ self.rules.append(r)
+ return
+
+ # Try again, but this time with chaining and without terminal (double-colon) rules
+
+ for r in newcandidates:
+ newrulestack = rulestack + [r.prule]
+
+ depfailed = None
+ for p in r.prerequisites:
+ t = makefile.gettarget(p)
+ try:
+ t.resolvedeps(makefile, targetstack, newrulestack, True)
+ except ResolutionError:
+ depfailed = p
+ break
+
+ if depfailed is not None:
+ _log.info("%s Rule at %s doesn't match: prerequisite '%s' could not be made.", indent, r.loc, depfailed)
+ continue
+
+ _log.info("%sFound implicit rule at %s for target '%s'", indent, r.loc, self.target)
+ self.rules.append(r)
+ return
+
+ _log.info("%sCouldn't find implicit rule to remake '%s'", indent, self.target)
+
+ def ruleswithcommands(self):
+ "The number of rules with commands"
+ return reduce(lambda i, rule: i + (len(rule.commands) > 0), self.rules, 0)
+
+ def resolvedeps(self, makefile, targetstack, rulestack, recursive):
+ """
+ Resolve the actual path of this target, using vpath if necessary.
+
+ Recursively resolve dependencies of this target. This means finding implicit
+ rules which match the target, if appropriate.
+
+ Figure out whether this target needs to be rebuild, and set self.outofdate
+ appropriately.
+
+ @param targetstack is the current stack of dependencies being resolved. If
+ this target is already in targetstack, bail to prevent infinite
+ recursion.
+ @param rulestack is the current stack of implicit rules being used to resolve
+ dependencies. A rule chain cannot use the same implicit rule twice.
+ """
+ assert makefile.parsingfinished
+
+ if self.target in targetstack:
+ raise ResolutionError("Recursive dependency: %s -> %s" % (
+ " -> ".join(targetstack), self.target))
+
+ targetstack = targetstack + [self.target]
+
+ indent = getindent(targetstack)
+
+ _log.info("%sConsidering target '%s'", indent, self.target)
+
+ self.resolvevpath(makefile)
+
+ # Sanity-check our rules. If we're single-colon, only one rule should have commands
+ ruleswithcommands = self.ruleswithcommands()
+ if len(self.rules) and not self.isdoublecolon():
+ if ruleswithcommands > 1:
+ # In GNU make this is a warning, not an error. I'm going to be stricter.
+ # TODO: provide locations
+ raise DataError("Target '%s' has multiple rules with commands." % self.target)
+
+ if ruleswithcommands == 0:
+ self.resolveimplicitrule(makefile, targetstack, rulestack)
+
+ # If a target is mentioned, but doesn't exist, has no commands and no
+ # prerequisites, it is special and exists just to say that targets which
+ # depend on it are always out of date. This is like .FORCE but more
+ # compatible with other makes.
+ # Otherwise, we don't know how to make it.
+ if not len(self.rules) and self.mtime is None and not util.any((len(rule.prerequisites) > 0
+ for rule in self.rules)):
+ raise ResolutionError("No rule to make target '%s' needed by %r" % (self.target,
+ targetstack))
+
+ if recursive:
+ for r in self.rules:
+ newrulestack = rulestack + [r]
+ for d in r.prerequisites:
+ dt = makefile.gettarget(d)
+ if dt.explicit:
+ continue
+
+ dt.resolvedeps(makefile, targetstack, newrulestack, True)
+
+ for v in makefile.getpatternvariablesfor(self.target):
+ self.variables.merge(v)
+
+ def resolvevpath(self, makefile):
+ if self.vpathtarget is not None:
+ return
+
+ if self.isphony(makefile):
+ self.vpathtarget = self.target
+ self.mtime = None
+ return
+
+ if self.target.startswith('-l'):
+ stem = self.target[2:]
+ f, s, e = makefile.variables.get('.LIBPATTERNS')
+ if e is not None:
+ libpatterns = [Pattern(stripdotslash(s)) for s in e.resolvesplit(makefile, makefile.variables)]
+ if len(libpatterns):
+ searchdirs = ['']
+ searchdirs.extend(makefile.getvpath(self.target))
+
+ for lp in libpatterns:
+ if not lp.ispattern():
+ raise DataError('.LIBPATTERNS contains a non-pattern')
+
+ libname = lp.resolve('', stem)
+
+ for dir in searchdirs:
+ libpath = util.normaljoin(dir, libname).replace('\\', '/')
+ fspath = util.normaljoin(makefile.workdir, libpath)
+ mtime = getmtime(fspath)
+ if mtime is not None:
+ self.vpathtarget = libpath
+ self.mtime = mtime
+ return
+
+ self.vpathtarget = self.target
+ self.mtime = None
+ return
+
+ search = [self.target]
+ if not os.path.isabs(self.target):
+ search += [util.normaljoin(dir, self.target).replace('\\', '/')
+ for dir in makefile.getvpath(self.target)]
+
+ targetandtime = self.searchinlocs(makefile, search)
+ if targetandtime is not None:
+ (self.vpathtarget, self.mtime) = targetandtime
+ return
+
+ self.vpathtarget = self.target
+ self.mtime = None
+
+ def searchinlocs(self, makefile, locs):
+ """
+ Look in the given locations relative to the makefile working directory
+ for a file. Return a pair of the target and the mtime if found, None
+ if not.
+ """
+ for t in locs:
+ fspath = util.normaljoin(makefile.workdir, t).replace('\\', '/')
+ mtime = getmtime(fspath)
+# _log.info("Searching %s ... checking %s ... mtime %r" % (t, fspath, mtime))
+ if mtime is not None:
+ return (t, mtime)
+
+ return None
+
+ def beingremade(self):
+ """
+ When we remake ourself, we have to drop any vpath prefixes.
+ """
+ self.vpathtarget = self.target
+ self.wasremade = True
+
+ def notifydone(self, makefile):
+ assert self._state == MAKESTATE_WORKING, "State was %s" % self._state
+ # If we were remade then resolve mtime again
+ if self.wasremade:
+ targetandtime = self.searchinlocs(makefile, [self.target])
+ if targetandtime is not None:
+ (_, self.mtime) = targetandtime
+ else:
+ self.mtime = None
+
+ self._state = MAKESTATE_FINISHED
+ for cb in self._callbacks:
+ makefile.context.defer(cb, error=self.error, didanything=self.didanything)
+ del self._callbacks
+
+ def make(self, makefile, targetstack, cb, avoidremakeloop=False, printerror=True):
+ """
+ If we are out of date, asynchronously make ourself. This is a multi-stage process, mostly handled
+ by the helper objects RemakeTargetSerially, RemakeTargetParallel,
+ RemakeRuleContext. These helper objects should keep us from developing
+ any cyclical dependencies.
+
+ * resolve dependencies (synchronous)
+ * gather a list of rules to execute and related dependencies (synchronous)
+ * for each rule (in parallel)
+ ** remake dependencies (asynchronous)
+ ** build list of commands to execute (synchronous)
+ ** execute each command (asynchronous)
+ * asynchronously notify when all rules are complete
+
+ @param cb A callback function to notify when remaking is finished. It is called
+ thusly: callback(error=True/False, didanything=True/False)
+ If there is no asynchronous activity to perform, the callback may be called directly.
+ """
+
+ serial = makefile.context.jcount == 1
+
+ if self._state == MAKESTATE_FINISHED:
+ cb(error=self.error, didanything=self.didanything)
+ return
+
+ if self._state == MAKESTATE_WORKING:
+ assert not serial
+ self._callbacks.append(cb)
+ return
+
+ assert self._state == MAKESTATE_NONE
+
+ self._state = MAKESTATE_WORKING
+ self._callbacks = [cb]
+ self.error = False
+ self.didanything = False
+
+ indent = getindent(targetstack)
+
+ try:
+ self.resolvedeps(makefile, targetstack, [], False)
+ except util.MakeError, e:
+ if printerror:
+ print e
+ self.error = True
+ self.notifydone(makefile)
+ return
+
+ assert self.vpathtarget is not None, "Target was never resolved!"
+ if not len(self.rules):
+ self.notifydone(makefile)
+ return
+
+ if self.isdoublecolon():
+ rulelist = [RemakeRuleContext(self, makefile, r, [(makefile.gettarget(p), False) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules]
+ else:
+ alldeps = []
+
+ commandrule = None
+ for r in self.rules:
+ rdeps = [(makefile.gettarget(p), r.weakdeps) for p in r.prerequisites]
+ if len(r.commands):
+ assert commandrule is None
+ commandrule = r
+ # The dependencies of the command rule are resolved before other dependencies,
+ # no matter the ordering of the other no-command rules
+ alldeps[0:0] = rdeps
+ else:
+ alldeps.extend(rdeps)
+
+ rulelist = [RemakeRuleContext(self, makefile, commandrule, alldeps, targetstack, avoidremakeloop)]
+
+ targetstack = targetstack + [self.target]
+
+ if serial:
+ RemakeTargetSerially(self, makefile, indent, rulelist)
+ else:
+ RemakeTargetParallel(self, makefile, indent, rulelist)
+
+def dirpart(p):
+ d, s, f = util.strrpartition(p, '/')
+ if d == '':
+ return '.'
+
+ return d
+
+def filepart(p):
+ d, s, f = util.strrpartition(p, '/')
+ return f
+
+def setautomatic(v, name, plist):
+ v.set(name, Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join(plist))
+ v.set(name + 'D', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((dirpart(p) for p in plist)))
+ v.set(name + 'F', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((filepart(p) for p in plist)))
+
+def setautomaticvariables(v, makefile, target, prerequisites):
+ prtargets = [makefile.gettarget(p) for p in prerequisites]
+ prall = [pt.vpathtarget for pt in prtargets]
+ proutofdate = [pt.vpathtarget for pt in withoutdups(prtargets)
+ if target.mtime is None or mtimeislater(pt.mtime, target.mtime)]
+
+ setautomatic(v, '@', [target.vpathtarget])
+ if len(prall):
+ setautomatic(v, '<', [prall[0]])
+
+ setautomatic(v, '?', proutofdate)
+ setautomatic(v, '^', list(withoutdups(prall)))
+ setautomatic(v, '+', prall)
+
+def splitcommand(command):
+ """
+ Using the esoteric rules, split command lines by unescaped newlines.
+ """
+ start = 0
+ i = 0
+ while i < len(command):
+ c = command[i]
+ if c == '\\':
+ i += 1
+ elif c == '\n':
+ yield command[start:i]
+ i += 1
+ start = i
+ continue
+
+ i += 1
+
+ if i > start:
+ yield command[start:i]
+
+def findmodifiers(command):
+ """
+ Find any of +-@% prefixed on the command.
+ @returns (command, isHidden, isRecursive, ignoreErrors, isNative)
+ """
+
+ isHidden = False
+ isRecursive = False
+ ignoreErrors = False
+ isNative = False
+
+ realcommand = command.lstrip(' \t\n@+-%')
+ modset = set(command[:-len(realcommand)])
+ return realcommand, '@' in modset, '+' in modset, '-' in modset, '%' in modset
+
+class _CommandWrapper(object):
+ def __init__(self, cline, ignoreErrors, loc, context, **kwargs):
+ self.ignoreErrors = ignoreErrors
+ self.loc = loc
+ self.cline = cline
+ self.kwargs = kwargs
+ self.context = context
+
+ def _cb(self, res):
+ if res != 0 and not self.ignoreErrors:
+ print "%s: command '%s' failed, return code %i" % (self.loc, self.cline, res)
+ self.usercb(error=True)
+ else:
+ self.usercb(error=False)
+
+ def __call__(self, cb):
+ self.usercb = cb
+ process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs)
+
+class _NativeWrapper(_CommandWrapper):
+ def __init__(self, cline, ignoreErrors, loc, context,
+ pycommandpath, **kwargs):
+ _CommandWrapper.__init__(self, cline, ignoreErrors, loc, context,
+ **kwargs)
+ if pycommandpath:
+ self.pycommandpath = re.split('[%s\s]+' % os.pathsep,
+ pycommandpath)
+ else:
+ self.pycommandpath = None
+
+ def __call__(self, cb):
+ # get the module and method to call
+ parts, badchar = process.clinetoargv(self.cline, self.kwargs['cwd'])
+ if parts is None:
+ raise DataError("native command '%s': shell metacharacter '%s' in command line" % (self.cline, badchar), self.loc)
+ if len(parts) < 2:
+ raise DataError("native command '%s': no method name specified" % self.cline, self.loc)
+ module = parts[0]
+ method = parts[1]
+ cline_list = parts[2:]
+ self.usercb = cb
+ process.call_native(module, method, cline_list,
+ loc=self.loc, cb=self._cb, context=self.context,
+ pycommandpath=self.pycommandpath, **self.kwargs)
+
+def getcommandsforrule(rule, target, makefile, prerequisites, stem):
+ v = Variables(parent=target.variables)
+ setautomaticvariables(v, makefile, target, prerequisites)
+ if stem is not None:
+ setautomatic(v, '*', [stem])
+
+ env = makefile.getsubenvironment(v)
+
+ for c in rule.commands:
+ cstring = c.resolvestr(makefile, v)
+ for cline in splitcommand(cstring):
+ cline, isHidden, isRecursive, ignoreErrors, isNative = findmodifiers(cline)
+ if (isHidden or makefile.silent) and not makefile.justprint:
+ echo = None
+ else:
+ echo = "%s$ %s" % (c.loc, cline)
+ if not isNative:
+ yield _CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context,
+ echo=echo, justprint=makefile.justprint)
+ else:
+ f, s, e = v.get("PYCOMMANDPATH", True)
+ if e:
+ e = e.resolvestr(makefile, v, ["PYCOMMANDPATH"])
+ yield _NativeWrapper(cline, ignoreErrors=ignoreErrors,
+ env=env, cwd=makefile.workdir,
+ loc=c.loc, context=makefile.context,
+ echo=echo, justprint=makefile.justprint,
+ pycommandpath=e)
+
+class Rule(object):
+ """
+ A rule contains a list of prerequisites and a list of commands. It may also
+ contain rule-specific variables. This rule may be associated with multiple targets.
+ """
+
+ def __init__(self, prereqs, doublecolon, loc, weakdeps):
+ self.prerequisites = prereqs
+ self.doublecolon = doublecolon
+ self.commands = []
+ self.loc = loc
+ self.weakdeps = weakdeps
+
+ def addcommand(self, c):
+ assert isinstance(c, (Expansion, StringExpansion))
+ self.commands.append(c)
+
+ def getcommands(self, target, makefile):
+ assert isinstance(target, Target)
+ # Prerequisites are merged if the target contains multiple rules and is
+ # not a terminal (double colon) rule. See
+ # https://www.gnu.org/software/make/manual/make.html#Multiple-Targets.
+ prereqs = []
+ prereqs.extend(self.prerequisites)
+
+ if not self.doublecolon:
+ for rule in target.rules:
+ # The current rule comes first, which is already in prereqs so
+ # we don't need to add it again.
+ if rule != self:
+ prereqs.extend(rule.prerequisites)
+
+ return getcommandsforrule(self, target, makefile, prereqs, stem=None)
+ # TODO: $* in non-pattern rules?
+
+class PatternRuleInstance(object):
+ weakdeps = False
+
+ """
+ A pattern rule instantiated for a particular target. It has the same API as Rule, but
+ different internals, forwarding most information on to the PatternRule.
+ """
+ def __init__(self, prule, dir, stem, ismatchany):
+ assert isinstance(prule, PatternRule)
+
+ self.dir = dir
+ self.stem = stem
+ self.prule = prule
+ self.prerequisites = prule.prerequisitesforstem(dir, stem)
+ self.doublecolon = prule.doublecolon
+ self.loc = prule.loc
+ self.ismatchany = ismatchany
+ self.commands = prule.commands
+
+ def getcommands(self, target, makefile):
+ assert isinstance(target, Target)
+ return getcommandsforrule(self, target, makefile, self.prerequisites, stem=self.dir + self.stem)
+
+ def __str__(self):
+ return "Pattern rule at %s with stem '%s', matchany: %s doublecolon: %s" % (self.loc,
+ self.dir + self.stem,
+ self.ismatchany,
+ self.doublecolon)
+
+class PatternRule(object):
+ """
+ An implicit rule or static pattern rule containing target patterns, prerequisite patterns,
+ and a list of commands.
+ """
+
+ def __init__(self, targetpatterns, prerequisites, doublecolon, loc):
+ self.targetpatterns = targetpatterns
+ self.prerequisites = prerequisites
+ self.doublecolon = doublecolon
+ self.loc = loc
+ self.commands = []
+
+ def addcommand(self, c):
+ assert isinstance(c, (Expansion, StringExpansion))
+ self.commands.append(c)
+
+ def ismatchany(self):
+ return util.any((t.ismatchany() for t in self.targetpatterns))
+
+ def hasspecificmatch(self, file):
+ for p in self.targetpatterns:
+ if not p.ismatchany() and p.match(file) is not None:
+ return True
+
+ return False
+
+ def matchesfor(self, dir, file, skipsinglecolonmatchany):
+ """
+ Determine all the target patterns of this rule that might match target t.
+ @yields a PatternRuleInstance for each.
+ """
+
+ for p in self.targetpatterns:
+ matchany = p.ismatchany()
+ if matchany:
+ if skipsinglecolonmatchany and not self.doublecolon:
+ continue
+
+ yield PatternRuleInstance(self, dir, file, True)
+ else:
+ stem = p.match(dir + file)
+ if stem is not None:
+ yield PatternRuleInstance(self, '', stem, False)
+ else:
+ stem = p.match(file)
+ if stem is not None:
+ yield PatternRuleInstance(self, dir, stem, False)
+
+ def prerequisitesforstem(self, dir, stem):
+ return [p.resolve(dir, stem) for p in self.prerequisites]
+
+class _RemakeContext(object):
+ def __init__(self, makefile, cb):
+ self.makefile = makefile
+ self.included = [(makefile.gettarget(f), required)
+ for f, required in makefile.included]
+ self.toremake = list(self.included)
+ self.cb = cb
+
+ self.remakecb(error=False, didanything=False)
+
+ def remakecb(self, error, didanything):
+ assert error in (True, False)
+
+ if error and self.required:
+ print "Error remaking makefiles (ignored)"
+
+ if len(self.toremake):
+ target, self.required = self.toremake.pop(0)
+ target.make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb, printerror=False)
+ else:
+ for t, required in self.included:
+ if t.wasremade:
+ _log.info("Included file %s was remade, restarting make", t.target)
+ self.cb(remade=True)
+ return
+ elif required and t.mtime is None:
+ self.cb(remade=False, error=DataError("No rule to remake missing include file %s" % t.target))
+ return
+
+ self.cb(remade=False)
+
+class Makefile(object):
+ """
+ The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
+ state data.
+ """
+
+ def __init__(self, workdir=None, env=None, restarts=0, make=None,
+ makeflags='', makeoverrides='',
+ makelevel=0, context=None, targets=(), keepgoing=False,
+ silent=False, justprint=False):
+ self.defaulttarget = None
+
+ if env is None:
+ env = os.environ
+ self.env = env
+
+ self.variables = Variables()
+ self.variables.readfromenvironment(env)
+
+ self.context = context
+ self.exportedvars = {}
+ self._targets = {}
+ self.keepgoing = keepgoing
+ self.silent = silent
+ self.justprint = justprint
+ self._patternvariables = [] # of (pattern, variables)
+ self.implicitrules = []
+ self.parsingfinished = False
+
+ self._patternvpaths = [] # of (pattern, [dir, ...])
+
+ if workdir is None:
+ workdir = os.getcwd()
+ workdir = os.path.realpath(workdir)
+ self.workdir = workdir
+ self.variables.set('CURDIR', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_AUTOMATIC, workdir.replace('\\','/'))
+
+ # the list of included makefiles, whether or not they existed
+ self.included = []
+
+ self.variables.set('MAKE_RESTARTS', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_AUTOMATIC, restarts > 0 and str(restarts) or '')
+
+ self.variables.set('.PYMAKE', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_MAKEFILE, "1")
+ if make is not None:
+ self.variables.set('MAKE', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_MAKEFILE, make)
+
+ if makeoverrides != '':
+ self.variables.set('-*-command-variables-*-', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_AUTOMATIC, makeoverrides)
+ makeflags += ' -- $(MAKEOVERRIDES)'
+
+ self.variables.set('MAKEOVERRIDES', Variables.FLAVOR_RECURSIVE,
+ Variables.SOURCE_ENVIRONMENT,
+ '${-*-command-variables-*-}')
+
+ self.variables.set('MAKEFLAGS', Variables.FLAVOR_RECURSIVE,
+ Variables.SOURCE_MAKEFILE, makeflags)
+ self.exportedvars['MAKEFLAGS'] = True
+
+ self.makelevel = makelevel
+ self.variables.set('MAKELEVEL', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_MAKEFILE, str(makelevel))
+
+ self.variables.set('MAKECMDGOALS', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_AUTOMATIC, ' '.join(targets))
+
+ for vname, val in implicit.variables.iteritems():
+ self.variables.set(vname,
+ Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_IMPLICIT, val)
+
+ def foundtarget(self, t):
+ """
+ Inform the makefile of a target which is a candidate for being the default target,
+ if there isn't already a default target.
+ """
+ flavor, source, value = self.variables.get('.DEFAULT_GOAL')
+ if self.defaulttarget is None and t != '.PHONY' and value is None:
+ self.defaulttarget = t
+ self.variables.set('.DEFAULT_GOAL', Variables.FLAVOR_SIMPLE,
+ Variables.SOURCE_AUTOMATIC, t)
+
+ def getpatternvariables(self, pattern):
+ assert isinstance(pattern, Pattern)
+
+ for p, v in self._patternvariables:
+ if p == pattern:
+ return v
+
+ v = Variables()
+ self._patternvariables.append( (pattern, v) )
+ return v
+
+ def getpatternvariablesfor(self, target):
+ for p, v in self._patternvariables:
+ if p.match(target):
+ yield v
+
+ def hastarget(self, target):
+ return target in self._targets
+
+ _globcheck = re.compile('[[*?]')
+ def gettarget(self, target):
+ assert isinstance(target, str_type)
+
+ target = target.rstrip('/')
+
+ assert target != '', "empty target?"
+
+ assert not self._globcheck.match(target)
+
+ t = self._targets.get(target, None)
+ if t is None:
+ t = Target(target, self)
+ self._targets[target] = t
+ return t
+
+ def appendimplicitrule(self, rule):
+ assert isinstance(rule, PatternRule)
+ self.implicitrules.append(rule)
+
+ def finishparsing(self):
+ """
+ Various activities, such as "eval", are not allowed after parsing is
+ finished. In addition, various warnings and errors can only be issued
+ after the parsing data model is complete. All dependency resolution
+ and rule execution requires that parsing be finished.
+ """
+ self.parsingfinished = True
+
+ flavor, source, value = self.variables.get('GPATH')
+ if value is not None and value.resolvestr(self, self.variables, ['GPATH']).strip() != '':
+ raise DataError('GPATH was set: pymake does not support GPATH semantics')
+
+ flavor, source, value = self.variables.get('VPATH')
+ if value is None:
+ self._vpath = []
+ else:
+ self._vpath = filter(lambda e: e != '',
+ re.split('[%s\s]+' % os.pathsep,
+ value.resolvestr(self, self.variables, ['VPATH'])))
+
+ targets = list(self._targets.itervalues())
+ for t in targets:
+ t.explicit = True
+ for r in t.rules:
+ for p in r.prerequisites:
+ self.gettarget(p).explicit = True
+
+ np = self.gettarget('.NOTPARALLEL')
+ if len(np.rules):
+ self.context = process.getcontext(1)
+
+ flavor, source, value = self.variables.get('.DEFAULT_GOAL')
+ if value is not None:
+ self.defaulttarget = value.resolvestr(self, self.variables, ['.DEFAULT_GOAL']).strip()
+
+ self.error = False
+
+ def include(self, path, required=True, weak=False, loc=None):
+ """
+ Include the makefile at `path`.
+ """
+ self.included.append((path, required))
+ fspath = util.normaljoin(self.workdir, path)
+ if os.path.exists(fspath):
+ if weak:
+ stmts = parser.parsedepfile(fspath)
+ else:
+ stmts = parser.parsefile(fspath)
+ self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None, self)
+ stmts.execute(self, weak=weak)
+ self.gettarget(path).explicit = True
+
+ def addvpath(self, pattern, dirs):
+ """
+ Add a directory to the vpath search for the given pattern.
+ """
+ self._patternvpaths.append((pattern, dirs))
+
+ def clearvpath(self, pattern):
+ """
+ Clear vpaths for the given pattern.
+ """
+ self._patternvpaths = [(p, dirs)
+ for p, dirs in self._patternvpaths
+ if not p.match(pattern)]
+
+ def clearallvpaths(self):
+ self._patternvpaths = []
+
+ def getvpath(self, target):
+ vp = list(self._vpath)
+ for p, dirs in self._patternvpaths:
+ if p.match(target):
+ vp.extend(dirs)
+
+ return withoutdups(vp)
+
+ def remakemakefiles(self, cb):
+ mlist = []
+ for f, required in self.included:
+ t = self.gettarget(f)
+ t.explicit = True
+ t.resolvevpath(self)
+ oldmtime = t.mtime
+
+ mlist.append((t, oldmtime))
+
+ _RemakeContext(self, cb)
+
+ def getsubenvironment(self, variables):
+ env = dict(self.env)
+ for vname, v in self.exportedvars.iteritems():
+ if v:
+ flavor, source, val = variables.get(vname)
+ if val is None:
+ strval = ''
+ else:
+ strval = val.resolvestr(self, variables, [vname])
+ env[vname] = strval
+ else:
+ env.pop(vname, None)
+
+ makeflags = ''
+
+ env['MAKELEVEL'] = str(self.makelevel + 1)
+ return env
diff --git a/build/pymake/pymake/functions.py b/build/pymake/pymake/functions.py
new file mode 100644
index 000000000..e53fb5472
--- /dev/null
+++ b/build/pymake/pymake/functions.py
@@ -0,0 +1,873 @@
+"""
+Makefile functions.
+"""
+
+import parser, util
+import subprocess, os, logging, sys
+from globrelative import glob
+from cStringIO import StringIO
+
+log = logging.getLogger('pymake.data')
+
+def emit_expansions(descend, *expansions):
+ """Helper function to emit all expansions within an input set."""
+ for expansion in expansions:
+ yield expansion
+
+ if not descend or not isinstance(expansion, list):
+ continue
+
+ for e, is_func in expansion:
+ if is_func:
+ for exp in e.expansions(True):
+ yield exp
+ else:
+ yield e
+
+class Function(object):
+ """
+ An object that represents a function call. This class is always subclassed
+ with the following methods and attributes:
+
+ minargs = minimum # of arguments
+ maxargs = maximum # of arguments (0 means unlimited)
+
+ def resolve(self, makefile, variables, fd, setting)
+ Calls the function
+ calls fd.write() with strings
+ """
+
+ __slots__ = ('_arguments', 'loc')
+
+ def __init__(self, loc):
+ self._arguments = []
+ self.loc = loc
+ assert self.minargs > 0
+
+ def __getitem__(self, key):
+ return self._arguments[key]
+
+ def setup(self):
+ argc = len(self._arguments)
+
+ if argc < self.minargs:
+ raise data.DataError("Not enough arguments to function %s, requires %s" % (self.name, self.minargs), self.loc)
+
+ assert self.maxargs == 0 or argc <= self.maxargs, "Parser screwed up, gave us too many args"
+
+ def append(self, arg):
+ assert isinstance(arg, (data.Expansion, data.StringExpansion))
+ self._arguments.append(arg)
+
+ def to_source(self):
+ """Convert the function back to make file "source" code."""
+ if not hasattr(self, 'name'):
+ raise Exception("%s must implement to_source()." % self.__class__)
+
+ # The default implementation simply prints the function name and all
+ # the arguments joined by a comma.
+ # According to the GNU make manual Section 8.1, whitespace around
+ # arguments is *not* part of the argument's value. So, we trim excess
+ # white space so we have consistent behavior.
+ args = []
+ curly = False
+ for i, arg in enumerate(self._arguments):
+ arg = arg.to_source()
+
+ if i == 0:
+ arg = arg.lstrip()
+
+ # Are balanced parens even OK?
+ if arg.count('(') != arg.count(')'):
+ curly = True
+
+ args.append(arg)
+
+ if curly:
+ return '${%s %s}' % (self.name, ','.join(args))
+
+ return '$(%s %s)' % (self.name, ','.join(args))
+
+ def expansions(self, descend=False):
+ """Obtain all expansions contained within this function.
+
+ By default, only expansions directly part of this function are
+ returned. If descend is True, we will descend into child expansions and
+ return all of the composite parts.
+
+ This is a generator for pymake.data.BaseExpansion instances.
+ """
+ # Our default implementation simply returns arguments. More advanced
+ # functions like variable references may need their own implementation.
+ return emit_expansions(descend, *self._arguments)
+
+ @property
+ def is_filesystem_dependent(self):
+ """Exposes whether this function depends on the filesystem for results.
+
+ If True, the function touches the filesystem as part of evaluation.
+
+ This only tests whether the function itself uses the filesystem. If
+ this function has arguments that are functions that touch the
+ filesystem, this will return False.
+ """
+ return False
+
+ def __len__(self):
+ return len(self._arguments)
+
+ def __repr__(self):
+ return "%s<%s>(%r)" % (
+ self.__class__.__name__, self.loc,
+ ','.join([repr(a) for a in self._arguments]),
+ )
+
+ def __eq__(self, other):
+ if not hasattr(self, 'name'):
+ raise Exception("%s must implement __eq__." % self.__class__)
+
+ if type(self) != type(other):
+ return False
+
+ if self.name != other.name:
+ return False
+
+ if len(self._arguments) != len(other._arguments):
+ return False
+
+ for i in xrange(len(self._arguments)):
+ # According to the GNU make manual Section 8.1, whitespace around
+ # arguments is *not* part of the argument's value. So, we do a
+ # whitespace-agnostic comparison.
+ if i == 0:
+ a = self._arguments[i]
+ a.lstrip()
+
+ b = other._arguments[i]
+ b.lstrip()
+
+ if a != b:
+ return False
+
+ continue
+
+ if self._arguments[i] != other._arguments[i]:
+ return False
+
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+class VariableRef(Function):
+ AUTOMATIC_VARIABLES = set(['@', '%', '<', '?', '^', '+', '|', '*'])
+
+ __slots__ = ('vname', 'loc')
+
+ def __init__(self, loc, vname):
+ self.loc = loc
+ assert isinstance(vname, (data.Expansion, data.StringExpansion))
+ self.vname = vname
+
+ def setup(self):
+ assert False, "Shouldn't get here"
+
+ def resolve(self, makefile, variables, fd, setting):
+ vname = self.vname.resolvestr(makefile, variables, setting)
+ if vname in setting:
+ raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
+
+ flavor, source, value = variables.get(vname)
+ if value is None:
+ log.debug("%s: variable '%s' was not set" % (self.loc, vname))
+ return
+
+ value.resolve(makefile, variables, fd, setting + [vname])
+
+ def to_source(self):
+ if isinstance(self.vname, data.StringExpansion):
+ if self.vname.s in self.AUTOMATIC_VARIABLES:
+ return '$%s' % self.vname.s
+
+ return '$(%s)' % self.vname.s
+
+ return '$(%s)' % self.vname.to_source()
+
+ def expansions(self, descend=False):
+ return emit_expansions(descend, self.vname)
+
+ def __repr__(self):
+ return "VariableRef<%s>(%r)" % (self.loc, self.vname)
+
+ def __eq__(self, other):
+ if not isinstance(other, VariableRef):
+ return False
+
+ return self.vname == other.vname
+
+class SubstitutionRef(Function):
+ """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)"""
+
+ __slots__ = ('loc', 'vname', 'substfrom', 'substto')
+
+ def __init__(self, loc, varname, substfrom, substto):
+ self.loc = loc
+ self.vname = varname
+ self.substfrom = substfrom
+ self.substto = substto
+
+ def setup(self):
+ assert False, "Shouldn't get here"
+
+ def resolve(self, makefile, variables, fd, setting):
+ vname = self.vname.resolvestr(makefile, variables, setting)
+ if vname in setting:
+ raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
+
+ substfrom = self.substfrom.resolvestr(makefile, variables, setting)
+ substto = self.substto.resolvestr(makefile, variables, setting)
+
+ flavor, source, value = variables.get(vname)
+ if value is None:
+ log.debug("%s: variable '%s' was not set" % (self.loc, vname))
+ return
+
+ f = data.Pattern(substfrom)
+ if not f.ispattern():
+ f = data.Pattern('%' + substfrom)
+ substto = '%' + substto
+
+ fd.write(' '.join([f.subst(substto, word, False)
+ for word in value.resolvesplit(makefile, variables, setting + [vname])]))
+
+ def to_source(self):
+ return '$(%s:%s=%s)' % (
+ self.vname.to_source(),
+ self.substfrom.to_source(),
+ self.substto.to_source())
+
+ def expansions(self, descend=False):
+ return emit_expansions(descend, self.vname, self.substfrom,
+ self.substto)
+
+ def __repr__(self):
+ return "SubstitutionRef<%s>(%r:%r=%r)" % (
+ self.loc, self.vname, self.substfrom, self.substto,)
+
+ def __eq__(self, other):
+ if not isinstance(other, SubstitutionRef):
+ return False
+
+ return self.vname == other.vname and self.substfrom == other.substfrom \
+ and self.substto == other.substto
+
+class SubstFunction(Function):
+ name = 'subst'
+ minargs = 3
+ maxargs = 3
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ s = self._arguments[0].resolvestr(makefile, variables, setting)
+ r = self._arguments[1].resolvestr(makefile, variables, setting)
+ d = self._arguments[2].resolvestr(makefile, variables, setting)
+ fd.write(d.replace(s, r))
+
+class PatSubstFunction(Function):
+ name = 'patsubst'
+ minargs = 3
+ maxargs = 3
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ s = self._arguments[0].resolvestr(makefile, variables, setting)
+ r = self._arguments[1].resolvestr(makefile, variables, setting)
+
+ p = data.Pattern(s)
+ fd.write(' '.join([p.subst(r, word, False)
+ for word in self._arguments[2].resolvesplit(makefile, variables, setting)]))
+
+class StripFunction(Function):
+ name = 'strip'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ util.joiniter(fd, self._arguments[0].resolvesplit(makefile, variables, setting))
+
+class FindstringFunction(Function):
+ name = 'findstring'
+ minargs = 2
+ maxargs = 2
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ s = self._arguments[0].resolvestr(makefile, variables, setting)
+ r = self._arguments[1].resolvestr(makefile, variables, setting)
+ if r.find(s) == -1:
+ return
+ fd.write(s)
+
+class FilterFunction(Function):
+ name = 'filter'
+ minargs = 2
+ maxargs = 2
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ plist = [data.Pattern(p)
+ for p in self._arguments[0].resolvesplit(makefile, variables, setting)]
+
+ fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
+ if util.any((p.match(w) for p in plist))]))
+
+class FilteroutFunction(Function):
+ name = 'filter-out'
+ minargs = 2
+ maxargs = 2
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ plist = [data.Pattern(p)
+ for p in self._arguments[0].resolvesplit(makefile, variables, setting)]
+
+ fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
+ if not util.any((p.match(w) for p in plist))]))
+
+class SortFunction(Function):
+ name = 'sort'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ d = set(self._arguments[0].resolvesplit(makefile, variables, setting))
+ util.joiniter(fd, sorted(d))
+
+class WordFunction(Function):
+ name = 'word'
+ minargs = 2
+ maxargs = 2
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ n = self._arguments[0].resolvestr(makefile, variables, setting)
+ # TODO: provide better error if this doesn't convert
+ n = int(n)
+ words = list(self._arguments[1].resolvesplit(makefile, variables, setting))
+ if n < 1 or n > len(words):
+ return
+ fd.write(words[n - 1])
+
+class WordlistFunction(Function):
+ name = 'wordlist'
+ minargs = 3
+ maxargs = 3
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ nfrom = self._arguments[0].resolvestr(makefile, variables, setting)
+ nto = self._arguments[1].resolvestr(makefile, variables, setting)
+ # TODO: provide better errors if this doesn't convert
+ nfrom = int(nfrom)
+ nto = int(nto)
+
+ words = list(self._arguments[2].resolvesplit(makefile, variables, setting))
+
+ if nfrom < 1:
+ nfrom = 1
+ if nto < 1:
+ nto = 1
+
+ util.joiniter(fd, words[nfrom - 1:nto])
+
+class WordsFunction(Function):
+ name = 'words'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ fd.write(str(len(self._arguments[0].resolvesplit(makefile, variables, setting))))
+
+class FirstWordFunction(Function):
+ name = 'firstword'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ l = self._arguments[0].resolvesplit(makefile, variables, setting)
+ if len(l):
+ fd.write(l[0])
+
+class LastWordFunction(Function):
+ name = 'lastword'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ l = self._arguments[0].resolvesplit(makefile, variables, setting)
+ if len(l):
+ fd.write(l[-1])
+
+def pathsplit(path, default='./'):
+ """
+ Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart
+ is ./
+ """
+ dir, slash, file = util.strrpartition(path, '/')
+ if dir == '':
+ return default, file
+
+ return dir + slash, file
+
+class DirFunction(Function):
+ name = 'dir'
+ minargs = 1
+ maxargs = 1
+
+ def resolve(self, makefile, variables, fd, setting):
+ fd.write(' '.join([pathsplit(path)[0]
+ for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class NotDirFunction(Function):
+ name = 'notdir'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ fd.write(' '.join([pathsplit(path)[1]
+ for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class SuffixFunction(Function):
+ name = 'suffix'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ @staticmethod
+ def suffixes(words):
+ for w in words:
+ dir, file = pathsplit(w)
+ base, dot, suffix = util.strrpartition(file, '.')
+ if base != '':
+ yield dot + suffix
+
+ def resolve(self, makefile, variables, fd, setting):
+ util.joiniter(fd, self.suffixes(self._arguments[0].resolvesplit(makefile, variables, setting)))
+
+class BasenameFunction(Function):
+ name = 'basename'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ @staticmethod
+ def basenames(words):
+ for w in words:
+ dir, file = pathsplit(w, '')
+ base, dot, suffix = util.strrpartition(file, '.')
+ if dot == '':
+ base = suffix
+
+ yield dir + base
+
+ def resolve(self, makefile, variables, fd, setting):
+ util.joiniter(fd, self.basenames(self._arguments[0].resolvesplit(makefile, variables, setting)))
+
+class AddSuffixFunction(Function):
+ name = 'addsuffix'
+ minargs = 2
+ maxargs = 2
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ suffix = self._arguments[0].resolvestr(makefile, variables, setting)
+
+ fd.write(' '.join([w + suffix for w in self._arguments[1].resolvesplit(makefile, variables, setting)]))
+
+class AddPrefixFunction(Function):
+ name = 'addprefix'
+ minargs = 2
+ maxargs = 2
+
+ def resolve(self, makefile, variables, fd, setting):
+ prefix = self._arguments[0].resolvestr(makefile, variables, setting)
+
+ fd.write(' '.join([prefix + w for w in self._arguments[1].resolvesplit(makefile, variables, setting)]))
+
+class JoinFunction(Function):
+ name = 'join'
+ minargs = 2
+ maxargs = 2
+
+ __slots__ = Function.__slots__
+
+ @staticmethod
+ def iterjoin(l1, l2):
+ for i in xrange(0, max(len(l1), len(l2))):
+ i1 = i < len(l1) and l1[i] or ''
+ i2 = i < len(l2) and l2[i] or ''
+ yield i1 + i2
+
+ def resolve(self, makefile, variables, fd, setting):
+ list1 = list(self._arguments[0].resolvesplit(makefile, variables, setting))
+ list2 = list(self._arguments[1].resolvesplit(makefile, variables, setting))
+
+ util.joiniter(fd, self.iterjoin(list1, list2))
+
+class WildcardFunction(Function):
+ name = 'wildcard'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ patterns = self._arguments[0].resolvesplit(makefile, variables, setting)
+
+ fd.write(' '.join([x.replace('\\','/')
+ for p in patterns
+ for x in glob(makefile.workdir, p)]))
+
+ @property
+ def is_filesystem_dependent(self):
+ return True
+
+class RealpathFunction(Function):
+ name = 'realpath'
+ minargs = 1
+ maxargs = 1
+
+ def resolve(self, makefile, variables, fd, setting):
+ fd.write(' '.join([os.path.realpath(os.path.join(makefile.workdir, path)).replace('\\', '/')
+ for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+ def is_filesystem_dependent(self):
+ return True
+
+class AbspathFunction(Function):
+ name = 'abspath'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ assert os.path.isabs(makefile.workdir)
+ fd.write(' '.join([util.normaljoin(makefile.workdir, path).replace('\\', '/')
+ for path in self._arguments[0].resolvesplit(makefile, variables, setting)]))
+
+class IfFunction(Function):
+ name = 'if'
+ minargs = 1
+ maxargs = 3
+
+ __slots__ = Function.__slots__
+
+ def setup(self):
+ Function.setup(self)
+ self._arguments[0].lstrip()
+ self._arguments[0].rstrip()
+
+ def resolve(self, makefile, variables, fd, setting):
+ condition = self._arguments[0].resolvestr(makefile, variables, setting)
+
+ if len(condition):
+ self._arguments[1].resolve(makefile, variables, fd, setting)
+ elif len(self._arguments) > 2:
+ return self._arguments[2].resolve(makefile, variables, fd, setting)
+
+class OrFunction(Function):
+ name = 'or'
+ minargs = 1
+ maxargs = 0
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ for arg in self._arguments:
+ r = arg.resolvestr(makefile, variables, setting)
+ if r != '':
+ fd.write(r)
+ return
+
+class AndFunction(Function):
+ name = 'and'
+ minargs = 1
+ maxargs = 0
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ r = ''
+
+ for arg in self._arguments:
+ r = arg.resolvestr(makefile, variables, setting)
+ if r == '':
+ return
+
+ fd.write(r)
+
+class ForEachFunction(Function):
+ name = 'foreach'
+ minargs = 3
+ maxargs = 3
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ vname = self._arguments[0].resolvestr(makefile, variables, setting)
+ e = self._arguments[2]
+
+ v = data.Variables(parent=variables)
+ firstword = True
+
+ for w in self._arguments[1].resolvesplit(makefile, variables, setting):
+ if firstword:
+ firstword = False
+ else:
+ fd.write(' ')
+
+ # The $(origin) of the local variable must be "automatic" to
+ # conform with GNU make. However, automatic variables have low
+ # priority. So, we must force its assignment to occur.
+ v.set(vname, data.Variables.FLAVOR_SIMPLE,
+ data.Variables.SOURCE_AUTOMATIC, w, force=True)
+ e.resolve(makefile, v, fd, setting)
+
+class CallFunction(Function):
+ name = 'call'
+ minargs = 1
+ maxargs = 0
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ vname = self._arguments[0].resolvestr(makefile, variables, setting)
+ if vname in setting:
+ raise data.DataError("Recursively setting variable '%s'" % (vname,))
+
+ v = data.Variables(parent=variables)
+ v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname)
+ for i in xrange(1, len(self._arguments)):
+ param = self._arguments[i].resolvestr(makefile, variables, setting)
+ v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param)
+
+ flavor, source, e = variables.get(vname)
+
+ if e is None:
+ return
+
+ if flavor == data.Variables.FLAVOR_SIMPLE:
+ log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname))
+
+ # but we'll do it anyway
+ e.resolve(makefile, v, fd, setting + [vname])
+
+class ValueFunction(Function):
+ name = 'value'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ varname = self._arguments[0].resolvestr(makefile, variables, setting)
+
+ flavor, source, value = variables.get(varname, expand=False)
+ if value is not None:
+ fd.write(value)
+
+class EvalFunction(Function):
+ name = 'eval'
+ minargs = 1
+ maxargs = 1
+
+ def resolve(self, makefile, variables, fd, setting):
+ if makefile.parsingfinished:
+ # GNU make allows variables to be set by recursive expansion during
+ # command execution. This seems really dumb to me, so I don't!
+ raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc)
+
+ stmts = parser.parsestring(self._arguments[0].resolvestr(makefile, variables, setting),
+ 'evaluation from %s' % self.loc)
+ stmts.execute(makefile)
+
+class OriginFunction(Function):
+ name = 'origin'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ vname = self._arguments[0].resolvestr(makefile, variables, setting)
+
+ flavor, source, value = variables.get(vname)
+ if source is None:
+ r = 'undefined'
+ elif source == data.Variables.SOURCE_OVERRIDE:
+ r = 'override'
+
+ elif source == data.Variables.SOURCE_MAKEFILE:
+ r = 'file'
+ elif source == data.Variables.SOURCE_ENVIRONMENT:
+ r = 'environment'
+ elif source == data.Variables.SOURCE_COMMANDLINE:
+ r = 'command line'
+ elif source == data.Variables.SOURCE_AUTOMATIC:
+ r = 'automatic'
+ elif source == data.Variables.SOURCE_IMPLICIT:
+ r = 'default'
+
+ fd.write(r)
+
+class FlavorFunction(Function):
+ name = 'flavor'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ varname = self._arguments[0].resolvestr(makefile, variables, setting)
+
+ flavor, source, value = variables.get(varname)
+ if flavor is None:
+ r = 'undefined'
+ elif flavor == data.Variables.FLAVOR_RECURSIVE:
+ r = 'recursive'
+ elif flavor == data.Variables.FLAVOR_SIMPLE:
+ r = 'simple'
+ fd.write(r)
+
+class ShellFunction(Function):
+ name = 'shell'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ from process import prepare_command
+ cline = self._arguments[0].resolvestr(makefile, variables, setting)
+ executable, cline = prepare_command(cline, makefile.workdir, self.loc)
+
+ # subprocess.Popen doesn't use the PATH set in the env argument for
+ # finding the executable on some platforms (but strangely it does on
+ # others!), so set os.environ['PATH'] explicitly.
+ oldpath = os.environ['PATH']
+ if makefile.env is not None and 'PATH' in makefile.env:
+ os.environ['PATH'] = makefile.env['PATH']
+
+ log.debug("%s: running command '%s'" % (self.loc, ' '.join(cline)))
+ try:
+ p = subprocess.Popen(cline, executable=executable, env=makefile.env, shell=False,
+ stdout=subprocess.PIPE, cwd=makefile.workdir)
+ except OSError, e:
+ print >>sys.stderr, "Error executing command %s" % cline[0], e
+ return
+ finally:
+ os.environ['PATH'] = oldpath
+
+ stdout, stderr = p.communicate()
+ stdout = stdout.replace('\r\n', '\n')
+ if stdout.endswith('\n'):
+ stdout = stdout[:-1]
+ stdout = stdout.replace('\n', ' ')
+
+ fd.write(stdout)
+
+class ErrorFunction(Function):
+ name = 'error'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ v = self._arguments[0].resolvestr(makefile, variables, setting)
+ raise data.DataError(v, self.loc)
+
+class WarningFunction(Function):
+ name = 'warning'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ v = self._arguments[0].resolvestr(makefile, variables, setting)
+ log.warning(v)
+
+class InfoFunction(Function):
+ name = 'info'
+ minargs = 1
+ maxargs = 1
+
+ __slots__ = Function.__slots__
+
+ def resolve(self, makefile, variables, fd, setting):
+ v = self._arguments[0].resolvestr(makefile, variables, setting)
+ print v
+
+functionmap = {
+ 'subst': SubstFunction,
+ 'patsubst': PatSubstFunction,
+ 'strip': StripFunction,
+ 'findstring': FindstringFunction,
+ 'filter': FilterFunction,
+ 'filter-out': FilteroutFunction,
+ 'sort': SortFunction,
+ 'word': WordFunction,
+ 'wordlist': WordlistFunction,
+ 'words': WordsFunction,
+ 'firstword': FirstWordFunction,
+ 'lastword': LastWordFunction,
+ 'dir': DirFunction,
+ 'notdir': NotDirFunction,
+ 'suffix': SuffixFunction,
+ 'basename': BasenameFunction,
+ 'addsuffix': AddSuffixFunction,
+ 'addprefix': AddPrefixFunction,
+ 'join': JoinFunction,
+ 'wildcard': WildcardFunction,
+ 'realpath': RealpathFunction,
+ 'abspath': AbspathFunction,
+ 'if': IfFunction,
+ 'or': OrFunction,
+ 'and': AndFunction,
+ 'foreach': ForEachFunction,
+ 'call': CallFunction,
+ 'value': ValueFunction,
+ 'eval': EvalFunction,
+ 'origin': OriginFunction,
+ 'flavor': FlavorFunction,
+ 'shell': ShellFunction,
+ 'error': ErrorFunction,
+ 'warning': WarningFunction,
+ 'info': InfoFunction,
+}
+
+import data
diff --git a/build/pymake/pymake/globrelative.py b/build/pymake/pymake/globrelative.py
new file mode 100644
index 000000000..37ca28e06
--- /dev/null
+++ b/build/pymake/pymake/globrelative.py
@@ -0,0 +1,68 @@
+"""
+Filename globbing like the python glob module with minor differences:
+
+* glob relative to an arbitrary directory
+* include . and ..
+* check that link targets exist, not just links
+"""
+
+import os, re, fnmatch
+import util
+
+_globcheck = re.compile('[[*?]')
+
+def hasglob(p):
+ return _globcheck.search(p) is not None
+
+def glob(fsdir, path):
+ """
+ Yield paths matching the path glob. Sorts as a bonus. Excludes '.' and '..'
+ """
+
+ dir, leaf = os.path.split(path)
+ if dir == '':
+ return globpattern(fsdir, leaf)
+
+ if hasglob(dir):
+ dirsfound = glob(fsdir, dir)
+ else:
+ dirsfound = [dir]
+
+ r = []
+
+ for dir in dirsfound:
+ fspath = util.normaljoin(fsdir, dir)
+ if not os.path.isdir(fspath):
+ continue
+
+ r.extend((util.normaljoin(dir, found) for found in globpattern(fspath, leaf)))
+
+ return r
+
+def globpattern(dir, pattern):
+ """
+ Return leaf names in the specified directory which match the pattern.
+ """
+
+ if not hasglob(pattern):
+ if pattern == '':
+ if os.path.isdir(dir):
+ return ['']
+ return []
+
+ if os.path.exists(util.normaljoin(dir, pattern)):
+ return [pattern]
+ return []
+
+ leaves = os.listdir(dir) + ['.', '..']
+
+ # "hidden" filenames are a bit special
+ if not pattern.startswith('.'):
+ leaves = [leaf for leaf in leaves
+ if not leaf.startswith('.')]
+
+ leaves = fnmatch.filter(leaves, pattern)
+ leaves = filter(lambda l: os.path.exists(util.normaljoin(dir, l)), leaves)
+
+ leaves.sort()
+ return leaves
diff --git a/build/pymake/pymake/implicit.py b/build/pymake/pymake/implicit.py
new file mode 100644
index 000000000..d73895cab
--- /dev/null
+++ b/build/pymake/pymake/implicit.py
@@ -0,0 +1,14 @@
+"""
+Implicit variables; perhaps in the future this will also include some implicit
+rules, at least match-anything cancellation rules.
+"""
+
+variables = {
+ 'MKDIR': '%pymake.builtins mkdir',
+ 'RM': '%pymake.builtins rm -f',
+ 'SLEEP': '%pymake.builtins sleep',
+ 'TOUCH': '%pymake.builtins touch',
+ '.LIBPATTERNS': 'lib%.so lib%.a',
+ '.PYMAKE': '1',
+ }
+
diff --git a/build/pymake/pymake/parser.py b/build/pymake/pymake/parser.py
new file mode 100644
index 000000000..4bff53368
--- /dev/null
+++ b/build/pymake/pymake/parser.py
@@ -0,0 +1,822 @@
+"""
+Module for parsing Makefile syntax.
+
+Makefiles use a line-based parsing system. Continuations and substitutions are handled differently based on the
+type of line being parsed:
+
+Lines with makefile syntax condense continuations to a single space, no matter the actual trailing whitespace
+of the first line or the leading whitespace of the continuation. In other situations, trailing whitespace is
+relevant.
+
+Lines with command syntax do not condense continuations: the backslash and newline are part of the command.
+(GNU Make is buggy in this regard, at least on mac).
+
+Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding).
+Otherwise, they are parsed as makefile syntax.
+
+This file parses into the data structures defined in the parserdata module. Those classes are what actually
+do the dirty work of "executing" the parsed data into a data.Makefile.
+
+Four iterator functions are available:
+* iterdata
+* itermakefilechars
+* itercommandchars
+
+The iterators handle line continuations and comments in different ways, but share a common calling
+convention:
+
+Called with (data, startoffset, tokenlist, finditer)
+
+yield 4-tuples (flatstr, token, tokenoffset, afteroffset)
+flatstr is data, guaranteed to have no tokens (may be '')
+token, tokenoffset, afteroffset *may be None*. That means there is more text
+coming.
+"""
+
+import logging, re, os, sys
+import data, functions, util, parserdata
+
+_log = logging.getLogger('pymake.parser')
+
+class SyntaxError(util.MakeError):
+ pass
+
+_skipws = re.compile('\S')
+class Data(object):
+ """
+ A single virtual "line", which can be multiple source lines joined with
+ continuations.
+ """
+
+ __slots__ = ('s', 'lstart', 'lend', 'loc')
+
+ def __init__(self, s, lstart, lend, loc):
+ self.s = s
+ self.lstart = lstart
+ self.lend = lend
+ self.loc = loc
+
+ @staticmethod
+ def fromstring(s, path):
+ return Data(s, 0, len(s), parserdata.Location(path, 1, 0))
+
+ def getloc(self, offset):
+ assert offset >= self.lstart and offset <= self.lend
+ return self.loc.offset(self.s, self.lstart, offset)
+
+ def skipwhitespace(self, offset):
+ """
+ Return the offset of the first non-whitespace character in data starting at offset, or None if there are
+ only whitespace characters remaining.
+ """
+ m = _skipws.search(self.s, offset, self.lend)
+ if m is None:
+ return self.lend
+
+ return m.start(0)
+
+_linere = re.compile(r'\\*\n')
+def enumeratelines(s, filename):
+ """
+ Enumerate lines in a string as Data objects, joining line
+ continuations.
+ """
+
+ off = 0
+ lineno = 1
+ curlines = 0
+ for m in _linere.finditer(s):
+ curlines += 1
+ start, end = m.span(0)
+
+ if (start - end) % 2 == 0:
+ # odd number of backslashes is a continuation
+ continue
+
+ yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0))
+
+ lineno += curlines
+ curlines = 0
+ off = end
+
+ yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0))
+
+_alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes
+ := |
+ \+= |
+ \?= |
+ :: |
+ (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character
+ :(?![\\/]) | # colon followed by anything except a slash (Windows path detection)
+ [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE)
+
+def iterdata(d, offset, tokenlist, it):
+ """
+ Iterate over flat data without line continuations, comments, or any special escaped characters.
+
+ Typically used to parse recursively-expanded variables.
+ """
+
+ assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!"
+ assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+
+ if offset == d.lend:
+ return
+
+ s = d.s
+ for m in it:
+ mstart, mend = m.span(0)
+ token = s[mstart:mend]
+ if token in tokenlist or (token[0] == '$' and '$' in tokenlist):
+ yield s[offset:mstart], token, mstart, mend
+ else:
+ yield s[offset:mend], None, None, mend
+ offset = mend
+
+ yield s[offset:d.lend], None, None, None
+
+# multiple backslashes before a newline are unescaped, halving their total number
+_makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*')
+def _replacemakecontinuations(m):
+ start, end = m.span(1)
+ if start == -1:
+ return ' '
+ return ' '.rjust((end - start) / 2 + 1, '\\')
+
+def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False):
+ """
+ Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines
+ are converted to single-space continuations.
+ """
+
+ assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+
+ if offset == d.lend:
+ return
+
+ s = d.s
+ for m in it:
+ mstart, mend = m.span(0)
+ token = s[mstart:mend]
+
+ starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart])
+
+ if token[-1] == '#' and not ignorecomments:
+ l = mend - mstart
+ # multiple backslashes before a hash are unescaped, halving their total number
+ if l % 2:
+ # found a comment
+ yield starttext + token[:(l - 1) / 2], None, None, None
+ return
+ else:
+ yield starttext + token[-l / 2:], None, None, mend
+ elif token in tokenlist or (token[0] == '$' and '$' in tokenlist):
+ yield starttext, token, mstart, mend
+ else:
+ yield starttext + token, None, None, mend
+ offset = mend
+
+ yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None
+
+_findcomment = re.compile(r'\\*\#')
+def flattenmakesyntax(d, offset):
+ """
+ A shortcut method for flattening line continuations and comments in makefile syntax without
+ looking for other tokens.
+ """
+
+ assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+ if offset == d.lend:
+ return ''
+
+ s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend])
+
+ elements = []
+ offset = 0
+ for m in _findcomment.finditer(s):
+ mstart, mend = m.span(0)
+ elements.append(s[offset:mstart])
+ if (mend - mstart) % 2:
+ # even number of backslashes... it's a comment
+ elements.append(''.ljust((mend - mstart - 1) / 2, '\\'))
+ return ''.join(elements)
+
+ # odd number of backslashes
+ elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#')
+ offset = mend
+
+ elements.append(s[offset:])
+ return ''.join(elements)
+
+def itercommandchars(d, offset, tokenlist, it):
+ """
+ Iterate over command syntax. # comment markers are not special, and escaped newlines are included
+ in the output text.
+ """
+
+ assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+
+ if offset == d.lend:
+ return
+
+ s = d.s
+ for m in it:
+ mstart, mend = m.span(0)
+ token = s[mstart:mend]
+ starttext = s[offset:mstart].replace('\n\t', '\n')
+
+ if token in tokenlist or (token[0] == '$' and '$' in tokenlist):
+ yield starttext, token, mstart, mend
+ else:
+ yield starttext + token, None, None, mend
+ offset = mend
+
+ yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None
+
+_redefines = re.compile('\s*define|\s*endef')
+def iterdefinelines(it, startloc):
+ """
+ Process the insides of a define. Most characters are included literally. Escaped newlines are treated
+ as they would be in makefile syntax. Internal define/endef pairs are ignored.
+ """
+
+ results = []
+
+ definecount = 1
+ for d in it:
+ m = _redefines.match(d.s, d.lstart, d.lend)
+ if m is not None:
+ directive = m.group(0).strip()
+ if directive == 'endef':
+ definecount -= 1
+ if definecount == 0:
+ return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results))
+ else:
+ definecount += 1
+
+ results.append(d.s[d.lstart:d.lend])
+
+ # Falling off the end is an unterminated define!
+ raise SyntaxError("define without matching endef", startloc)
+
+def _ensureend(d, offset, msg):
+ """
+ Ensure that only whitespace remains in this data.
+ """
+
+ s = flattenmakesyntax(d, offset)
+ if s != '' and not s.isspace():
+ raise SyntaxError(msg, d.getloc(offset))
+
+_eqargstokenlist = ('(', "'", '"')
+
+def ifeq(d, offset):
+ if offset > d.lend - 1:
+ raise SyntaxError("No arguments after conditional", d.getloc(offset))
+
+ # the variety of formats for this directive is rather maddening
+ token = d.s[offset]
+ if token not in _eqargstokenlist:
+ raise SyntaxError("No arguments after conditional", d.getloc(offset))
+
+ offset += 1
+
+ if token == '(':
+ arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars)
+ if t is None:
+ raise SyntaxError("Expected two arguments in conditional", d.getloc(d.lend))
+
+ arg1.rstrip()
+
+ offset = d.skipwhitespace(offset)
+ arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars)
+ if t is None:
+ raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
+
+ _ensureend(d, offset, "Unexpected text after conditional")
+ else:
+ arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars)
+ if t is None:
+ raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend))
+
+ offset = d.skipwhitespace(offset)
+ if offset == d.lend:
+ raise SyntaxError("Expected two arguments in conditional", d.getloc(offset))
+
+ token = d.s[offset]
+ if token not in '\'"':
+ raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
+
+ arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars)
+
+ _ensureend(d, offset, "Unexpected text after conditional")
+
+ return parserdata.EqCondition(arg1, arg2)
+
+def ifneq(d, offset):
+ c = ifeq(d, offset)
+ c.expected = False
+ return c
+
+def ifdef(d, offset):
+ e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
+ e.rstrip()
+
+ return parserdata.IfdefCondition(e)
+
+def ifndef(d, offset):
+ c = ifdef(d, offset)
+ c.expected = False
+ return c
+
+_conditionkeywords = {
+ 'ifeq': ifeq,
+ 'ifneq': ifneq,
+ 'ifdef': ifdef,
+ 'ifndef': ifndef
+ }
+
+_conditiontokens = tuple(_conditionkeywords.iterkeys())
+_conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens))
+
+_directivestokenlist = _conditiontokens + \
+ ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport')
+
+_directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist))
+
+_varsettokens = (':=', '+=', '?=', '=')
+
+def _parsefile(pathname):
+ fd = open(pathname, "rU")
+ stmts = parsestring(fd.read(), pathname)
+ stmts.mtime = os.fstat(fd.fileno()).st_mtime
+ fd.close()
+ return stmts
+
+def _checktime(path, stmts):
+ mtime = os.path.getmtime(path)
+ if mtime != stmts.mtime:
+ _log.debug("Re-parsing makefile '%s': mtimes differ", path)
+ return False
+
+ return True
+
+_parsecache = util.MostUsedCache(50, _parsefile, _checktime)
+
+def parsefile(pathname):
+ """
+ Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing
+ makefiles that have already been parsed and have not changed.
+ """
+
+ pathname = os.path.realpath(pathname)
+ return _parsecache.get(pathname)
+
+# colon followed by anything except a slash (Windows path detection)
+_depfilesplitter = re.compile(r':(?![\\/])')
+# simple variable references
+_vars = re.compile('\$\((\w+)\)')
+
+def parsedepfile(pathname):
+ """
+ Parse a filename listing only depencencies into a parserdata.StatementList.
+ Simple variable references are allowed in such files.
+ """
+ def continuation_iter(lines):
+ current_line = []
+ for line in lines:
+ line = line.rstrip()
+ if line.endswith("\\"):
+ current_line.append(line.rstrip("\\"))
+ continue
+ if not len(line):
+ continue
+ current_line.append(line)
+ yield ''.join(current_line)
+ current_line = []
+ if current_line:
+ yield ''.join(current_line)
+
+ def get_expansion(s):
+ if '$' in s:
+ expansion = data.Expansion()
+ # for an input like e.g. "foo $(bar) baz",
+ # _vars.split returns ["foo", "bar", "baz"]
+ # every other element is a variable name.
+ for i, element in enumerate(_vars.split(s)):
+ if i % 2:
+ expansion.appendfunc(functions.VariableRef(None,
+ data.StringExpansion(element, None)))
+ elif element:
+ expansion.appendstr(element)
+
+ return expansion
+
+ return data.StringExpansion(s, None)
+
+ pathname = os.path.realpath(pathname)
+ stmts = parserdata.StatementList()
+ for line in continuation_iter(open(pathname).readlines()):
+ target, deps = _depfilesplitter.split(line, 1)
+ stmts.append(parserdata.Rule(get_expansion(target),
+ get_expansion(deps), False))
+ return stmts
+
+def parsestring(s, filename):
+ """
+ Parse a string containing makefile data into a parserdata.StatementList.
+ """
+
+ currule = False
+ condstack = [parserdata.StatementList()]
+
+ fdlines = enumeratelines(s, filename)
+ for d in fdlines:
+ assert len(condstack) > 0
+
+ offset = d.lstart
+
+ if currule and offset < d.lend and d.s[offset] == '\t':
+ e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars)
+ assert token is None
+ assert offset is None
+ condstack[-1].append(parserdata.Command(e))
+ continue
+
+ # To parse Makefile syntax, we first strip leading whitespace and
+ # look for initial keywords. If there are no keywords, it's either
+ # setting a variable or writing a rule.
+
+ offset = d.skipwhitespace(offset)
+ if offset is None:
+ continue
+
+ m = _directivesre.match(d.s, offset, d.lend)
+ if m is not None:
+ kword = m.group(1)
+ offset = m.end(0)
+
+ if kword == 'endif':
+ _ensureend(d, offset, "Unexpected data after 'endif' directive")
+ if len(condstack) == 1:
+ raise SyntaxError("unmatched 'endif' directive",
+ d.getloc(offset))
+
+ condstack.pop().endloc = d.getloc(offset)
+ continue
+
+ if kword == 'else':
+ if len(condstack) == 1:
+ raise SyntaxError("unmatched 'else' directive",
+ d.getloc(offset))
+
+ m = _conditionre.match(d.s, offset, d.lend)
+ if m is None:
+ _ensureend(d, offset, "Unexpected data after 'else' directive.")
+ condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition())
+ else:
+ kword = m.group(1)
+ if kword not in _conditionkeywords:
+ raise SyntaxError("Unexpected condition after 'else' directive.",
+ d.getloc(offset))
+
+ startoffset = offset
+ offset = d.skipwhitespace(m.end(1))
+ c = _conditionkeywords[kword](d, offset)
+ condstack[-1].addcondition(d.getloc(startoffset), c)
+ continue
+
+ if kword in _conditionkeywords:
+ c = _conditionkeywords[kword](d, offset)
+ cb = parserdata.ConditionBlock(d.getloc(d.lstart), c)
+ condstack[-1].append(cb)
+ condstack.append(cb)
+ continue
+
+ if kword == 'endef':
+ raise SyntaxError("endef without matching define", d.getloc(offset))
+
+ if kword == 'define':
+ currule = False
+ vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars)
+ vname.rstrip()
+
+ startloc = d.getloc(d.lstart)
+ value = iterdefinelines(fdlines, startloc)
+ condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None))
+ continue
+
+ if kword in ('include', '-include', 'includedeps', '-includedeps'):
+ if kword.startswith('-'):
+ required = False
+ kword = kword[1:]
+ else:
+ required = True
+
+ deps = kword == 'includedeps'
+
+ currule = False
+ incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
+ condstack[-1].append(parserdata.Include(incfile, required, deps))
+
+ continue
+
+ if kword == 'vpath':
+ currule = False
+ e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
+ condstack[-1].append(parserdata.VPathDirective(e))
+ continue
+
+ if kword == 'override':
+ currule = False
+ vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars)
+ vname.lstrip()
+ vname.rstrip()
+
+ if token is None:
+ raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart))
+
+ value = flattenmakesyntax(d, offset).lstrip()
+
+ condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE))
+ continue
+
+ if kword == 'export':
+ currule = False
+ e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars)
+ e.lstrip()
+ e.rstrip()
+
+ if token is None:
+ condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=False))
+ else:
+ condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=True))
+
+ value = flattenmakesyntax(d, offset).lstrip()
+ condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None))
+
+ continue
+
+ if kword == 'unexport':
+ e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars)
+ condstack[-1].append(parserdata.UnexportDirective(e))
+ continue
+
+ e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars)
+ if token is None:
+ e.rstrip()
+ e.lstrip()
+ if not e.isempty():
+ condstack[-1].append(parserdata.EmptyDirective(e))
+ continue
+
+ # if we encountered real makefile syntax, the current rule is over
+ currule = False
+
+ if token in _varsettokens:
+ e.lstrip()
+ e.rstrip()
+
+ value = flattenmakesyntax(d, offset).lstrip()
+
+ condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None))
+ else:
+ doublecolon = token == '::'
+
+ # `e` is targets or target patterns, which can end up as
+ # * a rule
+ # * an implicit rule
+ # * a static pattern rule
+ # * a target-specific variable definition
+ # * a pattern-specific variable definition
+ # any of the rules may have order-only prerequisites
+ # delimited by |, and a command delimited by ;
+ targets = e
+
+ e, token, offset = parsemakesyntax(d, offset,
+ _varsettokens + (':', '|', ';'),
+ itermakefilechars)
+ if token in (None, ';'):
+ condstack[-1].append(parserdata.Rule(targets, e, doublecolon))
+ currule = True
+
+ if token == ';':
+ offset = d.skipwhitespace(offset)
+ e, t, offset = parsemakesyntax(d, offset, (), itercommandchars)
+ condstack[-1].append(parserdata.Command(e))
+
+ elif token in _varsettokens:
+ e.lstrip()
+ e.rstrip()
+
+ value = flattenmakesyntax(d, offset).lstrip()
+ condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets))
+ elif token == '|':
+ raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset))
+ else:
+ assert token == ':'
+ # static pattern rule
+
+ pattern = e
+
+ deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars)
+
+ condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon))
+ currule = True
+
+ if token == ';':
+ offset = d.skipwhitespace(offset)
+ e, token, offset = parsemakesyntax(d, offset, (), itercommandchars)
+ condstack[-1].append(parserdata.Command(e))
+
+ if len(condstack) != 1:
+ raise SyntaxError("Condition never terminated with endif", condstack[-1].loc)
+
+ return condstack[0]
+
+_PARSESTATE_TOPLEVEL = 0 # at the top level
+_PARSESTATE_FUNCTION = 1 # expanding a function call
+_PARSESTATE_VARNAME = 2 # expanding a variable expansion.
+_PARSESTATE_SUBSTFROM = 3 # expanding a variable expansion substitution "from" value
+_PARSESTATE_SUBSTTO = 4 # expanding a variable expansion substitution "to" value
+_PARSESTATE_PARENMATCH = 5 # inside nested parentheses/braces that must be matched
+
+class ParseStackFrame(object):
+ __slots__ = ('parsestate', 'parent', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom')
+
+ def __init__(self, parsestate, parent, expansion, tokenlist, openbrace, closebrace, function=None, loc=None):
+ self.parsestate = parsestate
+ self.parent = parent
+ self.expansion = expansion
+ self.tokenlist = tokenlist
+ self.openbrace = openbrace
+ self.closebrace = closebrace
+ self.function = function
+ self.loc = loc
+
+ def __str__(self):
+ return "<state=%i expansion=%s tokenlist=%s openbrace=%s closebrace=%s>" % (self.parsestate, self.expansion, self.tokenlist, self.openbrace, self.closebrace)
+
+_matchingbrace = {
+ '(': ')',
+ '{': '}',
+ }
+
+def parsemakesyntax(d, offset, stopon, iterfunc):
+ """
+ Given Data, parse it into a data.Expansion.
+
+ @param stopon (sequence)
+ Indicate characters where toplevel parsing should stop.
+
+ @param iterfunc (generator function)
+ A function which is used to iterate over d, yielding (char, offset, loc)
+ @see iterdata
+ @see itermakefilechars
+ @see itercommandchars
+
+ @return a tuple (expansion, token, offset). If all the data is consumed,
+ token and offset will be None
+ """
+
+ assert callable(iterfunc)
+
+ stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)),
+ tokenlist=stopon + ('$',),
+ openbrace=None, closebrace=None)
+
+ tokeniterator = _alltokens.finditer(d.s, offset, d.lend)
+
+ di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
+ while True: # this is not a for loop because `di` changes during the function
+ assert stacktop is not None
+ try:
+ s, token, tokenoffset, offset = di.next()
+ except StopIteration:
+ break
+
+ stacktop.expansion.appendstr(s)
+ if token is None:
+ continue
+
+ parsestate = stacktop.parsestate
+
+ if token[0] == '$':
+ if tokenoffset + 1 == d.lend:
+ # an unterminated $ expands to nothing
+ break
+
+ loc = d.getloc(tokenoffset)
+ c = token[1]
+ if c == '$':
+ assert len(token) == 2
+ stacktop.expansion.appendstr('$')
+ elif c in ('(', '{'):
+ closebrace = _matchingbrace[c]
+
+ if len(token) > 2:
+ fname = token[2:].rstrip()
+ fn = functions.functionmap[fname](loc)
+ e = data.Expansion()
+ if len(fn) + 1 == fn.maxargs:
+ tokenlist = (c, closebrace, '$')
+ else:
+ tokenlist = (',', c, closebrace, '$')
+
+ stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop,
+ e, tokenlist, function=fn,
+ openbrace=c, closebrace=closebrace)
+ else:
+ e = data.Expansion()
+ tokenlist = (':', c, closebrace, '$')
+ stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop,
+ e, tokenlist,
+ openbrace=c, closebrace=closebrace, loc=loc)
+ else:
+ assert len(token) == 2
+ e = data.Expansion.fromstring(c, loc)
+ stacktop.expansion.appendfunc(functions.VariableRef(loc, e))
+ elif token in ('(', '{'):
+ assert token == stacktop.openbrace
+
+ stacktop.expansion.appendstr(token)
+ stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop,
+ stacktop.expansion,
+ (token, stacktop.closebrace, '$'),
+ openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset))
+ elif parsestate == _PARSESTATE_PARENMATCH:
+ assert token == stacktop.closebrace
+ stacktop.expansion.appendstr(token)
+ stacktop = stacktop.parent
+ elif parsestate == _PARSESTATE_TOPLEVEL:
+ assert stacktop.parent is None
+ return stacktop.expansion.finish(), token, offset
+ elif parsestate == _PARSESTATE_FUNCTION:
+ if token == ',':
+ stacktop.function.append(stacktop.expansion.finish())
+
+ stacktop.expansion = data.Expansion()
+ if len(stacktop.function) + 1 == stacktop.function.maxargs:
+ tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
+ stacktop.tokenlist = tokenlist
+ elif token in (')', '}'):
+ fn = stacktop.function
+ fn.append(stacktop.expansion.finish())
+ fn.setup()
+
+ stacktop = stacktop.parent
+ stacktop.expansion.appendfunc(fn)
+ else:
+ assert False, "Not reached, _PARSESTATE_FUNCTION"
+ elif parsestate == _PARSESTATE_VARNAME:
+ if token == ':':
+ stacktop.varname = stacktop.expansion
+ stacktop.parsestate = _PARSESTATE_SUBSTFROM
+ stacktop.expansion = data.Expansion()
+ stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$')
+ elif token in (')', '}'):
+ fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish())
+ stacktop = stacktop.parent
+ stacktop.expansion.appendfunc(fn)
+ else:
+ assert False, "Not reached, _PARSESTATE_VARNAME"
+ elif parsestate == _PARSESTATE_SUBSTFROM:
+ if token == '=':
+ stacktop.substfrom = stacktop.expansion
+ stacktop.parsestate = _PARSESTATE_SUBSTTO
+ stacktop.expansion = data.Expansion()
+ stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
+ elif token in (')', '}'):
+ # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make
+ # parses it. Issue a warning. Combine the varname and substfrom expansions to
+ # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME
+ _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc)
+ stacktop.varname.appendstr(':')
+ stacktop.varname.concat(stacktop.expansion)
+ fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish())
+ stacktop = stacktop.parent
+ stacktop.expansion.appendfunc(fn)
+ else:
+ assert False, "Not reached, _PARSESTATE_SUBSTFROM"
+ elif parsestate == _PARSESTATE_SUBSTTO:
+ assert token in (')','}'), "Not reached, _PARSESTATE_SUBSTTO"
+
+ fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(),
+ stacktop.substfrom.finish(), stacktop.expansion.finish())
+ stacktop = stacktop.parent
+ stacktop.expansion.appendfunc(fn)
+ else:
+ assert False, "Unexpected parse state %s" % stacktop.parsestate
+
+ if stacktop.parent is not None and iterfunc == itercommandchars:
+ di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator,
+ ignorecomments=True)
+ else:
+ di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
+
+ if stacktop.parent is not None:
+ raise SyntaxError("Unterminated function call", d.getloc(offset))
+
+ assert stacktop.parsestate == _PARSESTATE_TOPLEVEL
+
+ return stacktop.expansion.finish(), None, None
diff --git a/build/pymake/pymake/parserdata.py b/build/pymake/pymake/parserdata.py
new file mode 100644
index 000000000..7b2e5443d
--- /dev/null
+++ b/build/pymake/pymake/parserdata.py
@@ -0,0 +1,1006 @@
+import logging, re, os
+import data, parser, functions, util
+from cStringIO import StringIO
+from pymake.globrelative import hasglob, glob
+
+_log = logging.getLogger('pymake.data')
+_tabwidth = 4
+
+class Location(object):
+ """
+ A location within a makefile.
+
+ For the moment, locations are just path/line/column, but in the future
+ they may reference parent locations for more accurate "included from"
+ or "evaled at" error reporting.
+ """
+ __slots__ = ('path', 'line', 'column')
+
+ def __init__(self, path, line, column):
+ self.path = path
+ self.line = line
+ self.column = column
+
+ def offset(self, s, start, end):
+ """
+ Returns a new location offset by
+ the specified string.
+ """
+
+ if start == end:
+ return self
+
+ skiplines = s.count('\n', start, end)
+ line = self.line + skiplines
+ if skiplines:
+ lastnl = s.rfind('\n', start, end)
+ assert lastnl != -1
+ start = lastnl + 1
+ column = 0
+ else:
+ column = self.column
+
+ while True:
+ j = s.find('\t', start, end)
+ if j == -1:
+ column += end - start
+ break
+
+ column += j - start
+ column += _tabwidth
+ column -= column % _tabwidth
+ start = j + 1
+
+ return Location(self.path, line, column)
+
+ def __str__(self):
+ return "%s:%s:%s" % (self.path, self.line, self.column)
+
+def _expandwildcards(makefile, tlist):
+ for t in tlist:
+ if not hasglob(t):
+ yield t
+ else:
+ l = glob(makefile.workdir, t)
+ for r in l:
+ yield r
+
+_flagescape = re.compile(r'([\s\\])')
+
+def parsecommandlineargs(args):
+ """
+ Given a set of arguments from a command-line invocation of make,
+ parse out the variable definitions and return (stmts, arglist, overridestr)
+ """
+
+ overrides = []
+ stmts = StatementList()
+ r = []
+ for i in xrange(0, len(args)):
+ a = args[i]
+
+ vname, t, val = util.strpartition(a, ':=')
+ if t == '':
+ vname, t, val = util.strpartition(a, '=')
+ if t != '':
+ overrides.append(_flagescape.sub(r'\\\1', a))
+
+ vname = vname.strip()
+ vnameexp = data.Expansion.fromstring(vname, "Command-line argument")
+
+ stmts.append(ExportDirective(vnameexp, concurrent_set=True))
+ stmts.append(SetVariable(vnameexp, token=t,
+ value=val, valueloc=Location('<command-line>', i, len(vname) + len(t)),
+ targetexp=None, source=data.Variables.SOURCE_COMMANDLINE))
+ else:
+ r.append(data.stripdotslash(a))
+
+ return stmts, r, ' '.join(overrides)
+
+class Statement(object):
+ """
+ Represents parsed make file syntax.
+
+ This is an abstract base class. Child classes are expected to implement
+ basic methods defined below.
+ """
+
+ def execute(self, makefile, context):
+ """Executes this Statement within a make file execution context."""
+ raise Exception("%s must implement execute()." % self.__class__)
+
+ def to_source(self):
+ """Obtain the make file "source" representation of the Statement.
+
+ This converts an individual Statement back to a string that can again
+ be parsed into this Statement.
+ """
+ raise Exception("%s must implement to_source()." % self.__class__)
+
+ def __eq__(self, other):
+ raise Exception("%s must implement __eq__." % self.__class__)
+
+ def __ne__(self, other):
+ return self.__eq__(other)
+
+class DummyRule(object):
+ __slots__ = ()
+
+ def addcommand(self, r):
+ pass
+
+class Rule(Statement):
+ """
+ Rules represent how to make specific targets.
+
+ See https://www.gnu.org/software/make/manual/make.html#Rules.
+
+ An individual rule is composed of a target, dependencies, and a recipe.
+ This class only contains references to the first 2. The recipe will be
+ contained in Command classes which follow this one in a stream of Statement
+ instances.
+
+ Instances also contain a boolean property `doublecolon` which says whether
+ this is a doublecolon rule. Doublecolon rules are rules that are always
+ executed, if they are evaluated. Normally, rules are only executed if their
+ target is out of date.
+ """
+ __slots__ = ('targetexp', 'depexp', 'doublecolon')
+
+ def __init__(self, targetexp, depexp, doublecolon):
+ assert isinstance(targetexp, (data.Expansion, data.StringExpansion))
+ assert isinstance(depexp, (data.Expansion, data.StringExpansion))
+
+ self.targetexp = targetexp
+ self.depexp = depexp
+ self.doublecolon = doublecolon
+
+ def execute(self, makefile, context):
+ if context.weak:
+ self._executeweak(makefile, context)
+ else:
+ self._execute(makefile, context)
+
+ def _executeweak(self, makefile, context):
+ """
+ If the context is weak (we're just handling dependencies) we can make a number of assumptions here.
+ This lets us go really fast and is generally good.
+ """
+ assert context.weak
+ deps = self.depexp.resolvesplit(makefile, makefile.variables)
+ # Skip targets with no rules and no dependencies
+ if not deps:
+ return
+ targets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))
+ rule = data.Rule(list(data.stripdotslashes(deps)), self.doublecolon, loc=self.targetexp.loc, weakdeps=True)
+ for target in targets:
+ makefile.gettarget(target).addrule(rule)
+ makefile.foundtarget(target)
+ context.currule = rule
+
+ def _execute(self, makefile, context):
+ assert not context.weak
+
+ atargets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))
+ targets = [data.Pattern(p) for p in _expandwildcards(makefile, atargets)]
+
+ if not len(targets):
+ context.currule = DummyRule()
+ return
+
+ ispatterns = set((t.ispattern() for t in targets))
+ if len(ispatterns) == 2:
+ raise data.DataError("Mixed implicit and normal rule", self.targetexp.loc)
+ ispattern, = ispatterns
+
+ deps = list(_expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables))))
+ if ispattern:
+ rule = data.PatternRule(targets, map(data.Pattern, deps), self.doublecolon, loc=self.targetexp.loc)
+ makefile.appendimplicitrule(rule)
+ else:
+ rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc, weakdeps=False)
+ for t in targets:
+ makefile.gettarget(t.gettarget()).addrule(rule)
+
+ makefile.foundtarget(targets[0].gettarget())
+
+ context.currule = rule
+
+ def dump(self, fd, indent):
+ print >>fd, "%sRule %s: %s" % (indent, self.targetexp, self.depexp)
+
+ def to_source(self):
+ sep = ':'
+
+ if self.doublecolon:
+ sep = '::'
+
+ deps = self.depexp.to_source()
+ if len(deps) > 0 and not deps[0].isspace():
+ sep += ' '
+
+ return '\n%s%s%s' % (
+ self.targetexp.to_source(escape_variables=True),
+ sep,
+ deps)
+
+ def __eq__(self, other):
+ if not isinstance(other, Rule):
+ return False
+
+ return self.targetexp == other.targetexp \
+ and self.depexp == other.depexp \
+ and self.doublecolon == other.doublecolon
+
+class StaticPatternRule(Statement):
+ """
+ Static pattern rules are rules which specify multiple targets based on a
+ string pattern.
+
+ See https://www.gnu.org/software/make/manual/make.html#Static-Pattern
+
+ They are like `Rule` instances except an added property, `patternexp` is
+ present. It contains the Expansion which represents the rule pattern.
+ """
+ __slots__ = ('targetexp', 'patternexp', 'depexp', 'doublecolon')
+
+ def __init__(self, targetexp, patternexp, depexp, doublecolon):
+ assert isinstance(targetexp, (data.Expansion, data.StringExpansion))
+ assert isinstance(patternexp, (data.Expansion, data.StringExpansion))
+ assert isinstance(depexp, (data.Expansion, data.StringExpansion))
+
+ self.targetexp = targetexp
+ self.patternexp = patternexp
+ self.depexp = depexp
+ self.doublecolon = doublecolon
+
+ def execute(self, makefile, context):
+ if context.weak:
+ raise data.DataError("Static pattern rules not allowed in includedeps", self.targetexp.loc)
+
+ targets = list(_expandwildcards(makefile, data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))))
+
+ if not len(targets):
+ context.currule = DummyRule()
+ return
+
+ patterns = list(data.stripdotslashes(self.patternexp.resolvesplit(makefile, makefile.variables)))
+ if len(patterns) != 1:
+ raise data.DataError("Static pattern rules must have a single pattern", self.patternexp.loc)
+ pattern = data.Pattern(patterns[0])
+
+ deps = [data.Pattern(p) for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))]
+
+ rule = data.PatternRule([pattern], deps, self.doublecolon, loc=self.targetexp.loc)
+
+ for t in targets:
+ if data.Pattern(t).ispattern():
+ raise data.DataError("Target '%s' of a static pattern rule must not be a pattern" % (t,), self.targetexp.loc)
+ stem = pattern.match(t)
+ if stem is None:
+ raise data.DataError("Target '%s' does not match the static pattern '%s'" % (t, pattern), self.targetexp.loc)
+ makefile.gettarget(t).addrule(data.PatternRuleInstance(rule, '', stem, pattern.ismatchany()))
+
+ makefile.foundtarget(targets[0])
+ context.currule = rule
+
+ def dump(self, fd, indent):
+ print >>fd, "%sStaticPatternRule %s: %s: %s" % (indent, self.targetexp, self.patternexp, self.depexp)
+
+ def to_source(self):
+ sep = ':'
+
+ if self.doublecolon:
+ sep = '::'
+
+ pattern = self.patternexp.to_source()
+ deps = self.depexp.to_source()
+
+ if len(pattern) > 0 and pattern[0] not in (' ', '\t'):
+ sep += ' '
+
+ return '\n%s%s%s:%s' % (
+ self.targetexp.to_source(escape_variables=True),
+ sep,
+ pattern,
+ deps)
+
+ def __eq__(self, other):
+ if not isinstance(other, StaticPatternRule):
+ return False
+
+ return self.targetexp == other.targetexp \
+ and self.patternexp == other.patternexp \
+ and self.depexp == other.depexp \
+ and self.doublecolon == other.doublecolon
+
+class Command(Statement):
+ """
+ Commands are things that get executed by a rule.
+
+ A rule's recipe is composed of 0 or more Commands.
+
+ A command is simply an expansion. Commands typically represent strings to
+ be executed in a shell (e.g. via system()). Although, since make files
+ allow arbitrary shells to be used for command execution, this isn't a
+ guarantee.
+ """
+ __slots__ = ('exp',)
+
+ def __init__(self, exp):
+ assert isinstance(exp, (data.Expansion, data.StringExpansion))
+ self.exp = exp
+
+ def execute(self, makefile, context):
+ assert context.currule is not None
+ if context.weak:
+ raise data.DataError("rules not allowed in includedeps", self.exp.loc)
+
+ context.currule.addcommand(self.exp)
+
+ def dump(self, fd, indent):
+ print >>fd, "%sCommand %s" % (indent, self.exp,)
+
+ def to_source(self):
+ # Commands have some interesting quirks when it comes to source
+ # formatting. First, they can be multi-line. Second, a tab needs to be
+ # inserted at the beginning of every line. Finally, there might be
+ # variable references inside the command. This means we need to escape
+ # variable references inside command strings. Luckily, this is handled
+ # by the Expansion.
+ s = self.exp.to_source(escape_variables=True)
+
+ return '\n'.join(['\t%s' % line for line in s.split('\n')])
+
+ def __eq__(self, other):
+ if not isinstance(other, Command):
+ return False
+
+ return self.exp == other.exp
+
+class SetVariable(Statement):
+ """
+ Represents a variable assignment.
+
+ Variable assignment comes in two different flavors.
+
+ Simple assignment has the form:
+
+ <Expansion> <Assignment Token> <string>
+
+ e.g. FOO := bar
+
+ These correspond to the fields `vnameexp`, `token`, and `value`. In
+ addition, `valueloc` will be a Location and `source` will be a
+ pymake.data.Variables.SOURCE_* constant.
+
+ There are also target-specific variables. These are variables that only
+ apply in the context of a specific target. They are like the aforementioned
+ assignment except the `targetexp` field is set to an Expansion representing
+ the target they apply to.
+ """
+ __slots__ = ('vnameexp', 'token', 'value', 'valueloc', 'targetexp', 'source')
+
+ def __init__(self, vnameexp, token, value, valueloc, targetexp, source=None):
+ assert isinstance(vnameexp, (data.Expansion, data.StringExpansion))
+ assert isinstance(value, str)
+ assert targetexp is None or isinstance(targetexp, (data.Expansion, data.StringExpansion))
+
+ if source is None:
+ source = data.Variables.SOURCE_MAKEFILE
+
+ self.vnameexp = vnameexp
+ self.token = token
+ self.value = value
+ self.valueloc = valueloc
+ self.targetexp = targetexp
+ self.source = source
+
+ def execute(self, makefile, context):
+ vname = self.vnameexp.resolvestr(makefile, makefile.variables)
+ if len(vname) == 0:
+ raise data.DataError("Empty variable name", self.vnameexp.loc)
+
+ if self.targetexp is None:
+ setvariables = [makefile.variables]
+ else:
+ setvariables = []
+
+ targets = [data.Pattern(t) for t in data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))]
+ for t in targets:
+ if t.ispattern():
+ setvariables.append(makefile.getpatternvariables(t))
+ else:
+ setvariables.append(makefile.gettarget(t.gettarget()).variables)
+
+ for v in setvariables:
+ if self.token == '+=':
+ v.append(vname, self.source, self.value, makefile.variables, makefile)
+ continue
+
+ if self.token == '?=':
+ flavor = data.Variables.FLAVOR_RECURSIVE
+ oldflavor, oldsource, oldval = v.get(vname, expand=False)
+ if oldval is not None:
+ continue
+ value = self.value
+ elif self.token == '=':
+ flavor = data.Variables.FLAVOR_RECURSIVE
+ value = self.value
+ else:
+ assert self.token == ':='
+
+ flavor = data.Variables.FLAVOR_SIMPLE
+ d = parser.Data.fromstring(self.value, self.valueloc)
+ e, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
+ value = e.resolvestr(makefile, makefile.variables)
+
+ v.set(vname, flavor, self.source, value)
+
+ def dump(self, fd, indent):
+ print >>fd, "%sSetVariable<%s> %s %s\n%s %r" % (indent, self.valueloc, self.vnameexp, self.token, indent, self.value)
+
+ def __eq__(self, other):
+ if not isinstance(other, SetVariable):
+ return False
+
+ return self.vnameexp == other.vnameexp \
+ and self.token == other.token \
+ and self.value == other.value \
+ and self.targetexp == other.targetexp \
+ and self.source == other.source
+
+ def to_source(self):
+ chars = []
+ for i in xrange(0, len(self.value)):
+ c = self.value[i]
+
+ # Literal # is escaped in variable assignment otherwise it would be
+ # a comment.
+ if c == '#':
+ # If a backslash precedes this, we need to escape it as well.
+ if i > 0 and self.value[i-1] == '\\':
+ chars.append('\\')
+
+ chars.append('\\#')
+ continue
+
+ chars.append(c)
+
+ value = ''.join(chars)
+
+ prefix = ''
+ if self.source == data.Variables.SOURCE_OVERRIDE:
+ prefix = 'override '
+
+ # SetVariable come in two flavors: simple and target-specific.
+
+ # We handle the target-specific syntax first.
+ if self.targetexp is not None:
+ return '%s: %s %s %s' % (
+ self.targetexp.to_source(),
+ self.vnameexp.to_source(),
+ self.token,
+ value)
+
+ # The variable could be multi-line or have leading whitespace. For
+ # regular variable assignment, whitespace after the token but before
+ # the value is ignored. If we see leading whitespace in the value here,
+ # the variable must have come from a define.
+ if value.count('\n') > 0 or (len(value) and value[0].isspace()):
+ # The parser holds the token in vnameexp for whatever reason.
+ return '%sdefine %s\n%s\nendef' % (
+ prefix,
+ self.vnameexp.to_source(),
+ value)
+
+ return '%s%s %s %s' % (
+ prefix,
+ self.vnameexp.to_source(),
+ self.token,
+ value)
+
+class Condition(object):
+ """
+ An abstract "condition", either ifeq or ifdef, perhaps negated.
+
+ See https://www.gnu.org/software/make/manual/make.html#Conditional-Syntax
+
+ Subclasses must implement:
+
+ def evaluate(self, makefile)
+ """
+
+ def __eq__(self, other):
+ raise Exception("%s must implement __eq__." % __class__)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+class EqCondition(Condition):
+ """
+ Represents an ifeq or ifneq conditional directive.
+
+ This directive consists of two Expansions which are compared for equality.
+
+ The `expected` field is a bool indicating what the condition must evaluate
+ to in order for its body to be executed. If True, this is an "ifeq"
+ conditional directive. If False, an "ifneq."
+ """
+ __slots__ = ('exp1', 'exp2', 'expected')
+
+ def __init__(self, exp1, exp2):
+ assert isinstance(exp1, (data.Expansion, data.StringExpansion))
+ assert isinstance(exp2, (data.Expansion, data.StringExpansion))
+
+ self.expected = True
+ self.exp1 = exp1
+ self.exp2 = exp2
+
+ def evaluate(self, makefile):
+ r1 = self.exp1.resolvestr(makefile, makefile.variables)
+ r2 = self.exp2.resolvestr(makefile, makefile.variables)
+ return (r1 == r2) == self.expected
+
+ def __str__(self):
+ return "ifeq (expected=%s) %s %s" % (self.expected, self.exp1, self.exp2)
+
+ def __eq__(self, other):
+ if not isinstance(other, EqCondition):
+ return False
+
+ return self.exp1 == other.exp1 \
+ and self.exp2 == other.exp2 \
+ and self.expected == other.expected
+
+class IfdefCondition(Condition):
+ """
+ Represents an ifdef or ifndef conditional directive.
+
+ This directive consists of a single expansion which represents the name of
+ a variable (without the leading '$') which will be checked for definition.
+
+ The `expected` field is a bool and has the same behavior as EqCondition.
+ If it is True, this represents a "ifdef" conditional. If False, "ifndef."
+ """
+ __slots__ = ('exp', 'expected')
+
+ def __init__(self, exp):
+ assert isinstance(exp, (data.Expansion, data.StringExpansion))
+ self.exp = exp
+ self.expected = True
+
+ def evaluate(self, makefile):
+ vname = self.exp.resolvestr(makefile, makefile.variables)
+ flavor, source, value = makefile.variables.get(vname, expand=False)
+
+ if value is None:
+ return not self.expected
+
+ return (len(value) > 0) == self.expected
+
+ def __str__(self):
+ return "ifdef (expected=%s) %s" % (self.expected, self.exp)
+
+ def __eq__(self, other):
+ if not isinstance(other, IfdefCondition):
+ return False
+
+ return self.exp == other.exp and self.expected == other.expected
+
+class ElseCondition(Condition):
+ """
+ Represents the transition between branches in a ConditionBlock.
+ """
+ __slots__ = ()
+
+ def evaluate(self, makefile):
+ return True
+
+ def __str__(self):
+ return "else"
+
+ def __eq__(self, other):
+ return isinstance(other, ElseCondition)
+
+class ConditionBlock(Statement):
+ """
+ A set of related Conditions.
+
+ This is essentially a list of 2-tuples of (Condition, list(Statement)).
+
+ The parser creates a ConditionBlock for all statements related to the same
+ conditional group. If iterating over the parser's output, where you think
+ you would see an ifeq, you will see a ConditionBlock containing an IfEq. In
+ other words, the parser collapses separate statements into this container
+ class.
+
+ ConditionBlock instances may exist within other ConditionBlock if the
+ conditional logic is multiple levels deep.
+ """
+ __slots__ = ('loc', '_groups')
+
+ def __init__(self, loc, condition):
+ self.loc = loc
+ self._groups = []
+ self.addcondition(loc, condition)
+
+ def getloc(self):
+ return self.loc
+
+ def addcondition(self, loc, condition):
+ assert isinstance(condition, Condition)
+ condition.loc = loc
+
+ if len(self._groups) and isinstance(self._groups[-1][0], ElseCondition):
+ raise parser.SyntaxError("Multiple else conditions for block starting at %s" % self.loc, loc)
+
+ self._groups.append((condition, StatementList()))
+
+ def append(self, statement):
+ self._groups[-1][1].append(statement)
+
+ def execute(self, makefile, context):
+ i = 0
+ for c, statements in self._groups:
+ if c.evaluate(makefile):
+ _log.debug("Condition at %s met by clause #%i", self.loc, i)
+ statements.execute(makefile, context)
+ return
+
+ i += 1
+
+ def dump(self, fd, indent):
+ print >>fd, "%sConditionBlock" % (indent,)
+
+ indent2 = indent + ' '
+ for c, statements in self._groups:
+ print >>fd, "%s Condition %s" % (indent, c)
+ statements.dump(fd, indent2)
+ print >>fd, "%s ~Condition" % (indent,)
+ print >>fd, "%s~ConditionBlock" % (indent,)
+
+ def to_source(self):
+ lines = []
+ index = 0
+ for condition, statements in self:
+ lines.append(ConditionBlock.condition_source(condition, index))
+ index += 1
+
+ for statement in statements:
+ lines.append(statement.to_source())
+
+ lines.append('endif')
+
+ return '\n'.join(lines)
+
+ def __eq__(self, other):
+ if not isinstance(other, ConditionBlock):
+ return False
+
+ if len(self) != len(other):
+ return False
+
+ for i in xrange(0, len(self)):
+ our_condition, our_statements = self[i]
+ other_condition, other_statements = other[i]
+
+ if our_condition != other_condition:
+ return False
+
+ if our_statements != other_statements:
+ return False
+
+ return True
+
+ @staticmethod
+ def condition_source(statement, index):
+ """Convert a condition to its source representation.
+
+ The index argument defines the index of this condition inside a
+ ConditionBlock. If it is greater than 0, an "else" will be prepended
+ to the result, if necessary.
+ """
+ prefix = ''
+ if isinstance(statement, (EqCondition, IfdefCondition)) and index > 0:
+ prefix = 'else '
+
+ if isinstance(statement, IfdefCondition):
+ s = statement.exp.s
+
+ if statement.expected:
+ return '%sifdef %s' % (prefix, s)
+
+ return '%sifndef %s' % (prefix, s)
+
+ if isinstance(statement, EqCondition):
+ args = [
+ statement.exp1.to_source(escape_comments=True),
+ statement.exp2.to_source(escape_comments=True)]
+
+ use_quotes = False
+ single_quote_present = False
+ double_quote_present = False
+ for i, arg in enumerate(args):
+ if len(arg) > 0 and (arg[0].isspace() or arg[-1].isspace()):
+ use_quotes = True
+
+ if "'" in arg:
+ single_quote_present = True
+
+ if '"' in arg:
+ double_quote_present = True
+
+ # Quote everything if needed.
+ if single_quote_present and double_quote_present:
+ raise Exception('Cannot format condition with multiple quotes.')
+
+ if use_quotes:
+ for i, arg in enumerate(args):
+ # Double to single quotes.
+ if single_quote_present:
+ args[i] = '"' + arg + '"'
+ else:
+ args[i] = "'" + arg + "'"
+
+ body = None
+ if use_quotes:
+ body = ' '.join(args)
+ else:
+ body = '(%s)' % ','.join(args)
+
+ if statement.expected:
+ return '%sifeq %s' % (prefix, body)
+
+ return '%sifneq %s' % (prefix, body)
+
+ if isinstance(statement, ElseCondition):
+ return 'else'
+
+ raise Exception('Unhandled Condition statement: %s' %
+ statement.__class__)
+
+ def __iter__(self):
+ return iter(self._groups)
+
+ def __len__(self):
+ return len(self._groups)
+
+ def __getitem__(self, i):
+ return self._groups[i]
+
+class Include(Statement):
+ """
+ Represents the include directive.
+
+ See https://www.gnu.org/software/make/manual/make.html#Include
+
+ The file to be included is represented by the Expansion defined in the
+ field `exp`. `required` is a bool indicating whether execution should fail
+ if the specified file could not be processed.
+ """
+ __slots__ = ('exp', 'required', 'deps')
+
+ def __init__(self, exp, required, weak):
+ assert isinstance(exp, (data.Expansion, data.StringExpansion))
+ self.exp = exp
+ self.required = required
+ self.weak = weak
+
+ def execute(self, makefile, context):
+ files = self.exp.resolvesplit(makefile, makefile.variables)
+ for f in files:
+ makefile.include(f, self.required, loc=self.exp.loc, weak=self.weak)
+
+ def dump(self, fd, indent):
+ print >>fd, "%sInclude %s" % (indent, self.exp)
+
+ def to_source(self):
+ prefix = ''
+
+ if not self.required:
+ prefix = '-'
+
+ return '%sinclude %s' % (prefix, self.exp.to_source())
+
+ def __eq__(self, other):
+ if not isinstance(other, Include):
+ return False
+
+ return self.exp == other.exp and self.required == other.required
+
+class VPathDirective(Statement):
+ """
+ Represents the vpath directive.
+
+ See https://www.gnu.org/software/make/manual/make.html#Selective-Search
+ """
+ __slots__ = ('exp',)
+
+ def __init__(self, exp):
+ assert isinstance(exp, (data.Expansion, data.StringExpansion))
+ self.exp = exp
+
+ def execute(self, makefile, context):
+ words = list(data.stripdotslashes(self.exp.resolvesplit(makefile, makefile.variables)))
+ if len(words) == 0:
+ makefile.clearallvpaths()
+ else:
+ pattern = data.Pattern(words[0])
+ mpaths = words[1:]
+
+ if len(mpaths) == 0:
+ makefile.clearvpath(pattern)
+ else:
+ dirs = []
+ for mpath in mpaths:
+ dirs.extend((dir for dir in mpath.split(os.pathsep)
+ if dir != ''))
+ if len(dirs):
+ makefile.addvpath(pattern, dirs)
+
+ def dump(self, fd, indent):
+ print >>fd, "%sVPath %s" % (indent, self.exp)
+
+ def to_source(self):
+ return 'vpath %s' % self.exp.to_source()
+
+ def __eq__(self, other):
+ if not isinstance(other, VPathDirective):
+ return False
+
+ return self.exp == other.exp
+
+class ExportDirective(Statement):
+ """
+ Represents the "export" directive.
+
+ This is used to control exporting variables to sub makes.
+
+ See https://www.gnu.org/software/make/manual/make.html#Variables_002fRecursion
+
+ The `concurrent_set` field defines whether this statement occurred with or
+ without a variable assignment. If False, no variable assignment was
+ present. If True, the SetVariable immediately following this statement
+ originally came from this export directive (the parser splits it into
+ multiple statements).
+ """
+
+ __slots__ = ('exp', 'concurrent_set')
+
+ def __init__(self, exp, concurrent_set):
+ assert isinstance(exp, (data.Expansion, data.StringExpansion))
+ self.exp = exp
+ self.concurrent_set = concurrent_set
+
+ def execute(self, makefile, context):
+ if self.concurrent_set:
+ vlist = [self.exp.resolvestr(makefile, makefile.variables)]
+ else:
+ vlist = list(self.exp.resolvesplit(makefile, makefile.variables))
+ if not len(vlist):
+ raise data.DataError("Exporting all variables is not supported", self.exp.loc)
+
+ for v in vlist:
+ makefile.exportedvars[v] = True
+
+ def dump(self, fd, indent):
+ print >>fd, "%sExport (single=%s) %s" % (indent, self.single, self.exp)
+
+ def to_source(self):
+ return ('export %s' % self.exp.to_source()).rstrip()
+
+ def __eq__(self, other):
+ if not isinstance(other, ExportDirective):
+ return False
+
+ # single is irrelevant because it just says whether the next Statement
+ # contains a variable definition.
+ return self.exp == other.exp
+
+class UnexportDirective(Statement):
+ """
+ Represents the "unexport" directive.
+
+ This is the opposite of ExportDirective.
+ """
+ __slots__ = ('exp',)
+
+ def __init__(self, exp):
+ self.exp = exp
+
+ def execute(self, makefile, context):
+ vlist = list(self.exp.resolvesplit(makefile, makefile.variables))
+ for v in vlist:
+ makefile.exportedvars[v] = False
+
+ def dump(self, fd, indent):
+ print >>fd, "%sUnexport %s" % (indent, self.exp)
+
+ def to_source(self):
+ return 'unexport %s' % self.exp.to_source()
+
+ def __eq__(self, other):
+ if not isinstance(other, UnexportDirective):
+ return False
+
+ return self.exp == other.exp
+
+class EmptyDirective(Statement):
+ """
+ Represents a standalone statement, usually an Expansion.
+
+ You will encounter EmptyDirective instances if there is a function
+ or similar at the top-level of a make file (e.g. outside of a rule or
+ variable assignment). You can also find them as the bodies of
+ ConditionBlock branches.
+ """
+ __slots__ = ('exp',)
+
+ def __init__(self, exp):
+ assert isinstance(exp, (data.Expansion, data.StringExpansion))
+ self.exp = exp
+
+ def execute(self, makefile, context):
+ v = self.exp.resolvestr(makefile, makefile.variables)
+ if v.strip() != '':
+ raise data.DataError("Line expands to non-empty value", self.exp.loc)
+
+ def dump(self, fd, indent):
+ print >>fd, "%sEmptyDirective: %s" % (indent, self.exp)
+
+ def to_source(self):
+ return self.exp.to_source()
+
+ def __eq__(self, other):
+ if not isinstance(other, EmptyDirective):
+ return False
+
+ return self.exp == other.exp
+
+class _EvalContext(object):
+ __slots__ = ('currule', 'weak')
+
+ def __init__(self, weak):
+ self.weak = weak
+
+class StatementList(list):
+ """
+ A list of Statement instances.
+
+ This is what is generated by the parser when a make file is parsed.
+
+ Consumers can iterate over all Statement instances in this collection to
+ statically inspect (and even modify) make files before they are executed.
+ """
+ __slots__ = ('mtime',)
+
+ def append(self, statement):
+ assert isinstance(statement, Statement)
+ list.append(self, statement)
+
+ def execute(self, makefile, context=None, weak=False):
+ if context is None:
+ context = _EvalContext(weak=weak)
+
+ for s in self:
+ s.execute(makefile, context)
+
+ def dump(self, fd, indent):
+ for s in self:
+ s.dump(fd, indent)
+
+ def __str__(self):
+ fd = StringIO()
+ self.dump(fd, '')
+ return fd.getvalue()
+
+ def to_source(self):
+ return '\n'.join([s.to_source() for s in self])
+
+def iterstatements(stmts):
+ for s in stmts:
+ yield s
+ if isinstance(s, ConditionBlock):
+ for c, sl in s:
+ for s2 in iterstatments(sl): yield s2
diff --git a/build/pymake/pymake/process.py b/build/pymake/pymake/process.py
new file mode 100644
index 000000000..01cadf5a9
--- /dev/null
+++ b/build/pymake/pymake/process.py
@@ -0,0 +1,556 @@
+"""
+Skipping shell invocations is good, when possible. This wrapper around subprocess does dirty work of
+parsing command lines into argv and making sure that no shell magic is being used.
+"""
+
+#TODO: ship pyprocessing?
+import multiprocessing
+import subprocess, shlex, re, logging, sys, traceback, os, imp, glob
+import site
+from collections import deque
+# XXXkhuey Work around http://bugs.python.org/issue1731717
+subprocess._cleanup = lambda: None
+import command, util
+if sys.platform=='win32':
+ import win32process
+
+_log = logging.getLogger('pymake.process')
+
+_escapednewlines = re.compile(r'\\\n')
+
+def tokens2re(tokens):
+ # Create a pattern for non-escaped tokens, in the form:
+ # (?<!\\)(?:a|b|c...)
+ # This is meant to match patterns a, b, or c, or ... if they are not
+ # preceded by a backslash.
+ # where a, b, c... are in the form
+ # (?P<name>pattern)
+ # which matches the pattern and captures it in a named match group.
+ # The group names and patterns come are given as a dict in the function
+ # argument.
+ nonescaped = r'(?<!\\)(?:%s)' % '|'.join('(?P<%s>%s)' % (name, value) for name, value in tokens.iteritems())
+ # The final pattern matches either the above pattern, or an escaped
+ # backslash, captured in the "escape" match group.
+ return re.compile('(?:%s|%s)' % (nonescaped, r'(?P<escape>\\\\)'))
+
+_unquoted_tokens = tokens2re({
+ 'whitespace': r'[\t\r\n ]+',
+ 'quote': r'[\'"]',
+ 'comment': '#',
+ 'special': r'[<>&|`~(){}$;]',
+ 'backslashed': r'\\[^\\]',
+ 'glob': r'[\*\?]',
+})
+
+_doubly_quoted_tokens = tokens2re({
+ 'quote': '"',
+ 'backslashedquote': r'\\"',
+ 'special': '\$',
+ 'backslashed': r'\\[^\\"]',
+})
+
+class MetaCharacterException(Exception):
+ def __init__(self, char):
+ self.char = char
+
+class ClineSplitter(list):
+ """
+ Parses a given command line string and creates a list of command
+ and arguments, with wildcard expansion.
+ """
+ def __init__(self, cline, cwd):
+ self.cwd = cwd
+ self.arg = None
+ self.cline = cline
+ self.glob = False
+ self._parse_unquoted()
+
+ def _push(self, str):
+ """
+ Push the given string as part of the current argument
+ """
+ if self.arg is None:
+ self.arg = ''
+ self.arg += str
+
+ def _next(self):
+ """
+ Finalize current argument, effectively adding it to the list.
+ Perform globbing if needed.
+ """
+ if self.arg is None:
+ return
+ if self.glob:
+ if os.path.isabs(self.arg):
+ path = self.arg
+ else:
+ path = os.path.join(self.cwd, self.arg)
+ globbed = glob.glob(path)
+ if not globbed:
+ # If globbing doesn't find anything, the literal string is
+ # used.
+ self.append(self.arg)
+ else:
+ self.extend(f[len(path)-len(self.arg):] for f in globbed)
+ self.glob = False
+ else:
+ self.append(self.arg)
+ self.arg = None
+
+ def _parse_unquoted(self):
+ """
+ Parse command line remainder in the context of an unquoted string.
+ """
+ while self.cline:
+ # Find the next token
+ m = _unquoted_tokens.search(self.cline)
+ # If we find none, the remainder of the string can be pushed to
+ # the current argument and the argument finalized
+ if not m:
+ self._push(self.cline)
+ break
+ # The beginning of the string, up to the found token, is part of
+ # the current argument
+ if m.start():
+ self._push(self.cline[:m.start()])
+ self.cline = self.cline[m.end():]
+
+ match = dict([(name, value) for name, value in m.groupdict().items() if value])
+ if 'quote' in match:
+ # " or ' start a quoted string
+ if match['quote'] == '"':
+ self._parse_doubly_quoted()
+ else:
+ self._parse_quoted()
+ elif 'comment' in match:
+ # Comments are ignored. The current argument can be finalized,
+ # and parsing stopped.
+ break
+ elif 'special' in match:
+ # Unquoted, non-escaped special characters need to be sent to a
+ # shell.
+ raise MetaCharacterException, match['special']
+ elif 'whitespace' in match:
+ # Whitespaces terminate current argument.
+ self._next()
+ elif 'escape' in match:
+ # Escaped backslashes turn into a single backslash
+ self._push('\\')
+ elif 'backslashed' in match:
+ # Backslashed characters are unbackslashed
+ # e.g. echo \a -> a
+ self._push(match['backslashed'][1])
+ elif 'glob' in match:
+ # ? or * will need globbing
+ self.glob = True
+ self._push(m.group(0))
+ else:
+ raise Exception, "Shouldn't reach here"
+ if self.arg:
+ self._next()
+
+ def _parse_quoted(self):
+ # Single quoted strings are preserved, except for the final quote
+ index = self.cline.find("'")
+ if index == -1:
+ raise Exception, 'Unterminated quoted string in command'
+ self._push(self.cline[:index])
+ self.cline = self.cline[index+1:]
+
+ def _parse_doubly_quoted(self):
+ if not self.cline:
+ raise Exception, 'Unterminated quoted string in command'
+ while self.cline:
+ m = _doubly_quoted_tokens.search(self.cline)
+ if not m:
+ raise Exception, 'Unterminated quoted string in command'
+ self._push(self.cline[:m.start()])
+ self.cline = self.cline[m.end():]
+ match = dict([(name, value) for name, value in m.groupdict().items() if value])
+ if 'quote' in match:
+ # a double quote ends the quoted string, so go back to
+ # unquoted parsing
+ return
+ elif 'special' in match:
+ # Unquoted, non-escaped special characters in a doubly quoted
+ # string still have a special meaning and need to be sent to a
+ # shell.
+ raise MetaCharacterException, match['special']
+ elif 'escape' in match:
+ # Escaped backslashes turn into a single backslash
+ self._push('\\')
+ elif 'backslashedquote' in match:
+ # Backslashed double quotes are un-backslashed
+ self._push('"')
+ elif 'backslashed' in match:
+ # Backslashed characters are kept backslashed
+ self._push(match['backslashed'])
+
+def clinetoargv(cline, cwd):
+ """
+ If this command line can safely skip the shell, return an argv array.
+ @returns argv, badchar
+ """
+ str = _escapednewlines.sub('', cline)
+ try:
+ args = ClineSplitter(str, cwd)
+ except MetaCharacterException, e:
+ return None, e.char
+
+ if len(args) and args[0].find('=') != -1:
+ return None, '='
+
+ return args, None
+
+# shellwords contains a set of shell builtin commands that need to be
+# executed within a shell. It also contains a set of commands that are known
+# to be giving problems when run directly instead of through the msys shell.
+shellwords = (':', '.', 'break', 'cd', 'continue', 'exec', 'exit', 'export',
+ 'getopts', 'hash', 'pwd', 'readonly', 'return', 'shift',
+ 'test', 'times', 'trap', 'umask', 'unset', 'alias',
+ 'set', 'bind', 'builtin', 'caller', 'command', 'declare',
+ 'echo', 'enable', 'help', 'let', 'local', 'logout',
+ 'printf', 'read', 'shopt', 'source', 'type', 'typeset',
+ 'ulimit', 'unalias', 'set', 'find')
+
+def prepare_command(cline, cwd, loc):
+ """
+ Returns a list of command and arguments for the given command line string.
+ If the command needs to be run through a shell for some reason, the
+ returned list contains the shell invocation.
+ """
+
+ #TODO: call this once up-front somewhere and save the result?
+ shell, msys = util.checkmsyscompat()
+
+ shellreason = None
+ executable = None
+ if msys and cline.startswith('/'):
+ shellreason = "command starts with /"
+ else:
+ argv, badchar = clinetoargv(cline, cwd)
+ if argv is None:
+ shellreason = "command contains shell-special character '%s'" % (badchar,)
+ elif len(argv) and argv[0] in shellwords:
+ shellreason = "command starts with shell primitive '%s'" % (argv[0],)
+ elif argv and (os.sep in argv[0] or os.altsep and os.altsep in argv[0]):
+ executable = util.normaljoin(cwd, argv[0])
+ # Avoid "%1 is not a valid Win32 application" errors, assuming
+ # that if the executable path is to be resolved with PATH, it will
+ # be a Win32 executable.
+ if sys.platform == 'win32' and os.path.isfile(executable) and open(executable, 'rb').read(2) == "#!":
+ shellreason = "command executable starts with a hashbang"
+
+ if shellreason is not None:
+ _log.debug("%s: using shell: %s: '%s'", loc, shellreason, cline)
+ if msys:
+ if len(cline) > 3 and cline[1] == ':' and cline[2] == '/':
+ cline = '/' + cline[0] + cline[2:]
+ argv = [shell, "-c", cline]
+ executable = None
+
+ return executable, argv
+
+def call(cline, env, cwd, loc, cb, context, echo, justprint=False):
+ executable, argv = prepare_command(cline, cwd, loc)
+
+ if not len(argv):
+ cb(res=0)
+ return
+
+ if argv[0] == command.makepypath:
+ command.main(argv[1:], env, cwd, cb)
+ return
+
+ if argv[0:2] == [sys.executable.replace('\\', '/'),
+ command.makepypath.replace('\\', '/')]:
+ command.main(argv[2:], env, cwd, cb)
+ return
+
+ context.call(argv, executable=executable, shell=False, env=env, cwd=cwd, cb=cb,
+ echo=echo, justprint=justprint)
+
+def call_native(module, method, argv, env, cwd, loc, cb, context, echo, justprint=False,
+ pycommandpath=None):
+ context.call_native(module, method, argv, env=env, cwd=cwd, cb=cb,
+ echo=echo, justprint=justprint, pycommandpath=pycommandpath)
+
+def statustoresult(status):
+ """
+ Convert the status returned from waitpid into a prettier numeric result.
+ """
+ sig = status & 0xFF
+ if sig:
+ return -sig
+
+ return status >>8
+
+class Job(object):
+ """
+ A single job to be executed on the process pool.
+ """
+ done = False # set to true when the job completes
+
+ def __init__(self):
+ self.exitcode = -127
+
+ def notify(self, condition, result):
+ condition.acquire()
+ self.done = True
+ self.exitcode = result
+ condition.notify()
+ condition.release()
+
+ def get_callback(self, condition):
+ return lambda result: self.notify(condition, result)
+
+class PopenJob(Job):
+ """
+ A job that executes a command using subprocess.Popen.
+ """
+ def __init__(self, argv, executable, shell, env, cwd):
+ Job.__init__(self)
+ self.argv = argv
+ self.executable = executable
+ self.shell = shell
+ self.env = env
+ self.cwd = cwd
+ self.parentpid = os.getpid()
+
+ def run(self):
+ assert os.getpid() != self.parentpid
+ # subprocess.Popen doesn't use the PATH set in the env argument for
+ # finding the executable on some platforms (but strangely it does on
+ # others!), so set os.environ['PATH'] explicitly. This is parallel-
+ # safe because pymake uses separate processes for parallelism, and
+ # each process is serial. See http://bugs.python.org/issue8557 for a
+ # general overview of "subprocess PATH semantics and portability".
+ oldpath = os.environ['PATH']
+ try:
+ if self.env is not None and self.env.has_key('PATH'):
+ os.environ['PATH'] = self.env['PATH']
+ p = subprocess.Popen(self.argv, executable=self.executable, shell=self.shell, env=self.env, cwd=self.cwd)
+ return p.wait()
+ except OSError, e:
+ print >>sys.stderr, e
+ return -127
+ finally:
+ os.environ['PATH'] = oldpath
+
+class PythonException(Exception):
+ def __init__(self, message, exitcode):
+ Exception.__init__(self)
+ self.message = message
+ self.exitcode = exitcode
+
+ def __str__(self):
+ return self.message
+
+
+class PythonJob(Job):
+ """
+ A job that calls a Python method.
+ """
+ def __init__(self, module, method, argv, env, cwd, pycommandpath=None):
+ self.module = module
+ self.method = method
+ self.argv = argv
+ self.env = env
+ self.cwd = cwd
+ self.pycommandpath = pycommandpath or []
+ self.parentpid = os.getpid()
+
+ def run(self):
+ assert os.getpid() != self.parentpid
+ # os.environ is a magic dictionary. Setting it to something else
+ # doesn't affect the environment of subprocesses, so use clear/update
+ oldenv = dict(os.environ)
+
+ # sys.path is adjusted for the entire lifetime of the command
+ # execution. This ensures any delayed imports will still work.
+ oldsyspath = list(sys.path)
+ try:
+ os.chdir(self.cwd)
+ os.environ.clear()
+ os.environ.update(self.env)
+
+ sys.path = []
+ for p in sys.path + self.pycommandpath:
+ site.addsitedir(p)
+ sys.path.extend(oldsyspath)
+
+ if self.module not in sys.modules:
+ try:
+ __import__(self.module)
+ except Exception as e:
+ print >>sys.stderr, 'Error importing %s: %s' % (
+ self.module, e)
+ return -127
+
+ m = sys.modules[self.module]
+ if self.method not in m.__dict__:
+ print >>sys.stderr, "No method named '%s' in module %s" % (self.method, self.module)
+ return -127
+ rv = m.__dict__[self.method](self.argv)
+ if rv != 0 and rv is not None:
+ print >>sys.stderr, (
+ "Native command '%s %s' returned value '%s'" %
+ (self.module, self.method, rv))
+ return (rv if isinstance(rv, int) else 1)
+
+ except PythonException, e:
+ print >>sys.stderr, e
+ return e.exitcode
+ except:
+ e = sys.exc_info()[1]
+ if isinstance(e, SystemExit) and (e.code == 0 or e.code is None):
+ pass # sys.exit(0) is not a failure
+ else:
+ print >>sys.stderr, e
+ traceback.print_exc()
+ return -127
+ finally:
+ os.environ.clear()
+ os.environ.update(oldenv)
+ sys.path = oldsyspath
+ # multiprocessing exits via os._exit, make sure that all output
+ # from command gets written out before that happens.
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ return 0
+
+def job_runner(job):
+ """
+ Run a job. Called in a Process pool.
+ """
+ return job.run()
+
+class ParallelContext(object):
+ """
+ Manages the parallel execution of processes.
+ """
+
+ _allcontexts = set()
+ _condition = multiprocessing.Condition()
+
+ def __init__(self, jcount):
+ self.jcount = jcount
+ self.exit = False
+
+ self.processpool = multiprocessing.Pool(processes=jcount)
+ self.pending = deque() # deque of (cb, args, kwargs)
+ self.running = [] # list of (subprocess, cb)
+
+ self._allcontexts.add(self)
+
+ def finish(self):
+ assert len(self.pending) == 0 and len(self.running) == 0, "pending: %i running: %i" % (len(self.pending), len(self.running))
+ self.processpool.close()
+ self.processpool.join()
+ self._allcontexts.remove(self)
+
+ def run(self):
+ while len(self.pending) and len(self.running) < self.jcount:
+ cb, args, kwargs = self.pending.popleft()
+ cb(*args, **kwargs)
+
+ def defer(self, cb, *args, **kwargs):
+ assert self.jcount > 1 or not len(self.pending), "Serial execution error defering %r %r %r: currently pending %r" % (cb, args, kwargs, self.pending)
+ self.pending.append((cb, args, kwargs))
+
+ def _docall_generic(self, pool, job, cb, echo, justprint):
+ if echo is not None:
+ print echo
+ processcb = job.get_callback(ParallelContext._condition)
+ if justprint:
+ processcb(0)
+ else:
+ pool.apply_async(job_runner, args=(job,), callback=processcb)
+ self.running.append((job, cb))
+
+ def call(self, argv, shell, env, cwd, cb, echo, justprint=False, executable=None):
+ """
+ Asynchronously call the process
+ """
+
+ job = PopenJob(argv, executable=executable, shell=shell, env=env, cwd=cwd)
+ self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint)
+
+ def call_native(self, module, method, argv, env, cwd, cb,
+ echo, justprint=False, pycommandpath=None):
+ """
+ Asynchronously call the native function
+ """
+
+ job = PythonJob(module, method, argv, env, cwd, pycommandpath)
+ self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint)
+
+ @staticmethod
+ def _waitany(condition):
+ def _checkdone():
+ jobs = []
+ for c in ParallelContext._allcontexts:
+ for i in xrange(0, len(c.running)):
+ if c.running[i][0].done:
+ jobs.append(c.running[i])
+ for j in jobs:
+ if j in c.running:
+ c.running.remove(j)
+ return jobs
+
+ # We must acquire the lock, and then check to see if any jobs have
+ # finished. If we don't check after acquiring the lock it's possible
+ # that all outstanding jobs will have completed before we wait and we'll
+ # wait for notifications that have already occurred.
+ condition.acquire()
+ jobs = _checkdone()
+
+ if jobs == []:
+ condition.wait()
+ jobs = _checkdone()
+
+ condition.release()
+
+ return jobs
+
+ @staticmethod
+ def spin():
+ """
+ Spin the 'event loop', and never return.
+ """
+
+ while True:
+ clist = list(ParallelContext._allcontexts)
+ for c in clist:
+ c.run()
+
+ dowait = util.any((len(c.running) for c in ParallelContext._allcontexts))
+ if dowait:
+ # Wait on local jobs first for perf
+ for job, cb in ParallelContext._waitany(ParallelContext._condition):
+ cb(job.exitcode)
+ else:
+ assert any(len(c.pending) for c in ParallelContext._allcontexts)
+
+def makedeferrable(usercb, **userkwargs):
+ def cb(*args, **kwargs):
+ kwargs.update(userkwargs)
+ return usercb(*args, **kwargs)
+
+ return cb
+
+_serialContext = None
+_parallelContext = None
+
+def getcontext(jcount):
+ global _serialContext, _parallelContext
+ if jcount == 1:
+ if _serialContext is None:
+ _serialContext = ParallelContext(1)
+ return _serialContext
+ else:
+ if _parallelContext is None:
+ _parallelContext = ParallelContext(jcount)
+ return _parallelContext
+
diff --git a/build/pymake/pymake/util.py b/build/pymake/pymake/util.py
new file mode 100644
index 000000000..c63f930cc
--- /dev/null
+++ b/build/pymake/pymake/util.py
@@ -0,0 +1,150 @@
+import os
+
+class MakeError(Exception):
+ def __init__(self, message, loc=None):
+ self.msg = message
+ self.loc = loc
+
+ def __str__(self):
+ locstr = ''
+ if self.loc is not None:
+ locstr = "%s:" % (self.loc,)
+
+ return "%s%s" % (locstr, self.msg)
+
+def normaljoin(path, suffix):
+ """
+ Combine the given path with the suffix, and normalize if necessary to shrink the path to avoid hitting path length limits
+ """
+ result = os.path.join(path, suffix)
+ if len(result) > 255:
+ result = os.path.normpath(result)
+ return result
+
+def joiniter(fd, it):
+ """
+ Given an iterator that returns strings, write the words with a space in between each.
+ """
+
+ it = iter(it)
+ for i in it:
+ fd.write(i)
+ break
+
+ for i in it:
+ fd.write(' ')
+ fd.write(i)
+
+def checkmsyscompat():
+ """For msys compatibility on windows, honor the SHELL environment variable,
+ and if $MSYSTEM == MINGW32, run commands through $SHELL -c instead of
+ letting Python use the system shell."""
+ if 'SHELL' in os.environ:
+ shell = os.environ['SHELL']
+ elif 'MOZILLABUILD' in os.environ:
+ shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+ elif 'COMSPEC' in os.environ:
+ shell = os.environ['COMSPEC']
+ else:
+ raise DataError("Can't find a suitable shell!")
+
+ msys = False
+ if 'MSYSTEM' in os.environ and os.environ['MSYSTEM'] == 'MINGW32':
+ msys = True
+ if not shell.lower().endswith(".exe"):
+ shell += ".exe"
+ return (shell, msys)
+
+if hasattr(str, 'partition'):
+ def strpartition(str, token):
+ return str.partition(token)
+
+ def strrpartition(str, token):
+ return str.rpartition(token)
+
+else:
+ def strpartition(str, token):
+ """Python 2.4 compatible str.partition"""
+
+ offset = str.find(token)
+ if offset == -1:
+ return str, '', ''
+
+ return str[:offset], token, str[offset + len(token):]
+
+ def strrpartition(str, token):
+ """Python 2.4 compatible str.rpartition"""
+
+ offset = str.rfind(token)
+ if offset == -1:
+ return '', '', str
+
+ return str[:offset], token, str[offset + len(token):]
+
+try:
+ from __builtin__ import any
+except ImportError:
+ def any(it):
+ for i in it:
+ if i:
+ return True
+ return False
+
+class _MostUsedItem(object):
+ __slots__ = ('key', 'o', 'count')
+
+ def __init__(self, key):
+ self.key = key
+ self.o = None
+ self.count = 1
+
+ def __repr__(self):
+ return "MostUsedItem(key=%r, count=%i, o=%r)" % (self.key, self.count, self.o)
+
+class MostUsedCache(object):
+ def __init__(self, capacity, creationfunc, verifyfunc):
+ self.capacity = capacity
+ self.cfunc = creationfunc
+ self.vfunc = verifyfunc
+
+ self.d = {}
+ self.active = [] # lazily sorted!
+
+ def setactive(self, item):
+ if item in self.active:
+ return
+
+ if len(self.active) == self.capacity:
+ self.active.sort(key=lambda i: i.count)
+ old = self.active.pop(0)
+ old.o = None
+ # print "Evicting %s" % old.key
+
+ self.active.append(item)
+
+ def get(self, key):
+ item = self.d.get(key, None)
+ if item is None:
+ item = _MostUsedItem(key)
+ self.d[key] = item
+ else:
+ item.count += 1
+
+ if item.o is not None and self.vfunc(key, item.o):
+ return item.o
+
+ item.o = self.cfunc(key)
+ self.setactive(item)
+ return item.o
+
+ def verify(self):
+ for k, v in self.d.iteritems():
+ if v.o:
+ assert v in self.active
+ else:
+ assert v not in self.active
+
+ def debugitems(self):
+ l = [i.key for i in self.active]
+ l.sort()
+ return l
diff --git a/build/pymake/pymake/win32process.py b/build/pymake/pymake/win32process.py
new file mode 100644
index 000000000..880a26a5b
--- /dev/null
+++ b/build/pymake/pymake/win32process.py
@@ -0,0 +1,28 @@
+from ctypes import windll, POINTER, byref, WinError
+from ctypes.wintypes import WINFUNCTYPE, HANDLE, DWORD, BOOL
+
+INFINITE = -1
+WAIT_FAILED = 0xFFFFFFFF
+
+LPDWORD = POINTER(DWORD)
+_GetExitCodeProcessProto = WINFUNCTYPE(BOOL, HANDLE, LPDWORD)
+_GetExitCodeProcess = _GetExitCodeProcessProto(("GetExitCodeProcess", windll.kernel32))
+def GetExitCodeProcess(h):
+ exitcode = DWORD()
+ r = _GetExitCodeProcess(h, byref(exitcode))
+ if r is 0:
+ raise WinError()
+ return exitcode.value
+
+_WaitForMultipleObjectsProto = WINFUNCTYPE(DWORD, DWORD, POINTER(HANDLE), BOOL, DWORD)
+_WaitForMultipleObjects = _WaitForMultipleObjectsProto(("WaitForMultipleObjects", windll.kernel32))
+
+def WaitForAnyProcess(processes):
+ arrtype = HANDLE * len(processes)
+ harray = arrtype(*(int(p._handle) for p in processes))
+
+ r = _WaitForMultipleObjects(len(processes), harray, False, INFINITE)
+ if r == WAIT_FAILED:
+ raise WinError()
+
+ return processes[r], GetExitCodeProcess(int(processes[r]._handle)) <<8
diff --git a/build/pymake/tests/automatic-variables.mk b/build/pymake/tests/automatic-variables.mk
new file mode 100644
index 000000000..5302c08ea
--- /dev/null
+++ b/build/pymake/tests/automatic-variables.mk
@@ -0,0 +1,79 @@
+$(shell \
+mkdir -p src/subd; \
+mkdir subd; \
+touch dummy; \
+sleep 2; \
+touch subd/test.out src/subd/test.in2; \
+sleep 2; \
+touch subd/test.out2 src/subd/test.in; \
+sleep 2; \
+touch subd/host_test.out subd/host_test.out2; \
+sleep 2; \
+touch host_prog; \
+)
+
+VPATH = src
+
+all: prog host_prog prog dir/
+ test "$@" = "all"
+ test "$<" = "prog"
+ test "$^" = "prog host_prog dir"
+ test "$?" = "prog host_prog dir"
+ test "$+" = "prog host_prog prog dir"
+ test "$(@D)" = "."
+ test "$(@F)" = "all"
+ test "$(<D)" = "."
+ test "$(<F)" = "prog"
+ test "$(^D)" = ". . ."
+ test "$(^F)" = "prog host_prog dir"
+ test "$(?D)" = ". . ."
+ test "$(?F)" = "prog host_prog dir"
+ test "$(+D)" = ". . . ."
+ test "$(+F)" = "prog host_prog prog dir"
+ @echo TEST-PASS
+
+dir/:
+ test "$@" = "dir"
+ test "$<" = ""
+ test "$^" = ""
+ test "$(@D)" = "."
+ test "$(@F)" = "dir"
+ mkdir $@
+
+prog: subd/test.out subd/test.out2
+ test "$@" = "prog"
+ test "$<" = "subd/test.out"
+ test "$^" = "subd/test.out subd/test.out2" # ^
+ test "$?" = "subd/test.out subd/test.out2" # ?
+ cat $<
+ test "$$(cat $<)" = "remade"
+ test "$$(cat $(word 2,$^))" = ""
+
+host_prog: subd/host_test.out subd/host_test.out2
+ @echo TEST-FAIL No need to remake
+
+%.out: %.in dummy
+ test "$@" = "subd/test.out"
+ test "$*" = "subd/test" # *
+ test "$<" = "src/subd/test.in" # <
+ test "$^" = "src/subd/test.in dummy" # ^
+ test "$?" = "src/subd/test.in" # ?
+ test "$+" = "src/subd/test.in dummy" # +
+ test "$(@D)" = "subd"
+ test "$(@F)" = "test.out"
+ test "$(*D)" = "subd"
+ test "$(*F)" = "test"
+ test "$(<D)" = "src/subd"
+ test "$(<F)" = "test.in"
+ test "$(^D)" = "src/subd ." # ^D
+ test "$(^F)" = "test.in dummy"
+ test "$(?D)" = "src/subd"
+ test "$(?F)" = "test.in"
+ test "$(+D)" = "src/subd ." # +D
+ test "$(+F)" = "test.in dummy"
+ printf "remade" >$@
+
+%.out2: %.in2 dummy
+ @echo TEST_FAIL No need to remake
+
+.PHONY: all
diff --git a/build/pymake/tests/bad-command-continuation.mk b/build/pymake/tests/bad-command-continuation.mk
new file mode 100644
index 000000000..d9ceccfc2
--- /dev/null
+++ b/build/pymake/tests/bad-command-continuation.mk
@@ -0,0 +1,3 @@
+all:
+ echo 'hello'\
+TEST-PASS
diff --git a/build/pymake/tests/call.mk b/build/pymake/tests/call.mk
new file mode 100644
index 000000000..9eeb7e00c
--- /dev/null
+++ b/build/pymake/tests/call.mk
@@ -0,0 +1,12 @@
+test = $0
+reverse = $2 $1
+twice = $1$1
+sideeffect = $(shell echo "called$1:" >>dummyfile)
+
+all:
+ test "$(call test)" = "test"
+ test "$(call reverse,1,2)" = "2 1"
+# expansion happens *before* substitution, thank sanity
+ test "$(call twice,$(sideeffect))" = ""
+ test `cat dummyfile` = "called:"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/cmd-stripdotslash.mk b/build/pymake/tests/cmd-stripdotslash.mk
new file mode 100644
index 000000000..ce5ed4244
--- /dev/null
+++ b/build/pymake/tests/cmd-stripdotslash.mk
@@ -0,0 +1,5 @@
+all:
+ $(MAKE) -f $(TESTPATH)/cmd-stripdotslash.mk ./foo
+
+./foo:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/cmdgoals.mk b/build/pymake/tests/cmdgoals.mk
new file mode 100644
index 000000000..a3b25e751
--- /dev/null
+++ b/build/pymake/tests/cmdgoals.mk
@@ -0,0 +1,9 @@
+default:
+ test "$(MAKECMDGOALS)" = ""
+ $(MAKE) -f $(TESTPATH)/cmdgoals.mk t1 t2
+ @echo TEST-PASS
+
+t1:
+ test "$(MAKECMDGOALS)" = "t1 t2"
+
+t2:
diff --git a/build/pymake/tests/commandmodifiers.mk b/build/pymake/tests/commandmodifiers.mk
new file mode 100644
index 000000000..8440462f3
--- /dev/null
+++ b/build/pymake/tests/commandmodifiers.mk
@@ -0,0 +1,21 @@
+define COMMAND
+$(1)
+ $(1)
+
+endef
+
+all:
+ $(call COMMAND,@true #TEST-FAIL)
+ $(call COMMAND,-exit 4)
+ $(call COMMAND,@-exit 1 # TEST-FAIL)
+ $(call COMMAND,-@exit 1 # TEST-FAIL)
+ $(call COMMAND,+exit 0)
+ $(call COMMAND,+-exit 1)
+ $(call COMMAND,@+exit 0 # TEST-FAIL)
+ $(call COMMAND,+@exit 0 # TEST-FAIL)
+ $(call COMMAND,-+@exit 1 # TEST-FAIL)
+ $(call COMMAND,+-@exit 1 # TEST-FAIL)
+ $(call COMMAND,@+-exit 1 # TEST-FAIL)
+ $(call COMMAND,@+-@+-exit 1 # TEST-FAIL)
+ $(call COMMAND,@@++exit 0 # TEST-FAIL)
+ @echo TEST-PASS
diff --git a/build/pymake/tests/comment-parsing.mk b/build/pymake/tests/comment-parsing.mk
new file mode 100644
index 000000000..d469e1aea
--- /dev/null
+++ b/build/pymake/tests/comment-parsing.mk
@@ -0,0 +1,29 @@
+# where do comments take effect?
+
+VAR = val1 # comment
+VAR2 = lit2\#hash
+VAR2_1 = lit2.1\\\#hash
+VAR3 = val3
+VAR4 = lit4\\#backslash
+VAR4_1 = lit4\\\\#backslash
+VAR5 = lit5\char
+VAR6 = lit6\\char
+VAR7 = lit7\\
+VAR8 = lit8\\\\
+VAR9 = lit9\\\\extra
+# This comment extends to the next line \
+VAR3 = ignored
+
+all:
+ test "$(VAR)" = "val1 "
+ test "$(VAR2)" = "lit2#hash"
+ test '$(VAR2_1)' = 'lit2.1\#hash'
+ test "$(VAR3)" = "val3"
+ test '$(VAR4)' = 'lit4\'
+ test '$(VAR4_1)' = 'lit4\\'
+ test '$(VAR5)' = 'lit5\char'
+ test '$(VAR6)' = 'lit6\\char'
+ test '$(VAR7)' = 'lit7\\'
+ test '$(VAR8)' = 'lit8\\\\'
+ test '$(VAR9)' = 'lit9\\\\extra'
+ @echo "TEST-PASS"
diff --git a/build/pymake/tests/continuations-in-functions.mk b/build/pymake/tests/continuations-in-functions.mk
new file mode 100644
index 000000000..533df6176
--- /dev/null
+++ b/build/pymake/tests/continuations-in-functions.mk
@@ -0,0 +1,6 @@
+all:
+ test 'Hello world.' = '$(if 1,Hello \
+ world.)'
+ test '(Hello world.)' != '(Hello \
+ world.)'
+ @echo TEST-PASS
diff --git a/build/pymake/tests/datatests.py b/build/pymake/tests/datatests.py
new file mode 100644
index 000000000..513028b0b
--- /dev/null
+++ b/build/pymake/tests/datatests.py
@@ -0,0 +1,237 @@
+import pymake.data, pymake.functions, pymake.util
+import unittest
+import re
+from cStringIO import StringIO
+
+def multitest(cls):
+ for name in cls.testdata.iterkeys():
+ def m(self, name=name):
+ return self.runSingle(*self.testdata[name])
+
+ setattr(cls, 'test_%s' % name, m)
+ return cls
+
+class SplitWordsTest(unittest.TestCase):
+ testdata = (
+ (' test test.c test.o ', ['test', 'test.c', 'test.o']),
+ ('\ttest\t test.c \ntest.o', ['test', 'test.c', 'test.o']),
+ )
+
+ def runTest(self):
+ for s, e in self.testdata:
+ w = s.split()
+ self.assertEqual(w, e, 'splitwords(%r)' % (s,))
+
+class GetPatSubstTest(unittest.TestCase):
+ testdata = (
+ ('%.c', '%.o', ' test test.c test.o ', 'test test.o test.o'),
+ ('%', '%.o', ' test.c test.o ', 'test.c.o test.o.o'),
+ ('foo', 'bar', 'test foo bar', 'test bar bar'),
+ ('foo', '%bar', 'test foo bar', 'test %bar bar'),
+ ('%', 'perc_%', 'path', 'perc_path'),
+ ('\\%', 'sub%', 'p %', 'p sub%'),
+ ('%.c', '\\%%.o', 'foo.c bar.o baz.cpp', '%foo.o bar.o baz.cpp'),
+ )
+
+ def runTest(self):
+ for s, r, d, e in self.testdata:
+ words = d.split()
+ p = pymake.data.Pattern(s)
+ a = ' '.join((p.subst(r, word, False)
+ for word in words))
+ self.assertEqual(a, e, 'Pattern(%r).subst(%r, %r)' % (s, r, d))
+
+class LRUTest(unittest.TestCase):
+ # getkey, expected, funccount, debugitems
+ expected = (
+ (0, '', 1, (0,)),
+ (0, '', 2, (0,)),
+ (1, ' ', 3, (1, 0)),
+ (1, ' ', 3, (1, 0)),
+ (0, '', 4, (0, 1)),
+ (2, ' ', 5, (2, 0, 1)),
+ (1, ' ', 5, (1, 2, 0)),
+ (3, ' ', 6, (3, 1, 2)),
+ )
+
+ def spaceFunc(self, l):
+ self.funccount += 1
+ return ''.ljust(l)
+
+ def runTest(self):
+ self.funccount = 0
+ c = pymake.util.LRUCache(3, self.spaceFunc, lambda k, v: k % 2)
+ self.assertEqual(tuple(c.debugitems()), ())
+
+ for i in xrange(0, len(self.expected)):
+ k, e, fc, di = self.expected[i]
+
+ v = c.get(k)
+ self.assertEqual(v, e)
+ self.assertEqual(self.funccount, fc,
+ "funccount, iteration %i, got %i expected %i" % (i, self.funccount, fc))
+ goti = tuple(c.debugitems())
+ self.assertEqual(goti, di,
+ "debugitems, iteration %i, got %r expected %r" % (i, goti, di))
+
+class EqualityTest(unittest.TestCase):
+ def test_string_expansion(self):
+ s1 = pymake.data.StringExpansion('foo bar', None)
+ s2 = pymake.data.StringExpansion('foo bar', None)
+
+ self.assertEqual(s1, s2)
+
+ def test_expansion_simple(self):
+ s1 = pymake.data.Expansion(None)
+ s2 = pymake.data.Expansion(None)
+
+ self.assertEqual(s1, s2)
+
+ s1.appendstr('foo')
+ s2.appendstr('foo')
+ self.assertEqual(s1, s2)
+
+ def test_expansion_string_finish(self):
+ """Adjacent strings should normalize to same value."""
+ s1 = pymake.data.Expansion(None)
+ s2 = pymake.data.Expansion(None)
+
+ s1.appendstr('foo')
+ s2.appendstr('foo')
+
+ s1.appendstr(' bar')
+ s1.appendstr(' baz')
+ s2.appendstr(' bar baz')
+
+ self.assertEqual(s1, s2)
+
+ def test_function(self):
+ s1 = pymake.data.Expansion(None)
+ s2 = pymake.data.Expansion(None)
+
+ n1 = pymake.data.StringExpansion('FOO', None)
+ n2 = pymake.data.StringExpansion('FOO', None)
+
+ v1 = pymake.functions.VariableRef(None, n1)
+ v2 = pymake.functions.VariableRef(None, n2)
+
+ s1.appendfunc(v1)
+ s2.appendfunc(v2)
+
+ self.assertEqual(s1, s2)
+
+
+class StringExpansionTest(unittest.TestCase):
+ def test_base_expansion_interface(self):
+ s1 = pymake.data.StringExpansion('FOO', None)
+
+ self.assertTrue(s1.is_static_string)
+
+ funcs = list(s1.functions())
+ self.assertEqual(len(funcs), 0)
+
+ funcs = list(s1.functions(True))
+ self.assertEqual(len(funcs), 0)
+
+ refs = list(s1.variable_references())
+ self.assertEqual(len(refs), 0)
+
+
+class ExpansionTest(unittest.TestCase):
+ def test_is_static_string(self):
+ e1 = pymake.data.Expansion()
+ e1.appendstr('foo')
+
+ self.assertTrue(e1.is_static_string)
+
+ e1.appendstr('bar')
+ self.assertTrue(e1.is_static_string)
+
+ vname = pymake.data.StringExpansion('FOO', None)
+ func = pymake.functions.VariableRef(None, vname)
+
+ e1.appendfunc(func)
+
+ self.assertFalse(e1.is_static_string)
+
+ def test_get_functions(self):
+ e1 = pymake.data.Expansion()
+ e1.appendstr('foo')
+
+ vname1 = pymake.data.StringExpansion('FOO', None)
+ vname2 = pymake.data.StringExpansion('BAR', None)
+
+ func1 = pymake.functions.VariableRef(None, vname1)
+ func2 = pymake.functions.VariableRef(None, vname2)
+
+ e1.appendfunc(func1)
+ e1.appendfunc(func2)
+
+ funcs = list(e1.functions())
+ self.assertEqual(len(funcs), 2)
+
+ func3 = pymake.functions.SortFunction(None)
+ func3.append(vname1)
+
+ e1.appendfunc(func3)
+
+ funcs = list(e1.functions())
+ self.assertEqual(len(funcs), 3)
+
+ refs = list(e1.variable_references())
+ self.assertEqual(len(refs), 2)
+
+ def test_get_functions_descend(self):
+ e1 = pymake.data.Expansion()
+ vname1 = pymake.data.StringExpansion('FOO', None)
+ func1 = pymake.functions.VariableRef(None, vname1)
+ e2 = pymake.data.Expansion()
+ e2.appendfunc(func1)
+
+ func2 = pymake.functions.SortFunction(None)
+ func2.append(e2)
+
+ e1.appendfunc(func2)
+
+ funcs = list(e1.functions())
+ self.assertEqual(len(funcs), 1)
+
+ funcs = list(e1.functions(True))
+ self.assertEqual(len(funcs), 2)
+
+ self.assertTrue(isinstance(funcs[0], pymake.functions.SortFunction))
+
+ def test_is_filesystem_dependent(self):
+ e = pymake.data.Expansion()
+ vname1 = pymake.data.StringExpansion('FOO', None)
+ func1 = pymake.functions.VariableRef(None, vname1)
+ e.appendfunc(func1)
+
+ self.assertFalse(e.is_filesystem_dependent)
+
+ func2 = pymake.functions.WildcardFunction(None)
+ func2.append(vname1)
+ e.appendfunc(func2)
+
+ self.assertTrue(e.is_filesystem_dependent)
+
+ def test_is_filesystem_dependent_descend(self):
+ sort = pymake.functions.SortFunction(None)
+ wildcard = pymake.functions.WildcardFunction(None)
+
+ e = pymake.data.StringExpansion('foo/*', None)
+ wildcard.append(e)
+
+ e = pymake.data.Expansion(None)
+ e.appendfunc(wildcard)
+
+ sort.append(e)
+
+ e = pymake.data.Expansion(None)
+ e.appendfunc(sort)
+
+ self.assertTrue(e.is_filesystem_dependent)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/build/pymake/tests/default-goal-set-first.mk b/build/pymake/tests/default-goal-set-first.mk
new file mode 100644
index 000000000..00a5b53a2
--- /dev/null
+++ b/build/pymake/tests/default-goal-set-first.mk
@@ -0,0 +1,7 @@
+.DEFAULT_GOAL := default
+
+not-default:
+ @echo TEST-FAIL did not run default rule
+
+default:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/default-goal.mk b/build/pymake/tests/default-goal.mk
new file mode 100644
index 000000000..699d6c0cd
--- /dev/null
+++ b/build/pymake/tests/default-goal.mk
@@ -0,0 +1,8 @@
+not-default:
+ @echo TEST-FAIL did not run default rule
+
+default:
+ @echo $(if $(filter not-default,$(INTERMEDIATE_DEFAULT_GOAL)),TEST-PASS,TEST-FAIL .DEFAULT_GOAL not set by $(MAKE))
+
+INTERMEDIATE_DEFAULT_GOAL := $(.DEFAULT_GOAL)
+.DEFAULT_GOAL := default
diff --git a/build/pymake/tests/default-target.mk b/build/pymake/tests/default-target.mk
new file mode 100644
index 000000000..701ac6916
--- /dev/null
+++ b/build/pymake/tests/default-target.mk
@@ -0,0 +1,14 @@
+test: VAR = value
+
+%.do:
+ @echo TEST-FAIL: ran target "$@", should have run "all"
+
+.PHONY: test
+
+all:
+ @echo TEST-PASS: the default target is all
+
+test:
+ @echo TEST-FAIL: ran target "$@", should have run "all"
+
+test.do:
diff --git a/build/pymake/tests/default-target2.mk b/build/pymake/tests/default-target2.mk
new file mode 100644
index 000000000..b5a4b1bbf
--- /dev/null
+++ b/build/pymake/tests/default-target2.mk
@@ -0,0 +1,6 @@
+test.foo: %.foo:
+ test "$@" = "test.foo"
+ @echo TEST-PASS made test.foo by default
+
+all:
+ @echo TEST-FAIL made $@, should have made test.foo
diff --git a/build/pymake/tests/define-directive.mk b/build/pymake/tests/define-directive.mk
new file mode 100644
index 000000000..789988666
--- /dev/null
+++ b/build/pymake/tests/define-directive.mk
@@ -0,0 +1,69 @@
+define COMMANDS
+shellvar=hello
+test "$$shellvar" != "hello"
+endef
+
+define COMMANDS2
+shellvar=hello; \
+ test "$$shellvar" = "hello"
+endef
+
+define VARWITHCOMMENT # comment
+value
+endef
+
+define TEST3
+ whitespace
+endef
+
+define TEST4
+define TEST5
+random
+endef
+ endef
+
+ifdef TEST5
+$(error TEST5 should not be set)
+endif
+
+define TEST6
+ define TEST7
+random
+endef
+endef
+
+ifdef TEST7
+$(error TEST7 should not be set)
+endif
+
+define TEST8
+is this # a comment?
+endef
+
+ifneq ($(TEST8),is this \# a comment?)
+$(error TEST8 value not expected: $(TEST8))
+endif
+
+# A backslash continuation "hides" the endef
+define TEST9
+value \
+endef
+endef
+
+# Test ridiculous spacing
+ define TEST10
+ define TEST11
+ baz
+endef
+define TEST12
+ foo
+ endef
+ endef
+
+all:
+ $(COMMANDS)
+ $(COMMANDS2)
+ test '$(VARWITHCOMMENT)' = 'value'
+ test '$(COMMANDS2)' = 'shellvar=hello; test "$$shellvar" = "hello"'
+ test "$(TEST3)" = " whitespace"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/depfailed.mk b/build/pymake/tests/depfailed.mk
new file mode 100644
index 000000000..ce4137c38
--- /dev/null
+++ b/build/pymake/tests/depfailed.mk
@@ -0,0 +1,4 @@
+#T returncode: 2
+
+all: foo.out foo.in
+ @echo TEST-PASS
diff --git a/build/pymake/tests/depfailedj.mk b/build/pymake/tests/depfailedj.mk
new file mode 100644
index 000000000..a94c74f6f
--- /dev/null
+++ b/build/pymake/tests/depfailedj.mk
@@ -0,0 +1,10 @@
+#T returncode: 2
+#T commandline: ['-j4']
+
+$(shell touch foo.in)
+
+all: foo.in foo.out missing
+ @echo TEST-PASS
+
+%.out: %.in
+ cp $< $@
diff --git a/build/pymake/tests/diamond-deps.mk b/build/pymake/tests/diamond-deps.mk
new file mode 100644
index 000000000..40a4176d9
--- /dev/null
+++ b/build/pymake/tests/diamond-deps.mk
@@ -0,0 +1,13 @@
+# If the dependency graph includes a diamond dependency, we should only remake
+# once!
+
+all: depA depB
+ cat testfile
+ test `cat testfile` = "data";
+ @echo TEST-PASS
+
+depA: testfile
+depB: testfile
+
+testfile:
+ printf "data" >>$@
diff --git a/build/pymake/tests/dotslash-dir.mk b/build/pymake/tests/dotslash-dir.mk
new file mode 100644
index 000000000..8b30d1e3c
--- /dev/null
+++ b/build/pymake/tests/dotslash-dir.mk
@@ -0,0 +1,8 @@
+#T grep-for: "dotslash-built"
+.PHONY: $(dir foo)
+
+all: $(dir foo)
+ @echo TEST-PASS
+
+$(dir foo):
+ @echo dotslash-built
diff --git a/build/pymake/tests/dotslash-parse.mk b/build/pymake/tests/dotslash-parse.mk
new file mode 100644
index 000000000..91461bedb
--- /dev/null
+++ b/build/pymake/tests/dotslash-parse.mk
@@ -0,0 +1,4 @@
+./:
+
+# This is merely a test to see that pymake doesn't choke on parsing ./
+$(info TEST-PASS)
diff --git a/build/pymake/tests/dotslash-phony.mk b/build/pymake/tests/dotslash-phony.mk
new file mode 100644
index 000000000..06b6ae78d
--- /dev/null
+++ b/build/pymake/tests/dotslash-phony.mk
@@ -0,0 +1,3 @@
+.PHONY: ./
+./:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/dotslash.mk b/build/pymake/tests/dotslash.mk
new file mode 100644
index 000000000..585db96b7
--- /dev/null
+++ b/build/pymake/tests/dotslash.mk
@@ -0,0 +1,9 @@
+$(shell touch foo.in)
+
+all: foo.out
+ test "$(wildcard ./*.in)" = "./foo.in"
+ @echo TEST-PASS
+
+./%.out: %.in
+ test "$@" = "foo.out"
+ cp $< $@
diff --git a/build/pymake/tests/doublecolon-exists.mk b/build/pymake/tests/doublecolon-exists.mk
new file mode 100644
index 000000000..5d99a1f6b
--- /dev/null
+++ b/build/pymake/tests/doublecolon-exists.mk
@@ -0,0 +1,16 @@
+$(shell touch foo.testfile1 foo.testfile2)
+
+# when a rule has commands and no prerequisites, should it be executed?
+# double-colon: yes
+# single-colon: no
+
+all: foo.testfile1 foo.testfile2
+ test "$$(cat foo.testfile1)" = ""
+ test "$$(cat foo.testfile2)" = "remade:foo.testfile2"
+ @echo TEST-PASS
+
+foo.testfile1:
+ @echo TEST-FAIL
+
+foo.testfile2::
+ printf "remade:$@"> $@
diff --git a/build/pymake/tests/doublecolon-priordeps.mk b/build/pymake/tests/doublecolon-priordeps.mk
new file mode 100644
index 000000000..6cdf3a8e7
--- /dev/null
+++ b/build/pymake/tests/doublecolon-priordeps.mk
@@ -0,0 +1,19 @@
+#T commandline: ['-j3']
+
+# All *prior* dependencies of a doublecolon rule must be satisfied before
+# subsequent commands are run.
+
+all:: target1
+
+all:: target2
+ test -f target1
+ @echo TEST-PASS
+
+target1:
+ touch starting-$@
+ sleep 1
+ touch $@
+
+target2:
+ sleep 0.1
+ test -f starting-target1
diff --git a/build/pymake/tests/doublecolon-remake.mk b/build/pymake/tests/doublecolon-remake.mk
new file mode 100644
index 000000000..52aa9265c
--- /dev/null
+++ b/build/pymake/tests/doublecolon-remake.mk
@@ -0,0 +1,4 @@
+$(shell touch somefile)
+
+all:: somefile
+ @echo TEST-PASS
diff --git a/build/pymake/tests/dynamic-var.mk b/build/pymake/tests/dynamic-var.mk
new file mode 100644
index 000000000..0993b9ccf
--- /dev/null
+++ b/build/pymake/tests/dynamic-var.mk
@@ -0,0 +1,18 @@
+# The *name* of variables can be constructed dynamically.
+
+VARNAME = FOOBAR
+
+$(VARNAME) = foovalue
+$(VARNAME)2 = foo2value
+
+$(VARNAME:%BAR=%BAM) = foobam
+
+all:
+ test "$(FOOBAR)" = "foovalue"
+ test "$(flavor FOOBAZ)" = "undefined"
+ test "$(FOOBAR2)" = "bazvalue"
+ test "$(FOOBAM)" = "foobam"
+ @echo TEST-PASS
+
+VARNAME = FOOBAZ
+FOOBAR2 = bazvalue
diff --git a/build/pymake/tests/empty-arg.mk b/build/pymake/tests/empty-arg.mk
new file mode 100644
index 000000000..616e5b694
--- /dev/null
+++ b/build/pymake/tests/empty-arg.mk
@@ -0,0 +1,2 @@
+all:
+ @ sh -c 'if [ $$# = 3 ] ; then echo TEST-PASS; else echo TEST-FAIL; fi' -- a "" b
diff --git a/build/pymake/tests/empty-command-semicolon.mk b/build/pymake/tests/empty-command-semicolon.mk
new file mode 100644
index 000000000..07789f3f1
--- /dev/null
+++ b/build/pymake/tests/empty-command-semicolon.mk
@@ -0,0 +1,5 @@
+all:
+ @echo TEST-PASS
+
+foo: ;
+
diff --git a/build/pymake/tests/empty-with-deps.mk b/build/pymake/tests/empty-with-deps.mk
new file mode 100644
index 000000000..284e5a113
--- /dev/null
+++ b/build/pymake/tests/empty-with-deps.mk
@@ -0,0 +1,4 @@
+default.test: default.c
+
+default.c:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/env-var-append.mk b/build/pymake/tests/env-var-append.mk
new file mode 100644
index 000000000..4db39c45f
--- /dev/null
+++ b/build/pymake/tests/env-var-append.mk
@@ -0,0 +1,7 @@
+#T environment: {'FOO': 'TEST'}
+
+FOO += $(BAR)
+BAR := PASS
+
+all:
+ @echo $(subst $(NULL) ,-,$(FOO))
diff --git a/build/pymake/tests/env-var-append2.mk b/build/pymake/tests/env-var-append2.mk
new file mode 100644
index 000000000..fc0735d88
--- /dev/null
+++ b/build/pymake/tests/env-var-append2.mk
@@ -0,0 +1,8 @@
+#T environment: {'FOO': '$(BAZ)'}
+
+FOO += $(BAR)
+BAR := PASS
+BAZ := TEST
+
+all:
+ @echo $(subst $(NULL) ,-,$(FOO))
diff --git a/build/pymake/tests/eof-continuation.mk b/build/pymake/tests/eof-continuation.mk
new file mode 100644
index 000000000..daeaabc3e
--- /dev/null
+++ b/build/pymake/tests/eof-continuation.mk
@@ -0,0 +1,5 @@
+all:
+ test '$(TESTVAR)' = 'testval\'
+ @echo TEST-PASS
+
+TESTVAR = testval\ \ No newline at end of file
diff --git a/build/pymake/tests/escape-chars.mk b/build/pymake/tests/escape-chars.mk
new file mode 100644
index 000000000..ebea33074
--- /dev/null
+++ b/build/pymake/tests/escape-chars.mk
@@ -0,0 +1,26 @@
+space = $(NULL) $(NULL)
+hello$(space)world$(space) = hellovalue
+
+A = aval
+
+VAR = value1\\
+VARAWFUL = value1\\#comment
+VAR2 = value2
+VAR3 = test\$A
+VAR4 = value4\\value5
+
+VAR5 = value1\\ \ \
+ value2
+
+EPERCENT = \%
+
+all:
+ test "$(hello world )" = "hellovalue"
+ test "$(VAR)" = "value1\\"
+ test '$(VARAWFUL)' = 'value1\'
+ test "$(VAR2)" = "value2"
+ test "$(VAR3)" = "test\aval"
+ test "$(VAR4)" = "value4\\value5"
+ test "$(VAR5)" = "value1\\ \ value2"
+ test "$(EPERCENT)" = "\%"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/escaped-continuation.mk b/build/pymake/tests/escaped-continuation.mk
new file mode 100644
index 000000000..537f7547f
--- /dev/null
+++ b/build/pymake/tests/escaped-continuation.mk
@@ -0,0 +1,6 @@
+#T returncode: 2
+
+all:
+ echo "Hello" \\
+ test "world" = "not!"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/eval-duringexecute.mk b/build/pymake/tests/eval-duringexecute.mk
new file mode 100644
index 000000000..dff848032
--- /dev/null
+++ b/build/pymake/tests/eval-duringexecute.mk
@@ -0,0 +1,12 @@
+#T returncode: 2
+
+# Once parsing is finished, recursive expansion in commands are not allowed to create any new rules (it may only set variables)
+
+define MORERULE
+all:
+ @echo TEST-FAIL
+endef
+
+all:
+ $(eval $(MORERULE))
+ @echo done
diff --git a/build/pymake/tests/eval.mk b/build/pymake/tests/eval.mk
new file mode 100644
index 000000000..de9759f02
--- /dev/null
+++ b/build/pymake/tests/eval.mk
@@ -0,0 +1,7 @@
+TESTVAR = val1
+
+$(eval TESTVAR = val2)
+
+all:
+ test "$(TESTVAR)" = "val2"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/exit-code.mk b/build/pymake/tests/exit-code.mk
new file mode 100644
index 000000000..84dcffcf9
--- /dev/null
+++ b/build/pymake/tests/exit-code.mk
@@ -0,0 +1,5 @@
+#T returncode: 2
+
+all:
+ exit 1
+ @echo TEST-PASS
diff --git a/build/pymake/tests/file-functions-symlinks.mk b/build/pymake/tests/file-functions-symlinks.mk
new file mode 100644
index 000000000..dcc0f6eef
--- /dev/null
+++ b/build/pymake/tests/file-functions-symlinks.mk
@@ -0,0 +1,22 @@
+#T returncode-on: {'win32': 2}
+$(shell \
+touch test.file; \
+ln -s test.file test.symlink; \
+ln -s test.missing missing.symlink; \
+touch .testhidden; \
+mkdir foo; \
+touch foo/testfile; \
+ln -s foo symdir; \
+)
+
+all:
+ test "$(abspath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.symlink"
+ test "$(realpath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.file"
+ test "$(sort $(wildcard *))" = "foo symdir test.file test.symlink"
+ test "$(sort $(wildcard .*))" = ". .. .testhidden"
+ test "$(sort $(wildcard test*))" = "test.file test.symlink"
+ test "$(sort $(wildcard foo/*))" = "foo/testfile"
+ test "$(sort $(wildcard ./*))" = "./foo ./symdir ./test.file ./test.symlink"
+ test "$(sort $(wildcard f?o/*))" = "foo/testfile"
+ test "$(sort $(wildcard */*))" = "foo/testfile symdir/testfile"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/file-functions.mk b/build/pymake/tests/file-functions.mk
new file mode 100644
index 000000000..7e4c68e85
--- /dev/null
+++ b/build/pymake/tests/file-functions.mk
@@ -0,0 +1,19 @@
+$(shell \
+touch test.file; \
+touch .testhidden; \
+mkdir foo; \
+touch foo/testfile; \
+)
+
+all:
+ test "$(abspath test.file)" = "$(CURDIR)/test.file"
+ test "$(realpath test.file)" = "$(CURDIR)/test.file"
+ test "$(sort $(wildcard *))" = "foo test.file"
+# commented out because GNU make matches . and .. while python doesn't, and I don't
+# care enough
+# test "$(sort $(wildcard .*))" = ". .. .testhidden"
+ test "$(sort $(wildcard test*))" = "test.file"
+ test "$(sort $(wildcard foo/*))" = "foo/testfile"
+ test "$(sort $(wildcard ./*))" = "./foo ./test.file"
+ test "$(sort $(wildcard f?o/*))" = "foo/testfile"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/foreach-local-variable.mk b/build/pymake/tests/foreach-local-variable.mk
new file mode 100644
index 000000000..2551621eb
--- /dev/null
+++ b/build/pymake/tests/foreach-local-variable.mk
@@ -0,0 +1,8 @@
+# This test ensures that a local variable in a $(foreach) is bound to
+# the local value, not a global value.
+i := dummy
+
+all:
+ test "$(foreach i,foo bar,found:$(i))" = "found:foo found:bar"
+ test "$(i)" = "dummy"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/formattingtests.py b/build/pymake/tests/formattingtests.py
new file mode 100644
index 000000000..7aad6d4cc
--- /dev/null
+++ b/build/pymake/tests/formattingtests.py
@@ -0,0 +1,289 @@
+# This file contains test code for the formatting of parsed statements back to
+# make file "source." It essentially verifies to to_source() functions
+# scattered across the tree.
+
+import glob
+import logging
+import os.path
+import unittest
+
+from pymake.data import Expansion
+from pymake.data import StringExpansion
+from pymake.functions import BasenameFunction
+from pymake.functions import SubstitutionRef
+from pymake.functions import VariableRef
+from pymake.functions import WordlistFunction
+from pymake.parserdata import Include
+from pymake.parserdata import SetVariable
+from pymake.parser import parsestring
+from pymake.parser import SyntaxError
+
+class TestBase(unittest.TestCase):
+ pass
+
+class VariableRefTest(TestBase):
+ def test_string_name(self):
+ e = StringExpansion('foo', None)
+ v = VariableRef(None, e)
+
+ self.assertEqual(v.to_source(), '$(foo)')
+
+ def test_special_variable(self):
+ e = StringExpansion('<', None)
+ v = VariableRef(None, e)
+
+ self.assertEqual(v.to_source(), '$<')
+
+ def test_expansion_simple(self):
+ e = Expansion()
+ e.appendstr('foo')
+ e.appendstr('bar')
+
+ v = VariableRef(None, e)
+
+ self.assertEqual(v.to_source(), '$(foobar)')
+
+class StandardFunctionTest(TestBase):
+ def test_basename(self):
+ e1 = StringExpansion('foo', None)
+ v = VariableRef(None, e1)
+ e2 = Expansion(None)
+ e2.appendfunc(v)
+
+ b = BasenameFunction(None)
+ b.append(e2)
+
+ self.assertEqual(b.to_source(), '$(basename $(foo))')
+
+ def test_wordlist(self):
+ e1 = StringExpansion('foo', None)
+ e2 = StringExpansion('bar ', None)
+ e3 = StringExpansion(' baz', None)
+
+ w = WordlistFunction(None)
+ w.append(e1)
+ w.append(e2)
+ w.append(e3)
+
+ self.assertEqual(w.to_source(), '$(wordlist foo,bar , baz)')
+
+ def test_curly_brackets(self):
+ e1 = Expansion(None)
+ e1.appendstr('foo')
+
+ e2 = Expansion(None)
+ e2.appendstr('foo ( bar')
+
+ f = WordlistFunction(None)
+ f.append(e1)
+ f.append(e2)
+
+ self.assertEqual(f.to_source(), '${wordlist foo,foo ( bar}')
+
+class StringExpansionTest(TestBase):
+ def test_simple(self):
+ e = StringExpansion('foobar', None)
+ self.assertEqual(e.to_source(), 'foobar')
+
+ e = StringExpansion('$var', None)
+ self.assertEqual(e.to_source(), '$var')
+
+ def test_escaping(self):
+ e = StringExpansion('$var', None)
+ self.assertEqual(e.to_source(escape_variables=True), '$$var')
+
+ e = StringExpansion('this is # not a comment', None)
+ self.assertEqual(e.to_source(escape_comments=True),
+ 'this is \# not a comment')
+
+ def test_empty(self):
+ e = StringExpansion('', None)
+ self.assertEqual(e.to_source(), '')
+
+ e = StringExpansion(' ', None)
+ self.assertEqual(e.to_source(), ' ')
+
+class ExpansionTest(TestBase):
+ def test_single_string(self):
+ e = Expansion()
+ e.appendstr('foo')
+
+ self.assertEqual(e.to_source(), 'foo')
+
+ def test_multiple_strings(self):
+ e = Expansion()
+ e.appendstr('hello')
+ e.appendstr('world')
+
+ self.assertEqual(e.to_source(), 'helloworld')
+
+ def test_string_escape(self):
+ e = Expansion()
+ e.appendstr('$var')
+ self.assertEqual(e.to_source(), '$var')
+ self.assertEqual(e.to_source(escape_variables=True), '$$var')
+
+ e = Expansion()
+ e.appendstr('foo')
+ e.appendstr(' $bar')
+ self.assertEqual(e.to_source(escape_variables=True), 'foo $$bar')
+
+class SubstitutionRefTest(TestBase):
+ def test_simple(self):
+ name = StringExpansion('foo', None)
+ c = StringExpansion('%.c', None)
+ o = StringExpansion('%.o', None)
+ s = SubstitutionRef(None, name, c, o)
+
+ self.assertEqual(s.to_source(), '$(foo:%.c=%.o)')
+
+class SetVariableTest(TestBase):
+ def test_simple(self):
+ v = SetVariable(StringExpansion('foo', None), '=', 'bar', None, None)
+ self.assertEqual(v.to_source(), 'foo = bar')
+
+ def test_multiline(self):
+ s = 'hello\nworld'
+ foo = StringExpansion('FOO', None)
+
+ v = SetVariable(foo, '=', s, None, None)
+
+ self.assertEqual(v.to_source(), 'define FOO\nhello\nworld\nendef')
+
+ def test_multiline_immediate(self):
+ source = 'define FOO :=\nhello\nworld\nendef'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements.to_source(), source)
+
+ def test_target_specific(self):
+ foo = StringExpansion('FOO', None)
+ bar = StringExpansion('BAR', None)
+
+ v = SetVariable(foo, '+=', 'value', None, bar)
+
+ self.assertEqual(v.to_source(), 'BAR: FOO += value')
+
+class IncludeTest(TestBase):
+ def test_include(self):
+ e = StringExpansion('rules.mk', None)
+ i = Include(e, True, False)
+ self.assertEqual(i.to_source(), 'include rules.mk')
+
+ i = Include(e, False, False)
+ self.assertEqual(i.to_source(), '-include rules.mk')
+
+class IfdefTest(TestBase):
+ def test_simple(self):
+ source = 'ifdef FOO\nbar := $(value)\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements[0].to_source(), source)
+
+ def test_nested(self):
+ source = 'ifdef FOO\nifdef BAR\nhello = world\nendif\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements[0].to_source(), source)
+
+ def test_negation(self):
+ source = 'ifndef FOO\nbar += value\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements[0].to_source(), source)
+
+class IfeqTest(TestBase):
+ def test_simple(self):
+ source = 'ifeq ($(foo),bar)\nhello = $(world)\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements[0].to_source(), source)
+
+ def test_negation(self):
+ source = 'ifneq (foo,bar)\nhello = world\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements.to_source(), source)
+
+class ConditionBlocksTest(TestBase):
+ def test_mixed_conditions(self):
+ source = 'ifdef FOO\nifeq ($(FOO),bar)\nvar += $(value)\nendif\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements.to_source(), source)
+
+ def test_extra_statements(self):
+ source = 'ifdef FOO\nF := 1\nifdef BAR\nB += 1\nendif\nC = 1\nendif'
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements.to_source(), source)
+
+ def test_whitespace_preservation(self):
+ source = "ifeq ' x' 'x '\n$(error stripping)\nendif"
+
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements.to_source(), source)
+
+ source = 'ifneq (x , x)\n$(error stripping)\nendif'
+ statements = parsestring(source, 'foo.mk')
+ self.assertEqual(statements.to_source(),
+ 'ifneq (x,x)\n$(error stripping)\nendif')
+
+class MakefileCorupusTest(TestBase):
+ """Runs the make files from the pymake corpus through the formatter.
+
+ All the above tests are child's play compared to this.
+ """
+
+ # Our reformatting isn't perfect. We ignore files with known failures until
+ # we make them work.
+ # TODO Address these formatting corner cases.
+ _IGNORE_FILES = [
+ # We are thrown off by backslashes at end of lines.
+ 'comment-parsing.mk',
+ 'escape-chars.mk',
+ 'include-notfound.mk',
+ ]
+
+ def _get_test_files(self):
+ ourdir = os.path.dirname(os.path.abspath(__file__))
+
+ for makefile in glob.glob(os.path.join(ourdir, '*.mk')):
+ if os.path.basename(makefile) in self._IGNORE_FILES:
+ continue
+
+ source = None
+ with open(makefile, 'rU') as fh:
+ source = fh.read()
+
+ try:
+ yield (makefile, source, parsestring(source, makefile))
+ except SyntaxError:
+ continue
+
+ def test_reparse_consistency(self):
+ for filename, source, statements in self._get_test_files():
+ reformatted = statements.to_source()
+
+ # We should be able to parse the reformatted source fine.
+ new_statements = parsestring(reformatted, filename)
+
+ # If we do the formatting again, the representation shouldn't
+ # change. i.e. the only lossy change should be the original
+ # (whitespace and some semantics aren't preserved).
+ reformatted_again = new_statements.to_source()
+ self.assertEqual(reformatted, reformatted_again,
+ '%s has lossless reformat.' % filename)
+
+ self.assertEqual(len(statements), len(new_statements))
+
+ for i in xrange(0, len(statements)):
+ original = statements[i]
+ formatted = new_statements[i]
+
+ self.assertEqual(original, formatted, '%s %d: %s != %s' % (filename,
+ i, original, formatted))
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.DEBUG)
+ unittest.main()
diff --git a/build/pymake/tests/func-refs.mk b/build/pymake/tests/func-refs.mk
new file mode 100644
index 000000000..82ab17ba8
--- /dev/null
+++ b/build/pymake/tests/func-refs.mk
@@ -0,0 +1,11 @@
+unknown var = uval
+
+all:
+ test "$(subst a,b,value)" = "vblue"
+ test "${subst a,b,va)lue}" = "vb)lue"
+ test "$(subst /,\,ab/c)" = "ab\c"
+ test '$(subst a,b,\\#)' = '\\#'
+ test "$( subst a,b,value)" = ""
+ test "$(Subst a,b,value)" = ""
+ test "$(unknown var)" = "uval"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/functions.mk b/build/pymake/tests/functions.mk
new file mode 100644
index 000000000..817be07aa
--- /dev/null
+++ b/build/pymake/tests/functions.mk
@@ -0,0 +1,36 @@
+all:
+ test "$(subst e,EE,hello)" = "hEEllo"
+ test "$(strip $(NULL) test data )" = "test data"
+ test "$(findstring hell,hello)" = "hell"
+ test "$(findstring heaven,hello)" = ""
+ test "$(filter foo/%.c b%,foo/a.c b.c foo/a.o)" = "foo/a.c b.c"
+ test "$(filter foo,foo bar)" = "foo"
+ test "$(filter-out foo/%.c b%,foo/a.c b.c foo/a.o)" = "foo/a.o"
+ test "$(filter-out %.c,foo,bar.c foo,bar.o)" = "foo,bar.o"
+ test "$(sort .go a b aa A c cc)" = ".go A a aa b c cc"
+ test "$(word 1, hello )" = "hello"
+ test "$(word 2, hello )" = ""
+ test "$(wordlist 1, 2, foo bar baz )" = "foo bar"
+ test "$(words 1 2 3)" = "3"
+ test "$(words )" = "0"
+ test "$(firstword $(NULL) foo bar baz)" = "foo"
+ test "$(firstword )" = ""
+ test "$(dir foo.c path/foo.o dir/dir2/)" = "./ path/ dir/dir2/"
+ test "$(notdir foo.c path/foo.o dir/dir2/)" = "foo.c foo.o "
+ test "$(suffix src/foo.c dir/my.dir/foo foo.o)" = ".c .o"
+ test "$(basename src/foo.c dir/my.dir/foo foo.c .c)" = "src/foo dir/my.dir/foo foo "
+ test "$(addprefix src/,foo bar.c dir/foo)" = "src/foo src/bar.c src/dir/foo"
+ test "$(addsuffix .c,foo dir/bar)" = "foo.c dir/bar.c"
+ test "$(join a b c, 1 2 3)" = "a1 b2 c3"
+ test "$(join a b, 1 2 3)" = "a1 b2 3"
+ test "$(join a b c, 1 2)" = "a1 b2 c"
+ test "$(if $(NULL) ,yes)" = ""
+ test "$(if 1,yes,no)" = "yes"
+ test "$(if ,yes,no )" = "no "
+ test "$(if ,$(error Short-circuit problem))" = ""
+ test "$(or $(NULL),1)" = "1"
+ test "$(or $(NULL),2,$(warning TEST-FAIL bad or short-circuit))" = "2"
+ test "$(and ,$(warning TEST-FAIL bad and short-circuit))" = ""
+ test "$(and 1,2)" = "2"
+ test "$(foreach i,foo bar,found:$(i))" = "found:foo found:bar"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/functiontests.py b/build/pymake/tests/functiontests.py
new file mode 100644
index 000000000..43a344a05
--- /dev/null
+++ b/build/pymake/tests/functiontests.py
@@ -0,0 +1,54 @@
+import unittest
+
+import pymake.data
+import pymake.functions
+
+class VariableRefTest(unittest.TestCase):
+ def test_get_expansions(self):
+ e = pymake.data.StringExpansion('FOO', None)
+ f = pymake.functions.VariableRef(None, e)
+
+ exps = list(f.expansions())
+ self.assertEqual(len(exps), 1)
+
+class GetExpansionsTest(unittest.TestCase):
+ def test_get_arguments(self):
+ f = pymake.functions.SubstFunction(None)
+
+ e1 = pymake.data.StringExpansion('FOO', None)
+ e2 = pymake.data.StringExpansion('BAR', None)
+ e3 = pymake.data.StringExpansion('BAZ', None)
+
+ f.append(e1)
+ f.append(e2)
+ f.append(e3)
+
+ exps = list(f.expansions())
+ self.assertEqual(len(exps), 3)
+
+ def test_descend(self):
+ f = pymake.functions.StripFunction(None)
+
+ e = pymake.data.Expansion(None)
+
+ e1 = pymake.data.StringExpansion('FOO', None)
+ f1 = pymake.functions.VariableRef(None, e1)
+ e.appendfunc(f1)
+
+ f2 = pymake.functions.WildcardFunction(None)
+ e2 = pymake.data.StringExpansion('foo/*', None)
+ f2.append(e2)
+ e.appendfunc(f2)
+
+ f.append(e)
+
+ exps = list(f.expansions())
+ self.assertEqual(len(exps), 1)
+
+ exps = list(f.expansions(True))
+ self.assertEqual(len(exps), 3)
+
+ self.assertFalse(f.is_filesystem_dependent)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/build/pymake/tests/if-syntaxerr.mk b/build/pymake/tests/if-syntaxerr.mk
new file mode 100644
index 000000000..c172492ef
--- /dev/null
+++ b/build/pymake/tests/if-syntaxerr.mk
@@ -0,0 +1,6 @@
+#T returncode: 2
+
+ifeq ($(FOO,VAR))
+all:
+ @echo TEST_FAIL
+endif
diff --git a/build/pymake/tests/ifdefs-nesting.mk b/build/pymake/tests/ifdefs-nesting.mk
new file mode 100644
index 000000000..340530ffa
--- /dev/null
+++ b/build/pymake/tests/ifdefs-nesting.mk
@@ -0,0 +1,13 @@
+ifdef RANDOM
+ifeq (,$(error Not evaluated!))
+endif
+endif
+
+ifdef RANDOM
+ifeq (,)
+else ifeq (,$(error Not evaluated!))
+endif
+endif
+
+all:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/ifdefs.mk b/build/pymake/tests/ifdefs.mk
new file mode 100644
index 000000000..a779d197b
--- /dev/null
+++ b/build/pymake/tests/ifdefs.mk
@@ -0,0 +1,127 @@
+ifdef FOO
+$(error FOO is not defined!)
+endif
+
+FOO = foo
+FOOFOUND = false
+BARFOUND = false
+BAZFOUND = false
+
+ifdef FOO
+FOOFOUND = true
+else ifdef BAR
+BARFOUND = true
+else
+BAZFOUND = true
+endif
+
+BAR2 = bar2
+FOO2FOUND = false
+BAR2FOUND = false
+BAZ2FOUND = false
+
+ifdef FOO2
+FOO2FOUND = true
+else ifdef BAR2
+BAR2FOUND = true
+else
+BAZ2FOUND = true
+endif
+
+FOO3FOUND = false
+BAR3FOUND = false
+BAZ3FOUND = false
+
+ifdef FOO3
+FOO3FOUND = true
+else ifdef BAR3
+BAR3FOUND = true
+else
+BAZ3FOUND = true
+endif
+
+ifdef RANDOM
+CONTINUATION = \
+else \
+endif
+endif
+
+ifndef ASDFJK
+else
+$(error ASFDJK was not set)
+endif
+
+TESTSET =
+
+ifdef TESTSET
+$(error TESTSET was not set)
+endif
+
+TESTEMPTY = $(NULL)
+ifndef TESTEMPTY
+$(error TEST-FAIL TESTEMPTY was probably expanded!)
+endif
+
+# ifneq ( a,a)
+# $(error Arguments to ifeq should be stripped before evaluation)
+# endif
+
+XSPACE = x # trick
+
+ifneq ($(NULL),$(NULL))
+$(error TEST-FAIL ifneq)
+endif
+
+ifneq (x , x)
+$(error argument-stripping1)
+endif
+
+ifeq ( x,x )
+$(error argument-stripping2)
+endif
+
+ifneq ($(XSPACE), x )
+$(error argument-stripping3)
+endif
+
+ifeq 'x ' ' x'
+$(error TEST-FAIL argument-stripping4)
+endif
+
+all:
+ test $(FOOFOUND) = true # FOOFOUND
+ test $(BARFOUND) = false # BARFOUND
+ test $(BAZFOUND) = false # BAZFOUND
+ test $(FOO2FOUND) = false # FOO2FOUND
+ test $(BAR2FOUND) = true # BAR2FOUND
+ test $(BAZ2FOUND) = false # BAZ2FOUND
+ test $(FOO3FOUND) = false # FOO3FOUND
+ test $(BAR3FOUND) = false # BAR3FOUND
+ test $(BAZ3FOUND) = true # BAZ3FOUND
+ifneq ($(FOO),foo)
+ echo TEST-FAIL 'FOO neq foo: "$(FOO)"'
+endif
+ifneq ($(FOO), foo) # Whitespace after the comma is stripped
+ echo TEST-FAIL 'FOO plus whitespace'
+endif
+ifeq ($(FOO), foo ) # But not trailing whitespace
+ echo TEST-FAIL 'FOO plus trailing whitespace'
+endif
+ifeq ( $(FOO),foo) # Not whitespace after the paren
+ echo TEST-FAIL 'FOO with leading whitespace'
+endif
+ifeq ($(FOO),$(NULL) foo) # Nor whitespace after expansion
+ echo TEST-FAIL 'FOO with embedded ws'
+endif
+ifeq ($(BAR2),bar)
+ echo TEST-FAIL 'BAR2 eq bar'
+endif
+ifeq '$(BAR3FOUND)' 'false'
+ echo BAR3FOUND is ok
+else
+ echo TEST-FAIL BAR3FOUND is not ok
+endif
+ifndef FOO
+ echo TEST-FAIL "foo not defined?"
+endif
+ @echo TEST-PASS
diff --git a/build/pymake/tests/ignore-error.mk b/build/pymake/tests/ignore-error.mk
new file mode 100644
index 000000000..dc8d3a72c
--- /dev/null
+++ b/build/pymake/tests/ignore-error.mk
@@ -0,0 +1,13 @@
+all:
+ -rm foo
+ +-rm bar
+ -+rm baz
+ @-rm bah
+ -@rm humbug
+ +-@rm sincere
+ +@-rm flattery
+ @+-rm will
+ @-+rm not
+ -+@rm save
+ -@+rm you
+ @echo TEST-PASS
diff --git a/build/pymake/tests/implicit-chain.mk b/build/pymake/tests/implicit-chain.mk
new file mode 100644
index 000000000..16288b3f5
--- /dev/null
+++ b/build/pymake/tests/implicit-chain.mk
@@ -0,0 +1,12 @@
+all: test.prog
+ test "$$(cat $<)" = "Program: Object: Source: test.source"
+ @echo TEST-PASS
+
+%.prog: %.object
+ printf "Program: %s" "$$(cat $<)" > $@
+
+%.object: %.source
+ printf "Object: %s" "$$(cat $<)" > $@
+
+%.source:
+ printf "Source: %s" $@ > $@
diff --git a/build/pymake/tests/implicit-dir.mk b/build/pymake/tests/implicit-dir.mk
new file mode 100644
index 000000000..c7f75e8d4
--- /dev/null
+++ b/build/pymake/tests/implicit-dir.mk
@@ -0,0 +1,16 @@
+# Implicit rules have special instructions to deal with directories, so that a pattern rule which doesn't directly apply
+# may still be used.
+
+all: dir/host_test.otest
+
+host_%.otest: %.osource extra.file
+ @echo making $@ from $<
+
+test.osource:
+ @echo TEST-FAIL should have made dir/test.osource
+
+dir/test.osource:
+ @echo TEST-PASS made the correct dependency
+
+extra.file:
+ @echo building $@
diff --git a/build/pymake/tests/implicit-terminal.mk b/build/pymake/tests/implicit-terminal.mk
new file mode 100644
index 000000000..db2e244ed
--- /dev/null
+++ b/build/pymake/tests/implicit-terminal.mk
@@ -0,0 +1,16 @@
+#T returncode: 2
+
+# the %.object rule is "terminal". This means that additional implicit rules cannot be chained to it.
+
+all: test.prog
+ test "$$(cat $<)" = "Program: Object: Source: test.source"
+ @echo TEST-FAIL
+
+%.prog: %.object
+ printf "Program: %s" "$$(cat $<)" > $@
+
+%.object:: %.source
+ printf "Object: %s" "$$(cat $<)" > $@
+
+%.source:
+ printf "Source: %s" $@ > $@
diff --git a/build/pymake/tests/implicitsubdir.mk b/build/pymake/tests/implicitsubdir.mk
new file mode 100644
index 000000000..b9d854a2a
--- /dev/null
+++ b/build/pymake/tests/implicitsubdir.mk
@@ -0,0 +1,12 @@
+$(shell \
+mkdir foo; \
+touch test.in \
+)
+
+all: foo/test.out
+ @echo TEST-PASS
+
+foo/%.out: %.in
+ cp $< $@
+
+
diff --git a/build/pymake/tests/include-dynamic.mk b/build/pymake/tests/include-dynamic.mk
new file mode 100644
index 000000000..571895dc3
--- /dev/null
+++ b/build/pymake/tests/include-dynamic.mk
@@ -0,0 +1,21 @@
+$(shell \
+if ! test -f include-dynamic.inc; then \
+ echo "TESTVAR = oldval" > include-dynamic.inc; \
+ sleep 2; \
+ echo "TESTVAR = newval" > include-dynamic.inc.in; \
+fi \
+)
+
+# before running the 'all' rule, we should be rebuilding include-dynamic.inc,
+# because there is a rule to do so
+
+all:
+ test $(TESTVAR) = newval
+ test "$(MAKE_RESTARTS)" = 1
+ @echo TEST-PASS
+
+include-dynamic.inc: include-dynamic.inc.in
+ test "$(MAKE_RESTARTS)" = ""
+ cp $< $@
+
+include include-dynamic.inc
diff --git a/build/pymake/tests/include-file.inc b/build/pymake/tests/include-file.inc
new file mode 100644
index 000000000..d5d495dec
--- /dev/null
+++ b/build/pymake/tests/include-file.inc
@@ -0,0 +1 @@
+INCLUDED = yes
diff --git a/build/pymake/tests/include-missing.mk b/build/pymake/tests/include-missing.mk
new file mode 100644
index 000000000..583d0a065
--- /dev/null
+++ b/build/pymake/tests/include-missing.mk
@@ -0,0 +1,9 @@
+#T returncode: 2
+
+# If an include file isn't present and doesn't have a rule to remake it, make
+# should fail.
+
+include notfound.mk
+
+all:
+ @echo TEST-FAIL
diff --git a/build/pymake/tests/include-notfound.mk b/build/pymake/tests/include-notfound.mk
new file mode 100644
index 000000000..1ee7e05b2
--- /dev/null
+++ b/build/pymake/tests/include-notfound.mk
@@ -0,0 +1,19 @@
+ifdef __WIN32__
+PS:=\\#
+else
+PS:=/
+endif
+
+ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk)
+$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk'))
+endif
+
+-include notfound.inc-dummy
+
+ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk)
+$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk'))
+endif
+
+all:
+ @echo TEST-PASS
+
diff --git a/build/pymake/tests/include-optional-warning.mk b/build/pymake/tests/include-optional-warning.mk
new file mode 100644
index 000000000..901938dff
--- /dev/null
+++ b/build/pymake/tests/include-optional-warning.mk
@@ -0,0 +1,4 @@
+-include TEST-FAIL.mk
+
+all:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/include-regen.mk b/build/pymake/tests/include-regen.mk
new file mode 100644
index 000000000..c86e0c78d
--- /dev/null
+++ b/build/pymake/tests/include-regen.mk
@@ -0,0 +1,10 @@
+# avoid infinite loops by not remaking makefiles with
+# double-colon no-dependency rules
+# http://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+-include notfound.mk
+
+all:
+ @echo TEST-PASS
+
+notfound.mk::
+ @echo TEST-FAIL
diff --git a/build/pymake/tests/include-regen2.mk b/build/pymake/tests/include-regen2.mk
new file mode 100644
index 000000000..fc7fef073
--- /dev/null
+++ b/build/pymake/tests/include-regen2.mk
@@ -0,0 +1,10 @@
+# make should make makefiles that it has rules for if they are
+# included
+include test.mk
+
+all:
+ test "$(X)" = "1"
+ @echo "TEST-PASS"
+
+test.mk:
+ @echo "X = 1" > $@
diff --git a/build/pymake/tests/include-regen3.mk b/build/pymake/tests/include-regen3.mk
new file mode 100644
index 000000000..878ce0adc
--- /dev/null
+++ b/build/pymake/tests/include-regen3.mk
@@ -0,0 +1,10 @@
+# make should make makefiles that it has rules for if they are
+# included
+-include test.mk
+
+all:
+ test "$(X)" = "1"
+ @echo "TEST-PASS"
+
+test.mk:
+ @echo "X = 1" > $@
diff --git a/build/pymake/tests/include-test.mk b/build/pymake/tests/include-test.mk
new file mode 100644
index 000000000..3608fc269
--- /dev/null
+++ b/build/pymake/tests/include-test.mk
@@ -0,0 +1,8 @@
+$(shell echo "INCLUDED2 = yes" >local-include.inc)
+
+include $(TESTPATH)/include-file.inc local-include.inc
+
+all:
+ test "$(INCLUDED)" = "yes"
+ test "$(INCLUDED2)" = "yes"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/includedeps-norebuild.mk b/build/pymake/tests/includedeps-norebuild.mk
new file mode 100644
index 000000000..e30abd439
--- /dev/null
+++ b/build/pymake/tests/includedeps-norebuild.mk
@@ -0,0 +1,15 @@
+#T gmake skip
+
+$(shell \
+touch filemissing; \
+sleep 2; \
+touch file1; \
+)
+
+all: file1
+ @echo TEST-PASS
+
+includedeps $(TESTPATH)/includedeps.deps
+
+file1:
+ @echo TEST-FAIL
diff --git a/build/pymake/tests/includedeps-sideeffects.mk b/build/pymake/tests/includedeps-sideeffects.mk
new file mode 100644
index 000000000..7e4ea30a2
--- /dev/null
+++ b/build/pymake/tests/includedeps-sideeffects.mk
@@ -0,0 +1,10 @@
+#T gmake skip
+#T returncode: 2
+
+all: file1 filemissing
+ @echo TEST-PASS
+
+includedeps $(TESTPATH)/includedeps.deps
+
+file:
+ touch $@
diff --git a/build/pymake/tests/includedeps-stripdotslash.deps b/build/pymake/tests/includedeps-stripdotslash.deps
new file mode 100644
index 000000000..352fca1bb
--- /dev/null
+++ b/build/pymake/tests/includedeps-stripdotslash.deps
@@ -0,0 +1 @@
+./test: TEST-PASS
diff --git a/build/pymake/tests/includedeps-stripdotslash.mk b/build/pymake/tests/includedeps-stripdotslash.mk
new file mode 100644
index 000000000..ee942e6db
--- /dev/null
+++ b/build/pymake/tests/includedeps-stripdotslash.mk
@@ -0,0 +1,8 @@
+#T gmake skip
+
+test:
+ @echo $<
+
+includedeps $(TESTPATH)/includedeps-stripdotslash.deps
+
+TEST-PASS:
diff --git a/build/pymake/tests/includedeps-variables.deps b/build/pymake/tests/includedeps-variables.deps
new file mode 100644
index 000000000..ba69e9b6c
--- /dev/null
+++ b/build/pymake/tests/includedeps-variables.deps
@@ -0,0 +1 @@
+$(FILE)1: filemissing
diff --git a/build/pymake/tests/includedeps-variables.mk b/build/pymake/tests/includedeps-variables.mk
new file mode 100644
index 000000000..314618da4
--- /dev/null
+++ b/build/pymake/tests/includedeps-variables.mk
@@ -0,0 +1,10 @@
+#T gmake skip
+
+FILE = includedeps-variables
+
+all: $(FILE)1
+
+includedeps $(TESTPATH)/includedeps-variables.deps
+
+filemissing:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/includedeps.deps b/build/pymake/tests/includedeps.deps
new file mode 100644
index 000000000..d3017c078
--- /dev/null
+++ b/build/pymake/tests/includedeps.deps
@@ -0,0 +1 @@
+file1: filemissing
diff --git a/build/pymake/tests/includedeps.mk b/build/pymake/tests/includedeps.mk
new file mode 100644
index 000000000..deaa71fe8
--- /dev/null
+++ b/build/pymake/tests/includedeps.mk
@@ -0,0 +1,9 @@
+#T gmake skip
+
+all: file1
+ @echo TEST-PASS
+
+includedeps $(TESTPATH)/includedeps.deps
+
+file1:
+ touch $@
diff --git a/build/pymake/tests/info.mk b/build/pymake/tests/info.mk
new file mode 100644
index 000000000..8dddfd815
--- /dev/null
+++ b/build/pymake/tests/info.mk
@@ -0,0 +1,8 @@
+#T grep-for: "info-printed\ninfo-nth"
+all:
+
+INFO = info-printed
+
+$(info $(INFO))
+$(info $(subst second,nth,info-second))
+$(info TEST-PASS)
diff --git a/build/pymake/tests/justprint-native.mk b/build/pymake/tests/justprint-native.mk
new file mode 100644
index 000000000..580e402e9
--- /dev/null
+++ b/build/pymake/tests/justprint-native.mk
@@ -0,0 +1,28 @@
+## $(TOUCH) and $(RM) are native commands in pymake.
+## Test that pymake --just-print just prints them.
+
+ifndef TOUCH
+TOUCH = touch
+endif
+
+all:
+ $(RM) justprint-native-file1.txt
+ $(TOUCH) justprint-native-file2.txt
+ $(MAKE) --just-print -f $(TESTPATH)/justprint-native.mk justprint_target > justprint.log
+# make --just-print shouldn't have actually done anything.
+ test ! -f justprint-native-file1.txt
+ test -f justprint-native-file2.txt
+# but it should have printed each command
+ grep -q 'touch justprint-native-file1.txt' justprint.log
+ grep -q 'rm -f justprint-native-file2.txt' justprint.log
+ grep -q 'this string is "unlikely to appear in the log by chance"' justprint.log
+# tidy up
+ $(RM) justprint-native-file2.txt
+ @echo TEST-PASS
+
+justprint_target:
+ $(TOUCH) justprint-native-file1.txt
+ $(RM) justprint-native-file2.txt
+ this string is "unlikely to appear in the log by chance"
+
+.PHONY: justprint_target
diff --git a/build/pymake/tests/justprint.mk b/build/pymake/tests/justprint.mk
new file mode 100644
index 000000000..be11ba8de
--- /dev/null
+++ b/build/pymake/tests/justprint.mk
@@ -0,0 +1,5 @@
+#T commandline: ['-n']
+
+all:
+ false # without -n, we wouldn't get past this
+ TEST-PASS # heh
diff --git a/build/pymake/tests/keep-going-doublecolon.mk b/build/pymake/tests/keep-going-doublecolon.mk
new file mode 100644
index 000000000..fa5b31df8
--- /dev/null
+++ b/build/pymake/tests/keep-going-doublecolon.mk
@@ -0,0 +1,16 @@
+#T commandline: ['-k']
+#T returncode: 2
+#T grep-for: "TEST-PASS"
+
+all:: t1
+ @echo TEST-FAIL "(t1)"
+
+all:: t2
+ @echo TEST-PASS
+
+t1:
+ @false
+
+t2:
+ touch $@
+
diff --git a/build/pymake/tests/keep-going-parallel.mk b/build/pymake/tests/keep-going-parallel.mk
new file mode 100644
index 000000000..a91d1a6ed
--- /dev/null
+++ b/build/pymake/tests/keep-going-parallel.mk
@@ -0,0 +1,11 @@
+#T commandline: ['-k', '-j2']
+#T returncode: 2
+#T grep-for: "TEST-PASS"
+
+all: t1 slow1 slow2 slow3 t2
+
+t2:
+ @echo TEST-PASS
+
+slow%:
+ sleep 1
diff --git a/build/pymake/tests/keep-going.mk b/build/pymake/tests/keep-going.mk
new file mode 100644
index 000000000..4c709288c
--- /dev/null
+++ b/build/pymake/tests/keep-going.mk
@@ -0,0 +1,14 @@
+#T commandline: ['-k']
+#T returncode: 2
+#T grep-for: "TEST-PASS"
+
+all: t2 t3
+
+t1:
+ @false
+
+t2: t1
+ @echo TEST-FAIL
+
+t3:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/line-continuations.mk b/build/pymake/tests/line-continuations.mk
new file mode 100644
index 000000000..8b44480ea
--- /dev/null
+++ b/build/pymake/tests/line-continuations.mk
@@ -0,0 +1,24 @@
+VAR = val1 \
+ val2
+
+VAR2 = val1space\
+val2
+
+VAR3 = val3 \\\
+ cont3
+
+all: otarget test.target
+ test "$(VAR)" = "val1 val2 "
+ test "$(VAR2)" = "val1space val2"
+ test '$(VAR3)' = 'val3 \ cont3'
+ test "hello \
+ world" = "hello world"
+ test "hello" = \
+"hello"
+ @echo TEST-PASS
+
+otarget: ; test "hello\
+ world" = "helloworld"
+
+test.target: %.target: ; test "hello\
+ world" = "helloworld"
diff --git a/build/pymake/tests/link-search.mk b/build/pymake/tests/link-search.mk
new file mode 100644
index 000000000..ea827f391
--- /dev/null
+++ b/build/pymake/tests/link-search.mk
@@ -0,0 +1,7 @@
+$(shell \
+touch libfoo.so \
+)
+
+all: -lfoo
+ test "$<" = "libfoo.so"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/makeflags.mk b/build/pymake/tests/makeflags.mk
new file mode 100644
index 000000000..288ff7866
--- /dev/null
+++ b/build/pymake/tests/makeflags.mk
@@ -0,0 +1,7 @@
+#T environment: {'MAKEFLAGS': 'OVAR=oval'}
+
+all:
+ test "$(OVAR)" = "oval"
+ test "$$OVAR" = "oval"
+ @echo TEST-PASS
+
diff --git a/build/pymake/tests/matchany.mk b/build/pymake/tests/matchany.mk
new file mode 100644
index 000000000..7876c90a3
--- /dev/null
+++ b/build/pymake/tests/matchany.mk
@@ -0,0 +1,14 @@
+#T returncode: 2
+
+# we should fail to make foo.ooo from foo.ooo.test
+all: foo.ooo
+ @echo TEST-FAIL
+
+%.ooo:
+
+# this match-anything pattern should not apply to %.ooo
+%: %.test
+ cp $< $@
+
+foo.ooo.test:
+ touch $@
diff --git a/build/pymake/tests/matchany2.mk b/build/pymake/tests/matchany2.mk
new file mode 100644
index 000000000..d21d9702c
--- /dev/null
+++ b/build/pymake/tests/matchany2.mk
@@ -0,0 +1,13 @@
+# we should succeed in making foo.ooo from foo.ooo.test
+all: foo.ooo
+ @echo TEST-PASS
+
+%.ooo: %.ccc
+ exit 1
+
+# this match-anything rule is terminal, and therefore applies
+%:: %.test
+ cp $< $@
+
+foo.ooo.test:
+ touch $@
diff --git a/build/pymake/tests/matchany3.mk b/build/pymake/tests/matchany3.mk
new file mode 100644
index 000000000..83de8af2b
--- /dev/null
+++ b/build/pymake/tests/matchany3.mk
@@ -0,0 +1,10 @@
+$(shell \
+echo "target" > target.in; \
+)
+
+all: target
+ test "$$(cat $^)" = "target"
+ @echo TEST-PASS
+
+%: %.in
+ cp $< $@
diff --git a/build/pymake/tests/mkdir-fail.mk b/build/pymake/tests/mkdir-fail.mk
new file mode 100644
index 000000000..b05734aa9
--- /dev/null
+++ b/build/pymake/tests/mkdir-fail.mk
@@ -0,0 +1,7 @@
+#T returncode: 2
+all:
+ mkdir newdir/subdir
+ test ! -d newdir/subdir
+ test ! -d newdir
+ rm -r newdir
+ @echo TEST-PASS
diff --git a/build/pymake/tests/mkdir.mk b/build/pymake/tests/mkdir.mk
new file mode 100644
index 000000000..413348f77
--- /dev/null
+++ b/build/pymake/tests/mkdir.mk
@@ -0,0 +1,27 @@
+MKDIR ?= mkdir
+
+all:
+ $(MKDIR) newdir
+ test -d newdir
+ # subdir, parent exists
+ $(MKDIR) newdir/subdir
+ test -d newdir/subdir
+ # -p, existing dir
+ $(MKDIR) -p newdir
+ # -p, existing subdir
+ $(MKDIR) -p newdir/subdir
+ # multiple subdirs, existing parent
+ $(MKDIR) newdir/subdir1 newdir/subdir2
+ test -d newdir/subdir1 -a -d newdir/subdir2
+ rm -r newdir
+ # -p, subdir, no existing parent
+ $(MKDIR) -p newdir/subdir
+ test -d newdir/subdir
+ rm -r newdir
+ # -p, multiple subdirs, no existing parent
+ $(MKDIR) -p newdir/subdir1 newdir/subdir2
+ test -d newdir/subdir1 -a -d newdir/subdir2
+ # -p, multiple existing subdirs
+ $(MKDIR) -p newdir/subdir1 newdir/subdir2
+ rm -r newdir
+ @echo TEST-PASS
diff --git a/build/pymake/tests/multiple-rules-prerequisite-merge.mk b/build/pymake/tests/multiple-rules-prerequisite-merge.mk
new file mode 100644
index 000000000..480d3b58c
--- /dev/null
+++ b/build/pymake/tests/multiple-rules-prerequisite-merge.mk
@@ -0,0 +1,25 @@
+# When a target is defined multiple times, the prerequisites should get
+# merged.
+
+default: foo bar baz
+
+foo:
+ test "$<" = "foo.in1"
+ @echo TEST-PASS
+
+foo: foo.in1
+
+bar: bar.in1
+ test "$<" = "bar.in1"
+ test "$^" = "bar.in1 bar.in2"
+ @echo TEST-PASS
+
+bar: bar.in2
+
+baz: baz.in2
+baz: baz.in1
+ test "$<" = "baz.in1"
+ test "$^" = "baz.in1 baz.in2"
+ @echo TEST-PASS
+
+foo.in1 bar.in1 bar.in2 baz.in1 baz.in2:
diff --git a/build/pymake/tests/native-command-delay-load.mk b/build/pymake/tests/native-command-delay-load.mk
new file mode 100644
index 000000000..a9f3774eb
--- /dev/null
+++ b/build/pymake/tests/native-command-delay-load.mk
@@ -0,0 +1,12 @@
+#T gmake skip
+
+# This test exists to verify that sys.path is adjusted during command
+# execution and that delay importing a module will work.
+
+CMD = %pycmd delayloadfn
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD)
+ @echo TEST-PASS
+
diff --git a/build/pymake/tests/native-command-raise.mk b/build/pymake/tests/native-command-raise.mk
new file mode 100644
index 000000000..d1b28b331
--- /dev/null
+++ b/build/pymake/tests/native-command-raise.mk
@@ -0,0 +1,9 @@
+#T gmake skip
+#T returncode: 2
+#T grep-for: "Exception: info-exception"
+
+CMD = %pycmd asplode_raise
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ @$(CMD) info-exception
diff --git a/build/pymake/tests/native-command-return-fail1.mk b/build/pymake/tests/native-command-return-fail1.mk
new file mode 100644
index 000000000..0cf085ae2
--- /dev/null
+++ b/build/pymake/tests/native-command-return-fail1.mk
@@ -0,0 +1,8 @@
+#T gmake skip
+#T returncode: 2
+
+CMD = %pycmd asplode_return
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) 1
diff --git a/build/pymake/tests/native-command-return-fail2.mk b/build/pymake/tests/native-command-return-fail2.mk
new file mode 100644
index 000000000..c071fc879
--- /dev/null
+++ b/build/pymake/tests/native-command-return-fail2.mk
@@ -0,0 +1,8 @@
+#T gmake skip
+#T returncode: 2
+
+CMD = %pycmd asplode_return
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) not-an-integer
diff --git a/build/pymake/tests/native-command-return.mk b/build/pymake/tests/native-command-return.mk
new file mode 100644
index 000000000..3e4d2e0c4
--- /dev/null
+++ b/build/pymake/tests/native-command-return.mk
@@ -0,0 +1,11 @@
+#T gmake skip
+
+CMD = %pycmd asplode_return
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) 0
+ -$(CMD) 1
+ $(CMD) None
+ -$(CMD) not-an-integer
+ @echo TEST-PASS
diff --git a/build/pymake/tests/native-command-shell-glob.mk b/build/pymake/tests/native-command-shell-glob.mk
new file mode 100644
index 000000000..4bcdad8b9
--- /dev/null
+++ b/build/pymake/tests/native-command-shell-glob.mk
@@ -0,0 +1,11 @@
+#T gmake skip
+all:
+ mkdir shell-glob-test
+ touch shell-glob-test/foo.txt
+ touch shell-glob-test/bar.txt
+ touch shell-glob-test/a.foo
+ touch shell-glob-test/b.foo
+ $(RM) shell-glob-test/*.txt
+ $(RM) shell-glob-test/?.foo
+ rmdir shell-glob-test
+ @echo TEST-PASS
diff --git a/build/pymake/tests/native-command-sys-exit-fail1.mk b/build/pymake/tests/native-command-sys-exit-fail1.mk
new file mode 100644
index 000000000..8e74800ed
--- /dev/null
+++ b/build/pymake/tests/native-command-sys-exit-fail1.mk
@@ -0,0 +1,8 @@
+#T gmake skip
+#T returncode: 2
+
+CMD = %pycmd asplode
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) 1
diff --git a/build/pymake/tests/native-command-sys-exit-fail2.mk b/build/pymake/tests/native-command-sys-exit-fail2.mk
new file mode 100644
index 000000000..0a04395ad
--- /dev/null
+++ b/build/pymake/tests/native-command-sys-exit-fail2.mk
@@ -0,0 +1,8 @@
+#T gmake skip
+#T returncode: 2
+
+CMD = %pycmd asplode
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) not-an-integer
diff --git a/build/pymake/tests/native-command-sys-exit.mk b/build/pymake/tests/native-command-sys-exit.mk
new file mode 100644
index 000000000..c04913aca
--- /dev/null
+++ b/build/pymake/tests/native-command-sys-exit.mk
@@ -0,0 +1,11 @@
+#T gmake skip
+
+CMD = %pycmd asplode
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) 0
+ -$(CMD) 1
+ $(CMD) None
+ -$(CMD) not-an-integer
+ @echo TEST-PASS
diff --git a/build/pymake/tests/native-environment.mk b/build/pymake/tests/native-environment.mk
new file mode 100644
index 000000000..36bd5894a
--- /dev/null
+++ b/build/pymake/tests/native-environment.mk
@@ -0,0 +1,11 @@
+#T gmake skip
+export EXPECTED := some data
+
+PYCOMMANDPATH = $(TESTPATH)
+
+all:
+ %pycmd writeenvtofile results EXPECTED
+ test "$$(cat results)" = "$(EXPECTED)"
+ %pycmd writesubprocessenvtofile results EXPECTED
+ test "$$(cat results)" = "$(EXPECTED)"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/native-pycommandpath-sep.mk b/build/pymake/tests/native-pycommandpath-sep.mk
new file mode 100644
index 000000000..b1c2c2b97
--- /dev/null
+++ b/build/pymake/tests/native-pycommandpath-sep.mk
@@ -0,0 +1,21 @@
+#T gmake skip
+EXPECTED := some data
+
+# verify that we can load native command modules from
+# multiple directories in PYCOMMANDPATH separated by the native
+# path separator
+ifdef __WIN32__
+PS:=;
+else
+PS:=:
+endif
+CMD = %pycmd writetofile
+CMD2 = %pymod writetofile
+PYCOMMANDPATH = $(TESTPATH)$(PS)$(TESTPATH)/subdir
+
+all:
+ $(CMD) results $(EXPECTED)
+ test "$$(cat results)" = "$(EXPECTED)"
+ $(CMD2) results2 $(EXPECTED)
+ test "$$(cat results2)" = "$(EXPECTED)"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/native-pycommandpath.mk b/build/pymake/tests/native-pycommandpath.mk
new file mode 100644
index 000000000..dd0fbc9f9
--- /dev/null
+++ b/build/pymake/tests/native-pycommandpath.mk
@@ -0,0 +1,15 @@
+#T gmake skip
+EXPECTED := some data
+
+# verify that we can load native command modules from
+# multiple space-separated directories in PYCOMMANDPATH
+CMD = %pycmd writetofile
+CMD2 = %pymod writetofile
+PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
+
+all:
+ $(CMD) results $(EXPECTED)
+ test "$$(cat results)" = "$(EXPECTED)"
+ $(CMD2) results2 $(EXPECTED)
+ test "$$(cat results2)" = "$(EXPECTED)"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/native-simple.mk b/build/pymake/tests/native-simple.mk
new file mode 100644
index 000000000..626a58670
--- /dev/null
+++ b/build/pymake/tests/native-simple.mk
@@ -0,0 +1,12 @@
+ifndef TOUCH
+TOUCH = touch
+endif
+
+all: testfile {testfile2} (testfile3)
+ test -f testfile
+ test -f {testfile2}
+ test -f "(testfile3)"
+ @echo TEST-PASS
+
+testfile {testfile2} (testfile3):
+ $(TOUCH) "$@"
diff --git a/build/pymake/tests/native-touch.mk b/build/pymake/tests/native-touch.mk
new file mode 100644
index 000000000..811161ece
--- /dev/null
+++ b/build/pymake/tests/native-touch.mk
@@ -0,0 +1,15 @@
+TOUCH ?= touch
+
+foo:
+ $(TOUCH) bar
+ $(TOUCH) baz
+ $(MAKE) -f $(TESTPATH)/native-touch.mk baz
+ $(TOUCH) -t 198007040802 baz
+ $(MAKE) -f $(TESTPATH)/native-touch.mk baz
+
+bar:
+ $(TOUCH) $@
+
+baz: bar
+ echo TEST-PASS
+ $(TOUCH) $@
diff --git a/build/pymake/tests/newlines.mk b/build/pymake/tests/newlines.mk
new file mode 100644
index 000000000..5d8195c94
--- /dev/null
+++ b/build/pymake/tests/newlines.mk
@@ -0,0 +1,30 @@
+#T gmake skip
+
+# Test that we handle \\\n properly
+
+all: dep1 dep2 dep3
+ cat testfile
+ test `cat testfile` = "data";
+ test "$$(cat results)" = "$(EXPECTED)";
+ @echo TEST-PASS
+
+# Test that something that still needs to go to the shell works
+testfile:
+ printf "data" \
+ >>$@
+
+dep1: testfile
+
+# Test that something that does not need to go to the shell works
+dep2:
+ $(echo foo) \
+ $(echo bar)
+
+export EXPECTED := some data
+
+CMD = %pycmd writeenvtofile
+PYCOMMANDPATH = $(TESTPATH)
+
+dep3:
+ $(CMD) \
+ results EXPECTED
diff --git a/build/pymake/tests/no-remake.mk b/build/pymake/tests/no-remake.mk
new file mode 100644
index 000000000..c8df81bc3
--- /dev/null
+++ b/build/pymake/tests/no-remake.mk
@@ -0,0 +1,7 @@
+$(shell date >testfile)
+
+all: testfile
+ @echo TEST-PASS
+
+testfile:
+ @echo TEST-FAIL "We shouldn't have remade this!"
diff --git a/build/pymake/tests/nosuchfile.mk b/build/pymake/tests/nosuchfile.mk
new file mode 100644
index 000000000..cca9ce1e9
--- /dev/null
+++ b/build/pymake/tests/nosuchfile.mk
@@ -0,0 +1,4 @@
+#T returncode: 2
+
+all:
+ reallythereisnosuchcommand
diff --git a/build/pymake/tests/notargets.mk b/build/pymake/tests/notargets.mk
new file mode 100644
index 000000000..8e55d944f
--- /dev/null
+++ b/build/pymake/tests/notargets.mk
@@ -0,0 +1,5 @@
+$(NULL): foo.c
+ @echo TEST-FAIL
+
+all:
+ @echo TEST-PASS
diff --git a/build/pymake/tests/notparallel.mk b/build/pymake/tests/notparallel.mk
new file mode 100644
index 000000000..4fd8b1a8d
--- /dev/null
+++ b/build/pymake/tests/notparallel.mk
@@ -0,0 +1,8 @@
+#T commandline: ['-j3']
+
+include $(TESTPATH)/serial-rule-execution.mk
+
+all::
+ $(MAKE) -f $(TESTPATH)/parallel-simple.mk
+
+.NOTPARALLEL:
diff --git a/build/pymake/tests/oneline-command-continuations.mk b/build/pymake/tests/oneline-command-continuations.mk
new file mode 100644
index 000000000..c11f3df52
--- /dev/null
+++ b/build/pymake/tests/oneline-command-continuations.mk
@@ -0,0 +1,5 @@
+all: test
+ @echo TEST-PASS
+
+test: ; test "Hello \
+ world" = "Hello world"
diff --git a/build/pymake/tests/override-propagate.mk b/build/pymake/tests/override-propagate.mk
new file mode 100644
index 000000000..a1663ff41
--- /dev/null
+++ b/build/pymake/tests/override-propagate.mk
@@ -0,0 +1,37 @@
+#T commandline: ['-w', 'OVAR=oval']
+
+OVAR=mval
+
+all: vartest run-override
+ $(MAKE) -f $(TESTPATH)/override-propagate.mk vartest
+ @echo TEST-PASS
+
+CLINE := OVAR=oval TESTPATH=$(TESTPATH) NATIVE_TESTPATH=$(NATIVE_TESTPATH)
+ifdef __WIN32__
+CLINE += __WIN32__=1
+endif
+
+SORTED_CLINE := $(subst \,\\,$(sort $(CLINE)))
+
+vartest:
+ @echo MAKELEVEL: '$(MAKELEVEL)'
+ test '$(value MAKEFLAGS)' = 'w -- $$(MAKEOVERRIDES)'
+ test '$(origin MAKEFLAGS)' = 'file'
+ test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}'
+ test "$(sort $(MAKEOVERRIDES))" = "$(SORTED_CLINE)"
+ test '$(origin MAKEOVERRIDES)' = 'environment'
+ test '$(origin -*-command-variables-*-)' = 'automatic'
+ test "$(origin OVAR)" = "command line"
+ test "$(OVAR)" = "oval"
+
+run-override: MAKEOVERRIDES=
+run-override:
+ test "$(OVAR)" = "oval"
+ $(MAKE) -f $(TESTPATH)/override-propagate.mk otest
+
+otest:
+ test '$(value MAKEFLAGS)' = 'w'
+ test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}'
+ test '$(MAKEOVERRIDES)' = ''
+ test '$(origin -*-command-variables-*-)' = 'undefined'
+ test "$(OVAR)" = "mval"
diff --git a/build/pymake/tests/parallel-dep-resolution.mk b/build/pymake/tests/parallel-dep-resolution.mk
new file mode 100644
index 000000000..7967eba2d
--- /dev/null
+++ b/build/pymake/tests/parallel-dep-resolution.mk
@@ -0,0 +1,8 @@
+#T commandline: ['-j3']
+#T returncode: 2
+
+all: t1 t2
+
+t1:
+ sleep 1
+ touch t1 t2
diff --git a/build/pymake/tests/parallel-dep-resolution2.mk b/build/pymake/tests/parallel-dep-resolution2.mk
new file mode 100644
index 000000000..7d61e6b3e
--- /dev/null
+++ b/build/pymake/tests/parallel-dep-resolution2.mk
@@ -0,0 +1,9 @@
+#T commandline: ['-j3']
+#T returncode: 2
+
+all::
+ sleep 1
+ touch somefile
+
+all:: somefile
+ @echo TEST-PASS
diff --git a/build/pymake/tests/parallel-native.mk b/build/pymake/tests/parallel-native.mk
new file mode 100644
index 000000000..d50cfbdbb
--- /dev/null
+++ b/build/pymake/tests/parallel-native.mk
@@ -0,0 +1,21 @@
+#T commandline: ['-j2']
+
+# ensure that calling python commands doesn't block other targets
+ifndef SLEEP
+SLEEP := sleep
+endif
+
+PRINTF = printf "$@:0:" >>results
+EXPECTED = target2:0:target1:0:
+
+all:: target1 target2
+ cat results
+ test "$$(cat results)" = "$(EXPECTED)"
+ @echo TEST-PASS
+
+target1:
+ $(SLEEP) 0.1
+ $(PRINTF)
+
+target2:
+ $(PRINTF)
diff --git a/build/pymake/tests/parallel-simple.mk b/build/pymake/tests/parallel-simple.mk
new file mode 100644
index 000000000..f1aafc5f1
--- /dev/null
+++ b/build/pymake/tests/parallel-simple.mk
@@ -0,0 +1,27 @@
+#T commandline: ['-j2']
+
+# CAUTION: this makefile is also used by serial-toparallel.mk
+
+define SLOWMAKE
+printf "$@:0:" >>results
+sleep 0.5
+printf "$@:1:" >>results
+sleep 0.5
+printf "$@:2:" >>results
+endef
+
+EXPECTED = target1:0:target2:0:target1:1:target2:1:target1:2:target2:2:
+
+all:: target1 target2
+ cat results
+ test "$$(cat results)" = "$(EXPECTED)"
+ @echo TEST-PASS
+
+target1:
+ $(SLOWMAKE)
+
+target2:
+ sleep 0.1
+ $(SLOWMAKE)
+
+.PHONY: all
diff --git a/build/pymake/tests/parallel-submake.mk b/build/pymake/tests/parallel-submake.mk
new file mode 100644
index 000000000..65cb2cf7c
--- /dev/null
+++ b/build/pymake/tests/parallel-submake.mk
@@ -0,0 +1,17 @@
+#T commandline: ['-j2']
+
+# A submake shouldn't return control to the parent until it has actually finished doing everything.
+
+all:
+ -$(MAKE) -f $(TESTPATH)/parallel-submake.mk subtarget
+ cat results
+ test "$$(cat results)" = "0123"
+ @echo TEST-PASS
+
+subtarget: succeed-slowly fail-quickly
+
+succeed-slowly:
+ printf 0 >>results; sleep 1; printf 1 >>results; sleep 1; printf 2 >>results; sleep 1; printf 3 >>results
+
+fail-quickly:
+ exit 1
diff --git a/build/pymake/tests/parallel-toserial.mk b/build/pymake/tests/parallel-toserial.mk
new file mode 100644
index 000000000..9a355eb33
--- /dev/null
+++ b/build/pymake/tests/parallel-toserial.mk
@@ -0,0 +1,31 @@
+#T commandline: ['-j4']
+
+# Test that -j1 in a submake has the proper effect.
+
+define SLOWCMD
+printf "$@:0:" >>$(RFILE)
+sleep 0.5
+printf "$@:1:" >>$(RFILE)
+endef
+
+all: p1 p2
+subtarget: s1 s2
+
+p1 p2: RFILE = presult
+s1 s2: RFILE = sresult
+
+p1 s1:
+ $(SLOWCMD)
+
+p2 s2:
+ sleep 0.1
+ $(SLOWCMD)
+
+all:
+ $(MAKE) -j1 -f $(TESTPATH)/parallel-toserial.mk subtarget
+ printf "presult: %s\n" "$$(cat presult)"
+ test "$$(cat presult)" = "p1:0:p2:0:p1:1:p2:1:"
+ printf "sresult: %s\n" "$$(cat sresult)"
+ test "$$(cat sresult)" = "s1:0:s1:1:s2:0:s2:1:"
+ @echo TEST-PASS
+
diff --git a/build/pymake/tests/parallel-waiting.mk b/build/pymake/tests/parallel-waiting.mk
new file mode 100644
index 000000000..40a6e0d50
--- /dev/null
+++ b/build/pymake/tests/parallel-waiting.mk
@@ -0,0 +1,21 @@
+#T commandline: ['-j2']
+
+EXPECTED = target1:before:target2:1:target2:2:target2:3:target1:after
+
+all:: target1 target2
+ cat results
+ test "$$(cat results)" = "$(EXPECTED)"
+ @echo TEST-PASS
+
+target1:
+ printf "$@:before:" >>results
+ sleep 4
+ printf "$@:after" >>results
+
+target2:
+ sleep 0.2
+ printf "$@:1:" >>results
+ sleep 0.1
+ printf "$@:2:" >>results
+ sleep 0.1
+ printf "$@:3:" >>results
diff --git a/build/pymake/tests/parentheses.mk b/build/pymake/tests/parentheses.mk
new file mode 100644
index 000000000..f207234ff
--- /dev/null
+++ b/build/pymake/tests/parentheses.mk
@@ -0,0 +1,2 @@
+all:
+ @(echo TEST-PASS)
diff --git a/build/pymake/tests/parsertests.py b/build/pymake/tests/parsertests.py
new file mode 100644
index 000000000..ab6406be0
--- /dev/null
+++ b/build/pymake/tests/parsertests.py
@@ -0,0 +1,314 @@
+import pymake.data, pymake.parser, pymake.parserdata, pymake.functions
+import unittest
+import logging
+
+from cStringIO import StringIO
+
+def multitest(cls):
+ for name in cls.testdata.iterkeys():
+ def m(self, name=name):
+ return self.runSingle(*self.testdata[name])
+
+ setattr(cls, 'test_%s' % name, m)
+ return cls
+
+class TestBase(unittest.TestCase):
+ def assertEqual(self, a, b, msg=""):
+ """Actually print the values which weren't equal, if things don't work out!"""
+ unittest.TestCase.assertEqual(self, a, b, "%s got %r expected %r" % (msg, a, b))
+
+class DataTest(TestBase):
+ testdata = {
+ 'oneline':
+ ("He\tllo", "f", 1, 0,
+ ((0, "f", 1, 0), (2, "f", 1, 2), (3, "f", 1, 4))),
+ 'twoline':
+ ("line1 \n\tl\tine2", "f", 1, 4,
+ ((0, "f", 1, 4), (5, "f", 1, 9), (6, "f", 1, 10), (7, "f", 2, 0), (8, "f", 2, 4), (10, "f", 2, 8), (13, "f", 2, 11))),
+ }
+
+ def runSingle(self, data, filename, line, col, results):
+ d = pymake.parser.Data(data, 0, len(data), pymake.parserdata.Location(filename, line, col))
+ for pos, file, lineno, col in results:
+ loc = d.getloc(pos)
+ self.assertEqual(loc.path, file, "data file offset %i" % pos)
+ self.assertEqual(loc.line, lineno, "data line offset %i" % pos)
+ self.assertEqual(loc.column, col, "data col offset %i" % pos)
+multitest(DataTest)
+
+class LineEnumeratorTest(TestBase):
+ testdata = {
+ 'simple': (
+ 'Hello, world', [
+ ('Hello, world', 1),
+ ]
+ ),
+ 'multi': (
+ 'Hello\nhappy \n\nworld\n', [
+ ('Hello', 1),
+ ('happy ', 2),
+ ('', 3),
+ ('world', 4),
+ ('', 5),
+ ]
+ ),
+ 'continuation': (
+ 'Hello, \\\n world\nJellybeans!', [
+ ('Hello, \\\n world', 1),
+ ('Jellybeans!', 3),
+ ]
+ ),
+ 'multislash': (
+ 'Hello, \\\\\n world', [
+ ('Hello, \\\\', 1),
+ (' world', 2),
+ ]
+ )
+ }
+
+ def runSingle(self, s, lines):
+ gotlines = [(d.s[d.lstart:d.lend], d.loc.line) for d in pymake.parser.enumeratelines(s, 'path')]
+ self.assertEqual(gotlines, lines)
+
+multitest(LineEnumeratorTest)
+
+class IterTest(TestBase):
+ testdata = {
+ 'plaindata': (
+ pymake.parser.iterdata,
+ "plaindata # test\n",
+ "plaindata # test\n"
+ ),
+ 'makecomment': (
+ pymake.parser.itermakefilechars,
+ "VAR = val # comment",
+ "VAR = val "
+ ),
+ 'makeescapedcomment': (
+ pymake.parser.itermakefilechars,
+ "VAR = val \# escaped hash",
+ "VAR = val # escaped hash"
+ ),
+ 'makeescapedslash': (
+ pymake.parser.itermakefilechars,
+ "VAR = val\\\\",
+ "VAR = val\\\\",
+ ),
+ 'makecontinuation': (
+ pymake.parser.itermakefilechars,
+ "VAR = VAL \\\n continuation # comment \\\n continuation",
+ "VAR = VAL continuation "
+ ),
+ 'makecontinuation2': (
+ pymake.parser.itermakefilechars,
+ "VAR = VAL \\ \\\n continuation",
+ "VAR = VAL \\ continuation"
+ ),
+ 'makeawful': (
+ pymake.parser.itermakefilechars,
+ "VAR = VAL \\\\# comment\n",
+ "VAR = VAL \\"
+ ),
+ 'command': (
+ pymake.parser.itercommandchars,
+ "echo boo # comment",
+ "echo boo # comment",
+ ),
+ 'commandcomment': (
+ pymake.parser.itercommandchars,
+ "echo boo \# comment",
+ "echo boo \# comment",
+ ),
+ 'commandcontinue': (
+ pymake.parser.itercommandchars,
+ "echo boo # \\\n\t command 2",
+ "echo boo # \\\n command 2"
+ ),
+ }
+
+ def runSingle(self, ifunc, idata, expected):
+ d = pymake.parser.Data.fromstring(idata, 'IterTest data')
+
+ it = pymake.parser._alltokens.finditer(d.s, 0, d.lend)
+ actual = ''.join( [c for c, t, o, oo in ifunc(d, 0, ('dummy-token',), it)] )
+ self.assertEqual(actual, expected)
+
+ if ifunc == pymake.parser.itermakefilechars:
+ print "testing %r" % expected
+ self.assertEqual(pymake.parser.flattenmakesyntax(d, 0), expected)
+
+multitest(IterTest)
+
+
+# 'define': (
+# pymake.parser.iterdefinechars,
+# "endef",
+# ""
+# ),
+# 'definenesting': (
+# pymake.parser.iterdefinechars,
+# """define BAR # comment
+#random text
+#endef not what you think!
+#endef # comment is ok\n""",
+# """define BAR # comment
+#random text
+#endef not what you think!"""
+# ),
+# 'defineescaped': (
+# pymake.parser.iterdefinechars,
+# """value \\
+#endef
+#endef\n""",
+# "value endef"
+# ),
+
+class MakeSyntaxTest(TestBase):
+ # (string, startat, stopat, stopoffset, expansion
+ testdata = {
+ 'text': ('hello world', 0, (), None, ['hello world']),
+ 'singlechar': ('hello $W', 0, (), None,
+ ['hello ',
+ {'type': 'VariableRef',
+ '.vname': ['W']}
+ ]),
+ 'stopat': ('hello: world', 0, (':', '='), 6, ['hello']),
+ 'funccall': ('h $(flavor FOO)', 0, (), None,
+ ['h ',
+ {'type': 'FlavorFunction',
+ '[0]': ['FOO']}
+ ]),
+ 'escapedollar': ('hello$$world', 0, (), None, ['hello$world']),
+ 'varref': ('echo $(VAR)', 0, (), None,
+ ['echo ',
+ {'type': 'VariableRef',
+ '.vname': ['VAR']}
+ ]),
+ 'dynamicvarname': ('echo $($(VARNAME):.c=.o)', 0, (':',), None,
+ ['echo ',
+ {'type': 'SubstitutionRef',
+ '.vname': [{'type': 'VariableRef',
+ '.vname': ['VARNAME']}
+ ],
+ '.substfrom': ['.c'],
+ '.substto': ['.o']}
+ ]),
+ 'substref': (' $(VAR:VAL) := $(VAL)', 0, (':=', '+=', '=', ':'), 15,
+ [' ',
+ {'type': 'VariableRef',
+ '.vname': ['VAR:VAL']},
+ ' ']),
+ 'vadsubstref': (' $(VAR:VAL) = $(VAL)', 15, (), None,
+ [{'type': 'VariableRef',
+ '.vname': ['VAL']},
+ ]),
+ }
+
+ def compareRecursive(self, actual, expected, path):
+ self.assertEqual(len(actual), len(expected),
+ "compareRecursive: %s %r" % (path, actual))
+ for i in xrange(0, len(actual)):
+ ipath = path + [i]
+
+ a, isfunc = actual[i]
+ e = expected[i]
+ if isinstance(e, str):
+ self.assertEqual(a, e, "compareRecursive: %s" % (ipath,))
+ else:
+ self.assertEqual(type(a), getattr(pymake.functions, e['type']),
+ "compareRecursive: %s" % (ipath,))
+ for k, v in e.iteritems():
+ if k == 'type':
+ pass
+ elif k[0] == '[':
+ item = int(k[1:-1])
+ proppath = ipath + [item]
+ self.compareRecursive(a[item], v, proppath)
+ elif k[0] == '.':
+ item = k[1:]
+ proppath = ipath + [item]
+ self.compareRecursive(getattr(a, item), v, proppath)
+ else:
+ raise Exception("Unexpected property at %s: %s" % (ipath, k))
+
+ def runSingle(self, s, startat, stopat, stopoffset, expansion):
+ d = pymake.parser.Data.fromstring(s, pymake.parserdata.Location('testdata', 1, 0))
+
+ a, t, offset = pymake.parser.parsemakesyntax(d, startat, stopat, pymake.parser.itermakefilechars)
+ self.compareRecursive(a, expansion, [])
+ self.assertEqual(offset, stopoffset)
+
+multitest(MakeSyntaxTest)
+
+class VariableTest(TestBase):
+ testdata = """
+ VAR = value
+ VARNAME = TESTVAR
+ $(VARNAME) = testvalue
+ $(VARNAME:VAR=VAL) = moretesting
+ IMM := $(VARNAME) # this is a comment
+ MULTIVAR = val1 \\
+ val2
+ VARNAME = newname
+ """
+ expected = {'VAR': 'value',
+ 'VARNAME': 'newname',
+ 'TESTVAR': 'testvalue',
+ 'TESTVAL': 'moretesting',
+ 'IMM': 'TESTVAR ',
+ 'MULTIVAR': 'val1 val2',
+ 'UNDEF': None}
+
+ def runTest(self):
+ stmts = pymake.parser.parsestring(self.testdata, 'VariableTest')
+
+ m = pymake.data.Makefile()
+ stmts.execute(m)
+ for k, v in self.expected.iteritems():
+ flavor, source, val = m.variables.get(k)
+ if val is None:
+ self.assertEqual(val, v, 'variable named %s' % k)
+ else:
+ self.assertEqual(val.resolvestr(m, m.variables), v, 'variable named %s' % k)
+
+class SimpleRuleTest(TestBase):
+ testdata = """
+ VAR = value
+TSPEC = dummy
+all: TSPEC = myrule
+all:: test test2 $(VAR)
+ echo "Hello, $(TSPEC)"
+
+%.o: %.c
+ $(CC) -o $@ $<
+"""
+
+ def runTest(self):
+ stmts = pymake.parser.parsestring(self.testdata, 'SimpleRuleTest')
+
+ m = pymake.data.Makefile()
+ stmts.execute(m)
+ self.assertEqual(m.defaulttarget, 'all', "Default target")
+
+ self.assertTrue(m.hastarget('all'), "Has 'all' target")
+ target = m.gettarget('all')
+ rules = target.rules
+ self.assertEqual(len(rules), 1, "Number of rules")
+ prereqs = rules[0].prerequisites
+ self.assertEqual(prereqs, ['test', 'test2', 'value'], "Prerequisites")
+ commands = rules[0].commands
+ self.assertEqual(len(commands), 1, "Number of commands")
+ expanded = commands[0].resolvestr(m, target.variables)
+ self.assertEqual(expanded, 'echo "Hello, myrule"')
+
+ irules = m.implicitrules
+ self.assertEqual(len(irules), 1, "Number of implicit rules")
+
+ irule = irules[0]
+ self.assertEqual(len(irule.targetpatterns), 1, "%.o target pattern count")
+ self.assertEqual(len(irule.prerequisites), 1, "%.o prerequisite count")
+ self.assertEqual(irule.targetpatterns[0].match('foo.o'), 'foo', "%.o stem")
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.DEBUG)
+ unittest.main()
diff --git a/build/pymake/tests/path-length.mk b/build/pymake/tests/path-length.mk
new file mode 100644
index 000000000..10c33b5ed
--- /dev/null
+++ b/build/pymake/tests/path-length.mk
@@ -0,0 +1,9 @@
+#T gmake skip
+
+$(shell \
+mkdir foo; \
+touch tfile; \
+)
+
+all: foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../tfile
+ @echo TEST-PASS
diff --git a/build/pymake/tests/pathdir/pathtest b/build/pymake/tests/pathdir/pathtest
new file mode 100755
index 000000000..17037159f
--- /dev/null
+++ b/build/pymake/tests/pathdir/pathtest
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo Called shell script: 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24
diff --git a/build/pymake/tests/pathdir/pathtest.exe b/build/pymake/tests/pathdir/pathtest.exe
new file mode 100644
index 000000000..3178db9a9
--- /dev/null
+++ b/build/pymake/tests/pathdir/pathtest.exe
Binary files differ
diff --git a/build/pymake/tests/pathdir/src/Makefile b/build/pymake/tests/pathdir/src/Makefile
new file mode 100644
index 000000000..6c24bd8f9
--- /dev/null
+++ b/build/pymake/tests/pathdir/src/Makefile
@@ -0,0 +1,2 @@
+pathtest.exe: pathtest.cpp
+ cl -EHsc -MT $^
diff --git a/build/pymake/tests/pathdir/src/pathtest.cpp b/build/pymake/tests/pathdir/src/pathtest.cpp
new file mode 100644
index 000000000..bef8d8a11
--- /dev/null
+++ b/build/pymake/tests/pathdir/src/pathtest.cpp
@@ -0,0 +1,6 @@
+#include <cstdio>
+
+int main() {
+ std::printf("Called Windows executable: 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24\n");
+ return 0;
+}
diff --git a/build/pymake/tests/patsubst.mk b/build/pymake/tests/patsubst.mk
new file mode 100644
index 000000000..0c3efdc4b
--- /dev/null
+++ b/build/pymake/tests/patsubst.mk
@@ -0,0 +1,7 @@
+all:
+ test "$(patsubst foo,%.bar,foo)" = "%.bar"
+ test "$(patsubst \%word,replace,word %word other)" = "word replace other"
+ test "$(patsubst %.c,\%%.o,foo.c bar.o baz.cpp)" = "%foo.o bar.o baz.cpp"
+ test "$(patsubst host_%.c,host_%.o,dir/host_foo.c host_bar.c)" = "dir/host_foo.c host_bar.o"
+ test "$(patsubst foo,bar,dir/foo foo baz)" = "dir/foo bar baz"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/phony.mk b/build/pymake/tests/phony.mk
new file mode 100644
index 000000000..36db4d121
--- /dev/null
+++ b/build/pymake/tests/phony.mk
@@ -0,0 +1,10 @@
+$(shell \
+touch dep; \
+sleep 2; \
+touch all; \
+)
+
+all:: dep
+ @echo TEST-PASS
+
+.PHONY: all
diff --git a/build/pymake/tests/pycmd.py b/build/pymake/tests/pycmd.py
new file mode 100644
index 000000000..83b9b966b
--- /dev/null
+++ b/build/pymake/tests/pycmd.py
@@ -0,0 +1,38 @@
+import os, sys, subprocess
+
+def writetofile(args):
+ with open(args[0], 'w') as f:
+ f.write(' '.join(args[1:]))
+
+def writeenvtofile(args):
+ with open(args[0], 'w') as f:
+ f.write(os.environ[args[1]])
+
+def writesubprocessenvtofile(args):
+ with open(args[0], 'w') as f:
+ p = subprocess.Popen([sys.executable, "-c",
+ "import os; print os.environ['%s']" % args[1]],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ assert p.returncode == 0
+ f.write(stdout)
+
+def convertasplode(arg):
+ try:
+ return int(arg)
+ except:
+ return (None if arg == "None" else arg)
+
+def asplode(args):
+ arg0 = convertasplode(args[0])
+ sys.exit(arg0)
+
+def asplode_return(args):
+ arg0 = convertasplode(args[0])
+ return arg0
+
+def asplode_raise(args):
+ raise Exception(args[0])
+
+def delayloadfn(args):
+ import delayload
diff --git a/build/pymake/tests/recursive-set.mk b/build/pymake/tests/recursive-set.mk
new file mode 100644
index 000000000..853f90463
--- /dev/null
+++ b/build/pymake/tests/recursive-set.mk
@@ -0,0 +1,7 @@
+#T returncode: 2
+
+FOO = $(FOO)
+
+all:
+ echo $(FOO)
+ @echo TEST-FAIL
diff --git a/build/pymake/tests/recursive-set2.mk b/build/pymake/tests/recursive-set2.mk
new file mode 100644
index 000000000..b68e34f0d
--- /dev/null
+++ b/build/pymake/tests/recursive-set2.mk
@@ -0,0 +1,8 @@
+#T returncode: 2
+
+FOO = $(BAR)
+BAR = $(FOO)
+
+all:
+ echo $(FOO)
+ @echo TEST-FAIL
diff --git a/build/pymake/tests/remake-mtime.mk b/build/pymake/tests/remake-mtime.mk
new file mode 100644
index 000000000..47c775b93
--- /dev/null
+++ b/build/pymake/tests/remake-mtime.mk
@@ -0,0 +1,14 @@
+# mtime(dep1) < mtime(target) so the target should not be made
+$(shell touch dep1; sleep 1; touch target)
+
+all: target
+ echo TEST-PASS
+
+target: dep1
+ echo TEST-FAIL target should not have been made
+
+dep1: dep2
+ @echo "Remaking dep1 (actually not)"
+
+dep2:
+ @echo "Making dep2 (actually not)"
diff --git a/build/pymake/tests/rm-fail.mk b/build/pymake/tests/rm-fail.mk
new file mode 100644
index 000000000..1a9aefb57
--- /dev/null
+++ b/build/pymake/tests/rm-fail.mk
@@ -0,0 +1,7 @@
+#T returncode: 2
+all:
+ mkdir newdir
+ test -d newdir
+ touch newdir/newfile
+ $(RM) newdir
+ @echo TEST-PASS
diff --git a/build/pymake/tests/rm.mk b/build/pymake/tests/rm.mk
new file mode 100644
index 000000000..6c7140e39
--- /dev/null
+++ b/build/pymake/tests/rm.mk
@@ -0,0 +1,21 @@
+all:
+# $(RM) defaults to -f
+ $(RM) nosuchfile
+ touch newfile
+ test -f newfile
+ $(RM) newfile
+ test ! -f newfile
+ mkdir newdir
+ test -d newdir
+ touch newdir/newfile
+ mkdir newdir/subdir
+ $(RM) -r newdir/subdir
+ test ! -d newdir/subdir
+ test -d newdir
+ mkdir newdir/subdir1 newdir/subdir2
+ $(RM) -r newdir/subdir1 newdir/subdir2
+ test ! -d newdir/subdir1 -a ! -d newdir/subdir2
+ test -d newdir
+ $(RM) -r newdir
+ test ! -d newdir
+ @echo TEST-PASS
diff --git a/build/pymake/tests/runtests.py b/build/pymake/tests/runtests.py
new file mode 100644
index 000000000..ab149ecfb
--- /dev/null
+++ b/build/pymake/tests/runtests.py
@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+"""
+Run the test(s) listed on the command line. If a directory is listed, the script will recursively
+walk the directory for files named .mk and run each.
+
+For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'.
+
+Each test is run in an empty directory.
+
+The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python:
+
+#T commandline: ['extra', 'params', 'here']
+#T returncode: 2
+#T returncode-on: {'win32': 2}
+#T environment: {'VAR': 'VALUE}
+#T grep-for: "text"
+"""
+
+from subprocess import Popen, PIPE, STDOUT
+from optparse import OptionParser
+import os, re, sys, shutil, glob
+
+class ParentDict(dict):
+ def __init__(self, parent, **kwargs):
+ self.d = dict(kwargs)
+ self.parent = parent
+
+ def __setitem__(self, k, v):
+ self.d[k] = v
+
+ def __getitem__(self, k):
+ if k in self.d:
+ return self.d[k]
+
+ return self.parent[k]
+
+thisdir = os.path.dirname(os.path.abspath(__file__))
+
+pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')]
+manifest = os.path.join(thisdir, 'tests.manifest')
+
+o = OptionParser()
+o.add_option('-g', '--gmake',
+ dest="gmake", default="gmake")
+o.add_option('-d', '--tempdir',
+ dest="tempdir", default="_mktests")
+opts, args = o.parse_args()
+
+if len(args) == 0:
+ args = [thisdir]
+
+makefiles = []
+for a in args:
+ if os.path.isfile(a):
+ makefiles.append(a)
+ elif os.path.isdir(a):
+ makefiles.extend(sorted(glob.glob(os.path.join(a, '*.mk'))))
+
+def runTest(makefile, make, logfile, options):
+ """
+ Given a makefile path, test it with a given `make` and return
+ (pass, message).
+ """
+
+ if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir)
+ os.mkdir(opts.tempdir, 0755)
+
+ logfd = open(logfile, 'w')
+ p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env'])
+ logfd.close()
+ retcode = p.wait()
+
+ if retcode != options['returncode']:
+ return False, "FAIL (returncode=%i)" % retcode
+
+ logfd = open(logfile)
+ stdout = logfd.read()
+ logfd.close()
+
+ if stdout.find('TEST-FAIL') != -1:
+ print stdout
+ return False, "FAIL (TEST-FAIL printed)"
+
+ if options['grepfor'] and stdout.find(options['grepfor']) == -1:
+ print stdout
+ return False, "FAIL (%s not in output)" % options['grepfor']
+
+ if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1:
+ print stdout
+ return False, 'FAIL (No TEST-PASS printed)'
+
+ if options['returncode'] != 0:
+ return True, 'PASS (retcode=%s)' % retcode
+
+ return True, 'PASS'
+
+print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:")
+
+gmakefails = 0
+pymakefails = 0
+
+tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$')
+
+for makefile in makefiles:
+ # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows
+ # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH
+ cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir]
+ if sys.platform == 'win32':
+ #XXX: hack so we can specialize the separator character on windows.
+ # we really shouldn't need this, but y'know
+ cline += ['__WIN32__=1']
+
+ options = {
+ 'returncode': 0,
+ 'grepfor': None,
+ 'env': dict(os.environ),
+ 'commandline': cline,
+ 'pass': True,
+ 'skip': False,
+ }
+
+ gmakeoptions = ParentDict(options)
+ pymakeoptions = ParentDict(options)
+
+ dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions}
+
+ mdata = open(makefile)
+ for line in mdata:
+ line = line.strip()
+ m = tre.search(line)
+ if m is None:
+ break
+
+ make, key, data = m.group(1, 2, 3)
+ d = dmap[make]
+ if data is not None:
+ data = eval(data)
+ if key == 'commandline':
+ assert make is None
+ d['commandline'].extend(data)
+ elif key == 'returncode':
+ d['returncode'] = data
+ elif key == 'returncode-on':
+ if sys.platform in data:
+ d['returncode'] = data[sys.platform]
+ elif key == 'environment':
+ for k, v in data.iteritems():
+ d['env'][k] = v
+ elif key == 'grep-for':
+ d['grepfor'] = data
+ elif key == 'fail':
+ d['pass'] = False
+ elif key == 'skip':
+ d['skip'] = True
+ else:
+ print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key)
+ sys.exit(1)
+
+ mdata.close()
+
+ if gmakeoptions['skip']:
+ gmakepass, gmakemsg = True, ''
+ else:
+ gmakepass, gmakemsg = runTest(makefile, [opts.gmake],
+ makefile + '.gmakelog', gmakeoptions)
+
+ if gmakeoptions['pass']:
+ if not gmakepass:
+ gmakefails += 1
+ else:
+ if gmakepass:
+ gmakefails += 1
+ gmakemsg = "UNEXPECTED PASS"
+ else:
+ gmakemsg = "KNOWN FAIL"
+
+ if pymakeoptions['skip']:
+ pymakepass, pymakemsg = True, ''
+ else:
+ pymakepass, pymakemsg = runTest(makefile, pymake,
+ makefile + '.pymakelog', pymakeoptions)
+
+ if pymakeoptions['pass']:
+ if not pymakepass:
+ pymakefails += 1
+ else:
+ if pymakepass:
+ pymakefails += 1
+ pymakemsg = "UNEXPECTED PASS"
+ else:
+ pymakemsg = "OK (known fail)"
+
+ print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile),
+ gmakemsg, pymakemsg)
+
+print
+print "Summary:"
+print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:")
+
+if gmakefails == 0:
+ gmakemsg = 'PASS'
+else:
+ gmakemsg = 'FAIL (%i failures)' % gmakefails
+
+if pymakefails == 0:
+ pymakemsg = 'PASS'
+else:
+ pymakemsg = 'FAIL (%i failures)' % pymakefails
+
+print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg)
+
+shutil.rmtree(opts.tempdir)
+
+if gmakefails or pymakefails:
+ sys.exit(1)
diff --git a/build/pymake/tests/serial-dep-resolution.mk b/build/pymake/tests/serial-dep-resolution.mk
new file mode 100644
index 000000000..e65f1ed03
--- /dev/null
+++ b/build/pymake/tests/serial-dep-resolution.mk
@@ -0,0 +1,5 @@
+all: t1 t2
+ @echo TEST-PASS
+
+t1:
+ touch t1 t2
diff --git a/build/pymake/tests/serial-doublecolon-execution.mk b/build/pymake/tests/serial-doublecolon-execution.mk
new file mode 100644
index 000000000..1871cb13a
--- /dev/null
+++ b/build/pymake/tests/serial-doublecolon-execution.mk
@@ -0,0 +1,18 @@
+#T commandline: ['-j3']
+
+# Commands of double-colon rules are always executed in order.
+
+all: dc
+ cat status
+ test "$$(cat status)" = "all1:all2:"
+ @echo TEST-PASS
+
+dc:: slowt
+ printf "all1:" >> status
+
+dc::
+ sleep 0.2
+ printf "all2:" >> status
+
+slowt:
+ sleep 1
diff --git a/build/pymake/tests/serial-rule-execution.mk b/build/pymake/tests/serial-rule-execution.mk
new file mode 100644
index 000000000..da5b177de
--- /dev/null
+++ b/build/pymake/tests/serial-rule-execution.mk
@@ -0,0 +1,5 @@
+all::
+ touch somefile
+
+all:: somefile
+ @echo TEST-PASS
diff --git a/build/pymake/tests/serial-rule-execution2.mk b/build/pymake/tests/serial-rule-execution2.mk
new file mode 100644
index 000000000..252a7df83
--- /dev/null
+++ b/build/pymake/tests/serial-rule-execution2.mk
@@ -0,0 +1,13 @@
+#T returncode: 2
+
+# The dependencies of the command rule of a single-colon target are resolved before the rules without commands.
+
+all: export
+
+export:
+ sleep 1
+ touch somefile
+
+all: somefile
+ test -f somefile
+ @echo TEST-PASS
diff --git a/build/pymake/tests/serial-toparallel.mk b/build/pymake/tests/serial-toparallel.mk
new file mode 100644
index 000000000..a980badc7
--- /dev/null
+++ b/build/pymake/tests/serial-toparallel.mk
@@ -0,0 +1,5 @@
+all::
+ $(MAKE) -j2 -f $(TESTPATH)/parallel-simple.mk
+
+all:: results
+ @echo TEST-PASS
diff --git a/build/pymake/tests/shellfunc.mk b/build/pymake/tests/shellfunc.mk
new file mode 100644
index 000000000..1e408dbac
--- /dev/null
+++ b/build/pymake/tests/shellfunc.mk
@@ -0,0 +1,7 @@
+all: testfile
+ test "$(shell cat $<)" = "Hello world"
+ test "$(shell printf "\n")" = ""
+ @echo TEST-PASS
+
+testfile:
+ printf "Hello\nworld\n" > $@
diff --git a/build/pymake/tests/simple-makeflags.mk b/build/pymake/tests/simple-makeflags.mk
new file mode 100644
index 000000000..c7c92ec9d
--- /dev/null
+++ b/build/pymake/tests/simple-makeflags.mk
@@ -0,0 +1,10 @@
+# There once was a time when MAKEFLAGS=w without any following spaces would
+# cause us to treat w as a target, not a flag. Silly!
+
+MAKEFLAGS=w
+
+all:
+ $(MAKE) -f $(TESTPATH)/simple-makeflags.mk subt
+ @echo TEST-PASS
+
+subt:
diff --git a/build/pymake/tests/sort.mk b/build/pymake/tests/sort.mk
new file mode 100644
index 000000000..e1313ad5c
--- /dev/null
+++ b/build/pymake/tests/sort.mk
@@ -0,0 +1,4 @@
+# sort should remove duplicates
+all:
+ @test "$(sort x a y b z c a z b x c y)" = "a b c x y z"
+ @echo "TEST-PASS"
diff --git a/build/pymake/tests/specified-target.mk b/build/pymake/tests/specified-target.mk
new file mode 100644
index 000000000..3b23fbf69
--- /dev/null
+++ b/build/pymake/tests/specified-target.mk
@@ -0,0 +1,7 @@
+#T commandline: ['VAR=all', '$(VAR)']
+
+all:
+ @echo TEST-FAIL: unexpected target 'all'
+
+$$(VAR):
+ @echo TEST-PASS: expected target '$$(VAR)'
diff --git a/build/pymake/tests/static-pattern.mk b/build/pymake/tests/static-pattern.mk
new file mode 100644
index 000000000..f613b8c9a
--- /dev/null
+++ b/build/pymake/tests/static-pattern.mk
@@ -0,0 +1,5 @@
+#T returncode: 2
+
+out/host_foo.o: host_%.o: host_%.c out
+ cp $< $@
+ @echo TEST-FAIL
diff --git a/build/pymake/tests/static-pattern2.mk b/build/pymake/tests/static-pattern2.mk
new file mode 100644
index 000000000..08ed834fd
--- /dev/null
+++ b/build/pymake/tests/static-pattern2.mk
@@ -0,0 +1,10 @@
+all: foo.out
+ test -f $^
+ @echo TEST-PASS
+
+foo.out: %.out: %.in
+ test "$*" = "foo"
+ cp $^ $@
+
+foo.in:
+ touch $@
diff --git a/build/pymake/tests/subdir/delayload.py b/build/pymake/tests/subdir/delayload.py
new file mode 100644
index 000000000..bdd6669db
--- /dev/null
+++ b/build/pymake/tests/subdir/delayload.py
@@ -0,0 +1 @@
+# This module exists to test delay importing of modules at run-time.
diff --git a/build/pymake/tests/subdir/pymod.py b/build/pymake/tests/subdir/pymod.py
new file mode 100644
index 000000000..1a47d8af2
--- /dev/null
+++ b/build/pymake/tests/subdir/pymod.py
@@ -0,0 +1,5 @@
+import testmodule
+
+def writetofile(args):
+ with open(args[0], 'w') as f:
+ f.write(' '.join(args[1:]))
diff --git a/build/pymake/tests/subdir/testmodule.py b/build/pymake/tests/subdir/testmodule.py
new file mode 100644
index 000000000..05b2f821a
--- /dev/null
+++ b/build/pymake/tests/subdir/testmodule.py
@@ -0,0 +1,3 @@
+# This is an empty module. It is imported by pymod.py to test that if a module
+# is loaded from the PYCOMMANDPATH, it can import other modules from the same
+# directory correctly.
diff --git a/build/pymake/tests/submake-path.makefile2 b/build/pymake/tests/submake-path.makefile2
new file mode 100644
index 000000000..1266db7d1
--- /dev/null
+++ b/build/pymake/tests/submake-path.makefile2
@@ -0,0 +1,11 @@
+# -*- Mode: Makefile -*-
+
+shellresult := $(shell pathtest)
+ifneq (2f7cdd0b-7277-48c1-beaf-56cb0dbacb24,$(filter $(shellresult),2f7cdd0b-7277-48c1-beaf-56cb0dbacb24))
+$(error pathtest not found in submake shell function)
+endif
+
+all:
+ @pathtest
+ @pathtest | grep -q 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24
+ @echo TEST-PASS
diff --git a/build/pymake/tests/submake-path.mk b/build/pymake/tests/submake-path.mk
new file mode 100644
index 000000000..b6432276d
--- /dev/null
+++ b/build/pymake/tests/submake-path.mk
@@ -0,0 +1,16 @@
+#T gmake skip
+#T grep-for: "2f7cdd0b-7277-48c1-beaf-56cb0dbacb24"
+
+ifdef __WIN32__
+PS:=;
+else
+PS:=:
+endif
+
+export PATH := $(TESTPATH)/pathdir$(PS)$(PATH)
+
+# This is similar to subprocess-path.mk, except we also check $(shell)
+# invocations since they're affected by exported environment variables too,
+# but only in submakes!
+all:
+ $(MAKE) -f $(TESTPATH)/submake-path.makefile2
diff --git a/build/pymake/tests/submake.makefile2 b/build/pymake/tests/submake.makefile2
new file mode 100644
index 000000000..12ce94834
--- /dev/null
+++ b/build/pymake/tests/submake.makefile2
@@ -0,0 +1,24 @@
+# -*- Mode: Makefile -*-
+
+$(info MAKEFLAGS = '$(MAKEFLAGS)')
+$(info MAKE = '$(MAKE)')
+$(info value MAKE = "$(value MAKE)")
+
+shellresult := $(shell echo -n $$EVAR)
+ifneq ($(shellresult),eval)
+$(error EVAR should be eval, is instead $(shellresult))
+endif
+
+all:
+ env
+ test "$(MAKELEVEL)" = "1"
+ echo "value(MAKE)" '$(value MAKE)'
+ echo "value(MAKE_COMMAND)" = '$(value MAKE_COMMAND)'
+ test "$(origin CVAR)" = "command line"
+ test "$(CVAR)" = "c val=spac\ed"
+ test "$(origin EVAR)" = "environment"
+ test "$(EVAR)" = "eval"
+ test "$(OVAL)" = "cline"
+ test "$(OVAL2)" = "cline2"
+ test "$(ALLVAR)" = "allspecific"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/submake.mk b/build/pymake/tests/submake.mk
new file mode 100644
index 000000000..41e47134b
--- /dev/null
+++ b/build/pymake/tests/submake.mk
@@ -0,0 +1,16 @@
+#T commandline: ['CVAR=c val=spac\\ed', 'OVAL=cline', 'OVAL2=cline2']
+
+export EVAR = eval
+override OVAL = makefile
+
+# exporting an override variable doesn't mean it's an override variable
+override OVAL2 = makefile2
+export OVAL2
+
+export ALLVAR
+ALLVAR = general
+all: ALLVAR = allspecific
+
+all:
+ test "$(MAKELEVEL)" = "0"
+ $(MAKE) -f $(TESTPATH)/submake.makefile2
diff --git a/build/pymake/tests/subprocess-path.mk b/build/pymake/tests/subprocess-path.mk
new file mode 100644
index 000000000..f63921414
--- /dev/null
+++ b/build/pymake/tests/subprocess-path.mk
@@ -0,0 +1,32 @@
+#T gmake skip
+#T grep-for: "2f7cdd0b-7277-48c1-beaf-56cb0dbacb24"
+
+ifdef __WIN32__
+PS:=;
+else
+PS:=:
+endif
+
+export PATH := $(TESTPATH)/pathdir$(PS)$(PATH)
+
+# Test two commands. The first one shouldn't go through the shell and the
+# second one should. The pathdir subdirectory has a Windows executable called
+# pathtest.exe and a shell script called pathtest. We don't care which one is
+# run, just that one of the two is (we use a uuid + grep-for to make sure
+# that happens).
+#
+# FAQ:
+# Q. Why skip GNU Make?
+# A. Because $(TESTPATH) is a Windows-style path, and MSYS make doesn't take
+# too kindly to Windows paths in the PATH environment variable.
+#
+# Q. Why use an exe and not a batch file?
+# A. The use cases here were all exe files without the extension. Batch file
+# lookup has broken semantics if the .bat extension isn't passed.
+#
+# Q. Why are the commands silent?
+# A. So that we don't pass the grep-for test by mistake.
+all:
+ @pathtest
+ @pathtest | grep -q 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24
+ @echo TEST-PASS
diff --git a/build/pymake/tests/tab-intro.mk b/build/pymake/tests/tab-intro.mk
new file mode 100644
index 000000000..1c25ce747
--- /dev/null
+++ b/build/pymake/tests/tab-intro.mk
@@ -0,0 +1,16 @@
+# Initial tab characters should be treated well.
+
+ THIS = a value
+
+ ifdef THIS
+ VAR = conditional value
+ endif
+
+all:
+ test "$(THIS)" = "another value"
+ test "$(VAR)" = "conditional value"
+ @echo TEST-PASS
+
+THAT = makefile syntax
+
+ THIS = another value
diff --git a/build/pymake/tests/target-specific.mk b/build/pymake/tests/target-specific.mk
new file mode 100644
index 000000000..217ed155e
--- /dev/null
+++ b/build/pymake/tests/target-specific.mk
@@ -0,0 +1,30 @@
+TESTVAR = anonval
+
+all: target.suffix target.suffix2 dummy host_test.py my.test1 my.test2
+ @echo TEST-PASS
+
+target.suffix: TESTVAR = testval
+
+%.suffix:
+ test "$(TESTVAR)" = "testval"
+
+%.suffix2: TESTVAR = testval2
+
+%.suffix2:
+ test "$(TESTVAR)" = "testval2"
+
+%my: TESTVAR = dummyval
+
+dummy:
+ test "$(TESTVAR)" = "dummyval"
+
+%.py: TESTVAR = pyval
+host_%.py: TESTVAR = hostval
+
+host_test.py:
+ test "$(TESTVAR)" = "hostval"
+
+%.test1 %.test2: TESTVAR = %val
+
+my.test1 my.test2:
+ test "$(TESTVAR)" = "%val"
diff --git a/build/pymake/tests/unexport.mk b/build/pymake/tests/unexport.mk
new file mode 100644
index 000000000..424411603
--- /dev/null
+++ b/build/pymake/tests/unexport.mk
@@ -0,0 +1,15 @@
+#T environment: {'ENVVAR': 'envval'}
+
+VAR1 = val1
+VAR2 = val2
+VAR3 = val3
+
+unexport VAR3
+export VAR1 VAR2 VAR3
+unexport VAR2 ENVVAR
+unexport
+
+all:
+ test "$(ENVVAR)" = "envval" # unexport.mk
+ $(MAKE) -f $(TESTPATH)/unexport.submk
+ @echo TEST-PASS
diff --git a/build/pymake/tests/unexport.submk b/build/pymake/tests/unexport.submk
new file mode 100644
index 000000000..8db6163de
--- /dev/null
+++ b/build/pymake/tests/unexport.submk
@@ -0,0 +1,15 @@
+# -@- Mode: Makefile -@-
+
+unexport VAR1
+
+all:
+ env
+ test "$(VAR1)" = "val1"
+ test "$(origin VAR1)" = "environment"
+ test "$(VAR2)" = "" # VAR2
+ test "$(VAR3)" = "val3"
+ test "$(ENVVAR)" = ""
+ $(MAKE) -f $(TESTPATH)/unexport.submk subt
+
+subt:
+ test "$(VAR1)" = ""
diff --git a/build/pymake/tests/unterminated-dollar.mk b/build/pymake/tests/unterminated-dollar.mk
new file mode 100644
index 000000000..dee9a207b
--- /dev/null
+++ b/build/pymake/tests/unterminated-dollar.mk
@@ -0,0 +1,6 @@
+VAR = value$
+VAR2 = other
+
+all:
+ test "$(VAR)" = "value"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/var-change-flavor.mk b/build/pymake/tests/var-change-flavor.mk
new file mode 100644
index 000000000..0cccf0bd6
--- /dev/null
+++ b/build/pymake/tests/var-change-flavor.mk
@@ -0,0 +1,12 @@
+VAR = value1
+VAR := value2
+
+VAR2 := val1
+VAR2 = val2
+
+default:
+ test "$(flavor VAR)" = "simple"
+ test "$(VAR)" = "value2"
+ test "$(flavor VAR2)" = "recursive"
+ test "$(VAR2)" = "val2"
+ @echo "TEST-PASS"
diff --git a/build/pymake/tests/var-commandline.mk b/build/pymake/tests/var-commandline.mk
new file mode 100644
index 000000000..e2cdad457
--- /dev/null
+++ b/build/pymake/tests/var-commandline.mk
@@ -0,0 +1,8 @@
+#T commandline: ['TESTVAR=$(MAKEVAL)', 'TESTVAR2:=$(MAKEVAL)']
+
+MAKEVAL=testvalue
+
+all:
+ test "$(TESTVAR)" = "testvalue"
+ test "$(TESTVAR2)" = ""
+ @echo "TEST-PASS" \ No newline at end of file
diff --git a/build/pymake/tests/var-overrides.mk b/build/pymake/tests/var-overrides.mk
new file mode 100644
index 000000000..bd0765d19
--- /dev/null
+++ b/build/pymake/tests/var-overrides.mk
@@ -0,0 +1,21 @@
+#T commandline: ['CLINEVAR=clineval', 'CLINEVAR2=clineval2']
+
+# this doesn't actually test overrides yet, because they aren't implemented in pymake,
+# but testing origins in general is important
+
+MVAR = mval
+CLINEVAR = deadbeef
+
+override CLINEVAR2 = mval2
+
+all:
+ test "$(origin NOVAR)" = "undefined"
+ test "$(CLINEVAR)" = "clineval"
+ test "$(origin CLINEVAR)" = "command line"
+ test "$(MVAR)" = "mval"
+ test "$(origin MVAR)" = "file"
+ test "$(@)" = "all"
+ test "$(origin @)" = "automatic"
+ test "$(origin CLINEVAR2)" = "override"
+ test "$(CLINEVAR2)" = "mval2"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/var-ref.mk b/build/pymake/tests/var-ref.mk
new file mode 100644
index 000000000..3bc1886f9
--- /dev/null
+++ b/build/pymake/tests/var-ref.mk
@@ -0,0 +1,19 @@
+VAR = value
+VAR2 == value
+
+VAR5 = $(NULL) $(NULL)
+VARC = value # comment
+
+$(VAR3)
+ $(VAR4)
+$(VAR5)
+
+VAR6$(VAR5) = val6
+
+all:
+ test "$( VAR)" = ""
+ test "$(VAR2)" = "= value"
+ test "${VAR2}" = "= value"
+ test "$(VAR6 )" = "val6"
+ test "$(VARC)" = "value "
+ @echo TEST-PASS
diff --git a/build/pymake/tests/var-set.mk b/build/pymake/tests/var-set.mk
new file mode 100644
index 000000000..1603e7a35
--- /dev/null
+++ b/build/pymake/tests/var-set.mk
@@ -0,0 +1,55 @@
+#T commandline: ['OBASIC=oval']
+
+BASIC = val
+
+TEST = $(TEST)
+
+TEST2 = $(TES
+TEST2 += T)
+
+TES T = val
+
+RECVAR = foo
+RECVAR += var baz
+
+IMMVAR := bloo
+IMMVAR += $(RECVAR)
+
+BASIC ?= notval
+
+all: BASIC = valall
+all: RECVAR += $(BASIC)
+all: IMMVAR += $(BASIC)
+all: UNSET += more
+all: OBASIC += allmore
+
+CHECKLIT = $(NULL) check
+all: CHECKLIT += appendliteral
+
+RECVAR = blimey
+
+TESTEMPTY = \
+ $(NULL)
+
+all: other
+ test "$(TEST2)" = "val"
+ test '$(value TEST2)' = '$$(TES T)'
+ test "$(RECVAR)" = "blimey valall"
+ test "$(IMMVAR)" = "bloo foo var baz valall"
+ test "$(UNSET)" = "more"
+ test "$(OBASIC)" = "oval"
+ test "$(CHECKLIT)" = " check appendliteral"
+ test "$(TESTEMPTY)" = ""
+ @echo TEST-PASS
+
+OVAR = oval
+OVAR ?= onotval
+
+other: OVAR ?= ooval
+other: LATERVAR ?= lateroverride
+
+LATERVAR = olater
+
+other:
+ test "$(OVAR)" = "oval"
+ test "$(LATERVAR)" = "lateroverride"
diff --git a/build/pymake/tests/var-substitutions.mk b/build/pymake/tests/var-substitutions.mk
new file mode 100644
index 000000000..d5627d7bd
--- /dev/null
+++ b/build/pymake/tests/var-substitutions.mk
@@ -0,0 +1,49 @@
+SIMPLEVAR = aabb.cc
+SIMPLEPERCENT = test_value%extra
+
+SIMPLE3SUBSTNAME = SIMPLEVAR:.dd
+$(SIMPLE3SUBSTNAME) = weirdval
+
+PERCENT = dummy
+
+SIMPLESUBST = $(SIMPLEVAR:.cc=.dd)
+SIMPLE2SUBST = $(SIMPLEVAR:.cc)
+SIMPLE3SUBST = $(SIMPLEVAR:.dd)
+SIMPLE4SUBST = $(SIMPLEVAR:.cc=.dd=.ee)
+SIMPLE5SUBST = $(SIMPLEVAR:.cc=%.dd)
+PERCENTSUBST = $(SIMPLEVAR:%.cc=%.ee)
+PERCENT2SUBST = $(SIMPLEVAR:aa%.cc=ff%.f)
+PERCENT3SUBST = $(SIMPLEVAR:aa%.dd=gg%.gg)
+PERCENT4SUBST = $(SIMPLEVAR:aa%.cc=gg)
+PERCENT5SUBST = $(SIMPLEVAR:aa)
+PERCENT6SUBST = $(SIMPLEVAR:%.cc=%.dd=%.ee)
+PERCENT7SUBST = $(SIMPLEVAR:$(PERCENT).cc=%.dd)
+PERCENT8SUBST = $(SIMPLEVAR:%.cc=$(PERCENT).dd)
+PERCENT9SUBST = $(SIMPLEVAR:$(PERCENT).cc=$(PERCENT).dd)
+PERCENT10SUBST = $(SIMPLEVAR:%%.bb.cc=zz.bb.cc)
+PERCENT11SUBST = $(SIMPLEPERCENT:test%value%extra=other%value%extra)
+
+SPACEDVAR = $(NULL) ex1.c ex2.c $(NULL)
+SPACEDSUBST = $(SPACEDVAR:.c=.o)
+
+all:
+ test "$(SIMPLESUBST)" = "aabb.dd"
+ test "$(SIMPLE2SUBST)" = ""
+ test "$(SIMPLE3SUBST)" = "weirdval"
+ test "$(SIMPLE4SUBST)" = "aabb.dd=.ee"
+ test "$(SIMPLE5SUBST)" = "aabb%.dd"
+ test "$(PERCENTSUBST)" = "aabb.ee"
+ test "$(PERCENT2SUBST)" = "ffbb.f"
+ test "$(PERCENT3SUBST)" = "aabb.cc"
+ test "$(PERCENT4SUBST)" = "gg"
+ test "$(PERCENT5SUBST)" = ""
+ test "$(PERCENT6SUBST)" = "aabb.dd=%.ee"
+ test "$(PERCENT7SUBST)" = "aabb.dd"
+ test "$(PERCENT8SUBST)" = "aabb.dd"
+ test "$(PERCENT9SUBST)" = "aabb.dd"
+ test "$(PERCENT10SUBST)" = "aabb.cc"
+ test "$(PERCENT11SUBST)" = "other_value%extra"
+ test "$(SPACEDSUBST)" = "ex1.o ex2.o"
+ @echo TEST-PASS
+
+PERCENT = %
diff --git a/build/pymake/tests/vpath-directive-dynamic.mk b/build/pymake/tests/vpath-directive-dynamic.mk
new file mode 100644
index 000000000..9aa1bf956
--- /dev/null
+++ b/build/pymake/tests/vpath-directive-dynamic.mk
@@ -0,0 +1,12 @@
+$(shell \
+mkdir subd1; \
+touch subd1/test.in; \
+)
+
+VVAR = %.in subd1
+
+vpath $(VVAR)
+
+all: test.in
+ test "$<" = "subd1/test.in"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/vpath-directive.mk b/build/pymake/tests/vpath-directive.mk
new file mode 100644
index 000000000..4c7d4bf39
--- /dev/null
+++ b/build/pymake/tests/vpath-directive.mk
@@ -0,0 +1,31 @@
+# On Windows, MSYS make takes Unix paths but Pymake takes Windows paths
+VPSEP := $(if $(and $(__WIN32__),$(.PYMAKE)),;,:)
+
+$(shell \
+mkdir subd1 subd2 subd3; \
+printf "reallybaddata" >subd1/foo.in; \
+printf "gooddata" >subd2/foo.in; \
+printf "baddata" >subd3/foo.in; \
+touch subd1/foo.in2 subd2/foo.in2 subd3/foo.in2; \
+)
+
+vpath %.in subd
+
+vpath
+vpath %.in subd2$(VPSEP)subd3
+
+vpath %.in2 subd0
+vpath f%.in2 subd1
+vpath %.in2 $(VPSEP)subd2
+
+%.out: %.in
+ test "$<" = "subd2/foo.in"
+ cp $< $@
+
+%.out2: %.in2
+ test "$<" = "subd1/foo.in2"
+ cp $< $@
+
+all: foo.out foo.out2
+ test "$$(cat foo.out)" = "gooddata"
+ @echo TEST-PASS
diff --git a/build/pymake/tests/vpath.mk b/build/pymake/tests/vpath.mk
new file mode 100644
index 000000000..06f52180c
--- /dev/null
+++ b/build/pymake/tests/vpath.mk
@@ -0,0 +1,18 @@
+VPATH = foo bar
+
+$(shell \
+mkdir foo; touch foo/tfile1; \
+mkdir bar; touch bar/tfile2 bar/tfile3 bar/test.objtest; \
+sleep 2; \
+touch bar/test.source; \
+)
+
+all: tfile1 tfile2 tfile3 test.objtest test.source
+ test "$^" = "foo/tfile1 bar/tfile2 tfile3 test.objtest bar/test.source"
+ @echo TEST-PASS
+
+tfile3: test.objtest
+
+%.objtest: %.source
+ test "$<" = bar/test.source
+ test "$@" = test.objtest
diff --git a/build/pymake/tests/vpath2.mk b/build/pymake/tests/vpath2.mk
new file mode 100644
index 000000000..be73ffe5c
--- /dev/null
+++ b/build/pymake/tests/vpath2.mk
@@ -0,0 +1,18 @@
+VPATH = foo bar
+
+$(shell \
+mkdir bar; touch bar/test.source; \
+sleep 2; \
+mkdir foo; touch foo/tfile1; \
+touch bar/tfile2 bar/tfile3 bar/test.objtest; \
+)
+
+all: tfile1 tfile2 tfile3 test.objtest test.source
+ test "$^" = "foo/tfile1 bar/tfile2 bar/tfile3 bar/test.objtest bar/test.source"
+ @echo TEST-PASS
+
+tfile3: test.objtest
+
+%.objtest: %.source
+ test "$<" = bar/test.source
+ test "$@" = test.objtest
diff --git a/build/pymake/tests/wildcards.mk b/build/pymake/tests/wildcards.mk
new file mode 100644
index 000000000..24ff3f14c
--- /dev/null
+++ b/build/pymake/tests/wildcards.mk
@@ -0,0 +1,22 @@
+$(shell \
+mkdir foo; \
+touch a.c b.c c.out foo/d.c; \
+sleep 2; \
+touch c.in; \
+)
+
+VPATH = foo
+
+all: c.out prog
+ cat $<
+ test "$$(cat $<)" = "remadec.out"
+ @echo TEST-PASS
+
+*.out: %.out: %.in
+ test "$@" = c.out
+ test "$<" = c.in
+ printf "remade$@" >$@
+
+prog: *.c
+ test "$^" = "a.c b.c"
+ touch $@
diff --git a/build/pymake/tests/windows-paths.mk b/build/pymake/tests/windows-paths.mk
new file mode 100644
index 000000000..5f33a9050
--- /dev/null
+++ b/build/pymake/tests/windows-paths.mk
@@ -0,0 +1,5 @@
+all:
+ touch file.in
+ printf "%s: %s\n\ttrue" '$(CURDIR)/file.out' '$(CURDIR)/file.in' >test.mk
+ $(MAKE) -f test.mk $(CURDIR)/file.out
+ @echo TEST-PASS