summaryrefslogtreecommitdiffstats
path: root/addon-sdk/source/python-lib/simplejson/scanner.py
diff options
context:
space:
mode:
Diffstat (limited to 'addon-sdk/source/python-lib/simplejson/scanner.py')
-rw-r--r--addon-sdk/source/python-lib/simplejson/scanner.py67
1 files changed, 67 insertions, 0 deletions
diff --git a/addon-sdk/source/python-lib/simplejson/scanner.py b/addon-sdk/source/python-lib/simplejson/scanner.py
new file mode 100644
index 000000000..2a18390d0
--- /dev/null
+++ b/addon-sdk/source/python-lib/simplejson/scanner.py
@@ -0,0 +1,67 @@
+"""
+Iterator based sre token scanner
+"""
+import re
+from re import VERBOSE, MULTILINE, DOTALL
+import sre_parse
+import sre_compile
+import sre_constants
+from sre_constants import BRANCH, SUBPATTERN
+
+__all__ = ['Scanner', 'pattern']
+
+FLAGS = (VERBOSE | MULTILINE | DOTALL)
+
+class Scanner(object):
+ def __init__(self, lexicon, flags=FLAGS):
+ self.actions = [None]
+ # Combine phrases into a compound pattern
+ s = sre_parse.Pattern()
+ s.flags = flags
+ p = []
+ for idx, token in enumerate(lexicon):
+ phrase = token.pattern
+ try:
+ subpattern = sre_parse.SubPattern(s,
+ [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
+ except sre_constants.error:
+ raise
+ p.append(subpattern)
+ self.actions.append(token)
+
+ s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
+ p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
+ self.scanner = sre_compile.compile(p)
+
+ def iterscan(self, string, idx=0, context=None):
+ """
+ Yield match, end_idx for each match
+ """
+ match = self.scanner.scanner(string, idx).match
+ actions = self.actions
+ lastend = idx
+ end = len(string)
+ while True:
+ m = match()
+ if m is None:
+ break
+ matchbegin, matchend = m.span()
+ if lastend == matchend:
+ break
+ action = actions[m.lastindex]
+ if action is not None:
+ rval, next_pos = action(m, context)
+ if next_pos is not None and next_pos != matchend:
+ # "fast forward" the scanner
+ matchend = next_pos
+ match = self.scanner.scanner(string, matchend).match
+ yield rval, matchend
+ lastend = matchend
+
+
+def pattern(pattern, flags=FLAGS):
+ def decorator(fn):
+ fn.pattern = pattern
+ fn.regex = re.compile(pattern, flags)
+ return fn
+ return decorator \ No newline at end of file