diff options
author | fmoga <fmoga@13f79535-47bb-0310-9956-ffa450edef68> | 2011-02-12 09:19:33 +0000 |
---|---|---|
committer | fmoga <fmoga@13f79535-47bb-0310-9956-ffa450edef68> | 2011-02-12 09:19:33 +0000 |
commit | f0527e13c01fe1fecbd195a70192b6caa9e5f7d0 (patch) | |
tree | 6a5859781e4f7ab190ce781d60744df738b37349 /sca-java-2.x/tags/2.0-Beta2-RC3/modules/implementation-python-runtime/src/main/resources/django/utils/simplejson/scanner.py | |
parent | 4f62b1f3e877a6ca26cfdd1670a550bc6a77477f (diff) |
Create the 2.0-Beta2-RC3 tag.
git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1070049 13f79535-47bb-0310-9956-ffa450edef68
Diffstat (limited to 'sca-java-2.x/tags/2.0-Beta2-RC3/modules/implementation-python-runtime/src/main/resources/django/utils/simplejson/scanner.py')
-rw-r--r-- | sca-java-2.x/tags/2.0-Beta2-RC3/modules/implementation-python-runtime/src/main/resources/django/utils/simplejson/scanner.py | 63 |
1 files changed, 63 insertions, 0 deletions
diff --git a/sca-java-2.x/tags/2.0-Beta2-RC3/modules/implementation-python-runtime/src/main/resources/django/utils/simplejson/scanner.py b/sca-java-2.x/tags/2.0-Beta2-RC3/modules/implementation-python-runtime/src/main/resources/django/utils/simplejson/scanner.py new file mode 100644 index 0000000000..64f4999fb5 --- /dev/null +++ b/sca-java-2.x/tags/2.0-Beta2-RC3/modules/implementation-python-runtime/src/main/resources/django/utils/simplejson/scanner.py @@ -0,0 +1,63 @@ +""" +Iterator based sre token scanner +""" +import sre_parse, sre_compile, sre_constants +from sre_constants import BRANCH, SUBPATTERN +from re import VERBOSE, MULTILINE, DOTALL +import re + +__all__ = ['Scanner', 'pattern'] + +FLAGS = (VERBOSE | MULTILINE | DOTALL) +class Scanner(object): + def __init__(self, lexicon, flags=FLAGS): + self.actions = [None] + # combine phrases into a compound pattern + s = sre_parse.Pattern() + s.flags = flags + p = [] + for idx, token in enumerate(lexicon): + phrase = token.pattern + try: + subpattern = sre_parse.SubPattern(s, + [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) + except sre_constants.error: + raise + p.append(subpattern) + self.actions.append(token) + + p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.scanner = sre_compile.compile(p) + + + def iterscan(self, string, idx=0, context=None): + """ + Yield match, end_idx for each match + """ + match = self.scanner.scanner(string, idx).match + actions = self.actions + lastend = idx + end = len(string) + while True: + m = match() + if m is None: + break + matchbegin, matchend = m.span() + if lastend == matchend: + break + action = actions[m.lastindex] + if action is not None: + rval, next_pos = action(m, context) + if next_pos is not None and next_pos != matchend: + # "fast forward" the scanner + matchend = next_pos + match = self.scanner.scanner(string, matchend).match + yield rval, matchend + lastend = matchend + +def pattern(pattern, flags=FLAGS): + def decorator(fn): + fn.pattern = pattern + fn.regex = re.compile(pattern, flags) + return fn + return decorator |