blob: 8e647de876f69d292919349cf3d7272514f7d0ef (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
|
import re
import string
class InlineTokenizer:
def __init__(self, data):
self.data = data
def punctuation(self):
puncts = [ '[', ']', '{', '}' ]
for punct in puncts:
if self.data[0] == punct:
self.data = self.data[1:]
return punct
def up_to_comma(self):
match = re.match('(.*?)\s*, (.*)', self.data)
if match:
self.data = match.groups()[1]
return match.groups()[0]
def up_to_end_brace(self):
match = re.match('(.*?)(\s*[\]}].*)', self.data)
if match:
self.data = match.groups()[1]
return match.groups()[0]
def next(self):
self.data = string.strip(self.data)
productions = [
self.punctuation,
self.up_to_comma,
self.up_to_end_brace
]
for production in productions:
token = production()
if token:
return token
|