diff options
Diffstat (limited to 'python/pytoml')
| -rw-r--r-- | python/pytoml/PKG-INFO | 10 | ||||
| -rw-r--r-- | python/pytoml/pytoml/__init__.py | 3 | ||||
| -rw-r--r-- | python/pytoml/pytoml/core.py | 13 | ||||
| -rw-r--r-- | python/pytoml/pytoml/parser.py | 366 | ||||
| -rw-r--r-- | python/pytoml/pytoml/writer.py | 120 | ||||
| -rw-r--r-- | python/pytoml/setup.cfg | 5 | ||||
| -rw-r--r-- | python/pytoml/setup.py | 17 | ||||
| -rw-r--r-- | python/pytoml/test/test.py | 100 |
8 files changed, 634 insertions, 0 deletions
diff --git a/python/pytoml/PKG-INFO b/python/pytoml/PKG-INFO new file mode 100644 index 000000000..844436f95 --- /dev/null +++ b/python/pytoml/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0
+Name: pytoml
+Version: 0.1.10
+Summary: A parser for TOML-0.4.0
+Home-page: https://github.com/avakar/pytoml
+Author: Martin Vejnár
+Author-email: avakar@ratatanek.cz
+License: MIT
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/python/pytoml/pytoml/__init__.py b/python/pytoml/pytoml/__init__.py new file mode 100644 index 000000000..222a1967f --- /dev/null +++ b/python/pytoml/pytoml/__init__.py @@ -0,0 +1,3 @@ +from .core import TomlError
+from .parser import load, loads
+from .writer import dump, dumps
diff --git a/python/pytoml/pytoml/core.py b/python/pytoml/pytoml/core.py new file mode 100644 index 000000000..0fcada48c --- /dev/null +++ b/python/pytoml/pytoml/core.py @@ -0,0 +1,13 @@ +class TomlError(RuntimeError):
+ def __init__(self, message, line, col, filename):
+ RuntimeError.__init__(self, message, line, col, filename)
+ self.message = message
+ self.line = line
+ self.col = col
+ self.filename = filename
+
+ def __str__(self):
+ return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message)
+
+ def __repr__(self):
+ return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename)
diff --git a/python/pytoml/pytoml/parser.py b/python/pytoml/pytoml/parser.py new file mode 100644 index 000000000..d4c4ac187 --- /dev/null +++ b/python/pytoml/pytoml/parser.py @@ -0,0 +1,366 @@ +import string, re, sys, datetime
+from .core import TomlError
+
+if sys.version_info[0] == 2:
+ _chr = unichr
+else:
+ _chr = chr
+
+def load(fin, translate=lambda t, x, v: v):
+ return loads(fin.read(), translate=translate, filename=fin.name)
+
+def loads(s, filename='<string>', translate=lambda t, x, v: v):
+ if isinstance(s, bytes):
+ s = s.decode('utf-8')
+
+ s = s.replace('\r\n', '\n')
+
+ root = {}
+ tables = {}
+ scope = root
+
+ src = _Source(s, filename=filename)
+ ast = _p_toml(src)
+
+ def error(msg):
+ raise TomlError(msg, pos[0], pos[1], filename)
+
+ def process_value(v):
+ kind, text, value, pos = v
+ if kind == 'str' and value.startswith('\n'):
+ value = value[1:]
+ if kind == 'array':
+ if value and any(k != value[0][0] for k, t, v, p in value[1:]):
+ error('array-type-mismatch')
+ value = [process_value(item) for item in value]
+ elif kind == 'table':
+ value = dict([(k, process_value(value[k])) for k in value])
+ return translate(kind, text, value)
+
+ for kind, value, pos in ast:
+ if kind == 'kv':
+ k, v = value
+ if k in scope:
+ error('duplicate_keys. Key "{0}" was used more than once.'.format(k))
+ scope[k] = process_value(v)
+ else:
+ is_table_array = (kind == 'table_array')
+ cur = tables
+ for name in value[:-1]:
+ if isinstance(cur.get(name), list):
+ d, cur = cur[name][-1]
+ else:
+ d, cur = cur.setdefault(name, (None, {}))
+
+ scope = {}
+ name = value[-1]
+ if name not in cur:
+ if is_table_array:
+ cur[name] = [(scope, {})]
+ else:
+ cur[name] = (scope, {})
+ elif isinstance(cur[name], list):
+ if not is_table_array:
+ error('table_type_mismatch')
+ cur[name].append((scope, {}))
+ else:
+ if is_table_array:
+ error('table_type_mismatch')
+ old_scope, next_table = cur[name]
+ if old_scope is not None:
+ error('duplicate_tables')
+ cur[name] = (scope, next_table)
+
+ def merge_tables(scope, tables):
+ if scope is None:
+ scope = {}
+ for k in tables:
+ if k in scope:
+ error('key_table_conflict')
+ v = tables[k]
+ if isinstance(v, list):
+ scope[k] = [merge_tables(sc, tbl) for sc, tbl in v]
+ else:
+ scope[k] = merge_tables(v[0], v[1])
+ return scope
+
+ return merge_tables(root, tables)
+
+class _Source:
+ def __init__(self, s, filename=None):
+ self.s = s
+ self._pos = (1, 1)
+ self._last = None
+ self._filename = filename
+ self.backtrack_stack = []
+
+ def last(self):
+ return self._last
+
+ def pos(self):
+ return self._pos
+
+ def fail(self):
+ return self._expect(None)
+
+ def consume_dot(self):
+ if self.s:
+ self._last = self.s[0]
+ self.s = self[1:]
+ self._advance(self._last)
+ return self._last
+ return None
+
+ def expect_dot(self):
+ return self._expect(self.consume_dot())
+
+ def consume_eof(self):
+ if not self.s:
+ self._last = ''
+ return True
+ return False
+
+ def expect_eof(self):
+ return self._expect(self.consume_eof())
+
+ def consume(self, s):
+ if self.s.startswith(s):
+ self.s = self.s[len(s):]
+ self._last = s
+ self._advance(s)
+ return True
+ return False
+
+ def expect(self, s):
+ return self._expect(self.consume(s))
+
+ def consume_re(self, re):
+ m = re.match(self.s)
+ if m:
+ self.s = self.s[len(m.group(0)):]
+ self._last = m
+ self._advance(m.group(0))
+ return m
+ return None
+
+ def expect_re(self, re):
+ return self._expect(self.consume_re(re))
+
+ def __enter__(self):
+ self.backtrack_stack.append((self.s, self._pos))
+
+ def __exit__(self, type, value, traceback):
+ if type is None:
+ self.backtrack_stack.pop()
+ else:
+ self.s, self._pos = self.backtrack_stack.pop()
+ return type == TomlError
+
+ def commit(self):
+ self.backtrack_stack[-1] = (self.s, self._pos)
+
+ def _expect(self, r):
+ if not r:
+ raise TomlError('msg', self._pos[0], self._pos[1], self._filename)
+ return r
+
+ def _advance(self, s):
+ suffix_pos = s.rfind('\n')
+ if suffix_pos == -1:
+ self._pos = (self._pos[0], self._pos[1] + len(s))
+ else:
+ self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos)
+
+_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*')
+def _p_ews(s):
+ s.expect_re(_ews_re)
+
+_ws_re = re.compile(r'[ \t]*')
+def _p_ws(s):
+ s.expect_re(_ws_re)
+
+_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'',
+ '\\': '\\', '/': '/', 'f': '\f' }
+
+_basicstr_re = re.compile(r'[^"\\\000-\037]*')
+_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})')
+_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})')
+_escapes_re = re.compile('[bnrt"\'\\\\/f]')
+_newline_esc_re = re.compile('\n[ \t\n]*')
+def _p_basicstr_content(s, content=_basicstr_re):
+ res = []
+ while True:
+ res.append(s.expect_re(content).group(0))
+ if not s.consume('\\'):
+ break
+ if s.consume_re(_newline_esc_re):
+ pass
+ elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re):
+ res.append(_chr(int(s.last().group(1), 16)))
+ else:
+ s.expect_re(_escapes_re)
+ res.append(_escapes[s.last().group(0)])
+ return ''.join(res)
+
+_key_re = re.compile(r'[0-9a-zA-Z-_]+')
+def _p_key(s):
+ with s:
+ s.expect('"')
+ r = _p_basicstr_content(s, _basicstr_re)
+ s.expect('"')
+ return r
+ return s.expect_re(_key_re).group(0)
+
+_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?')
+_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))')
+
+_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*')
+_litstr_re = re.compile(r"[^'\000-\037]*")
+_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*")
+def _p_value(s):
+ pos = s.pos()
+
+ if s.consume('true'):
+ return 'bool', s.last(), True, pos
+ if s.consume('false'):
+ return 'bool', s.last(), False, pos
+
+ if s.consume('"'):
+ if s.consume('""'):
+ r = _p_basicstr_content(s, _basicstr_ml_re)
+ s.expect('"""')
+ else:
+ r = _p_basicstr_content(s, _basicstr_re)
+ s.expect('"')
+ return 'str', r, r, pos
+
+ if s.consume('\''):
+ if s.consume('\'\''):
+ r = s.expect_re(_litstr_ml_re).group(0)
+ s.expect('\'\'\'')
+ else:
+ r = s.expect_re(_litstr_re).group(0)
+ s.expect('\'')
+ return 'str', r, r, pos
+
+ if s.consume_re(_datetime_re):
+ m = s.last()
+ s0 = m.group(0)
+ r = map(int, m.groups()[:6])
+ if m.group(7):
+ micro = float(m.group(7))
+ else:
+ micro = 0
+
+ if m.group(8):
+ g = int(m.group(8), 10) * 60 + int(m.group(9), 10)
+ tz = _TimeZone(datetime.timedelta(0, g * 60))
+ else:
+ tz = _TimeZone(datetime.timedelta(0, 0))
+
+ y, m, d, H, M, S = r
+ dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz)
+ return 'datetime', s0, dt, pos
+
+ if s.consume_re(_float_re):
+ m = s.last().group(0)
+ r = m.replace('_','')
+ if '.' in m or 'e' in m or 'E' in m:
+ return 'float', m, float(r), pos
+ else:
+ return 'int', m, int(r, 10), pos
+
+ if s.consume('['):
+ items = []
+ with s:
+ while True:
+ _p_ews(s)
+ items.append(_p_value(s))
+ s.commit()
+ _p_ews(s)
+ s.expect(',')
+ s.commit()
+ _p_ews(s)
+ s.expect(']')
+ return 'array', None, items, pos
+
+ if s.consume('{'):
+ _p_ws(s)
+ items = {}
+ if not s.consume('}'):
+ k = _p_key(s)
+ _p_ws(s)
+ s.expect('=')
+ _p_ws(s)
+ items[k] = _p_value(s)
+ _p_ws(s)
+ while s.consume(','):
+ _p_ws(s)
+ k = _p_key(s)
+ _p_ws(s)
+ s.expect('=')
+ _p_ws(s)
+ items[k] = _p_value(s)
+ _p_ws(s)
+ s.expect('}')
+ return 'table', None, items, pos
+
+ s.fail()
+
+def _p_stmt(s):
+ pos = s.pos()
+ if s.consume( '['):
+ is_array = s.consume('[')
+ _p_ws(s)
+ keys = [_p_key(s)]
+ _p_ws(s)
+ while s.consume('.'):
+ _p_ws(s)
+ keys.append(_p_key(s))
+ _p_ws(s)
+ s.expect(']')
+ if is_array:
+ s.expect(']')
+ return 'table_array' if is_array else 'table', keys, pos
+
+ key = _p_key(s)
+ _p_ws(s)
+ s.expect('=')
+ _p_ws(s)
+ value = _p_value(s)
+ return 'kv', (key, value), pos
+
+_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*')
+def _p_toml(s):
+ stmts = []
+ _p_ews(s)
+ with s:
+ stmts.append(_p_stmt(s))
+ while True:
+ s.commit()
+ s.expect_re(_stmtsep_re)
+ stmts.append(_p_stmt(s))
+ _p_ews(s)
+ s.expect_eof()
+ return stmts
+
+class _TimeZone(datetime.tzinfo):
+ def __init__(self, offset):
+ self._offset = offset
+
+ def utcoffset(self, dt):
+ return self._offset
+
+ def dst(self, dt):
+ return None
+
+ def tzname(self, dt):
+ m = self._offset.total_seconds() // 60
+ if m < 0:
+ res = '-'
+ m = -m
+ else:
+ res = '+'
+ h = m // 60
+ m = m - h * 60
+ return '{}{:.02}{:.02}'.format(res, h, m)
diff --git a/python/pytoml/pytoml/writer.py b/python/pytoml/pytoml/writer.py new file mode 100644 index 000000000..2c9f7c69d --- /dev/null +++ b/python/pytoml/pytoml/writer.py @@ -0,0 +1,120 @@ +from __future__ import unicode_literals
+import io, datetime, sys
+
+if sys.version_info[0] == 3:
+ long = int
+ unicode = str
+
+
+def dumps(obj, sort_keys=False):
+ fout = io.StringIO()
+ dump(fout, obj, sort_keys=sort_keys)
+ return fout.getvalue()
+
+
+_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'}
+
+
+def _escape_string(s):
+ res = []
+ start = 0
+
+ def flush():
+ if start != i:
+ res.append(s[start:i])
+ return i + 1
+
+ i = 0
+ while i < len(s):
+ c = s[i]
+ if c in '"\\\n\r\t\b\f':
+ start = flush()
+ res.append('\\' + _escapes[c])
+ elif ord(c) < 0x20:
+ start = flush()
+ res.append('\\u%04x' % ord(c))
+ i += 1
+
+ flush()
+ return '"' + ''.join(res) + '"'
+
+
+def _escape_id(s):
+ if any(not c.isalnum() and c not in '-_' for c in s):
+ return _escape_string(s)
+ return s
+
+
+def _format_list(v):
+ return '[{0}]'.format(', '.join(_format_value(obj) for obj in v))
+
+# Formula from:
+# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds
+# Once support for py26 is dropped, this can be replaced by td.total_seconds()
+def _total_seconds(td):
+ return ((td.microseconds
+ + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6)
+
+def _format_value(v):
+ if isinstance(v, bool):
+ return 'true' if v else 'false'
+ if isinstance(v, int) or isinstance(v, long):
+ return unicode(v)
+ if isinstance(v, float):
+ return '{0:.17f}'.format(v)
+ elif isinstance(v, unicode) or isinstance(v, bytes):
+ return _escape_string(v)
+ elif isinstance(v, datetime.datetime):
+ offs = v.utcoffset()
+ offs = _total_seconds(offs) // 60 if offs is not None else 0
+
+ if offs == 0:
+ suffix = 'Z'
+ else:
+ if offs > 0:
+ suffix = '+'
+ else:
+ suffix = '-'
+ offs = -offs
+ suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60)
+
+ if v.microsecond:
+ return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix
+ else:
+ return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix
+ elif isinstance(v, list):
+ return _format_list(v)
+ else:
+ raise RuntimeError(v)
+
+
+def dump(fout, obj, sort_keys=False):
+ tables = [((), obj, False)]
+
+ while tables:
+ if sort_keys:
+ tables.sort(key=lambda tup: tup[0], reverse=True)
+ name, table, is_array = tables.pop()
+ if name:
+ section_name = '.'.join(_escape_id(c) for c in name)
+ if is_array:
+ fout.write('[[{0}]]\n'.format(section_name))
+ else:
+ fout.write('[{0}]\n'.format(section_name))
+
+ table_keys = sorted(table.keys()) if sort_keys else table.keys()
+ for k in table_keys:
+ v = table[k]
+ if isinstance(v, dict):
+ tables.append((name + (k,), v, False))
+ elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v):
+ tables.extend((name + (k,), d, True) for d in reversed(v))
+ elif v is None:
+ # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344
+ fout.write(
+ '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k)))
+ else:
+ fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v)))
+
+ if tables:
+ fout.write('\n')
diff --git a/python/pytoml/setup.cfg b/python/pytoml/setup.cfg new file mode 100644 index 000000000..b14b0bc3d --- /dev/null +++ b/python/pytoml/setup.cfg @@ -0,0 +1,5 @@ +[egg_info]
+tag_build =
+tag_date = 0
+tag_svn_revision = 0
+
diff --git a/python/pytoml/setup.py b/python/pytoml/setup.py new file mode 100644 index 000000000..98c08a540 --- /dev/null +++ b/python/pytoml/setup.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python
+# coding: utf-8
+
+from setuptools import setup
+
+setup(
+ name='pytoml',
+ version='0.1.10',
+
+ description='A parser for TOML-0.4.0',
+ author='Martin Vejnár',
+ author_email='avakar@ratatanek.cz',
+ url='https://github.com/avakar/pytoml',
+ license='MIT',
+
+ packages=['pytoml'],
+ )
diff --git a/python/pytoml/test/test.py b/python/pytoml/test/test.py new file mode 100644 index 000000000..53fcd229d --- /dev/null +++ b/python/pytoml/test/test.py @@ -0,0 +1,100 @@ +import os, json, sys, io, traceback, argparse
+import pytoml as toml
+
+# Formula from:
+# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds
+# Once support for py26 is dropped, this can be replaced by td.total_seconds()
+def _total_seconds(td):
+ return ((td.microseconds
+ + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6)
+
+def _testbench_literal(type, text, value):
+ if type == 'table':
+ return value
+ if type == 'array':
+ return { 'type': 'array', 'value': value }
+ if type == 'datetime':
+ offs = _total_seconds(value.tzinfo.utcoffset(value)) // 60
+ offs = 'Z' if offs == 0 else '{}{}:{}'.format('-' if offs < 0 else '-', abs(offs) // 60, abs(offs) % 60)
+ v = '{0:04}-{1:02}-{2:02}T{3:02}:{4:02}:{5:02}{6}'.format(value.year, value.month, value.day, value.hour, value.minute, value.second, offs)
+ return { 'type': 'datetime', 'value': v }
+ if type == 'bool':
+ return { 'type': 'bool', 'value': 'true' if value else 'false' }
+ if type == 'float':
+ return { 'type': 'float', 'value': value }
+ if type == 'str':
+ return { 'type': 'string', 'value': value }
+ if type == 'int':
+ return { 'type': 'integer', 'value': str(value) }
+
+def adjust_bench(v):
+ if isinstance(v, dict):
+ if v.get('type') == 'float':
+ v['value'] = float(v['value'])
+ return v
+ return dict([(k, adjust_bench(v[k])) for k in v])
+ if isinstance(v, list):
+ return [adjust_bench(v) for v in v]
+ return v
+
+def _main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('-d', '--dir', action='append')
+ ap.add_argument('testcase', nargs='*')
+ args = ap.parse_args()
+
+ if not args.dir:
+ args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')]
+
+ succeeded = []
+ failed = []
+
+ for path in args.dir:
+ if not os.path.isdir(path):
+ print('error: not a dir: {0}'.format(path))
+ return 2
+ for top, dirnames, fnames in os.walk(path):
+ for fname in fnames:
+ if not fname.endswith('.toml'):
+ continue
+
+ if args.testcase and not any(arg in fname for arg in args.testcase):
+ continue
+
+ parse_error = None
+ try:
+ with open(os.path.join(top, fname), 'rb') as fin:
+ parsed = toml.load(fin)
+ except toml.TomlError:
+ parsed = None
+ parse_error = sys.exc_info()
+ else:
+ dumped = toml.dumps(parsed)
+ parsed2 = toml.loads(dumped)
+ if parsed != parsed2:
+ failed.append((fname, None))
+ continue
+
+ with open(os.path.join(top, fname), 'rb') as fin:
+ parsed = toml.load(fin, translate=_testbench_literal)
+
+ try:
+ with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
+ bench = json.load(fin)
+ except IOError:
+ bench = None
+
+ if parsed != adjust_bench(bench):
+ failed.append((fname, parsed, bench, parse_error))
+ else:
+ succeeded.append(fname)
+
+ for f, parsed, bench, e in failed:
+ print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
+ if e:
+ traceback.print_exception(*e)
+ print('succeeded: {0}'.format(len(succeeded)))
+ return 1 if failed or not succeeded else 0
+
+if __name__ == '__main__':
+ sys.exit(_main())
|