173 lines
5.2 KiB
Python
173 lines
5.2 KiB
Python
from __future__ import unicode_literals, print_function, absolute_import, division, generators, nested_scopes
|
|
import sys
|
|
import logging
|
|
|
|
import ply.lex
|
|
|
|
from jsonpath_ng.exceptions import JsonPathLexerError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class JsonPathLexer(object):
|
|
'''
|
|
A Lexical analyzer for JsonPath.
|
|
'''
|
|
|
|
def __init__(self, debug=False):
|
|
self.debug = debug
|
|
if self.__doc__ is None:
|
|
raise JsonPathLexerError('Docstrings have been removed! By design of PLY, jsonpath-rw requires docstrings. You must not use PYTHONOPTIMIZE=2 or python -OO.')
|
|
|
|
def tokenize(self, string):
|
|
'''
|
|
Maps a string to an iterator over tokens. In other words: [char] -> [token]
|
|
'''
|
|
|
|
new_lexer = ply.lex.lex(module=self, debug=self.debug, errorlog=logger)
|
|
new_lexer.latest_newline = 0
|
|
new_lexer.string_value = None
|
|
new_lexer.input(string)
|
|
|
|
while True:
|
|
t = new_lexer.token()
|
|
if t is None:
|
|
break
|
|
t.col = t.lexpos - new_lexer.latest_newline
|
|
yield t
|
|
|
|
if new_lexer.string_value is not None:
|
|
raise JsonPathLexerError('Unexpected EOF in string literal or identifier')
|
|
|
|
# ============== PLY Lexer specification ==================
|
|
#
|
|
# This probably should be private but:
|
|
# - the parser requires access to `tokens` (perhaps they should be defined in a third, shared dependency)
|
|
# - things like `literals` might be a legitimate part of the public interface.
|
|
#
|
|
# Anyhow, it is pythonic to give some rope to hang oneself with :-)
|
|
|
|
literals = ['*', '.', '[', ']', '(', ')', '$', ',', ':', '|', '&', '~']
|
|
|
|
reserved_words = { 'where': 'WHERE' }
|
|
|
|
tokens = ['DOUBLEDOT', 'NUMBER', 'ID', 'NAMED_OPERATOR'] + list(reserved_words.values())
|
|
|
|
states = [ ('singlequote', 'exclusive'),
|
|
('doublequote', 'exclusive'),
|
|
('backquote', 'exclusive') ]
|
|
|
|
# Normal lexing, rather easy
|
|
t_DOUBLEDOT = r'\.\.'
|
|
t_ignore = ' \t'
|
|
|
|
def t_ID(self, t):
|
|
r'[a-zA-Z_@][a-zA-Z0-9_@\-]*'
|
|
t.type = self.reserved_words.get(t.value, 'ID')
|
|
return t
|
|
|
|
def t_NUMBER(self, t):
|
|
r'-?\d+'
|
|
t.value = int(t.value)
|
|
return t
|
|
|
|
|
|
# Single-quoted strings
|
|
t_singlequote_ignore = ''
|
|
def t_singlequote(self, t):
|
|
r"'"
|
|
t.lexer.string_start = t.lexer.lexpos
|
|
t.lexer.string_value = ''
|
|
t.lexer.push_state('singlequote')
|
|
|
|
def t_singlequote_content(self, t):
|
|
r"[^'\\]+"
|
|
t.lexer.string_value += t.value
|
|
|
|
def t_singlequote_escape(self, t):
|
|
r'\\.'
|
|
t.lexer.string_value += t.value[1]
|
|
|
|
def t_singlequote_end(self, t):
|
|
r"'"
|
|
t.value = t.lexer.string_value
|
|
t.type = 'ID'
|
|
t.lexer.string_value = None
|
|
t.lexer.pop_state()
|
|
return t
|
|
|
|
def t_singlequote_error(self, t):
|
|
raise JsonPathLexerError('Error on line %s, col %s while lexing singlequoted field: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.lexer.latest_newline, t.value[0]))
|
|
|
|
|
|
# Double-quoted strings
|
|
t_doublequote_ignore = ''
|
|
def t_doublequote(self, t):
|
|
r'"'
|
|
t.lexer.string_start = t.lexer.lexpos
|
|
t.lexer.string_value = ''
|
|
t.lexer.push_state('doublequote')
|
|
|
|
def t_doublequote_content(self, t):
|
|
r'[^"\\]+'
|
|
t.lexer.string_value += t.value
|
|
|
|
def t_doublequote_escape(self, t):
|
|
r'\\.'
|
|
t.lexer.string_value += t.value[1]
|
|
|
|
def t_doublequote_end(self, t):
|
|
r'"'
|
|
t.value = t.lexer.string_value
|
|
t.type = 'ID'
|
|
t.lexer.string_value = None
|
|
t.lexer.pop_state()
|
|
return t
|
|
|
|
def t_doublequote_error(self, t):
|
|
raise JsonPathLexerError('Error on line %s, col %s while lexing doublequoted field: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.lexer.latest_newline, t.value[0]))
|
|
|
|
|
|
# Back-quoted "magic" operators
|
|
t_backquote_ignore = ''
|
|
def t_backquote(self, t):
|
|
r'`'
|
|
t.lexer.string_start = t.lexer.lexpos
|
|
t.lexer.string_value = ''
|
|
t.lexer.push_state('backquote')
|
|
|
|
def t_backquote_escape(self, t):
|
|
r'\\.'
|
|
t.lexer.string_value += t.value[1]
|
|
|
|
def t_backquote_content(self, t):
|
|
r"[^`\\]+"
|
|
t.lexer.string_value += t.value
|
|
|
|
def t_backquote_end(self, t):
|
|
r'`'
|
|
t.value = t.lexer.string_value
|
|
t.type = 'NAMED_OPERATOR'
|
|
t.lexer.string_value = None
|
|
t.lexer.pop_state()
|
|
return t
|
|
|
|
def t_backquote_error(self, t):
|
|
raise JsonPathLexerError('Error on line %s, col %s while lexing backquoted operator: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.lexer.latest_newline, t.value[0]))
|
|
|
|
|
|
# Counting lines, handling errors
|
|
def t_newline(self, t):
|
|
r'\n'
|
|
t.lexer.lineno += 1
|
|
t.lexer.latest_newline = t.lexpos
|
|
|
|
def t_error(self, t):
|
|
raise JsonPathLexerError('Error on line %s, col %s: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.lexer.latest_newline, t.value[0]))
|
|
|
|
if __name__ == '__main__':
|
|
logging.basicConfig()
|
|
lexer = JsonPathLexer(debug=True)
|
|
for token in lexer.tokenize(sys.stdin.read()):
|
|
print('%-20s%s' % (token.value, token.type))
|