From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- third_party/python/json_e/jsone/parser.py | 254 ++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 third_party/python/json_e/jsone/parser.py (limited to 'third_party/python/json_e/jsone/parser.py') diff --git a/third_party/python/json_e/jsone/parser.py b/third_party/python/json_e/jsone/parser.py new file mode 100644 index 0000000000..090928af8d --- /dev/null +++ b/third_party/python/json_e/jsone/parser.py @@ -0,0 +1,254 @@ +from .AST import Primitive, UnaryOp, ContextValue, BinOp, FunctionCall, ValueAccess, Object, List +from collections import namedtuple +import re +from .shared import TemplateError + +Token = namedtuple('Token', ['kind', 'value', 'start', 'end']) + +expectedTokens = ["!", "(", "+", "-", "[", "false", "identifier", "null", "number", "string", "true", "{"] + + +class SyntaxError(TemplateError): + + @classmethod + def unexpected(cls, got, exp): + exp = ', '.join(sorted(exp)) + return cls('Found: {} token, expected one of: {}'.format(got.value, exp)) + + +class Parser(object): + def __init__(self, source, tokenizer): + self.tokens = tokenizer.generate_tokens(source) + self.source = source + self.current_token = next(self.tokens) + self.unaryOpTokens = ["-", "+", "!"] + self.primitivesTokens = ["number", "null", "true", "false", "string"] + self.operatorsByPriority = [["||"], ["&&"], ["in"], ["==", "!="], ["<", ">", "<=", ">="], ["+", "-"], + ["*", "/"], ["**"]] + + def take_token(self, *kinds): + if not self.current_token: + raise SyntaxError('Unexpected end of input') + if kinds and self.current_token.kind not in kinds: + raise SyntaxError.unexpected(self.current_token, kinds) + try: + self.current_token = next(self.tokens) + except StopIteration: + self.current_token = None + except SyntaxError as exc: + raise exc + + def parse(self, level=0): + """ expr : logicalAnd (OR logicalAnd)* """ + """ logicalAnd : inStatement (AND inStatement)* """ + """ inStatement : equality (IN equality)* """ + """ equality : comparison (EQUALITY | INEQUALITY comparison)* """ + """ comparison : addition (LESS | GREATER | LESSEQUAL | GREATEREQUAL addition)* """ + """ addition : multiplication (PLUS | MINUS multiplication)* """ + """ multiplication : exponentiation (MUL | DIV exponentiation)* """ + """ exponentiation : propertyAccessOrFunc (EXP exponentiation)* """ + if level == len(self.operatorsByPriority) - 1: + node = self.parse_property_access_or_func() + token = self.current_token + + while token is not None and token.kind in self.operatorsByPriority[level]: + self.take_token(token.kind) + node = BinOp(token, self.parse(level), node) + token = self.current_token + else: + node = self.parse(level + 1) + token = self.current_token + + while token is not None and token.kind in self.operatorsByPriority[level]: + self.take_token(token.kind) + node = BinOp(token, node, self.parse(level + 1)) + token = self.current_token + + return node + + def parse_property_access_or_func(self): + """ propertyAccessOrFunc : unit (accessWithBrackets | DOT id | functionCall)* """ + node = self.parse_unit() + token = self.current_token + operators = ["[", "(", "."] + while token is not None and token.kind in operators: + if token.kind == "[": + node = self.parse_access_with_brackets(node) + elif token.kind == ".": + token = self.current_token + self.take_token(".") + right_part = Primitive(self.current_token) + self.take_token("identifier") + node = BinOp(token, node, right_part) + elif token.kind == "(": + node = self.parse_function_call(node) + token = self.current_token + return node + + def parse_unit(self): + # unit : unaryOp unit | primitives | contextValue | LPAREN expr RPAREN | list | object + token = self.current_token + if self.current_token is None: + raise SyntaxError('Unexpected end of input') + node = None + + if token.kind in self.unaryOpTokens: + self.take_token(token.kind) + node = UnaryOp(token, self.parse_unit()) + elif token.kind in self.primitivesTokens: + self.take_token(token.kind) + node = Primitive(token) + elif token.kind == "identifier": + self.take_token(token.kind) + node = ContextValue(token) + elif token.kind == "(": + self.take_token("(") + node = self.parse() + if node is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + self.take_token(")") + elif token.kind == "[": + node = self.parse_list() + elif token.kind == "{": + node = self.parse_object() + + return node + + def parse_function_call(self, name): + """functionCall: LPAREN (expr ( COMMA expr)*)? RPAREN""" + args = [] + token = self.current_token + self.take_token("(") + + if self.current_token.kind != ")": + node = self.parse() + args.append(node) + + while self.current_token is not None and self.current_token.kind == ",": + if args[-1] is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + self.take_token(",") + node = self.parse() + args.append(node) + + self.take_token(")") + node = FunctionCall(token, name, args) + + return node + + def parse_list(self): + """ list: LSQAREBRAKET (expr (COMMA expr)*)? RSQAREBRAKET """ + arr = [] + token = self.current_token + self.take_token("[") + + if self.current_token != "]": + node = self.parse() + arr.append(node) + + while self.current_token and self.current_token.kind == ",": + if arr[-1] is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + self.take_token(",") + node = self.parse() + arr.append(node) + + self.take_token("]") + node = List(token, arr) + + return node + + def parse_access_with_brackets(self, node): + """ valueAccess : LSQAREBRAKET expr |(expr? COLON expr?) RSQAREBRAKET)""" + left = None + right = None + is_interval = False + token = self.current_token + self.take_token("[") + if self.current_token.kind == "]": + raise SyntaxError.unexpected(self.current_token, expectedTokens) + if self.current_token.kind != ":": + left = self.parse() + if self.current_token.kind == ":": + is_interval = True + self.take_token(":") + if self.current_token.kind != "]": + right = self.parse() + + if is_interval and right is None and self.current_token.kind != "]": + raise SyntaxError.unexpected(self.current_token, expectedTokens) + + self.take_token("]") + node = ValueAccess(token, node, is_interval, left, right) + + return node + + def parse_object(self): + # """ object : LCURLYBRACE ( STR | ID COLON expr (COMMA STR | ID COLON expr)*)? + # RCURLYBRACE """ + obj = {} + objToken = self.current_token + self.take_token("{") + token = self.current_token + + while token is not None and (token.kind == "string" or token.kind == "identifier"): + key = token.value + if token.kind == "string": + key = parse_string(key) + self.take_token(token.kind) + self.take_token(":") + value = self.parse() + if value is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + obj[key] = value + if self.current_token and self.current_token.kind == "}": + break + else: + self.take_token(",") + token = self.current_token + + self.take_token("}") + node = Object(objToken, obj) + + return node + + +def parse_string(string): + return string[1:-1] + + +class Tokenizer(object): + def __init__(self, ignore, patterns, tokens): + self.ignore = ignore + self.patterns = patterns + self.tokens = tokens + # build a regular expression to generate a sequence of tokens + token_patterns = [ + '({})'.format(self.patterns.get(t, re.escape(t))) + for t in self.tokens] + if self.ignore: + token_patterns.append('(?:{})'.format(self.ignore)) + self.token_re = re.compile('^(?:' + '|'.join(token_patterns) + ')') + + def generate_tokens(self, source): + offset = 0 + while True: + start = offset + remainder = source[offset:] + mo = self.token_re.match(remainder) + if not mo: + if remainder: + raise SyntaxError( + "Unexpected input for '{}' at '{}'".format(source, remainder)) + break + offset += mo.end() + + # figure out which token matched (note that idx is 0-based) + indexes = [idx for idx, grp in enumerate(mo.groups()) if grp is not None] + if indexes: + idx = indexes[0] + yield Token( + kind=self.tokens[idx], + value=mo.group(idx + 1), # (mo.group is 1-based) + start=start, + end=offset) -- cgit v1.2.3