From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- .../python/json_e/json_e-4.5.3.dist-info/METADATA | 17 + .../python/json_e/json_e-4.5.3.dist-info/RECORD | 12 + .../python/json_e/json_e-4.5.3.dist-info/WHEEL | 5 + .../json_e/json_e-4.5.3.dist-info/top_level.txt | 1 + third_party/python/json_e/jsone/AST.py | 52 +++ third_party/python/json_e/jsone/__init__.py | 25 ++ third_party/python/json_e/jsone/builtins.py | 155 +++++++ third_party/python/json_e/jsone/interpreter.py | 211 ++++++++++ third_party/python/json_e/jsone/parser.py | 254 ++++++++++++ third_party/python/json_e/jsone/render.py | 450 +++++++++++++++++++++ third_party/python/json_e/jsone/shared.py | 133 ++++++ third_party/python/json_e/jsone/six.py | 23 ++ 12 files changed, 1338 insertions(+) create mode 100644 third_party/python/json_e/json_e-4.5.3.dist-info/METADATA create mode 100644 third_party/python/json_e/json_e-4.5.3.dist-info/RECORD create mode 100644 third_party/python/json_e/json_e-4.5.3.dist-info/WHEEL create mode 100644 third_party/python/json_e/json_e-4.5.3.dist-info/top_level.txt create mode 100644 third_party/python/json_e/jsone/AST.py create mode 100644 third_party/python/json_e/jsone/__init__.py create mode 100644 third_party/python/json_e/jsone/builtins.py create mode 100644 third_party/python/json_e/jsone/interpreter.py create mode 100644 third_party/python/json_e/jsone/parser.py create mode 100644 third_party/python/json_e/jsone/render.py create mode 100644 third_party/python/json_e/jsone/shared.py create mode 100644 third_party/python/json_e/jsone/six.py (limited to 'third_party/python/json_e') diff --git a/third_party/python/json_e/json_e-4.5.3.dist-info/METADATA b/third_party/python/json_e/json_e-4.5.3.dist-info/METADATA new file mode 100644 index 0000000000..3e49370776 --- /dev/null +++ b/third_party/python/json_e/json_e-4.5.3.dist-info/METADATA @@ -0,0 +1,17 @@ +Metadata-Version: 2.1 +Name: json-e +Version: 4.5.3 +Summary: A data-structure parameterization system written for embedding context in JSON objects +Home-page: https://json-e.js.org +Author: Dustin J. Mitchell +Author-email: dustin@mozilla.com +License: MPL2 +Platform: UNKNOWN +Provides-Extra: release +Requires-Dist: towncrier ; extra == 'release' + +A data-structure parameterization system written for embedding context in JSON objects. + +See https://json-e.js.org for usage information. + + diff --git a/third_party/python/json_e/json_e-4.5.3.dist-info/RECORD b/third_party/python/json_e/json_e-4.5.3.dist-info/RECORD new file mode 100644 index 0000000000..acd709f3d9 --- /dev/null +++ b/third_party/python/json_e/json_e-4.5.3.dist-info/RECORD @@ -0,0 +1,12 @@ +jsone/AST.py,sha256=cJ91hvKhwjHEV6XiwjFBMua2TNIWNY8V_WvpfWDMQ1E,1135 +jsone/__init__.py,sha256=vuEVBfu9njheEO60w85t0UmZFT8nWUwMpurzTa0Cs6c,936 +jsone/builtins.py,sha256=lWUibi4zIkpw6WfpAFxNVd-xL9f2CYE3P0hOc926b2E,4922 +jsone/interpreter.py,sha256=B19sSEgjDfN3UC12sEFLn7kN45n_73IuQjf7azf8iXY,7623 +jsone/parser.py,sha256=FpGX-3S4BkppPkWXUBKIMFOpposLZqykL0BQciYm8vM,9405 +jsone/render.py,sha256=3iM8BUlAzaMiRrMdK4YMhSrSsJbw5IUb1PCkZUAJbSc,14545 +jsone/shared.py,sha256=MY7c-pwN90zJ6h7bGaO5MFfVpWRqI97KSUy11Lo8Rw4,3560 +jsone/six.py,sha256=u2Ak2U5WK4-RIHgmA9GbfD7IwTFGQZEzeQHGxOunhDk,793 +json_e-4.5.3.dist-info/METADATA,sha256=z9rOlUDXI6K4aw4Ma6g5jSeWqxxpkt977yC7syZu5Fo,482 +json_e-4.5.3.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92 +json_e-4.5.3.dist-info/top_level.txt,sha256=SLNeKVK9ibpBIQPdn0rxszb15jdnJpdsEgppw0CFHi4,6 +json_e-4.5.3.dist-info/RECORD,, diff --git a/third_party/python/json_e/json_e-4.5.3.dist-info/WHEEL b/third_party/python/json_e/json_e-4.5.3.dist-info/WHEEL new file mode 100644 index 0000000000..d272f6ed55 --- /dev/null +++ b/third_party/python/json_e/json_e-4.5.3.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.41.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/third_party/python/json_e/json_e-4.5.3.dist-info/top_level.txt b/third_party/python/json_e/json_e-4.5.3.dist-info/top_level.txt new file mode 100644 index 0000000000..afe8caa74c --- /dev/null +++ b/third_party/python/json_e/json_e-4.5.3.dist-info/top_level.txt @@ -0,0 +1 @@ +jsone diff --git a/third_party/python/json_e/jsone/AST.py b/third_party/python/json_e/jsone/AST.py new file mode 100644 index 0000000000..3503f79dfc --- /dev/null +++ b/third_party/python/json_e/jsone/AST.py @@ -0,0 +1,52 @@ +class ASTNode(object): + def __init__(self, token): + self.token = token + + +Primitive = ASTNode + + +class BinOp(ASTNode): + def __init__(self, token, left, right): + ASTNode.__init__(self, token) + self.left = left + self.right = right + + +class UnaryOp(ASTNode): + def __init__(self, token, expr): + ASTNode.__init__(self, token) + self.expr = expr + + +class FunctionCall(ASTNode): + def __init__(self, token, name, args): + ASTNode.__init__(self, token) + self.name = name + self.args = args + + +class ContextValue(ASTNode): + def __init__(self, token): + ASTNode.__init__(self, token) + + +class List(ASTNode): + def __init__(self, token, list): + ASTNode.__init__(self, token) + self.list = list + + +class ValueAccess(ASTNode): + def __init__(self, token, arr, isInterval, left, right): + ASTNode.__init__(self, token) + self.arr = arr + self.isInterval = isInterval + self.left = left + self.right = right + + +class Object(ASTNode): + def __init__(self, token, obj): + ASTNode.__init__(self, token) + self.obj = obj diff --git a/third_party/python/json_e/jsone/__init__.py b/third_party/python/json_e/jsone/__init__.py new file mode 100644 index 0000000000..e8720e7f98 --- /dev/null +++ b/third_party/python/json_e/jsone/__init__.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +from .render import renderValue, containsFunctions +from .shared import JSONTemplateError, DeleteMarker, TemplateError, fromNow +from . import builtins + +_context_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$') + + +def render(template, context): + if type(context) != dict: + raise TemplateError("context must be a dictionary") + if not all(_context_re.match(c) for c in context): + raise TemplateError('top level keys of context must follow ' + '/[a-zA-Z_][a-zA-Z0-9_]*/') + full_context = {'now': fromNow('0 seconds', None)} + full_context.update(builtins.build()) + full_context.update(context) + rv = renderValue(template, full_context) + if rv is DeleteMarker: + return None + if containsFunctions(rv): + raise TemplateError('evaluated template contained uncalled functions') + return rv diff --git a/third_party/python/json_e/jsone/builtins.py b/third_party/python/json_e/jsone/builtins.py new file mode 100644 index 0000000000..fb70266d8f --- /dev/null +++ b/third_party/python/json_e/jsone/builtins.py @@ -0,0 +1,155 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import math +from .shared import string, to_str, fromNow, JSONTemplateError + + +class BuiltinError(JSONTemplateError): + pass + + +def build(): + builtins = {} + + def builtin(name, variadic=None, argument_tests=None, minArgs=None, needs_context=False): + def wrap(fn): + if variadic: + def invoke(context, *args): + if minArgs: + if len(args) < minArgs: + raise BuiltinError( + 'invalid arguments to builtin: {}: expected at least {} arguments'.format(name, minArgs) + ) + for arg in args: + if not variadic(arg): + raise BuiltinError('invalid arguments to builtin: {}'.format(name)) + if needs_context is True: + return fn(context, *args) + return fn(*args) + + elif argument_tests: + def invoke(context, *args): + if len(args) != len(argument_tests): + raise BuiltinError('invalid arguments to builtin: {}'.format(name)) + for t, arg in zip(argument_tests, args): + if not t(arg): + raise BuiltinError('invalid arguments to builtin: {}'.format(name)) + if needs_context is True: + return fn(context, *args) + return fn(*args) + + else: + def invoke(context, *args): + if needs_context is True: + return fn(context, *args) + return fn(*args) + + invoke._jsone_builtin = True + builtins[name] = invoke + return fn + + return wrap + + def is_number(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + def is_string(v): + return isinstance(v, string) + + def is_string_or_number(v): + return is_string(v) or is_number(v) + + def is_array(v): + return isinstance(v, list) + + def is_string_or_array(v): + return isinstance(v, (string, list)) + + def anything_except_array(v): + return isinstance(v, (string, int, float, bool)) or v is None + + def anything(v): + return isinstance(v, (string, int, float, list, dict)) or v is None or callable(v) + + # --- + + builtin('min', variadic=is_number, minArgs=1)(min) + builtin('max', variadic=is_number, minArgs=1)(max) + builtin('sqrt', argument_tests=[is_number])(math.sqrt) + builtin('abs', argument_tests=[is_number])(abs) + + @builtin('ceil', argument_tests=[is_number]) + def ceil(v): + return int(math.ceil(v)) + + @builtin('floor', argument_tests=[is_number]) + def floor(v): + return int(math.floor(v)) + + @builtin('lowercase', argument_tests=[is_string]) + def lowercase(v): + return v.lower() + + @builtin('uppercase', argument_tests=[is_string]) + def lowercase(v): + return v.upper() + + builtin('len', argument_tests=[is_string_or_array])(len) + builtin('str', argument_tests=[anything_except_array])(to_str) + builtin('number', variadic=is_string, minArgs=1)(float) + + @builtin('strip', argument_tests=[is_string]) + def strip(s): + return s.strip() + + @builtin('rstrip', argument_tests=[is_string]) + def rstrip(s): + return s.rstrip() + + @builtin('lstrip', argument_tests=[is_string]) + def lstrip(s): + return s.lstrip() + + @builtin('join', argument_tests=[is_array, is_string_or_number]) + def join(list, separator): + # convert potential numbers into strings + string_list = [str(int) for int in list] + + return str(separator).join(string_list) + + @builtin('split', variadic=is_string_or_number, minArgs=1) + def split(s, d=''): + if not d and is_string(s): + return list(s) + + return s.split(to_str(d)) + + @builtin('fromNow', variadic=is_string, minArgs=1, needs_context=True) + def fromNow_builtin(context, offset, reference=None): + return fromNow(offset, reference or context.get('now')) + + @builtin('typeof', argument_tests=[anything]) + def typeof(v): + if isinstance(v, bool): + return 'boolean' + elif isinstance(v, string): + return 'string' + elif isinstance(v, (int, float)): + return 'number' + elif isinstance(v, list): + return 'array' + elif isinstance(v, dict): + return 'object' + elif v is None: + return 'null' + elif callable(v): + return 'function' + + @builtin('defined', argument_tests=[is_string], needs_context=True) + def defined(context, s): + if s not in context: + return False + else: + return True + + return builtins diff --git a/third_party/python/json_e/jsone/interpreter.py b/third_party/python/json_e/jsone/interpreter.py new file mode 100644 index 0000000000..5ae09a53d7 --- /dev/null +++ b/third_party/python/json_e/jsone/interpreter.py @@ -0,0 +1,211 @@ +from .shared import InterpreterError, string +import operator + + +def infixExpectationError(operator, expected): + return InterpreterError('infix: {} expects {} {} {}'. + format(operator, expected, operator, expected)) + + +class Interpreter: + def __init__(self, context): + self.context = context + + def visit(self, node): + method_name = 'visit_' + type(node).__name__ + visitor = getattr(self, method_name) + return visitor(node) + + def visit_ASTNode(self, node): + if node.token.kind == "number": + v = node.token.value + return float(v) if '.' in v else int(v) + elif node.token.kind == "null": + return None + elif node.token.kind == "string": + return node.token.value[1:-1] + elif node.token.kind == "true": + return True + elif node.token.kind == "false": + return False + elif node.token.kind == "identifier": + return node.token.value + + def visit_UnaryOp(self, node): + value = self.visit(node.expr) + if node.token.kind == "+": + if not is_number(value): + raise InterpreterError('{} expects {}'.format('unary +', 'number')) + return value + elif node.token.kind == "-": + if not is_number(value): + raise InterpreterError('{} expects {}'.format('unary -', 'number')) + return -value + elif node.token.kind == "!": + return not self.visit(node.expr) + + def visit_BinOp(self, node): + left = self.visit(node.left) + if node.token.kind == "||": + return bool(left or self.visit(node.right)) + elif node.token.kind == "&&": + return bool(left and self.visit(node.right)) + else: + right = self.visit(node.right) + + if node.token.kind == "+": + if not isinstance(left, (string, int, float)) or isinstance(left, bool): + raise infixExpectationError('+', 'numbers/strings') + if not isinstance(right, (string, int, float)) or isinstance(right, bool): + raise infixExpectationError('+', 'numbers/strings') + if type(right) != type(left) and \ + (isinstance(left, string) or isinstance(right, string)): + raise infixExpectationError('+', 'numbers/strings') + return left + right + elif node.token.kind == "-": + test_math_operands("-", left, right) + return left - right + elif node.token.kind == "/": + test_math_operands("/", left, right) + return operator.truediv(left, right) + elif node.token.kind == "*": + test_math_operands("*", left, right) + return left * right + elif node.token.kind == ">": + test_comparison_operands(">", left, right) + return left > right + elif node.token.kind == "<": + test_comparison_operands("<", left, right) + return left < right + elif node.token.kind == ">=": + test_comparison_operands(">=", left, right) + return left >= right + elif node.token.kind == "<=": + test_comparison_operands("<=", left, right) + return left <= right + elif node.token.kind == "!=": + return left != right + elif node.token.kind == "==": + return left == right + elif node.token.kind == "**": + test_math_operands("**", left, right) + return right ** left + elif node.token.value == "in": + if isinstance(right, dict): + if not isinstance(left, string): + raise infixExpectationError('in-object', 'string on left side') + elif isinstance(right, string): + if not isinstance(left, string): + raise infixExpectationError('in-string', 'string on left side') + elif not isinstance(right, list): + raise infixExpectationError( + 'in', 'Array, string, or object on right side') + try: + return left in right + except TypeError: + raise infixExpectationError('in', 'scalar value, collection') + + elif node.token.kind == ".": + if not isinstance(left, dict): + raise InterpreterError('infix: {} expects {}'.format(".", 'objects')) + try: + return left[right] + except KeyError: + raise InterpreterError( + 'object has no property "{}"'.format(right)) + + def visit_List(self, node): + list = [] + + if node.list[0] is not None: + for item in node.list: + list.append(self.visit(item)) + + return list + + def visit_ValueAccess(self, node): + value = self.visit(node.arr) + left = 0 + right = None + + if node.left: + left = self.visit(node.left) + if node.right: + right = self.visit(node.right) + + if isinstance(value, (list, string)): + if node.isInterval: + if right is None: + right = len(value) + try: + return value[left:right] + except TypeError: + raise InterpreterError('cannot perform interval access with non-integers') + else: + try: + return value[left] + except IndexError: + raise InterpreterError('index out of bounds') + except TypeError: + raise InterpreterError('should only use integers to access arrays or strings') + + if not isinstance(value, dict): + raise InterpreterError('infix: {} expects {}'.format('"[..]"', 'object, array, or string')) + if not isinstance(left, string): + raise InterpreterError('object keys must be strings') + + try: + return value[left] + except KeyError: + return None + + def visit_ContextValue(self, node): + try: + contextValue = self.context[node.token.value] + except KeyError: + raise InterpreterError( + 'unknown context value {}'.format(node.token.value)) + return contextValue + + def visit_FunctionCall(self, node): + args = [] + func_name = self.visit(node.name) + if callable(func_name): + if node.args is not None: + for item in node.args: + args.append(self.visit(item)) + if hasattr(func_name, "_jsone_builtin"): + return func_name(self.context, *args) + else: + return func_name(*args) + else: + raise InterpreterError( + '{} is not callable'.format(func_name)) + + def visit_Object(self, node): + obj = {} + for key in node.obj: + obj[key] = self.visit(node.obj[key]) + return obj + + def interpret(self, tree): + return self.visit(tree) + + +def test_math_operands(op, left, right): + if not is_number(left): + raise infixExpectationError(op, 'number') + if not is_number(right): + raise infixExpectationError(op, 'number') + return + + +def test_comparison_operands(op, left, right): + if type(left) != type(right) or \ + not (isinstance(left, (int, float, string)) and not isinstance(left, bool)): + raise infixExpectationError(op, 'numbers/strings') + return + + +def is_number(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) diff --git a/third_party/python/json_e/jsone/parser.py b/third_party/python/json_e/jsone/parser.py new file mode 100644 index 0000000000..090928af8d --- /dev/null +++ b/third_party/python/json_e/jsone/parser.py @@ -0,0 +1,254 @@ +from .AST import Primitive, UnaryOp, ContextValue, BinOp, FunctionCall, ValueAccess, Object, List +from collections import namedtuple +import re +from .shared import TemplateError + +Token = namedtuple('Token', ['kind', 'value', 'start', 'end']) + +expectedTokens = ["!", "(", "+", "-", "[", "false", "identifier", "null", "number", "string", "true", "{"] + + +class SyntaxError(TemplateError): + + @classmethod + def unexpected(cls, got, exp): + exp = ', '.join(sorted(exp)) + return cls('Found: {} token, expected one of: {}'.format(got.value, exp)) + + +class Parser(object): + def __init__(self, source, tokenizer): + self.tokens = tokenizer.generate_tokens(source) + self.source = source + self.current_token = next(self.tokens) + self.unaryOpTokens = ["-", "+", "!"] + self.primitivesTokens = ["number", "null", "true", "false", "string"] + self.operatorsByPriority = [["||"], ["&&"], ["in"], ["==", "!="], ["<", ">", "<=", ">="], ["+", "-"], + ["*", "/"], ["**"]] + + def take_token(self, *kinds): + if not self.current_token: + raise SyntaxError('Unexpected end of input') + if kinds and self.current_token.kind not in kinds: + raise SyntaxError.unexpected(self.current_token, kinds) + try: + self.current_token = next(self.tokens) + except StopIteration: + self.current_token = None + except SyntaxError as exc: + raise exc + + def parse(self, level=0): + """ expr : logicalAnd (OR logicalAnd)* """ + """ logicalAnd : inStatement (AND inStatement)* """ + """ inStatement : equality (IN equality)* """ + """ equality : comparison (EQUALITY | INEQUALITY comparison)* """ + """ comparison : addition (LESS | GREATER | LESSEQUAL | GREATEREQUAL addition)* """ + """ addition : multiplication (PLUS | MINUS multiplication)* """ + """ multiplication : exponentiation (MUL | DIV exponentiation)* """ + """ exponentiation : propertyAccessOrFunc (EXP exponentiation)* """ + if level == len(self.operatorsByPriority) - 1: + node = self.parse_property_access_or_func() + token = self.current_token + + while token is not None and token.kind in self.operatorsByPriority[level]: + self.take_token(token.kind) + node = BinOp(token, self.parse(level), node) + token = self.current_token + else: + node = self.parse(level + 1) + token = self.current_token + + while token is not None and token.kind in self.operatorsByPriority[level]: + self.take_token(token.kind) + node = BinOp(token, node, self.parse(level + 1)) + token = self.current_token + + return node + + def parse_property_access_or_func(self): + """ propertyAccessOrFunc : unit (accessWithBrackets | DOT id | functionCall)* """ + node = self.parse_unit() + token = self.current_token + operators = ["[", "(", "."] + while token is not None and token.kind in operators: + if token.kind == "[": + node = self.parse_access_with_brackets(node) + elif token.kind == ".": + token = self.current_token + self.take_token(".") + right_part = Primitive(self.current_token) + self.take_token("identifier") + node = BinOp(token, node, right_part) + elif token.kind == "(": + node = self.parse_function_call(node) + token = self.current_token + return node + + def parse_unit(self): + # unit : unaryOp unit | primitives | contextValue | LPAREN expr RPAREN | list | object + token = self.current_token + if self.current_token is None: + raise SyntaxError('Unexpected end of input') + node = None + + if token.kind in self.unaryOpTokens: + self.take_token(token.kind) + node = UnaryOp(token, self.parse_unit()) + elif token.kind in self.primitivesTokens: + self.take_token(token.kind) + node = Primitive(token) + elif token.kind == "identifier": + self.take_token(token.kind) + node = ContextValue(token) + elif token.kind == "(": + self.take_token("(") + node = self.parse() + if node is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + self.take_token(")") + elif token.kind == "[": + node = self.parse_list() + elif token.kind == "{": + node = self.parse_object() + + return node + + def parse_function_call(self, name): + """functionCall: LPAREN (expr ( COMMA expr)*)? RPAREN""" + args = [] + token = self.current_token + self.take_token("(") + + if self.current_token.kind != ")": + node = self.parse() + args.append(node) + + while self.current_token is not None and self.current_token.kind == ",": + if args[-1] is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + self.take_token(",") + node = self.parse() + args.append(node) + + self.take_token(")") + node = FunctionCall(token, name, args) + + return node + + def parse_list(self): + """ list: LSQAREBRAKET (expr (COMMA expr)*)? RSQAREBRAKET """ + arr = [] + token = self.current_token + self.take_token("[") + + if self.current_token != "]": + node = self.parse() + arr.append(node) + + while self.current_token and self.current_token.kind == ",": + if arr[-1] is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + self.take_token(",") + node = self.parse() + arr.append(node) + + self.take_token("]") + node = List(token, arr) + + return node + + def parse_access_with_brackets(self, node): + """ valueAccess : LSQAREBRAKET expr |(expr? COLON expr?) RSQAREBRAKET)""" + left = None + right = None + is_interval = False + token = self.current_token + self.take_token("[") + if self.current_token.kind == "]": + raise SyntaxError.unexpected(self.current_token, expectedTokens) + if self.current_token.kind != ":": + left = self.parse() + if self.current_token.kind == ":": + is_interval = True + self.take_token(":") + if self.current_token.kind != "]": + right = self.parse() + + if is_interval and right is None and self.current_token.kind != "]": + raise SyntaxError.unexpected(self.current_token, expectedTokens) + + self.take_token("]") + node = ValueAccess(token, node, is_interval, left, right) + + return node + + def parse_object(self): + # """ object : LCURLYBRACE ( STR | ID COLON expr (COMMA STR | ID COLON expr)*)? + # RCURLYBRACE """ + obj = {} + objToken = self.current_token + self.take_token("{") + token = self.current_token + + while token is not None and (token.kind == "string" or token.kind == "identifier"): + key = token.value + if token.kind == "string": + key = parse_string(key) + self.take_token(token.kind) + self.take_token(":") + value = self.parse() + if value is None: + raise SyntaxError.unexpected(self.current_token, expectedTokens) + obj[key] = value + if self.current_token and self.current_token.kind == "}": + break + else: + self.take_token(",") + token = self.current_token + + self.take_token("}") + node = Object(objToken, obj) + + return node + + +def parse_string(string): + return string[1:-1] + + +class Tokenizer(object): + def __init__(self, ignore, patterns, tokens): + self.ignore = ignore + self.patterns = patterns + self.tokens = tokens + # build a regular expression to generate a sequence of tokens + token_patterns = [ + '({})'.format(self.patterns.get(t, re.escape(t))) + for t in self.tokens] + if self.ignore: + token_patterns.append('(?:{})'.format(self.ignore)) + self.token_re = re.compile('^(?:' + '|'.join(token_patterns) + ')') + + def generate_tokens(self, source): + offset = 0 + while True: + start = offset + remainder = source[offset:] + mo = self.token_re.match(remainder) + if not mo: + if remainder: + raise SyntaxError( + "Unexpected input for '{}' at '{}'".format(source, remainder)) + break + offset += mo.end() + + # figure out which token matched (note that idx is 0-based) + indexes = [idx for idx, grp in enumerate(mo.groups()) if grp is not None] + if indexes: + idx = indexes[0] + yield Token( + kind=self.tokens[idx], + value=mo.group(idx + 1), # (mo.group is 1-based) + start=start, + end=offset) diff --git a/third_party/python/json_e/jsone/render.py b/third_party/python/json_e/jsone/render.py new file mode 100644 index 0000000000..7c749b752b --- /dev/null +++ b/third_party/python/json_e/jsone/render.py @@ -0,0 +1,450 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +import json as json +from .shared import JSONTemplateError, TemplateError, DeleteMarker, string, to_str +from . import shared +from .six import viewitems +from .parser import Parser, Tokenizer +from .interpreter import Interpreter +import functools +from inspect import isfunction, isbuiltin + +operators = {} +IDENTIFIER_RE = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$') + + +class SyntaxError(TemplateError): + + @classmethod + def unexpected(cls, got): + return cls('Found: {} token, expected one of: !=, &&, (, *, **, +, -, ., /, <, <=, ==, >, >=, [, in,' + ' ||'.format(got.value)) + + +def operator(name): + def wrap(fn): + operators[name] = fn + return fn + return wrap + + +tokenizer = Tokenizer( + '\\s+', + { + 'number': '[0-9]+(?:\\.[0-9]+)?', + 'identifier': '[a-zA-Z_][a-zA-Z_0-9]*', + 'string': '\'[^\']*\'|"[^"]*"', + # avoid matching these as prefixes of identifiers e.g., `insinutations` + 'true': 'true(?![a-zA-Z_0-9])', + 'false': 'false(?![a-zA-Z_0-9])', + 'in': 'in(?![a-zA-Z_0-9])', + 'null': 'null(?![a-zA-Z_0-9])', + }, + [ + '**', '+', '-', '*', '/', '[', ']', '.', '(', ')', '{', '}', ':', ',', + '>=', '<=', '<', '>', '==', '!=', '!', '&&', '||', 'true', 'false', 'in', + 'null', 'number', 'identifier', 'string', + ], +) + + +def parse(source, context): + parser = Parser(source, tokenizer) + tree = parser.parse() + if parser.current_token is not None: + raise SyntaxError.unexpected(parser.current_token) + + interp = Interpreter(context) + result = interp.interpret(tree) + return result + + +def parse_until_terminator(source, context, terminator): + parser = Parser(source, tokenizer) + tree = parser.parse() + if parser.current_token.kind != terminator: + raise SyntaxError.unexpected(parser.current_token) + interp = Interpreter(context) + result = interp.interpret(tree) + return result, parser.current_token.start + + +_interpolation_start_re = re.compile(r'\$?\${') + + +def interpolate(string, context): + mo = _interpolation_start_re.search(string) + if not mo: + return string + + result = [] + + while True: + result.append(string[:mo.start()]) + if mo.group() != '$${': + string = string[mo.end():] + parsed, offset = parse_until_terminator(string, context, '}') + if isinstance(parsed, (list, dict)): + raise TemplateError( + "interpolation of '{}' produced an array or object".format(string[:offset])) + if parsed is None: + result.append("") + else: + result.append(to_str(parsed)) + string = string[offset + 1:] + else: # found `$${` + result.append('${') + string = string[mo.end():] + + mo = _interpolation_start_re.search(string) + if not mo: + result.append(string) + break + return ''.join(result) + + +def checkUndefinedProperties(template, allowed): + unknownKeys = [] + combined = "|".join(allowed) + "$" + unknownKeys = [key for key in sorted(template) + if not re.match(combined, key)] + if unknownKeys: + raise TemplateError(allowed[0].replace('\\', '') + + " has undefined properties: " + " ".join(unknownKeys)) + + +@operator('$eval') +def eval(template, context): + checkUndefinedProperties(template, [r'\$eval']) + if not isinstance(template['$eval'], string): + raise TemplateError("$eval must be given a string expression") + return parse(template['$eval'], context) + + +@operator('$flatten') +def flatten(template, context): + checkUndefinedProperties(template, [r'\$flatten']) + value = renderValue(template['$flatten'], context) + if not isinstance(value, list): + raise TemplateError('$flatten value must evaluate to an array') + + def gen(): + for e in value: + if isinstance(e, list): + for e2 in e: + yield e2 + else: + yield e + return list(gen()) + + +@operator('$flattenDeep') +def flattenDeep(template, context): + checkUndefinedProperties(template, [r'\$flattenDeep']) + value = renderValue(template['$flattenDeep'], context) + if not isinstance(value, list): + raise TemplateError('$flattenDeep value must evaluate to an array') + + def gen(value): + if isinstance(value, list): + for e in value: + for sub in gen(e): + yield sub + else: + yield value + + return list(gen(value)) + + +@operator('$fromNow') +def fromNow(template, context): + checkUndefinedProperties(template, [r'\$fromNow', 'from']) + offset = renderValue(template['$fromNow'], context) + reference = renderValue( + template['from'], context) if 'from' in template else context.get('now') + + if not isinstance(offset, string): + raise TemplateError("$fromNow expects a string") + return shared.fromNow(offset, reference) + + +@operator('$if') +def ifConstruct(template, context): + checkUndefinedProperties(template, [r'\$if', 'then', 'else']) + condition = parse(template['$if'], context) + try: + if condition: + rv = template['then'] + else: + rv = template['else'] + except KeyError: + return DeleteMarker + return renderValue(rv, context) + + +@operator('$json') +def jsonConstruct(template, context): + checkUndefinedProperties(template, [r'\$json']) + value = renderValue(template['$json'], context) + if containsFunctions(value): + raise TemplateError('evaluated template contained uncalled functions') + return json.dumps(value, separators=(',', ':'), sort_keys=True, ensure_ascii=False) + + +@operator('$let') +def let(template, context): + checkUndefinedProperties(template, [r'\$let', 'in']) + if not isinstance(template['$let'], dict): + raise TemplateError("$let value must be an object") + + subcontext = context.copy() + initial_result = renderValue(template['$let'], context) + if not isinstance(initial_result, dict): + raise TemplateError("$let value must be an object") + for k, v in initial_result.items(): + if not IDENTIFIER_RE.match(k): + raise TemplateError("top level keys of $let must follow /[a-zA-Z_][a-zA-Z0-9_]*/") + else: + subcontext[k] = v + try: + in_expression = template['in'] + except KeyError: + raise TemplateError("$let operator requires an `in` clause") + return renderValue(in_expression, subcontext) + + +@operator('$map') +def map(template, context): + EACH_RE = r'each\([a-zA-Z_][a-zA-Z0-9_]*(,\s*([a-zA-Z_][a-zA-Z0-9_]*))?\)' + checkUndefinedProperties(template, [r'\$map', EACH_RE]) + value = renderValue(template['$map'], context) + if not isinstance(value, list) and not isinstance(value, dict): + raise TemplateError("$map value must evaluate to an array or object") + + is_obj = isinstance(value, dict) + + each_keys = [k for k in template if k.startswith('each(')] + if len(each_keys) != 1: + raise TemplateError( + "$map requires exactly one other property, each(..)") + each_key = each_keys[0] + each_args = [x.strip() for x in each_key[5:-1].split(',')] + each_var = each_args[0] + each_idx = each_args[1] if len(each_args) > 1 else None + + each_template = template[each_key] + + def gen(val): + subcontext = context.copy() + for i, elt in enumerate(val): + if each_idx is None: + subcontext[each_var] = elt + else: + subcontext[each_var] = elt['val'] if is_obj else elt + subcontext[each_idx] = elt['key'] if is_obj else i + elt = renderValue(each_template, subcontext) + if elt is not DeleteMarker: + yield elt + if is_obj: + value = [{'key': v[0], 'val': v[1]} for v in value.items()] + v = dict() + for e in gen(value): + if not isinstance(e, dict): + raise TemplateError( + "$map on objects expects {0} to evaluate to an object".format(each_key)) + v.update(e) + return v + else: + return list(gen(value)) + + +@operator('$match') +def matchConstruct(template, context): + checkUndefinedProperties(template, [r'\$match']) + + if not isinstance(template['$match'], dict): + raise TemplateError("$match can evaluate objects only") + + result = [] + for condition in sorted(template['$match']): + if parse(condition, context): + result.append(renderValue(template['$match'][condition], context)) + + return result + + +@operator('$switch') +def switch(template, context): + checkUndefinedProperties(template, [r'\$switch']) + + if not isinstance(template['$switch'], dict): + raise TemplateError("$switch can evaluate objects only") + + result = [] + for condition in template['$switch']: + if not condition == '$default' and parse(condition, context): + result.append(renderValue(template['$switch'][condition], context)) + + if len(result) > 1: + raise TemplateError("$switch can only have one truthy condition") + + if len(result) == 0: + if '$default' in template['$switch']: + result.append(renderValue(template['$switch']['$default'], context)) + + return result[0] if len(result) > 0 else DeleteMarker + + +@operator('$merge') +def merge(template, context): + checkUndefinedProperties(template, [r'\$merge']) + value = renderValue(template['$merge'], context) + if not isinstance(value, list) or not all(isinstance(e, dict) for e in value): + raise TemplateError( + "$merge value must evaluate to an array of objects") + v = dict() + for e in value: + v.update(e) + return v + + +@operator('$mergeDeep') +def merge(template, context): + checkUndefinedProperties(template, [r'\$mergeDeep']) + value = renderValue(template['$mergeDeep'], context) + if not isinstance(value, list) or not all(isinstance(e, dict) for e in value): + raise TemplateError( + "$mergeDeep value must evaluate to an array of objects") + + def merge(l, r): + if isinstance(l, list) and isinstance(r, list): + return l + r + if isinstance(l, dict) and isinstance(r, dict): + res = l.copy() + for k, v in viewitems(r): + if k in l: + res[k] = merge(l[k], v) + else: + res[k] = v + return res + return r + if len(value) == 0: + return {} + return functools.reduce(merge, value[1:], value[0]) + + +@operator('$reverse') +def reverse(template, context): + checkUndefinedProperties(template, [r'\$reverse']) + value = renderValue(template['$reverse'], context) + if not isinstance(value, list): + raise TemplateError("$reverse value must evaluate to an array of objects") + return list(reversed(value)) + + +@operator('$sort') +def sort(template, context): + BY_RE = r'by\([a-zA-Z_][a-zA-Z0-9_]*\)' + checkUndefinedProperties(template, [r'\$sort', BY_RE]) + value = renderValue(template['$sort'], context) + if not isinstance(value, list): + raise TemplateError('$sorted values to be sorted must have the same type') + + # handle by(..) if given, applying the schwartzian transform + by_keys = [k for k in template if k.startswith('by(')] + if len(by_keys) == 1: + by_key = by_keys[0] + by_var = by_key[3:-1] + by_expr = template[by_key] + + def xform(): + subcontext = context.copy() + for e in value: + subcontext[by_var] = e + yield parse(by_expr, subcontext), e + to_sort = list(xform()) + elif len(by_keys) == 0: + to_sort = [(e, e) for e in value] + else: + raise TemplateError('only one by(..) is allowed') + + # check types + try: + eltype = type(to_sort[0][0]) + except IndexError: + return [] + if eltype in (list, dict, bool, type(None)): + raise TemplateError('$sorted values to be sorted must have the same type') + if not all(isinstance(e[0], eltype) for e in to_sort): + raise TemplateError('$sorted values to be sorted must have the same type') + + # unzip the schwartzian transform + return list(e[1] for e in sorted(to_sort)) + + +def containsFunctions(rendered): + if hasattr(rendered, '__call__'): + return True + elif isinstance(rendered, list): + for e in rendered: + if containsFunctions(e): + return True + return False + elif isinstance(rendered, dict): + for k, v in viewitems(rendered): + if containsFunctions(v): + return True + return False + else: + return False + + +def renderValue(template, context): + if isinstance(template, string): + return interpolate(template, context) + + elif isinstance(template, dict): + matches = [k for k in template if k in operators] + if matches: + if len(matches) > 1: + raise TemplateError("only one operator allowed") + return operators[matches[0]](template, context) + + def updated(): + for k, v in viewitems(template): + if k.startswith('$$'): + k = k[1:] + elif k.startswith('$') and IDENTIFIER_RE.match(k[1:]): + raise TemplateError( + '$ is reserved; use $$') + else: + k = interpolate(k, context) + + try: + v = renderValue(v, context) + except JSONTemplateError as e: + if IDENTIFIER_RE.match(k): + e.add_location('.{}'.format(k)) + else: + e.add_location('[{}]'.format(json.dumps(k))) + raise + if v is not DeleteMarker: + yield k, v + return dict(updated()) + + elif isinstance(template, list): + def updated(): + for i, e in enumerate(template): + try: + v = renderValue(e, context) + if v is not DeleteMarker: + yield v + except JSONTemplateError as e: + e.add_location('[{}]'.format(i)) + raise + + return list(updated()) + + else: + return template diff --git a/third_party/python/json_e/jsone/shared.py b/third_party/python/json_e/jsone/shared.py new file mode 100644 index 0000000000..220d16ce41 --- /dev/null +++ b/third_party/python/json_e/jsone/shared.py @@ -0,0 +1,133 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +import datetime + + +class DeleteMarker: + pass + + +class JSONTemplateError(Exception): + def __init__(self, message): + super(JSONTemplateError, self).__init__(message) + self.location = [] + + def add_location(self, loc): + self.location.insert(0, loc) + + def __str__(self): + location = ' at template' + ''.join(self.location) + return "{}{}: {}".format( + self.__class__.__name__, + location if self.location else '', + self.args[0]) + + +class TemplateError(JSONTemplateError): + pass + + +class InterpreterError(JSONTemplateError): + pass + + +# Regular expression matching: X days Y hours Z minutes +# todo: support hr, wk, yr +FROMNOW_RE = re.compile(''.join([ + r'^(\s*(?P\d+)\s*y(ears?)?)?', + r'(\s*(?P\d+)\s*mo(nths?)?)?', + r'(\s*(?P\d+)\s*w(eeks?)?)?', + r'(\s*(?P\d+)\s*d(ays?)?)?', + r'(\s*(?P\d+)\s*h(ours?)?)?', + r'(\s*(?P\d+)\s*m(in(utes?)?)?)?\s*', + r'(\s*(?P\d+)\s*s(ec(onds?)?)?)?\s*$', +])) + + +def fromNow(offset, reference): + # copied from taskcluster-client.py + # We want to handle past dates as well as future + future = True + offset = offset.lstrip() + if offset.startswith('-'): + future = False + offset = offset[1:].lstrip() + if offset.startswith('+'): + offset = offset[1:].lstrip() + + # Parse offset + m = FROMNOW_RE.match(offset) + if m is None: + raise ValueError("offset string: '%s' does not parse" % offset) + + # In order to calculate years and months we need to calculate how many days + # to offset the offset by, since timedelta only goes as high as weeks + days = 0 + hours = 0 + minutes = 0 + seconds = 0 + if m.group('years'): + # forget leap years, a year is 365 days + years = int(m.group('years')) + days += 365 * years + if m.group('months'): + # assume "month" means 30 days + months = int(m.group('months')) + days += 30 * months + days += int(m.group('days') or 0) + hours += int(m.group('hours') or 0) + minutes += int(m.group('minutes') or 0) + seconds += int(m.group('seconds') or 0) + + # Offset datetime from utc + delta = datetime.timedelta( + weeks=int(m.group('weeks') or 0), + days=days, + hours=hours, + minutes=minutes, + seconds=seconds, + ) + + if isinstance(reference, string): + reference = datetime.datetime.strptime( + reference, '%Y-%m-%dT%H:%M:%S.%fZ') + elif reference is None: + reference = datetime.datetime.utcnow() + return stringDate(reference + delta if future else reference - delta) + + +datefmt_re = re.compile(r'(\.[0-9]{3})[0-9]*(\+00:00)?') + + +def to_str(v): + if isinstance(v, bool): + return {True: 'true', False: 'false'}[v] + elif isinstance(v, list): + return ','.join(to_str(e) for e in v) + elif v is None: + return 'null' + elif isinstance(v, string): + return v + else: + return str(v) + + +def stringDate(date): + # Convert to isoFormat + try: + string = date.isoformat(timespec='microseconds') + # py2.7 to py3.5 does not have timespec + except TypeError as e: + string = date.isoformat() + if string.find('.') == -1: + string += '.000' + string = datefmt_re.sub(r'\1Z', string) + return string + + +# the base class for strings, regardless of python version +try: + string = basestring +except NameError: + string = str diff --git a/third_party/python/json_e/jsone/six.py b/third_party/python/json_e/jsone/six.py new file mode 100644 index 0000000000..1ab9cd2d7d --- /dev/null +++ b/third_party/python/json_e/jsone/six.py @@ -0,0 +1,23 @@ +import sys +import operator + +# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L818 + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L578 +if sys.version_info[0] == 3: + viewitems = operator.methodcaller("items") +else: + viewitems = operator.methodcaller("viewitems") -- cgit v1.2.3