diff options
Diffstat (limited to 'third_party/python/json-e/jsone')
-rw-r--r-- | third_party/python/json-e/jsone/__init__.py | 21 | ||||
-rw-r--r-- | third_party/python/json-e/jsone/builtins.py | 121 | ||||
-rw-r--r-- | third_party/python/json-e/jsone/interpreter.py | 289 | ||||
-rw-r--r-- | third_party/python/json-e/jsone/prattparser.py | 191 | ||||
-rw-r--r-- | third_party/python/json-e/jsone/render.py | 354 | ||||
-rw-r--r-- | third_party/python/json-e/jsone/shared.py | 131 | ||||
-rw-r--r-- | third_party/python/json-e/jsone/six.py | 23 |
7 files changed, 1130 insertions, 0 deletions
diff --git a/third_party/python/json-e/jsone/__init__.py b/third_party/python/json-e/jsone/__init__.py new file mode 100644 index 0000000000..943674e672 --- /dev/null +++ b/third_party/python/json-e/jsone/__init__.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +from .render import renderValue +from .shared import JSONTemplateError, DeleteMarker, TemplateError, fromNow +from . import builtins + +_context_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$') + + +def render(template, context): + if not all(_context_re.match(c) for c in context): + raise TemplateError('top level keys of context must follow ' + '/[a-zA-Z_][a-zA-Z0-9_]*/') + full_context = {'now': fromNow('0 seconds', None)} + full_context.update(builtins.build(full_context)) + full_context.update(context) + rv = renderValue(template, full_context) + if rv is DeleteMarker: + return None + return rv diff --git a/third_party/python/json-e/jsone/builtins.py b/third_party/python/json-e/jsone/builtins.py new file mode 100644 index 0000000000..751ee2dc04 --- /dev/null +++ b/third_party/python/json-e/jsone/builtins.py @@ -0,0 +1,121 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import math +from .shared import string, to_str, fromNow, JSONTemplateError + + +class BuiltinError(JSONTemplateError): + pass + + +def build(context): + builtins = {} + + def builtin(name, variadic=None, argument_tests=None, minArgs=None): + def wrap(fn): + def bad(reason=None): + raise BuiltinError( + (reason or 'invalid arguments to builtin: {}').format(name)) + if variadic: + def invoke(*args): + if minArgs: + if len(args) < minArgs: + bad("too few arguments to {}") + for arg in args: + if not variadic(arg): + bad() + return fn(*args) + + elif argument_tests: + def invoke(*args): + if len(args) != len(argument_tests): + bad() + for t, arg in zip(argument_tests, args): + if not t(arg): + bad() + return fn(*args) + + else: + def invoke(*args): + return fn(*args) + + builtins[name] = invoke + return fn + return wrap + + def is_number(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + def is_string(v): + return isinstance(v, string) + + def is_string_or_array(v): + return isinstance(v, (string, list)) + + def anything_except_array(v): + return isinstance(v, (string, int, float, bool)) or v is None + + def anything(v): + return isinstance(v, (string, int, float, list, dict)) or v is None or callable(v) + + # --- + + builtin('min', variadic=is_number, minArgs=1)(min) + builtin('max', variadic=is_number, minArgs=1)(max) + builtin('sqrt', argument_tests=[is_number])(math.sqrt) + builtin('abs', argument_tests=[is_number])(abs) + + @builtin('ceil', argument_tests=[is_number]) + def ceil(v): + return int(math.ceil(v)) + + @builtin('floor', argument_tests=[is_number]) + def floor(v): + return int(math.floor(v)) + + @builtin('lowercase', argument_tests=[is_string]) + def lowercase(v): + return v.lower() + + @builtin('uppercase', argument_tests=[is_string]) + def lowercase(v): + return v.upper() + + builtin('len', argument_tests=[is_string_or_array])(len) + builtin('str', argument_tests=[anything_except_array])(to_str) + builtin('number', variadic=is_string, minArgs=1)(float) + + @builtin('strip', argument_tests=[is_string]) + def strip(s): + return s.strip() + + @builtin('rstrip', argument_tests=[is_string]) + def rstrip(s): + return s.rstrip() + + @builtin('lstrip', argument_tests=[is_string]) + def lstrip(s): + return s.lstrip() + + @builtin('fromNow', variadic=is_string, minArgs=1) + def fromNow_builtin(offset, reference=None): + return fromNow(offset, reference or context.get('now')) + + @builtin('typeof', argument_tests=[anything]) + def typeof(v): + if isinstance(v, bool): + return 'boolean' + elif isinstance(v, string): + return 'string' + elif isinstance(v, (int, float)): + return 'number' + elif isinstance(v, list): + return 'array' + elif isinstance(v, dict): + return 'object' + elif v is None: + return None + elif callable(v): + return 'function' + + return builtins diff --git a/third_party/python/json-e/jsone/interpreter.py b/third_party/python/json-e/jsone/interpreter.py new file mode 100644 index 0000000000..eb38a9c85b --- /dev/null +++ b/third_party/python/json-e/jsone/interpreter.py @@ -0,0 +1,289 @@ +from __future__ import absolute_import, print_function, unicode_literals + +from .prattparser import PrattParser, infix, prefix +from .shared import TemplateError, InterpreterError, string +import operator +import json + +OPERATORS = { + '-': operator.sub, + '*': operator.mul, + '/': operator.truediv, + '**': operator.pow, + '==': operator.eq, + '!=': operator.ne, + '<=': operator.le, + '<': operator.lt, + '>': operator.gt, + '>=': operator.ge, + '&&': lambda a, b: bool(a and b), + '||': lambda a, b: bool(a or b), +} + + +def infixExpectationError(operator, expected): + return InterpreterError('infix: {} expects {} {} {}'. + format(operator, expected, operator, expected)) + + +class ExpressionEvaluator(PrattParser): + + ignore = '\\s+' + patterns = { + 'number': '[0-9]+(?:\\.[0-9]+)?', + 'identifier': '[a-zA-Z_][a-zA-Z_0-9]*', + 'string': '\'[^\']*\'|"[^"]*"', + # avoid matching these as prefixes of identifiers e.g., `insinutations` + 'true': 'true(?![a-zA-Z_0-9])', + 'false': 'false(?![a-zA-Z_0-9])', + 'in': 'in(?![a-zA-Z_0-9])', + 'null': 'null(?![a-zA-Z_0-9])', + } + tokens = [ + '**', '+', '-', '*', '/', '[', ']', '.', '(', ')', '{', '}', ':', ',', + '>=', '<=', '<', '>', '==', '!=', '!', '&&', '||', 'true', 'false', 'in', + 'null', 'number', 'identifier', 'string', + ] + precedence = [ + ['||'], + ['&&'], + ['in'], + ['==', '!='], + ['>=', '<=', '<', '>'], + ['+', '-'], + ['*', '/'], + ['**-right-associative'], + ['**'], + ['[', '.'], + ['('], + ['unary'], + ] + + def __init__(self, context): + super(ExpressionEvaluator, self).__init__() + self.context = context + + def parse(self, expression): + if not isinstance(expression, string): + raise TemplateError('expression to be evaluated must be a string') + return super(ExpressionEvaluator, self).parse(expression) + + @prefix('number') + def number(self, token, pc): + v = token.value + return float(v) if '.' in v else int(v) + + @prefix("!") + def bang(self, token, pc): + return not pc.parse('unary') + + @prefix("-") + def uminus(self, token, pc): + v = pc.parse('unary') + if not isNumber(v): + raise InterpreterError('{} expects {}'.format('unary -', 'number')) + return -v + + @prefix("+") + def uplus(self, token, pc): + v = pc.parse('unary') + if not isNumber(v): + raise InterpreterError('{} expects {}'.format('unary +', 'number')) + return v + + @prefix("identifier") + def identifier(self, token, pc): + try: + return self.context[token.value] + except KeyError: + raise InterpreterError( + 'unknown context value {}'.format(token.value)) + + @prefix("null") + def null(self, token, pc): + return None + + @prefix("[") + def array_bracket(self, token, pc): + return parseList(pc, ',', ']') + + @prefix("(") + def grouping_paren(self, token, pc): + rv = pc.parse() + pc.require(')') + return rv + + @prefix("{") + def object_brace(self, token, pc): + return parseObject(pc) + + @prefix("string") + def string(self, token, pc): + return parseString(token.value) + + @prefix("true") + def true(self, token, pc): + return True + + @prefix("false") + def false(self, token, ps): + return False + + @infix("+") + def plus(self, left, token, pc): + if not isinstance(left, (string, int, float)) or isinstance(left, bool): + raise infixExpectationError('+', 'number/string') + right = pc.parse(token.kind) + if not isinstance(right, (string, int, float)) or isinstance(right, bool): + raise infixExpectationError('+', 'number/string') + if type(right) != type(left) and \ + (isinstance(left, string) or isinstance(right, string)): + raise infixExpectationError('+', 'numbers/strings') + return left + right + + @infix('-', '*', '/', '**') + def arith(self, left, token, pc): + op = token.kind + if not isNumber(left): + raise infixExpectationError(op, 'number') + right = pc.parse({'**': '**-right-associative'}.get(op)) + if not isNumber(right): + raise infixExpectationError(op, 'number') + return OPERATORS[op](left, right) + + @infix("[") + def index_slice(self, left, token, pc): + a = None + b = None + is_interval = False + if pc.attempt(':'): + a = 0 + is_interval = True + else: + a = pc.parse() + if pc.attempt(':'): + is_interval = True + + if is_interval and not pc.attempt(']'): + b = pc.parse() + pc.require(']') + + if not is_interval: + pc.require(']') + + return accessProperty(left, a, b, is_interval) + + @infix(".") + def property_dot(self, left, token, pc): + if not isinstance(left, dict): + raise infixExpectationError('.', 'object') + k = pc.require('identifier').value + try: + return left[k] + except KeyError: + raise TemplateError( + '{} not found in {}'.format(k, json.dumps(left))) + + @infix("(") + def function_call(self, left, token, pc): + if not callable(left): + raise TemplateError('function call', 'callable') + args = parseList(pc, ',', ')') + return left(*args) + + @infix('==', '!=', '||', '&&') + def equality_and_logic(self, left, token, pc): + op = token.kind + right = pc.parse(op) + return OPERATORS[op](left, right) + + @infix('<=', '<', '>', '>=') + def inequality(self, left, token, pc): + op = token.kind + right = pc.parse(op) + if type(left) != type(right) or \ + not (isinstance(left, (int, float, string)) and not isinstance(left, bool)): + raise infixExpectationError(op, 'numbers/strings') + return OPERATORS[op](left, right) + + @infix("in") + def contains(self, left, token, pc): + right = pc.parse(token.kind) + if isinstance(right, dict): + if not isinstance(left, string): + raise infixExpectationError('in-object', 'string on left side') + elif isinstance(right, string): + if not isinstance(left, string): + raise infixExpectationError('in-string', 'string on left side') + elif not isinstance(right, list): + raise infixExpectationError( + 'in', 'Array, string, or object on right side') + try: + return left in right + except TypeError: + raise infixExpectationError('in', 'scalar value, collection') + + +def isNumber(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def parseString(v): + return v[1:-1] + + +def parseList(pc, separator, terminator): + rv = [] + if not pc.attempt(terminator): + while True: + rv.append(pc.parse()) + if not pc.attempt(separator): + break + pc.require(terminator) + return rv + + +def parseObject(pc): + rv = {} + if not pc.attempt('}'): + while True: + k = pc.require('identifier', 'string') + if k.kind == 'string': + k = parseString(k.value) + else: + k = k.value + pc.require(':') + v = pc.parse() + rv[k] = v + if not pc.attempt(','): + break + pc.require('}') + return rv + + +def accessProperty(value, a, b, is_interval): + if isinstance(value, (list, string)): + if is_interval: + if b is None: + b = len(value) + try: + return value[a:b] + except TypeError: + raise infixExpectationError('[..]', 'integer') + else: + try: + return value[a] + except IndexError: + raise TemplateError('index out of bounds') + except TypeError: + raise infixExpectationError('[..]', 'integer') + + if not isinstance(value, dict): + raise infixExpectationError('[..]', 'object, array, or string') + if not isinstance(a, string): + raise infixExpectationError('[..]', 'string index') + + try: + return value[a] + except KeyError: + return None diff --git a/third_party/python/json-e/jsone/prattparser.py b/third_party/python/json-e/jsone/prattparser.py new file mode 100644 index 0000000000..5bf250a816 --- /dev/null +++ b/third_party/python/json-e/jsone/prattparser.py @@ -0,0 +1,191 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +from collections import namedtuple +from .shared import TemplateError +from .six import with_metaclass, viewitems + + +class SyntaxError(TemplateError): + + @classmethod + def unexpected(cls, got, exp): + exp = ', '.join(sorted(exp)) + return cls('Found {}, expected {}'.format(got.value, exp)) + + +Token = namedtuple('Token', ['kind', 'value', 'start', 'end']) + + +def prefix(*kinds): + """Decorate a method as handling prefix tokens of the given kinds""" + def wrap(fn): + try: + fn.prefix_kinds.extend(kinds) + except AttributeError: + fn.prefix_kinds = list(kinds) + return fn + return wrap + + +def infix(*kinds): + """Decorate a method as handling infix tokens of the given kinds""" + def wrap(fn): + try: + fn.infix_kinds.extend(kinds) + except AttributeError: + fn.infix_kinds = list(kinds) + return fn + return wrap + + +class PrattParserMeta(type): + + def __init__(cls, name, bases, body): + # set up rules based on decorated methods + infix_rules = cls.infix_rules = {} + prefix_rules = cls.prefix_rules = {} + for prop, value in viewitems(body): + if hasattr(value, 'prefix_kinds'): + for kind in value.prefix_kinds: + prefix_rules[kind] = value + delattr(cls, prop) + if hasattr(value, 'infix_kinds'): + for kind in value.infix_kinds: + infix_rules[kind] = value + delattr(cls, prop) + + # build a regular expression to generate a sequence of tokens + token_patterns = [ + '({})'.format(cls.patterns.get(t, re.escape(t))) + for t in cls.tokens] + if cls.ignore: + token_patterns.append('(?:{})'.format(cls.ignore)) + cls.token_re = re.compile('^(?:' + '|'.join(token_patterns) + ')') + + # build a map from token kind to precedence level + cls.precedence_map = { + kind: prec + 1 + for (prec, row) in enumerate(cls.precedence) + for kind in row + } + + +class PrattParser(with_metaclass(PrattParserMeta, object)): + + # regular expression for ignored input (e.g., whitespace) + ignore = None + + # regular expressions for tokens that do not match themselves + patterns = {} + + # all token kinds (note that order matters - the first matching token + # will be returned) + tokens = [] + + # precedence of tokens, as a list of lists, from lowest to highest + precedence = [] + + def parse(self, source): + pc = ParseContext(self, source, self._generate_tokens(source)) + result = pc.parse() + # if there are any tokens remaining, that's an error.. + token = pc.attempt() + if token: + raise SyntaxError.unexpected(token, self.infix_rules) + return result + + def parseUntilTerminator(self, source, terminator): + pc = ParseContext(self, source, self._generate_tokens(source)) + result = pc.parse() + token = pc.attempt() + if token.kind != terminator: + raise SyntaxError.unexpected(token, [terminator]) + return (result, token.start) + + def _generate_tokens(self, source): + offset = 0 + while True: + start = offset + remainder = source[offset:] + mo = self.token_re.match(remainder) + if not mo: + if remainder: + raise SyntaxError( + "Unexpected input: '{}'".format(remainder)) + break + offset += mo.end() + + # figure out which token matched (note that idx is 0-based) + indexes = list( + filter(lambda x: x[1] is not None, enumerate(mo.groups()))) + if indexes: + idx = indexes[0][0] + yield Token( + kind=self.tokens[idx], + value=mo.group(idx + 1), # (mo.group is 1-based) + start=start, + end=offset) + + +class ParseContext(object): + + def __init__(self, parser, source, token_generator): + self.parser = parser + self.source = source + + self._tokens = token_generator + self._error = None + + self._advance() + + def _advance(self): + try: + self.next_token = next(self._tokens) + except StopIteration: + self.next_token = None + except SyntaxError as exc: + self._error = exc + + def attempt(self, *kinds): + """Try to get the next token if it matches one of the kinds given, + otherwise returning None. If no kinds are given, any kind is + accepted.""" + if self._error: + raise self._error + token = self.next_token + if not token: + return None + if kinds and token.kind not in kinds: + return None + self._advance() + return token + + def require(self, *kinds): + """Get the next token, raising an exception if it doesn't match one of + the given kinds, or the input ends. If no kinds are given, returns the + next token of any kind.""" + token = self.attempt() + if not token: + raise SyntaxError('Unexpected end of input') + if kinds and token.kind not in kinds: + raise SyntaxError.unexpected(token, kinds) + return token + + def parse(self, precedence=None): + parser = self.parser + precedence = parser.precedence_map[precedence] if precedence else 0 + token = self.require() + prefix_rule = parser.prefix_rules.get(token.kind) + if not prefix_rule: + raise SyntaxError.unexpected(token, parser.prefix_rules) + left = prefix_rule(parser, token, self) + while self.next_token: + kind = self.next_token.kind + if kind not in parser.infix_rules: + break + if precedence >= parser.precedence_map[kind]: + break + token = self.require() + left = parser.infix_rules[kind](parser, left, token, self) + return left diff --git a/third_party/python/json-e/jsone/render.py b/third_party/python/json-e/jsone/render.py new file mode 100644 index 0000000000..e820da1ec2 --- /dev/null +++ b/third_party/python/json-e/jsone/render.py @@ -0,0 +1,354 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +import json as json +from .shared import JSONTemplateError, TemplateError, DeleteMarker, string, to_str +from . import shared +from .interpreter import ExpressionEvaluator +from .six import viewitems +import functools + +operators = {} +IDENTIFIER_RE = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$') + + +def operator(name): + def wrap(fn): + operators[name] = fn + return fn + return wrap + + +def evaluateExpression(expr, context): + evaluator = ExpressionEvaluator(context) + return evaluator.parse(expr) + + +_interpolation_start_re = re.compile(r'\$?\${') + + +def interpolate(string, context): + mo = _interpolation_start_re.search(string) + if not mo: + return string + + result = [] + evaluator = ExpressionEvaluator(context) + + while True: + result.append(string[:mo.start()]) + if mo.group() != '$${': + string = string[mo.end():] + parsed, offset = evaluator.parseUntilTerminator(string, '}') + if isinstance(parsed, (list, dict)): + raise TemplateError( + "interpolation of '{}' produced an array or object".format(string[:offset])) + if to_str(parsed) == "null": + result.append("") + else: + result.append(to_str(parsed)) + string = string[offset + 1:] + else: # found `$${` + result.append('${') + string = string[mo.end():] + + mo = _interpolation_start_re.search(string) + if not mo: + result.append(string) + break + + return ''.join(result) + + +def checkUndefinedProperties(template, allowed): + unknownKeys = [] + combined = "|".join(allowed) + "$" + unknownKeys = [key for key in sorted(template) + if not re.match(combined, key)] + if unknownKeys: + raise TemplateError(allowed[0].replace('\\', '') + + " has undefined properties: " + " ".join(unknownKeys)) + + +@operator('$eval') +def eval(template, context): + checkUndefinedProperties(template, ['\$eval']) + if not isinstance(template['$eval'], string): + raise TemplateError("$eval must be given a string expression") + return evaluateExpression(template['$eval'], context) + + +@operator('$flatten') +def flatten(template, context): + checkUndefinedProperties(template, ['\$flatten']) + value = renderValue(template['$flatten'], context) + if not isinstance(value, list): + raise TemplateError('$flatten value must evaluate to an array') + + def gen(): + for e in value: + if isinstance(e, list): + for e2 in e: + yield e2 + else: + yield e + return list(gen()) + + +@operator('$flattenDeep') +def flattenDeep(template, context): + checkUndefinedProperties(template, ['\$flattenDeep']) + value = renderValue(template['$flattenDeep'], context) + if not isinstance(value, list): + raise TemplateError('$flattenDeep value must evaluate to an array') + + def gen(value): + if isinstance(value, list): + for e in value: + for sub in gen(e): + yield sub + else: + yield value + + return list(gen(value)) + + +@operator('$fromNow') +def fromNow(template, context): + checkUndefinedProperties(template, ['\$fromNow', 'from']) + offset = renderValue(template['$fromNow'], context) + reference = renderValue( + template['from'], context) if 'from' in template else context.get('now') + + if not isinstance(offset, string): + raise TemplateError("$fromNow expects a string") + return shared.fromNow(offset, reference) + + +@operator('$if') +def ifConstruct(template, context): + checkUndefinedProperties(template, ['\$if', 'then', 'else']) + condition = evaluateExpression(template['$if'], context) + try: + if condition: + rv = template['then'] + else: + rv = template['else'] + except KeyError: + return DeleteMarker + return renderValue(rv, context) + + +@operator('$json') +def jsonConstruct(template, context): + checkUndefinedProperties(template, ['\$json']) + value = renderValue(template['$json'], context) + return json.dumps(value, separators=(',', ':'), sort_keys=True, ensure_ascii=False) + + +@operator('$let') +def let(template, context): + checkUndefinedProperties(template, ['\$let', 'in']) + if not isinstance(template['$let'], dict): + raise TemplateError("$let value must be an object") + + subcontext = context.copy() + for k, v in template['$let'].items(): + if not IDENTIFIER_RE.match(k): + raise TemplateError('top level keys of $let must follow /[a-zA-Z_][a-zA-Z0-9_]*/') + subcontext[k] = renderValue(v, context) + + try: + in_expression = template['in'] + except KeyError: + raise TemplateError("$let operator requires an `in` clause") + return renderValue(in_expression, subcontext) + + +@operator('$map') +def map(template, context): + EACH_RE = 'each\([a-zA-Z_][a-zA-Z0-9_]*\)' + checkUndefinedProperties(template, ['\$map', EACH_RE]) + value = renderValue(template['$map'], context) + if not isinstance(value, list) and not isinstance(value, dict): + raise TemplateError("$map value must evaluate to an array or object") + + is_obj = isinstance(value, dict) + + each_keys = [k for k in template if k.startswith('each(')] + if len(each_keys) != 1: + raise TemplateError( + "$map requires exactly one other property, each(..)") + each_key = each_keys[0] + each_var = each_key[5:-1] + each_template = template[each_key] + + def gen(val): + subcontext = context.copy() + for elt in val: + subcontext[each_var] = elt + elt = renderValue(each_template, subcontext) + if elt is not DeleteMarker: + yield elt + if is_obj: + value = [{'key': v[0], 'val': v[1]} for v in value.items()] + v = dict() + for e in gen(value): + if not isinstance(e, dict): + raise TemplateError( + "$map on objects expects {0} to evaluate to an object".format(each_key)) + v.update(e) + return v + else: + return list(gen(value)) + + +@operator('$match') +def matchConstruct(template, context): + checkUndefinedProperties(template, ['\$match']) + + if not isinstance(template['$match'], dict): + raise TemplateError("$match can evaluate objects only") + + result = [] + for condition in template['$match']: + if evaluateExpression(condition, context): + result.append(renderValue(template['$match'][condition], context)) + + return result + + +@operator('$merge') +def merge(template, context): + checkUndefinedProperties(template, ['\$merge']) + value = renderValue(template['$merge'], context) + if not isinstance(value, list) or not all(isinstance(e, dict) for e in value): + raise TemplateError( + "$merge value must evaluate to an array of objects") + v = dict() + for e in value: + v.update(e) + return v + + +@operator('$mergeDeep') +def merge(template, context): + checkUndefinedProperties(template, ['\$mergeDeep']) + value = renderValue(template['$mergeDeep'], context) + if not isinstance(value, list) or not all(isinstance(e, dict) for e in value): + raise TemplateError( + "$mergeDeep value must evaluate to an array of objects") + + def merge(l, r): + if isinstance(l, list) and isinstance(r, list): + return l + r + if isinstance(l, dict) and isinstance(r, dict): + res = l.copy() + for k, v in viewitems(r): + if k in l: + res[k] = merge(l[k], v) + else: + res[k] = v + return res + return r + if len(value) == 0: + return {} + return functools.reduce(merge, value[1:], value[0]) + + +@operator('$reverse') +def reverse(template, context): + checkUndefinedProperties(template, ['\$reverse']) + value = renderValue(template['$reverse'], context) + if not isinstance(value, list): + raise TemplateError("$reverse value must evaluate to an array of objects") + return list(reversed(value)) + + +@operator('$sort') +def sort(template, context): + BY_RE = 'by\([a-zA-Z_][a-zA-Z0-9_]*\)' + checkUndefinedProperties(template, ['\$sort', BY_RE]) + value = renderValue(template['$sort'], context) + if not isinstance(value, list): + raise TemplateError('$sorted values to be sorted must have the same type') + + # handle by(..) if given, applying the schwartzian transform + by_keys = [k for k in template if k.startswith('by(')] + if len(by_keys) == 1: + by_key = by_keys[0] + by_var = by_key[3:-1] + by_expr = template[by_key] + + def xform(): + subcontext = context.copy() + for e in value: + subcontext[by_var] = e + yield evaluateExpression(by_expr, subcontext), e + to_sort = list(xform()) + elif len(by_keys) == 0: + to_sort = [(e, e) for e in value] + else: + raise TemplateError('only one by(..) is allowed') + + # check types + try: + eltype = type(to_sort[0][0]) + except IndexError: + return [] + if eltype in (list, dict, bool, type(None)): + raise TemplateError('$sorted values to be sorted must have the same type') + if not all(isinstance(e[0], eltype) for e in to_sort): + raise TemplateError('$sorted values to be sorted must have the same type') + + # unzip the schwartzian transform + return list(e[1] for e in sorted(to_sort)) + + +def renderValue(template, context): + if isinstance(template, string): + return interpolate(template, context) + + elif isinstance(template, dict): + matches = [k for k in template if k in operators] + if matches: + if len(matches) > 1: + raise TemplateError("only one operator allowed") + return operators[matches[0]](template, context) + + def updated(): + for k, v in viewitems(template): + if k.startswith('$$'): + k = k[1:] + elif k.startswith('$') and IDENTIFIER_RE.match(k[1:]): + raise TemplateError( + '$<identifier> is reserved; ues $$<identifier>') + else: + k = interpolate(k, context) + + try: + v = renderValue(v, context) + except JSONTemplateError as e: + if IDENTIFIER_RE.match(k): + e.add_location('.{}'.format(k)) + else: + e.add_location('[{}]'.format(json.dumps(k))) + raise + if v is not DeleteMarker: + yield k, v + return dict(updated()) + + elif isinstance(template, list): + def updated(): + for i, e in enumerate(template): + try: + v = renderValue(e, context) + if v is not DeleteMarker: + yield v + except JSONTemplateError as e: + e.add_location('[{}]'.format(i)) + raise + return list(updated()) + + else: + return template diff --git a/third_party/python/json-e/jsone/shared.py b/third_party/python/json-e/jsone/shared.py new file mode 100644 index 0000000000..0e70e21f81 --- /dev/null +++ b/third_party/python/json-e/jsone/shared.py @@ -0,0 +1,131 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +import datetime + + +class DeleteMarker: + pass + + +class JSONTemplateError(Exception): + def __init__(self, message): + super(JSONTemplateError, self).__init__(message) + self.location = [] + + def add_location(self, loc): + self.location.insert(0, loc) + + def __str__(self): + location = ' at template' + ''.join(self.location) + return "{}{}: {}".format( + self.__class__.__name__, + location if self.location else '', + self.args[0]) + + +class TemplateError(JSONTemplateError): + pass + + +class InterpreterError(JSONTemplateError): + pass + + +# Regular expression matching: X days Y hours Z minutes +# todo: support hr, wk, yr +FROMNOW_RE = re.compile(''.join([ + '^(\s*(?P<years>\d+)\s*y(ears?)?)?', + '(\s*(?P<months>\d+)\s*mo(nths?)?)?', + '(\s*(?P<weeks>\d+)\s*w(eeks?)?)?', + '(\s*(?P<days>\d+)\s*d(ays?)?)?', + '(\s*(?P<hours>\d+)\s*h(ours?)?)?', + '(\s*(?P<minutes>\d+)\s*m(in(utes?)?)?)?\s*', + '(\s*(?P<seconds>\d+)\s*s(ec(onds?)?)?)?\s*$', +])) + + +def fromNow(offset, reference): + # copied from taskcluster-client.py + # We want to handle past dates as well as future + future = True + offset = offset.lstrip() + if offset.startswith('-'): + future = False + offset = offset[1:].lstrip() + if offset.startswith('+'): + offset = offset[1:].lstrip() + + # Parse offset + m = FROMNOW_RE.match(offset) + if m is None: + raise ValueError("offset string: '%s' does not parse" % offset) + + # In order to calculate years and months we need to calculate how many days + # to offset the offset by, since timedelta only goes as high as weeks + days = 0 + hours = 0 + minutes = 0 + seconds = 0 + if m.group('years'): + # forget leap years, a year is 365 days + years = int(m.group('years')) + days += 365 * years + if m.group('months'): + # assume "month" means 30 days + months = int(m.group('months')) + days += 30 * months + days += int(m.group('days') or 0) + hours += int(m.group('hours') or 0) + minutes += int(m.group('minutes') or 0) + seconds += int(m.group('seconds') or 0) + + # Offset datetime from utc + delta = datetime.timedelta( + weeks=int(m.group('weeks') or 0), + days=days, + hours=hours, + minutes=minutes, + seconds=seconds, + ) + + if isinstance(reference, string): + reference = datetime.datetime.strptime( + reference, '%Y-%m-%dT%H:%M:%S.%fZ') + elif reference is None: + reference = datetime.datetime.utcnow() + return stringDate(reference + delta if future else reference - delta) + + +datefmt_re = re.compile(r'(\.[0-9]{3})[0-9]*(\+00:00)?') + + +def to_str(v): + if isinstance(v, bool): + return {True: 'true', False: 'false'}[v] + elif isinstance(v, list): + return ','.join(to_str(e) for e in v) + elif v is None: + return 'null' + else: + return str(v) + + +def stringDate(date): + # Convert to isoFormat + try: + string = date.isoformat(timespec='microseconds') + # py2.7 to py3.5 does not have timespec + except TypeError as e: + string = date.isoformat() + if string.find('.') == -1: + string += '.000' + string = datefmt_re.sub(r'\1Z', string) + return string + + +# the base class for strings, regardless of python version +try: + string = basestring +except NameError: + string = str diff --git a/third_party/python/json-e/jsone/six.py b/third_party/python/json-e/jsone/six.py new file mode 100644 index 0000000000..1ab9cd2d7d --- /dev/null +++ b/third_party/python/json-e/jsone/six.py @@ -0,0 +1,23 @@ +import sys +import operator + +# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L818 + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L578 +if sys.version_info[0] == 3: + viewitems = operator.methodcaller("items") +else: + viewitems = operator.methodcaller("viewitems") |