diff options
Diffstat (limited to 'third_party/python/json-e')
18 files changed, 1981 insertions, 0 deletions
diff --git a/third_party/python/json-e/MANIFEST.in b/third_party/python/json-e/MANIFEST.in new file mode 100644 index 0000000000..a6995977cb --- /dev/null +++ b/third_party/python/json-e/MANIFEST.in @@ -0,0 +1,3 @@ +include jsone *.py +include package.json +recursive-exclude test * diff --git a/third_party/python/json-e/PKG-INFO b/third_party/python/json-e/PKG-INFO new file mode 100644 index 0000000000..bf41f82167 --- /dev/null +++ b/third_party/python/json-e/PKG-INFO @@ -0,0 +1,11 @@ +Metadata-Version: 2.1 +Name: json-e +Version: 2.7.0 +Summary: A data-structure parameterization system written for embedding context in JSON objects +Home-page: https://taskcluster.github.io/json-e/ +Author: Dustin J. Mitchell +Author-email: dustin@mozilla.com +License: MPL2 +Description: UNKNOWN +Platform: UNKNOWN +Provides-Extra: release diff --git a/third_party/python/json-e/README.md b/third_party/python/json-e/README.md new file mode 100644 index 0000000000..155b2e6ded --- /dev/null +++ b/third_party/python/json-e/README.md @@ -0,0 +1,730 @@ +* [Full documentation](https://taskcluster.github.io/json-e) + +# JSON-e + +JSON-e is a data-structure parameterization system for embedding context in +JSON objects. + +The central idea is to treat a data structure as a "template" and transform it, +using another data structure as context, to produce an output data structure. + +There are countless libraries to do this with strings, such as +[mustache](https://mustache.github.io/). What makes JSON-e unique is that it +operates on data structures, not on their textual representation. This allows +input to be written in a number of formats (JSON, YAML, etc.) or even generated +dynamically. It also means that the output cannot be "invalid", even when +including large chunks of contextual data. + +JSON-e is also designed to be safe for use on untrusted data. It never uses +`eval` or any other function that might result in arbitrary code execution. It +also disallows unbounded iteration, so any JSON-e rendering operation will +finish in finite time. + +## Changes + +See +[CHANGELOG.rst](https://github.com/taskcluster/json-e/blob/master/CHANGELOG.rst) +for the changes in each version of this library. + +# Interface + +## JavaScript + +The JS module is installed with either of + +```shell +npm install --save json-e +yarn add json-e +``` + +The module exposes following interface: + +```javascript +import jsone from 'json-e'; + +var template = {a: {$eval: "foo.bar"}}; +var context = {foo: {bar: "zoo"}}; +console.log(jsone(template, context)); +// -> { a: 'zoo' } +``` + +Note that the context can contain functions, and those functions can be called +from the template: + +```javascript +var template = {$eval: "foo(1)"}; +var context = {"foo": function(x) { return x + 2; }}; +console.log(jsone(template, context)); // -> 3 +``` + +*NOTE*: Context functions are called synchronously. Any complex asynchronous +operations should be handled before rendering the template. + +*NOTE*: If the template is untrusted, it can pass arbitrary data to functions +in the context, which must guard against such behavior. + +### Browser + +JSON-e is distributed as a CommonJS package is not designed to be included +directly in a browser with `<script>`. Instead, it must be incorproated using a +tool that understands CommonJS such as Webpack. See +[Neutrino](https://neutrino.js.org/) for an easy, configuration-free way to +build such applications. + +## Python + +The Python distribution exposes a `render` function: + +```python +import jsone + +template = {"a": {"$eval": "foo.bar"}} +context = {"foo": {"bar": "zoo"}} +print(jsone.render(template, context)) # -> {"a": "zoo"} +``` + +and also allows custom functions in the context: + +```python +template = {"$eval": "foo(1)"} +context = {"foo": lambda x: x + 2} +print(jsone.render(template, context)) # -> 3 +``` + +## Go (golang) + +The [golang package for json-e](https://godoc.org/github.com/taskcluster/json-e) exposes a `Render` function: + +```golang +import ( + "fmt" + "github.com/taskcluster/json-e" +) + +// Template must be given using types: +// map[string]interface{}, []interface{}, float64, string, bool, nil +// The same types that json.Unmarshal() will create when targeting an interface{} +template := map[string]interface{}{ + "result": map[string]interface{}{ + "$eval": "f() + 5", + }, +} +// Context can be JSON types just like template, but may also contain functions +// these can JSON types as arguments, and return a value and optionally an error. +context := map[string]interface{}{ + "f": func() int { return 37 }, +} + +func main() { + value, err := jsone.Render(template, context) + fmt.Printf("%#v\n", value) +} +``` + +## CLI + +You can use the 3rd party package [rjsone](https://wryun.github.io/rjsone/) to template +JSON-e from the command line, passing templates/contexts as files or arguments and using +stdout for the result. + + +# Language Reference + +The examples here are given in YAML for ease of reading. Of course, the +rendering operation takes place on the parsed data, so the input format is +irrelevant to its operation. + +## Simple Operations + +All JSON-e directives involve the `$` character, so a template without any directives is +rendered unchanged: + +```yaml +template: {key: [1,2,{key2: 'val', key3: 1}, true], f: false} +context: {} +result: {key: [1,2,{key2: 'val', key3: 1}, true], f: false} +``` + +## String Interpolation + +The simplest form of substitution occurs within strings, using `${..}`: + +```yaml +template: {message: 'hello ${key}', 'k=${num}': true} +context: {key: 'world', num: 1} +result: {message: 'hello world', 'k=1': true} +``` + +The bit inside the `${..}` is an expression, and must evaluate to something +that interpolates obviously into a string (so, a string, number, boolean,). +If it is null, then the expression interpolates into an empty string. +The expression syntax is described in more detail below. + +Values interpolate as their JSON literal values: + +```yaml +template: ["number: ${num}", "booleans: ${t} ${f}", "null: ${nil}"] +context: {num: 3, t: true, f: false, nil: null} +result: ["number: 3", "booleans: true false", "null: "] +``` + +Note that object keys can be interpolated, too: + +```yaml +template: {"tc_${name}": "${value}"} +context: {name: 'foo', value: 'bar'} +result: {"tc_foo": "bar"} +``` + +The string `${` can be escaped as `$${`. + +## Operators + +JSON-e defines a bunch of operators. Each is represented as an object with a +property beginning with `$`. This object can be buried deeply within the +template. Some operators take additional arguments as properties of the same +object. + +### `$eval` + +The `$eval` operator evaluates the given expression and is replaced with the +result of that evaluation. Unlike with string interpolation, the result need +not be a string, but can be an arbitrary data structure. + +```yaml +template: {config: {$eval: 'settings.staging'}} +context: + settings: + staging: + transactionBackend: mock + production: + transactionBackend: customerdb +result: {config: {transactionBackend: 'mock'}} +``` + +The expression syntax is described in more detail below. + +Note that `$eval`'s value must be a string. "Metaprogramming" by providing a +calculated value to eval is not allowed. For example, `{$eval: {$eval: +"${var1} + ${var2}"}}` is not valid JSON-e. + +### `$json` + +The `$json` operator formats the given value as JSON with sorted keys. It does +not evaluate the value (use `$eval` for that). While this can be useful in some +cases, it is an unusual case to include a JSON string in a larger data +structure. + +```yaml +template: {$json: [a, b, {$eval: 'a+b'}, 4]} +context: {a: 1, b: 2} +result: '["a", "b", 3, 4]' +``` + +### `$if` - `then` - `else` + +The `$if` operator supports conditionals. It evaluates the given value, and +replaces itself with the `then` or `else` properties. If either property is +omitted, then the expression is omitted from the parent object. + +```yaml +template: {key: {$if: 'cond', then: 1}, k2: 3} +context: {cond: true} +result: {key: 1, k2: 3} +``` + +```yaml +template: {$if: 'x > 5', then: 1, else: -1} +context: {x: 10} +result: 1 +``` + +```yaml +template: [1, {$if: 'cond', else: 2}, 3] +context: {cond: false} +result: [1,2,3] +``` + +```yaml +template: {key: {$if: 'cond', then: 2}, other: 3} +context: {cond: false} +result: {other: 3} +``` + +### `$flatten` + +The `$flatten` operator flattens an array of arrays into one array. + +```yaml +template: {$flatten: [[1, 2], [3, 4], [5]]} +context: {} +result: [1, 2, 3, 4, 5] +``` + +### `$flattenDeep` + +The `$flattenDeep` operator deeply flattens an array of arrays into one array. + +```yaml +template: {$flattenDeep: [[1, [2, [3]]]]} +context: {} +result: [1, 2, 3] +``` + +### `$fromNow` + +The `$fromNow` operator is a shorthand for the built-in function `fromNow`. It +creates a JSON (ISO 8601) datestamp for a time relative to the current time +(see the `now` builtin, below) or, if `from` is given, relative to that time. +The offset is specified by a sequence of number/unit pairs in a string. For +example: + +```yaml +template: {$fromNow: '2 days 1 hour'} +context: {} +result: '2017-01-19T16:27:20.974Z' +``` + +```yaml +template: {$fromNow: '1 hour', from: '2017-01-19T16:27:20.974Z'} +context: {} +result: '2017-01-19T17:27:20.974Z' +``` + +The available units are `day`, `hour`, and `minute`, for all of which a plural +is also accepted. + +### `$let` + +The `$let` operator evaluates an expression using a context amended with the +given values. It is analogous to the Haskell `where` clause. + +```yaml +template: {$let: {ts: 100, foo: 200}, + in: [{$eval: "ts+foo"}, {$eval: "ts-foo"}, {$eval: "ts*foo"}]} +context: {} +result: [300, -100, 20000] +``` + +The `$let` operator here added the `ts` and `foo` variables to the scope of +the context and accordingly evaluated the `in` clause using those variables +to return the correct result. + +The variable names in the `$let` value must be valid context variable names and +must be written literally. That is, an expression like `{$let: {$eval: +"extraVariables"}, in : ..}` is not allowed. + +### `$map` + +The `$map` operator evaluates an expression for each value of the given array or object, +constructing the result as an array or object of the evaluated values. + +When given an array, map always returns an array. + +```yaml +template: + $map: [2, 4, 6] + each(x): {$eval: 'x + a'} +context: {a: 1} +result: [3, 5, 7] +``` +The array or object is the value of the `$map` property, and the expression to evaluate +is given by `each(var)` where `var` is the name of the variable containing each +element. In the case of iterating over an object, `var` will be an object with two keys: +`key` and `val`. These keys correspond to a key in the object and its corresponding value. + +When $map is given an object, the expression defined by `each(var)` must evaluate to an +object for each key/value pair (`key` and `val`).The objects constructed by each 'each(var)' +can then be merged internally to give the resulting object with later keys overwriting +the previous ones.Otherwise the expression becomes invalid for the $map operator + +```yaml +template: + $map: {a: 1, b: 2, c: 3} + each(y): {'${y.key}x': {$eval: 'y.val + 1'}} +context: {} +result: {ax: 2, bx: 3, cx: 4} +``` + +### `$match` + +The `$match` operator is not dissimilar to pattern matching operators. It gets an object, in which every key is a string expression(s) to evaluate to `true` or `false` based on the context. The result will be an array of things (all types are supported) that were values corresponding to the keys that were evaluated to `true`. The order of the things in the array will be arbitrary. If there are no matches, the result is an empty array. + +```yaml +template: {$match: {"x == 10": "ten", "x == 20": "twenty"}} +context: {x: 10} +result: ["ten"] +``` + +```yaml +template: {$match: {"x == 10 || x == 20": "tens", "x == 10": "ten"}} +context: {x: 10} +one possible result: ["tens", "ten"] +another possible result: ["ten", "tens"] +``` +```yaml +template: {$match: {"x < 10": "tens"}} +context: {x: 10} +result: [] +``` + +### `$merge` + +The `$merge` operator merges an array of objects, returning a single object +that combines all of the objects in the array, where the right-side objects +overwrite the values of the left-side ones. + +```yaml +template: {$merge: [{a: 1, b: 1}, {b: 2, c: 3}, {d: 4}]} +context: {} +result: {a: 1, b: 2, c: 3, d: 4} +``` + +### `$mergeDeep` + +The `$mergeDeep` operator is like `$merge`, but it recurses into objects to +combine their contents property by property. Arrays are concatenated. + +```yaml +template: + $mergeDeep: + - task: + payload: + command: [a, b] + - task: + extra: + foo: bar + - task: + payload: + command: [c] +context: {} +result: + task: + extra: + foo: bar + payload: + command: [a, b, c] +``` + +### `$sort` + +The `$sort` operator sorts the given array. It takes a `by(var)` property which +should evaluate to a comparable value for each element. The `by(var)` property +defaults to the identity function. + +```yaml +template: + $sort: [{a: 2}, {a: 1, b: []}, {a: 3}] + by(x): 'x.a' +context: {} +result: [{a: 1, b: []}, {a: 2}, {a: 3}] +``` + +### `$reverse` + +The `$reverse` operator simply reverses the given array. + +```yaml +template: {$reverse: [3, 4, 1, 2]} +context: {} +result: [2, 1, 4, 3] +``` + +### Escaping operators + +All property names starting with `$` are reserved for JSON-e. +You can use `$$` to escape such properties: + +```yaml +template: {$$reverse: [3, 2, {$$eval: '2 - 1'}, 0]} +context: {} +result: {$reverse: [3, 2, {$eval: '2 - 1'}, 0]} +``` + +## Truthiness + +Many values can be evaluated in context where booleans are required, +not just booleans themselves. JSON-e defines the following values as false. +Anything else will be true. + +```yaml +template: {$if: 'a || b || c || d || e || f', then: "uh oh", else: "falsy" } +context: {a: null, b: [], c: {}, d: "", e: 0, f: false} +result: "falsy" +``` + +## Expression Syntax + +Expression are given in a simple Python- or JavaScript-like expression +language. Its data types are limited to JSON types plus function objects. + +### Literals + +Literals are similar to those for JSON. Numeric literals only accept integer +and decimal notation. Strings do not support any kind of escaping. The use of +`\n` and `\t` in the example below depends on the YAML parser to expand the +escapes. + +```yaml +template: + - {$eval: "1.3"} + - {$eval: "'abc'"} + - {$eval: '"abc"'} + - {$eval: "'\n\t'"} +context: {} +result: + - 1.3 + - "abc" + - "abc" + - "\n\t" +``` + +Array and object literals also look much like JSON, with bare identifiers +allowed as keys like in Javascript: + +```yaml +template: + - {$eval: '[1, 2, "three"]'} + - {$eval: '{foo: 1, "bar": 2}'} +context: {} +result: + - [1, 2, "three"] + - {"foo": 1, "bar": 2} +``` + +### Context References + +Bare identifiers refer to items from the context or to built-ins (described below). + +```yaml +template: {$eval: '[x, z, x+z]'} +context: {x: 'quick', z: 'sort'} +reslut: ['quick', 'sort', 'quicksort'] +``` + +### Arithmetic Operations + +The usual arithmetic operators are all defined, with typical associativity and +precedence: + +```yaml +template: + - {$eval: 'x + z'} + - {$eval: 's + t'} + - {$eval: 'z - x'} + - {$eval: 'x * z'} + - {$eval: 'z / x'} + - {$eval: 'z ** 2'} + - {$eval: '(z / x) ** 2'} +context: {x: 10, z: 20, s: "face", t: "plant"} +result: + - 30 + - "faceplant" + - 10 + - 200 + - 2 + - 400 + - 4 +``` + +Note that strings can be concatenated with `+`, but none of the other operators +apply. + +### Comparison Operations + +Comparisons work as expected. Equality is "deep" in the sense of doing +comparisons of the contents of data structures. + +```yaml +template: + - {$eval: 'x < z'} + - {$eval: 'x <= z'} + - {$eval: 'x > z'} + - {$eval: 'x >= z'} + - {$eval: 'deep == [1, [3, {a: 5}]]'} + - {$eval: 'deep != [1, [3, {a: 5}]]'} +context: {x: -10, z: 10, deep: [1, [3, {a: 5}]]} +result: [true, true, false, false, true, false] +``` + +### Boolean Operations + +Boolean operations use C- and Javascript-style symbls `||`, `&&`, and `!`: + +```yaml +template: {$eval: '!(false || false) && true'} +context: {} +result: true +``` + +### Object Property Access + +Like Javascript, object properties can be accessed either with array-index +syntax or with dot syntax. Unlike Javascript, `obj.prop` is an error if `obj` +does not have `prop`, while `obj['prop']` will evaulate to `null`. + +```yaml +template: {$eval: 'v.a + v["b"]'} +context: {v: {a: 'apple', b: 'bananna', c: 'carrot'}} +result: 'applebananna' +```` + +### Indexing and Slicing + +Strings and arrays can be indexed and sliced using a Python-like indexing +scheme. Negative indexes are counted from the end of the value. Slices are +treated as "half-open", meaning that the result contains the first index and +does not contain the second index. A "backward" slice with the start index +greater than the end index is treated as empty. + +```yaml +template: + - {$eval: '[array[1], string[1]]'} + - {$eval: '[array[1:4], string[1:4]]'} + - {$eval: '[array[2:], string[2:]]'} + - {$eval: '[array[:2], string[:2]]'} + - {$eval: '[array[4:2], string[4:2]]'} + - {$eval: '[array[-2], string[-2]]'} + - {$eval: '[array[-2:], string[-2:]]'} + - {$eval: '[array[:-3], string[:-3]]'} +context: {array: ['a', 'b', 'c', 'd', 'e'], string: 'abcde'} +result: + - ['b', 'b'] + - [['b', 'c', 'd'], 'bcd'] + - [['c', 'd', 'e'], 'cde'] + - [['a', 'b'], 'ab'] + - [[], ''] + - ['d', 'd'] + - [['d', 'e'], 'de'] + - [['a', 'b'], 'ab'] +``` + +### Containment Operation + +The `in` keyword can be used to check for containment: a property in an object, +an element in an array, or a substring in a string. + +```yaml +template: + - {$eval: '"foo" in {foo: 1, bar: 2}'} + - {$eval: '"foo" in ["foo", "bar"]'} + - {$eval: '"foo" in "foobar"'} +context: {} +result: [true, true, true] +``` + +### Function Invocation + +Function calls are made with the usual `fn(arg1, arg2)` syntax. Functions are +not JSON data, so they cannot be created in JSON-e, but they can be provided as +built-ins or supplied in the context and called from JSON-e. + +### Built-In Functions and Variables + +The expression language provides a laundry-list of built-in functions/variables. Library +users can easily add additional functions/variables, or override the built-ins, as part +of the context. + +#### Time + +The built-in context value `now` is set to the current time at the start of +evaluation of the template, and used as the default "from" value for `$fromNow` +and the built-in `fromNow()`. + +```yaml +template: + - {$eval: 'now'} + - {$eval: 'fromNow("1 minute")'} + - {$eval: 'fromNow("1 minute", "2017-01-19T16:27:20.974Z")'} +context: {} +result: + - '2017-01-19T16:27:20.974Z', + - '2017-01-19T16:28:20.974Z', + - '2017-01-19T16:28:20.974Z', +``` + +#### Math + +```yaml +template: + # the smallest of the arguments + - {$eval: 'min(1, 3, 5)'} + # the largest of the arguments + - {$eval: 'max(2, 4, 6)'} + # mathematical functions + - {$eval: 'sqrt(16)'} + - {$eval: 'ceil(0.3)'} + - {$eval: 'floor(0.3)'} + - {$eval: 'abs(-0.3)'} +context: {} +result: + - 1 + - 6 + - 4 + - 1 + - 0 + - 0.3 +``` + +#### Strings + +```yaml +template: + # convert string case + - {$eval: 'lowercase("Fools!")'} + - {$eval: 'uppercase("Fools!")'} + # convert string, number, boolean, or array to string + - {$eval: 'str(130)'} + # strip whitespace from left, right, or both ends of a string + - {$eval: 'lstrip(" room ")'} + - {$eval: 'rstrip(" room ")'} + - {$eval: 'strip(" room ")'} +context: {} +result: + - "fools!" + - "FOOLS!" + - "130" + - "room " + - " room" + - room +``` + +#### Type + +The `typeof()` built-in returns the type of an object. Its behavior around +`null` is reminiscent of JavaScript. + +```yaml +template: + - "${typeof('abc')}" + - "${typeof(42)}" + - "${typeof(42.0)}" + - "${typeof(true)}" + - "${typeof([])}" + - "${typeof({})}" + - "${typeof(typeof)}" + - {$eval: "typeof(null)"} + - "${typeof(null)}" +context: {} +result: + - string + - number + - number + - boolean + - array + - object + - function + - null # note: the value null, not the string "null" + - '' # .. which interpolates to an empty string +``` + +#### Length + +The `len()` built-in returns the length of a string or array. + +```yaml +template: {$eval: 'len([1, 2, 3])'} +context: {} +result: 3 +``` + diff --git a/third_party/python/json-e/json_e.egg-info/PKG-INFO b/third_party/python/json-e/json_e.egg-info/PKG-INFO new file mode 100644 index 0000000000..bf41f82167 --- /dev/null +++ b/third_party/python/json-e/json_e.egg-info/PKG-INFO @@ -0,0 +1,11 @@ +Metadata-Version: 2.1 +Name: json-e +Version: 2.7.0 +Summary: A data-structure parameterization system written for embedding context in JSON objects +Home-page: https://taskcluster.github.io/json-e/ +Author: Dustin J. Mitchell +Author-email: dustin@mozilla.com +License: MPL2 +Description: UNKNOWN +Platform: UNKNOWN +Provides-Extra: release diff --git a/third_party/python/json-e/json_e.egg-info/SOURCES.txt b/third_party/python/json-e/json_e.egg-info/SOURCES.txt new file mode 100644 index 0000000000..ec2e4f666b --- /dev/null +++ b/third_party/python/json-e/json_e.egg-info/SOURCES.txt @@ -0,0 +1,17 @@ +MANIFEST.in +README.md +package.json +setup.cfg +setup.py +json_e.egg-info/PKG-INFO +json_e.egg-info/SOURCES.txt +json_e.egg-info/dependency_links.txt +json_e.egg-info/requires.txt +json_e.egg-info/top_level.txt +jsone/__init__.py +jsone/builtins.py +jsone/interpreter.py +jsone/prattparser.py +jsone/render.py +jsone/shared.py +jsone/six.py
\ No newline at end of file diff --git a/third_party/python/json-e/json_e.egg-info/dependency_links.txt b/third_party/python/json-e/json_e.egg-info/dependency_links.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/third_party/python/json-e/json_e.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/third_party/python/json-e/json_e.egg-info/requires.txt b/third_party/python/json-e/json_e.egg-info/requires.txt new file mode 100644 index 0000000000..f25a637d71 --- /dev/null +++ b/third_party/python/json-e/json_e.egg-info/requires.txt @@ -0,0 +1,3 @@ + +[release] +towncrier diff --git a/third_party/python/json-e/json_e.egg-info/top_level.txt b/third_party/python/json-e/json_e.egg-info/top_level.txt new file mode 100644 index 0000000000..afe8caa74c --- /dev/null +++ b/third_party/python/json-e/json_e.egg-info/top_level.txt @@ -0,0 +1 @@ +jsone diff --git a/third_party/python/json-e/jsone/__init__.py b/third_party/python/json-e/jsone/__init__.py new file mode 100644 index 0000000000..943674e672 --- /dev/null +++ b/third_party/python/json-e/jsone/__init__.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +from .render import renderValue +from .shared import JSONTemplateError, DeleteMarker, TemplateError, fromNow +from . import builtins + +_context_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$') + + +def render(template, context): + if not all(_context_re.match(c) for c in context): + raise TemplateError('top level keys of context must follow ' + '/[a-zA-Z_][a-zA-Z0-9_]*/') + full_context = {'now': fromNow('0 seconds', None)} + full_context.update(builtins.build(full_context)) + full_context.update(context) + rv = renderValue(template, full_context) + if rv is DeleteMarker: + return None + return rv diff --git a/third_party/python/json-e/jsone/builtins.py b/third_party/python/json-e/jsone/builtins.py new file mode 100644 index 0000000000..751ee2dc04 --- /dev/null +++ b/third_party/python/json-e/jsone/builtins.py @@ -0,0 +1,121 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import math +from .shared import string, to_str, fromNow, JSONTemplateError + + +class BuiltinError(JSONTemplateError): + pass + + +def build(context): + builtins = {} + + def builtin(name, variadic=None, argument_tests=None, minArgs=None): + def wrap(fn): + def bad(reason=None): + raise BuiltinError( + (reason or 'invalid arguments to builtin: {}').format(name)) + if variadic: + def invoke(*args): + if minArgs: + if len(args) < minArgs: + bad("too few arguments to {}") + for arg in args: + if not variadic(arg): + bad() + return fn(*args) + + elif argument_tests: + def invoke(*args): + if len(args) != len(argument_tests): + bad() + for t, arg in zip(argument_tests, args): + if not t(arg): + bad() + return fn(*args) + + else: + def invoke(*args): + return fn(*args) + + builtins[name] = invoke + return fn + return wrap + + def is_number(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + def is_string(v): + return isinstance(v, string) + + def is_string_or_array(v): + return isinstance(v, (string, list)) + + def anything_except_array(v): + return isinstance(v, (string, int, float, bool)) or v is None + + def anything(v): + return isinstance(v, (string, int, float, list, dict)) or v is None or callable(v) + + # --- + + builtin('min', variadic=is_number, minArgs=1)(min) + builtin('max', variadic=is_number, minArgs=1)(max) + builtin('sqrt', argument_tests=[is_number])(math.sqrt) + builtin('abs', argument_tests=[is_number])(abs) + + @builtin('ceil', argument_tests=[is_number]) + def ceil(v): + return int(math.ceil(v)) + + @builtin('floor', argument_tests=[is_number]) + def floor(v): + return int(math.floor(v)) + + @builtin('lowercase', argument_tests=[is_string]) + def lowercase(v): + return v.lower() + + @builtin('uppercase', argument_tests=[is_string]) + def lowercase(v): + return v.upper() + + builtin('len', argument_tests=[is_string_or_array])(len) + builtin('str', argument_tests=[anything_except_array])(to_str) + builtin('number', variadic=is_string, minArgs=1)(float) + + @builtin('strip', argument_tests=[is_string]) + def strip(s): + return s.strip() + + @builtin('rstrip', argument_tests=[is_string]) + def rstrip(s): + return s.rstrip() + + @builtin('lstrip', argument_tests=[is_string]) + def lstrip(s): + return s.lstrip() + + @builtin('fromNow', variadic=is_string, minArgs=1) + def fromNow_builtin(offset, reference=None): + return fromNow(offset, reference or context.get('now')) + + @builtin('typeof', argument_tests=[anything]) + def typeof(v): + if isinstance(v, bool): + return 'boolean' + elif isinstance(v, string): + return 'string' + elif isinstance(v, (int, float)): + return 'number' + elif isinstance(v, list): + return 'array' + elif isinstance(v, dict): + return 'object' + elif v is None: + return None + elif callable(v): + return 'function' + + return builtins diff --git a/third_party/python/json-e/jsone/interpreter.py b/third_party/python/json-e/jsone/interpreter.py new file mode 100644 index 0000000000..eb38a9c85b --- /dev/null +++ b/third_party/python/json-e/jsone/interpreter.py @@ -0,0 +1,289 @@ +from __future__ import absolute_import, print_function, unicode_literals + +from .prattparser import PrattParser, infix, prefix +from .shared import TemplateError, InterpreterError, string +import operator +import json + +OPERATORS = { + '-': operator.sub, + '*': operator.mul, + '/': operator.truediv, + '**': operator.pow, + '==': operator.eq, + '!=': operator.ne, + '<=': operator.le, + '<': operator.lt, + '>': operator.gt, + '>=': operator.ge, + '&&': lambda a, b: bool(a and b), + '||': lambda a, b: bool(a or b), +} + + +def infixExpectationError(operator, expected): + return InterpreterError('infix: {} expects {} {} {}'. + format(operator, expected, operator, expected)) + + +class ExpressionEvaluator(PrattParser): + + ignore = '\\s+' + patterns = { + 'number': '[0-9]+(?:\\.[0-9]+)?', + 'identifier': '[a-zA-Z_][a-zA-Z_0-9]*', + 'string': '\'[^\']*\'|"[^"]*"', + # avoid matching these as prefixes of identifiers e.g., `insinutations` + 'true': 'true(?![a-zA-Z_0-9])', + 'false': 'false(?![a-zA-Z_0-9])', + 'in': 'in(?![a-zA-Z_0-9])', + 'null': 'null(?![a-zA-Z_0-9])', + } + tokens = [ + '**', '+', '-', '*', '/', '[', ']', '.', '(', ')', '{', '}', ':', ',', + '>=', '<=', '<', '>', '==', '!=', '!', '&&', '||', 'true', 'false', 'in', + 'null', 'number', 'identifier', 'string', + ] + precedence = [ + ['||'], + ['&&'], + ['in'], + ['==', '!='], + ['>=', '<=', '<', '>'], + ['+', '-'], + ['*', '/'], + ['**-right-associative'], + ['**'], + ['[', '.'], + ['('], + ['unary'], + ] + + def __init__(self, context): + super(ExpressionEvaluator, self).__init__() + self.context = context + + def parse(self, expression): + if not isinstance(expression, string): + raise TemplateError('expression to be evaluated must be a string') + return super(ExpressionEvaluator, self).parse(expression) + + @prefix('number') + def number(self, token, pc): + v = token.value + return float(v) if '.' in v else int(v) + + @prefix("!") + def bang(self, token, pc): + return not pc.parse('unary') + + @prefix("-") + def uminus(self, token, pc): + v = pc.parse('unary') + if not isNumber(v): + raise InterpreterError('{} expects {}'.format('unary -', 'number')) + return -v + + @prefix("+") + def uplus(self, token, pc): + v = pc.parse('unary') + if not isNumber(v): + raise InterpreterError('{} expects {}'.format('unary +', 'number')) + return v + + @prefix("identifier") + def identifier(self, token, pc): + try: + return self.context[token.value] + except KeyError: + raise InterpreterError( + 'unknown context value {}'.format(token.value)) + + @prefix("null") + def null(self, token, pc): + return None + + @prefix("[") + def array_bracket(self, token, pc): + return parseList(pc, ',', ']') + + @prefix("(") + def grouping_paren(self, token, pc): + rv = pc.parse() + pc.require(')') + return rv + + @prefix("{") + def object_brace(self, token, pc): + return parseObject(pc) + + @prefix("string") + def string(self, token, pc): + return parseString(token.value) + + @prefix("true") + def true(self, token, pc): + return True + + @prefix("false") + def false(self, token, ps): + return False + + @infix("+") + def plus(self, left, token, pc): + if not isinstance(left, (string, int, float)) or isinstance(left, bool): + raise infixExpectationError('+', 'number/string') + right = pc.parse(token.kind) + if not isinstance(right, (string, int, float)) or isinstance(right, bool): + raise infixExpectationError('+', 'number/string') + if type(right) != type(left) and \ + (isinstance(left, string) or isinstance(right, string)): + raise infixExpectationError('+', 'numbers/strings') + return left + right + + @infix('-', '*', '/', '**') + def arith(self, left, token, pc): + op = token.kind + if not isNumber(left): + raise infixExpectationError(op, 'number') + right = pc.parse({'**': '**-right-associative'}.get(op)) + if not isNumber(right): + raise infixExpectationError(op, 'number') + return OPERATORS[op](left, right) + + @infix("[") + def index_slice(self, left, token, pc): + a = None + b = None + is_interval = False + if pc.attempt(':'): + a = 0 + is_interval = True + else: + a = pc.parse() + if pc.attempt(':'): + is_interval = True + + if is_interval and not pc.attempt(']'): + b = pc.parse() + pc.require(']') + + if not is_interval: + pc.require(']') + + return accessProperty(left, a, b, is_interval) + + @infix(".") + def property_dot(self, left, token, pc): + if not isinstance(left, dict): + raise infixExpectationError('.', 'object') + k = pc.require('identifier').value + try: + return left[k] + except KeyError: + raise TemplateError( + '{} not found in {}'.format(k, json.dumps(left))) + + @infix("(") + def function_call(self, left, token, pc): + if not callable(left): + raise TemplateError('function call', 'callable') + args = parseList(pc, ',', ')') + return left(*args) + + @infix('==', '!=', '||', '&&') + def equality_and_logic(self, left, token, pc): + op = token.kind + right = pc.parse(op) + return OPERATORS[op](left, right) + + @infix('<=', '<', '>', '>=') + def inequality(self, left, token, pc): + op = token.kind + right = pc.parse(op) + if type(left) != type(right) or \ + not (isinstance(left, (int, float, string)) and not isinstance(left, bool)): + raise infixExpectationError(op, 'numbers/strings') + return OPERATORS[op](left, right) + + @infix("in") + def contains(self, left, token, pc): + right = pc.parse(token.kind) + if isinstance(right, dict): + if not isinstance(left, string): + raise infixExpectationError('in-object', 'string on left side') + elif isinstance(right, string): + if not isinstance(left, string): + raise infixExpectationError('in-string', 'string on left side') + elif not isinstance(right, list): + raise infixExpectationError( + 'in', 'Array, string, or object on right side') + try: + return left in right + except TypeError: + raise infixExpectationError('in', 'scalar value, collection') + + +def isNumber(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def parseString(v): + return v[1:-1] + + +def parseList(pc, separator, terminator): + rv = [] + if not pc.attempt(terminator): + while True: + rv.append(pc.parse()) + if not pc.attempt(separator): + break + pc.require(terminator) + return rv + + +def parseObject(pc): + rv = {} + if not pc.attempt('}'): + while True: + k = pc.require('identifier', 'string') + if k.kind == 'string': + k = parseString(k.value) + else: + k = k.value + pc.require(':') + v = pc.parse() + rv[k] = v + if not pc.attempt(','): + break + pc.require('}') + return rv + + +def accessProperty(value, a, b, is_interval): + if isinstance(value, (list, string)): + if is_interval: + if b is None: + b = len(value) + try: + return value[a:b] + except TypeError: + raise infixExpectationError('[..]', 'integer') + else: + try: + return value[a] + except IndexError: + raise TemplateError('index out of bounds') + except TypeError: + raise infixExpectationError('[..]', 'integer') + + if not isinstance(value, dict): + raise infixExpectationError('[..]', 'object, array, or string') + if not isinstance(a, string): + raise infixExpectationError('[..]', 'string index') + + try: + return value[a] + except KeyError: + return None diff --git a/third_party/python/json-e/jsone/prattparser.py b/third_party/python/json-e/jsone/prattparser.py new file mode 100644 index 0000000000..5bf250a816 --- /dev/null +++ b/third_party/python/json-e/jsone/prattparser.py @@ -0,0 +1,191 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +from collections import namedtuple +from .shared import TemplateError +from .six import with_metaclass, viewitems + + +class SyntaxError(TemplateError): + + @classmethod + def unexpected(cls, got, exp): + exp = ', '.join(sorted(exp)) + return cls('Found {}, expected {}'.format(got.value, exp)) + + +Token = namedtuple('Token', ['kind', 'value', 'start', 'end']) + + +def prefix(*kinds): + """Decorate a method as handling prefix tokens of the given kinds""" + def wrap(fn): + try: + fn.prefix_kinds.extend(kinds) + except AttributeError: + fn.prefix_kinds = list(kinds) + return fn + return wrap + + +def infix(*kinds): + """Decorate a method as handling infix tokens of the given kinds""" + def wrap(fn): + try: + fn.infix_kinds.extend(kinds) + except AttributeError: + fn.infix_kinds = list(kinds) + return fn + return wrap + + +class PrattParserMeta(type): + + def __init__(cls, name, bases, body): + # set up rules based on decorated methods + infix_rules = cls.infix_rules = {} + prefix_rules = cls.prefix_rules = {} + for prop, value in viewitems(body): + if hasattr(value, 'prefix_kinds'): + for kind in value.prefix_kinds: + prefix_rules[kind] = value + delattr(cls, prop) + if hasattr(value, 'infix_kinds'): + for kind in value.infix_kinds: + infix_rules[kind] = value + delattr(cls, prop) + + # build a regular expression to generate a sequence of tokens + token_patterns = [ + '({})'.format(cls.patterns.get(t, re.escape(t))) + for t in cls.tokens] + if cls.ignore: + token_patterns.append('(?:{})'.format(cls.ignore)) + cls.token_re = re.compile('^(?:' + '|'.join(token_patterns) + ')') + + # build a map from token kind to precedence level + cls.precedence_map = { + kind: prec + 1 + for (prec, row) in enumerate(cls.precedence) + for kind in row + } + + +class PrattParser(with_metaclass(PrattParserMeta, object)): + + # regular expression for ignored input (e.g., whitespace) + ignore = None + + # regular expressions for tokens that do not match themselves + patterns = {} + + # all token kinds (note that order matters - the first matching token + # will be returned) + tokens = [] + + # precedence of tokens, as a list of lists, from lowest to highest + precedence = [] + + def parse(self, source): + pc = ParseContext(self, source, self._generate_tokens(source)) + result = pc.parse() + # if there are any tokens remaining, that's an error.. + token = pc.attempt() + if token: + raise SyntaxError.unexpected(token, self.infix_rules) + return result + + def parseUntilTerminator(self, source, terminator): + pc = ParseContext(self, source, self._generate_tokens(source)) + result = pc.parse() + token = pc.attempt() + if token.kind != terminator: + raise SyntaxError.unexpected(token, [terminator]) + return (result, token.start) + + def _generate_tokens(self, source): + offset = 0 + while True: + start = offset + remainder = source[offset:] + mo = self.token_re.match(remainder) + if not mo: + if remainder: + raise SyntaxError( + "Unexpected input: '{}'".format(remainder)) + break + offset += mo.end() + + # figure out which token matched (note that idx is 0-based) + indexes = list( + filter(lambda x: x[1] is not None, enumerate(mo.groups()))) + if indexes: + idx = indexes[0][0] + yield Token( + kind=self.tokens[idx], + value=mo.group(idx + 1), # (mo.group is 1-based) + start=start, + end=offset) + + +class ParseContext(object): + + def __init__(self, parser, source, token_generator): + self.parser = parser + self.source = source + + self._tokens = token_generator + self._error = None + + self._advance() + + def _advance(self): + try: + self.next_token = next(self._tokens) + except StopIteration: + self.next_token = None + except SyntaxError as exc: + self._error = exc + + def attempt(self, *kinds): + """Try to get the next token if it matches one of the kinds given, + otherwise returning None. If no kinds are given, any kind is + accepted.""" + if self._error: + raise self._error + token = self.next_token + if not token: + return None + if kinds and token.kind not in kinds: + return None + self._advance() + return token + + def require(self, *kinds): + """Get the next token, raising an exception if it doesn't match one of + the given kinds, or the input ends. If no kinds are given, returns the + next token of any kind.""" + token = self.attempt() + if not token: + raise SyntaxError('Unexpected end of input') + if kinds and token.kind not in kinds: + raise SyntaxError.unexpected(token, kinds) + return token + + def parse(self, precedence=None): + parser = self.parser + precedence = parser.precedence_map[precedence] if precedence else 0 + token = self.require() + prefix_rule = parser.prefix_rules.get(token.kind) + if not prefix_rule: + raise SyntaxError.unexpected(token, parser.prefix_rules) + left = prefix_rule(parser, token, self) + while self.next_token: + kind = self.next_token.kind + if kind not in parser.infix_rules: + break + if precedence >= parser.precedence_map[kind]: + break + token = self.require() + left = parser.infix_rules[kind](parser, left, token, self) + return left diff --git a/third_party/python/json-e/jsone/render.py b/third_party/python/json-e/jsone/render.py new file mode 100644 index 0000000000..e820da1ec2 --- /dev/null +++ b/third_party/python/json-e/jsone/render.py @@ -0,0 +1,354 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +import json as json +from .shared import JSONTemplateError, TemplateError, DeleteMarker, string, to_str +from . import shared +from .interpreter import ExpressionEvaluator +from .six import viewitems +import functools + +operators = {} +IDENTIFIER_RE = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$') + + +def operator(name): + def wrap(fn): + operators[name] = fn + return fn + return wrap + + +def evaluateExpression(expr, context): + evaluator = ExpressionEvaluator(context) + return evaluator.parse(expr) + + +_interpolation_start_re = re.compile(r'\$?\${') + + +def interpolate(string, context): + mo = _interpolation_start_re.search(string) + if not mo: + return string + + result = [] + evaluator = ExpressionEvaluator(context) + + while True: + result.append(string[:mo.start()]) + if mo.group() != '$${': + string = string[mo.end():] + parsed, offset = evaluator.parseUntilTerminator(string, '}') + if isinstance(parsed, (list, dict)): + raise TemplateError( + "interpolation of '{}' produced an array or object".format(string[:offset])) + if to_str(parsed) == "null": + result.append("") + else: + result.append(to_str(parsed)) + string = string[offset + 1:] + else: # found `$${` + result.append('${') + string = string[mo.end():] + + mo = _interpolation_start_re.search(string) + if not mo: + result.append(string) + break + + return ''.join(result) + + +def checkUndefinedProperties(template, allowed): + unknownKeys = [] + combined = "|".join(allowed) + "$" + unknownKeys = [key for key in sorted(template) + if not re.match(combined, key)] + if unknownKeys: + raise TemplateError(allowed[0].replace('\\', '') + + " has undefined properties: " + " ".join(unknownKeys)) + + +@operator('$eval') +def eval(template, context): + checkUndefinedProperties(template, ['\$eval']) + if not isinstance(template['$eval'], string): + raise TemplateError("$eval must be given a string expression") + return evaluateExpression(template['$eval'], context) + + +@operator('$flatten') +def flatten(template, context): + checkUndefinedProperties(template, ['\$flatten']) + value = renderValue(template['$flatten'], context) + if not isinstance(value, list): + raise TemplateError('$flatten value must evaluate to an array') + + def gen(): + for e in value: + if isinstance(e, list): + for e2 in e: + yield e2 + else: + yield e + return list(gen()) + + +@operator('$flattenDeep') +def flattenDeep(template, context): + checkUndefinedProperties(template, ['\$flattenDeep']) + value = renderValue(template['$flattenDeep'], context) + if not isinstance(value, list): + raise TemplateError('$flattenDeep value must evaluate to an array') + + def gen(value): + if isinstance(value, list): + for e in value: + for sub in gen(e): + yield sub + else: + yield value + + return list(gen(value)) + + +@operator('$fromNow') +def fromNow(template, context): + checkUndefinedProperties(template, ['\$fromNow', 'from']) + offset = renderValue(template['$fromNow'], context) + reference = renderValue( + template['from'], context) if 'from' in template else context.get('now') + + if not isinstance(offset, string): + raise TemplateError("$fromNow expects a string") + return shared.fromNow(offset, reference) + + +@operator('$if') +def ifConstruct(template, context): + checkUndefinedProperties(template, ['\$if', 'then', 'else']) + condition = evaluateExpression(template['$if'], context) + try: + if condition: + rv = template['then'] + else: + rv = template['else'] + except KeyError: + return DeleteMarker + return renderValue(rv, context) + + +@operator('$json') +def jsonConstruct(template, context): + checkUndefinedProperties(template, ['\$json']) + value = renderValue(template['$json'], context) + return json.dumps(value, separators=(',', ':'), sort_keys=True, ensure_ascii=False) + + +@operator('$let') +def let(template, context): + checkUndefinedProperties(template, ['\$let', 'in']) + if not isinstance(template['$let'], dict): + raise TemplateError("$let value must be an object") + + subcontext = context.copy() + for k, v in template['$let'].items(): + if not IDENTIFIER_RE.match(k): + raise TemplateError('top level keys of $let must follow /[a-zA-Z_][a-zA-Z0-9_]*/') + subcontext[k] = renderValue(v, context) + + try: + in_expression = template['in'] + except KeyError: + raise TemplateError("$let operator requires an `in` clause") + return renderValue(in_expression, subcontext) + + +@operator('$map') +def map(template, context): + EACH_RE = 'each\([a-zA-Z_][a-zA-Z0-9_]*\)' + checkUndefinedProperties(template, ['\$map', EACH_RE]) + value = renderValue(template['$map'], context) + if not isinstance(value, list) and not isinstance(value, dict): + raise TemplateError("$map value must evaluate to an array or object") + + is_obj = isinstance(value, dict) + + each_keys = [k for k in template if k.startswith('each(')] + if len(each_keys) != 1: + raise TemplateError( + "$map requires exactly one other property, each(..)") + each_key = each_keys[0] + each_var = each_key[5:-1] + each_template = template[each_key] + + def gen(val): + subcontext = context.copy() + for elt in val: + subcontext[each_var] = elt + elt = renderValue(each_template, subcontext) + if elt is not DeleteMarker: + yield elt + if is_obj: + value = [{'key': v[0], 'val': v[1]} for v in value.items()] + v = dict() + for e in gen(value): + if not isinstance(e, dict): + raise TemplateError( + "$map on objects expects {0} to evaluate to an object".format(each_key)) + v.update(e) + return v + else: + return list(gen(value)) + + +@operator('$match') +def matchConstruct(template, context): + checkUndefinedProperties(template, ['\$match']) + + if not isinstance(template['$match'], dict): + raise TemplateError("$match can evaluate objects only") + + result = [] + for condition in template['$match']: + if evaluateExpression(condition, context): + result.append(renderValue(template['$match'][condition], context)) + + return result + + +@operator('$merge') +def merge(template, context): + checkUndefinedProperties(template, ['\$merge']) + value = renderValue(template['$merge'], context) + if not isinstance(value, list) or not all(isinstance(e, dict) for e in value): + raise TemplateError( + "$merge value must evaluate to an array of objects") + v = dict() + for e in value: + v.update(e) + return v + + +@operator('$mergeDeep') +def merge(template, context): + checkUndefinedProperties(template, ['\$mergeDeep']) + value = renderValue(template['$mergeDeep'], context) + if not isinstance(value, list) or not all(isinstance(e, dict) for e in value): + raise TemplateError( + "$mergeDeep value must evaluate to an array of objects") + + def merge(l, r): + if isinstance(l, list) and isinstance(r, list): + return l + r + if isinstance(l, dict) and isinstance(r, dict): + res = l.copy() + for k, v in viewitems(r): + if k in l: + res[k] = merge(l[k], v) + else: + res[k] = v + return res + return r + if len(value) == 0: + return {} + return functools.reduce(merge, value[1:], value[0]) + + +@operator('$reverse') +def reverse(template, context): + checkUndefinedProperties(template, ['\$reverse']) + value = renderValue(template['$reverse'], context) + if not isinstance(value, list): + raise TemplateError("$reverse value must evaluate to an array of objects") + return list(reversed(value)) + + +@operator('$sort') +def sort(template, context): + BY_RE = 'by\([a-zA-Z_][a-zA-Z0-9_]*\)' + checkUndefinedProperties(template, ['\$sort', BY_RE]) + value = renderValue(template['$sort'], context) + if not isinstance(value, list): + raise TemplateError('$sorted values to be sorted must have the same type') + + # handle by(..) if given, applying the schwartzian transform + by_keys = [k for k in template if k.startswith('by(')] + if len(by_keys) == 1: + by_key = by_keys[0] + by_var = by_key[3:-1] + by_expr = template[by_key] + + def xform(): + subcontext = context.copy() + for e in value: + subcontext[by_var] = e + yield evaluateExpression(by_expr, subcontext), e + to_sort = list(xform()) + elif len(by_keys) == 0: + to_sort = [(e, e) for e in value] + else: + raise TemplateError('only one by(..) is allowed') + + # check types + try: + eltype = type(to_sort[0][0]) + except IndexError: + return [] + if eltype in (list, dict, bool, type(None)): + raise TemplateError('$sorted values to be sorted must have the same type') + if not all(isinstance(e[0], eltype) for e in to_sort): + raise TemplateError('$sorted values to be sorted must have the same type') + + # unzip the schwartzian transform + return list(e[1] for e in sorted(to_sort)) + + +def renderValue(template, context): + if isinstance(template, string): + return interpolate(template, context) + + elif isinstance(template, dict): + matches = [k for k in template if k in operators] + if matches: + if len(matches) > 1: + raise TemplateError("only one operator allowed") + return operators[matches[0]](template, context) + + def updated(): + for k, v in viewitems(template): + if k.startswith('$$'): + k = k[1:] + elif k.startswith('$') and IDENTIFIER_RE.match(k[1:]): + raise TemplateError( + '$<identifier> is reserved; ues $$<identifier>') + else: + k = interpolate(k, context) + + try: + v = renderValue(v, context) + except JSONTemplateError as e: + if IDENTIFIER_RE.match(k): + e.add_location('.{}'.format(k)) + else: + e.add_location('[{}]'.format(json.dumps(k))) + raise + if v is not DeleteMarker: + yield k, v + return dict(updated()) + + elif isinstance(template, list): + def updated(): + for i, e in enumerate(template): + try: + v = renderValue(e, context) + if v is not DeleteMarker: + yield v + except JSONTemplateError as e: + e.add_location('[{}]'.format(i)) + raise + return list(updated()) + + else: + return template diff --git a/third_party/python/json-e/jsone/shared.py b/third_party/python/json-e/jsone/shared.py new file mode 100644 index 0000000000..0e70e21f81 --- /dev/null +++ b/third_party/python/json-e/jsone/shared.py @@ -0,0 +1,131 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import re +import datetime + + +class DeleteMarker: + pass + + +class JSONTemplateError(Exception): + def __init__(self, message): + super(JSONTemplateError, self).__init__(message) + self.location = [] + + def add_location(self, loc): + self.location.insert(0, loc) + + def __str__(self): + location = ' at template' + ''.join(self.location) + return "{}{}: {}".format( + self.__class__.__name__, + location if self.location else '', + self.args[0]) + + +class TemplateError(JSONTemplateError): + pass + + +class InterpreterError(JSONTemplateError): + pass + + +# Regular expression matching: X days Y hours Z minutes +# todo: support hr, wk, yr +FROMNOW_RE = re.compile(''.join([ + '^(\s*(?P<years>\d+)\s*y(ears?)?)?', + '(\s*(?P<months>\d+)\s*mo(nths?)?)?', + '(\s*(?P<weeks>\d+)\s*w(eeks?)?)?', + '(\s*(?P<days>\d+)\s*d(ays?)?)?', + '(\s*(?P<hours>\d+)\s*h(ours?)?)?', + '(\s*(?P<minutes>\d+)\s*m(in(utes?)?)?)?\s*', + '(\s*(?P<seconds>\d+)\s*s(ec(onds?)?)?)?\s*$', +])) + + +def fromNow(offset, reference): + # copied from taskcluster-client.py + # We want to handle past dates as well as future + future = True + offset = offset.lstrip() + if offset.startswith('-'): + future = False + offset = offset[1:].lstrip() + if offset.startswith('+'): + offset = offset[1:].lstrip() + + # Parse offset + m = FROMNOW_RE.match(offset) + if m is None: + raise ValueError("offset string: '%s' does not parse" % offset) + + # In order to calculate years and months we need to calculate how many days + # to offset the offset by, since timedelta only goes as high as weeks + days = 0 + hours = 0 + minutes = 0 + seconds = 0 + if m.group('years'): + # forget leap years, a year is 365 days + years = int(m.group('years')) + days += 365 * years + if m.group('months'): + # assume "month" means 30 days + months = int(m.group('months')) + days += 30 * months + days += int(m.group('days') or 0) + hours += int(m.group('hours') or 0) + minutes += int(m.group('minutes') or 0) + seconds += int(m.group('seconds') or 0) + + # Offset datetime from utc + delta = datetime.timedelta( + weeks=int(m.group('weeks') or 0), + days=days, + hours=hours, + minutes=minutes, + seconds=seconds, + ) + + if isinstance(reference, string): + reference = datetime.datetime.strptime( + reference, '%Y-%m-%dT%H:%M:%S.%fZ') + elif reference is None: + reference = datetime.datetime.utcnow() + return stringDate(reference + delta if future else reference - delta) + + +datefmt_re = re.compile(r'(\.[0-9]{3})[0-9]*(\+00:00)?') + + +def to_str(v): + if isinstance(v, bool): + return {True: 'true', False: 'false'}[v] + elif isinstance(v, list): + return ','.join(to_str(e) for e in v) + elif v is None: + return 'null' + else: + return str(v) + + +def stringDate(date): + # Convert to isoFormat + try: + string = date.isoformat(timespec='microseconds') + # py2.7 to py3.5 does not have timespec + except TypeError as e: + string = date.isoformat() + if string.find('.') == -1: + string += '.000' + string = datefmt_re.sub(r'\1Z', string) + return string + + +# the base class for strings, regardless of python version +try: + string = basestring +except NameError: + string = str diff --git a/third_party/python/json-e/jsone/six.py b/third_party/python/json-e/jsone/six.py new file mode 100644 index 0000000000..1ab9cd2d7d --- /dev/null +++ b/third_party/python/json-e/jsone/six.py @@ -0,0 +1,23 @@ +import sys +import operator + +# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L818 + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L578 +if sys.version_info[0] == 3: + viewitems = operator.methodcaller("items") +else: + viewitems = operator.methodcaller("viewitems") diff --git a/third_party/python/json-e/package.json b/third_party/python/json-e/package.json new file mode 100644 index 0000000000..0c388d57db --- /dev/null +++ b/third_party/python/json-e/package.json @@ -0,0 +1,35 @@ +{ + "name": "json-e", + "version": "2.7.0", + "description": "json parameterization module inspired from json-parameterization", + "main": "./src/index.js", + "scripts": { + "lint": "eslint src/*.js test/*.js", + "test": "yarn lint && mocha test/*_test.js", + "build-demo": "cd demo && yarn && yarn build", + "start-demo": "cd demo && yarn && yarn start" + }, + "files": [ + "src" + ], + "repository": { + "type": "git", + "url": "https://github.com/taskcluster/json-e" + }, + "author": "", + "license": "MPL-2.0", + "dependencies": { + "json-stable-stringify": "^1.0.1" + }, + "devDependencies": { + "assume": "^1.5.2", + "browserify": "^14.5.0", + "eslint-config-taskcluster": "^3.0.0", + "mocha": "^4.0.1", + "source-map-support": "^0.5.0", + "timekeeper": "^2.0.0" + }, + "engines": { + "node": ">=6.4.0" + } +} diff --git a/third_party/python/json-e/setup.cfg b/third_party/python/json-e/setup.cfg new file mode 100644 index 0000000000..6410597b69 --- /dev/null +++ b/third_party/python/json-e/setup.cfg @@ -0,0 +1,8 @@ +[pep8] +max-line-length = 100 +select = E,W + +[egg_info] +tag_build = +tag_date = 0 + diff --git a/third_party/python/json-e/setup.py b/third_party/python/json-e/setup.py new file mode 100644 index 0000000000..76299df9be --- /dev/null +++ b/third_party/python/json-e/setup.py @@ -0,0 +1,31 @@ +import json +import os +from setuptools import setup, find_packages + +package_json = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json') +with open(package_json) as f: + version = json.load(f)['version'] + +setup(name='json-e', + version=version, + description='A data-structure parameterization system written for embedding context in JSON objects', + author='Dustin J. Mitchell', + url='https://taskcluster.github.io/json-e/', + author_email='dustin@mozilla.com', + packages=['jsone'], + test_suite='nose.collector', + license='MPL2', + extras_require={ + 'release': [ + 'towncrier', + ], + }, + tests_require=[ + "freezegun", + "hypothesis", + "nose", + "PyYAML", + "python-dateutil", + 'pep8', + ] +) |