diff options
Diffstat (limited to 'testing/web-platform/tests/annotation-model/scripts/JSONtest.js')
| -rw-r--r-- | testing/web-platform/tests/annotation-model/scripts/JSONtest.js | 803 |
1 files changed, 803 insertions, 0 deletions
diff --git a/testing/web-platform/tests/annotation-model/scripts/JSONtest.js b/testing/web-platform/tests/annotation-model/scripts/JSONtest.js new file mode 100644 index 0000000000..3ee49b86a8 --- /dev/null +++ b/testing/web-platform/tests/annotation-model/scripts/JSONtest.js @@ -0,0 +1,803 @@ +/* globals add_completion_callback, Promise, showdown, done, assert_true, Ajv, on_event */ + +/** + * Creates a JSONtest object. If the parameters are supplied + * it also loads a referenced testFile, processes that file, loads any + * referenced external assertions, and sets up event listeners to process the + * user's test data. The loading is done asynchronously via Promises. The test + * button's text is changed to Loading while it is processing, and to "Check + * JSON" once the data is loaded. + * + * @constructor + * @param {object} params + * @param {string} [params.test] - object containing JSON test definition + * @param {string} [params.testFile] - URI of a file with JSON test definition + * @param {string} params.runTest - IDREF of an element that when clicked will run the test + * @param {string} params.testInput - IDREF of an element that contains the JSON(-LD) to evaluate against the assertions in the test / testFile + * @event DOMContentLoaded Calls init once DOM is fully loaded + * @returns {object} Reference to the new object + */ + +function JSONtest(params) { + 'use strict'; + + this.Assertions = []; // object that will contain the assertions to process + this.AssertionText = ""; // string that holds the titles of all the assertions in use + this.DescriptionText = ""; + this.Base = null; // URI "base" for the test suite being run + this.TestDir = null; // URI "base" for the test case being run + this.Params = null; // paramaters passed in + this.Promise = null; // master Promise that resolves when intialization is complete + this.Properties = null; // testharness_properties from the opening window + this.SkipFailures = []; // list of assertionType values that should be skipped if their test would fail + this.Test = null; // test being run + this.AssertionCounter = 0;// keeps track of which assertion is being processed + + this._assertionCache = [];// Array to put loaded assertions into + this._assertionText = []; // Array of text or nested arrays of assertions + this._loading = true; + + showdown.extension('strip', function() { + return [ + { type: 'output', + regex: /<p>/, + replace: '' + }, + { type: 'output', + regex: /<\/p>$/, + replace: '' + } + ]; + }); + + + this.markdown = new showdown.Converter({ extensions: [ 'strip' ] }) ; + + var pending = [] ; + + // set up in case DOM finishes loading early + pending.push(new Promise(function(resolve) { + on_event(document, "DOMContentLoaded", function() { + resolve(true); + }.bind(this)); + }.bind(this))); + + // create an ajv object that will stay around so that caching + // of schema that are compiled just works + this.ajv = new Ajv({allErrors: true, validateSchema: false}) ; + + // determine the base URI for the test collection. This is + // the top level folder in the test "document.location" + + var l = document.location; + var p = l.pathname; + this.TestDir = p.substr(0, 1+p.lastIndexOf('/')); + this.Base = p.substr(0, 1+p.indexOf('/', 1)); + + // if we are under runner, then there are props in the parent window + // + // if "output" is set in that, then pause at the end of running so the output + // can be analyzed. @@@TODO@@@ + if (window && window.opener && window.opener.testharness_properties) { + this.Properties = window.opener.testharness_properties; + } + + this.Params = params; + + // if there is a list of definitions in the params, + // include them + if (this.Params.schemaDefs) { + var defPromise = new Promise(function(resolve, reject) { + var promisedSchema = this.Params.schemaDefs.map(function(item) { + return this.loadDefinition(item); + }.bind(this)); + + // Once all the loadAssertion promises resolve... + Promise.all(promisedSchema) + .then(function (schemaContents) { + this.ajv.addSchema(schemaContents); + resolve(true); + }.bind(this)) + .catch(function(err) { + reject(err); + }.bind(this)); + }.bind(this)); + // these schema need to load up too + pending.push(defPromise) ; + } + + // start by loading the test (it might be inline, but + // loadTest deals with that + pending.push(this.loadTest(params) + .then(function(test) { + // if the test is NOT an object, turn it into one + if (typeof test === 'string') { + test = JSON.parse(test) ; + } + + this.Test = test; + + // Test should have information that we can put in the template + + if (test.description) { + this.DescriptionText = test.description; + } + + if (test.hasOwnProperty("skipFailures") && Array.isArray(test.skipFailures) ) { + this.SkipFailures = test.skipFailures; + } + + if (test.content) { + // we have content + if (typeof test.content === "string") { + // the test content is a string - meaning it is a reference to a file of content + var cPromise = new Promise(function(resolve, reject) { + this.loadDefinition(test.content) + .then(function(content) { + if (typeof content === 'string') { + content = JSON.parse(content) ; + } + test.content = content; + resolve(true); + }.bind(this)) + .catch(function(err) { + reject("Loading " + test.content + ": " + JSON.stringify(err)); + }); + + }.bind(this)); + pending.push(cPromise); + } + } + + return new Promise(function(resolve, reject) { + if (test.assertions && + typeof test.assertions === "object") { + // we have at least one assertion + // get the inline contents and the references to external files + var assertFiles = this._assertionRefs(test.assertions); + + var promisedAsserts = assertFiles.map(function(item) { + return this.loadAssertion(item); + }.bind(this)); + + // Once all the loadAssertion promises resolve... + Promise.all(promisedAsserts) + .then(function (assertContents) { + // assertContents has assertions in document order + + var typeMap = { + 'must' : "<b>[MANDATORY]</b> ", + 'may' : "<b>[OPTIONAL]</b> ", + 'should' : "<b>[RECOMMENDED]</b> " + }; + + var assertIdx = 0; + + // populate the display of assertions that are being exercised + // returns the list of top level assertions to walk through + + var buildList = function(assertions, level) { + if (level === undefined) { + level = 1; + } + + // accumulate the assertions - but only when level is 0 + var list = [] ; + + var type = ""; + if (assertions) { + if (typeof assertions === "object" && assertions.hasOwnProperty('assertions')) { + // this is a conditionObject + if (level === 0) { + list.push(assertContents[assertIdx]); + } + type = assertContents[assertIdx].hasOwnProperty('assertionType') ? + assertContents[assertIdx].assertionType : "must" ; + + // ensure type defaults to must + if (!typeMap.hasOwnProperty(type)) { + type = "must"; + } + + this.AssertionText += "<li>" + typeMap[type] + this.markdown.makeHtml(assertContents[assertIdx++].title); + this.AssertionText += "<ol>"; + buildList(assertions.assertions, level+1) ; + this.AssertionText += "</ol></li>\n"; + } else { + // it is NOT a conditionObject - must be an array + assertions.forEach( function(assert) { + if (typeof assert === "object" && Array.isArray(assert)) { + this.AssertionText += "<ol>"; + // it is a nested list - recurse + buildList(assert, level+1) ; + this.AssertionText += "</ol>\n"; + } else if (typeof assert === "object" && + !Array.isArray(assert) && + assert.hasOwnProperty('assertions')) { + if (level === 0) { + list.push(assertContents[assertIdx]); + } + type = assertContents[assertIdx].hasOwnProperty('assertionType') ? + assertContents[assertIdx].assertionType : "must" ; + + // ensure type defaults to must + if (!typeMap.hasOwnProperty(type)) { + type = "must"; + } + + // there is a condition object in the array + this.AssertionText += "<li>" + typeMap[type] + this.markdown.makeHtml(assertContents[assertIdx++].title); + this.AssertionText += "<ol>"; + buildList(assert, level+1) ; // capture the children too + this.AssertionText += "</ol></li>\n"; + } else { + if (level === 0) { + list.push(assertContents[assertIdx]); + } + type = assertContents[assertIdx].hasOwnProperty('assertionType') ? + assertContents[assertIdx].assertionType : "must" ; + + // ensure type defaults to must + if (!typeMap.hasOwnProperty(type)) { + type = "must"; + } + + this.AssertionText += "<li>" + typeMap[type] + this.markdown.makeHtml(assertContents[assertIdx++].title) + "</li>\n"; + } + }.bind(this)); + } + } + return list; + }.bind(this); + + // Assertions will ONLY contain the top level assertions + this.Assertions = buildList(test.assertions, 0); + resolve(true); + }.bind(this)) + .catch(function(err) { + reject(err); + }.bind(this)); + } else { + if (!test.assertions) { + reject("Test has no assertion property"); + } else { + reject("Test assertion property is not an Array"); + } + } + }.bind(this)); + }.bind(this))); + + this.Promise = new Promise(function(resolve, reject) { + // once the DOM and the test / assertions are loaded... set us up + Promise.all(pending) + .then(function() { + this.loading = false; + this.init(); + resolve(this); + }.bind(this)) + .catch(function(err) { + // loading the components failed somehow - report the errors and mark the test failed + test( function() { + assert_true(false, "Loading of test components failed: " +JSON.stringify(err)) ; + }, "Loading test components"); + done() ; + reject("Loading of test components failed: "+JSON.stringify(err)); + return ; + }.bind(this)); + }.bind(this)); + + return this; +} + +JSONtest.prototype = { + + /** + * @listens click + */ + init: function() { + 'use strict'; + // set up a handler + var runButton = document.getElementById(this.Params.runTest) ; + var closeButton = document.getElementById(this.Params.closeWindow) ; + var testInput = document.getElementById(this.Params.testInput) ; + var assertion = document.getElementById("assertion") ; + var desc = document.getElementById("testDescription") ; + + if (!this.loading) { + if (runButton) { + runButton.disabled = false; + runButton.value = "Check JSON"; + } + if (desc) { + desc.innerHTML = this.DescriptionText; + } + if (assertion) { + assertion.innerHTML = "<ol>" + this.AssertionText + "</ol>\n"; + } + } else { + window.alert("Loading did not finish before init handler was called!"); + } + + // @@@TODO@@@ implement the output showing handler + if (0 && this.Properties && this.Properties.output && closeButton) { + // set up a callback + add_completion_callback( function() { + var p = new Promise(function(resolve) { + closeButton.style.display = "inline"; + closeButton.disabled = false; + on_event(closeButton, "click", function() { + resolve(true); + }); + }.bind(this)); + p.then(); + }.bind(this)); + } + + if (runButton) { + on_event(runButton, "click", function() { + // user clicked + var content = testInput.value; + runButton.disabled = true; + + // make sure content is an object + if (typeof content === "string") { + try { + content = JSON.parse(content) ; + } catch(err) { + // if the parsing failed, create a special test and mark it failed + test( function() { + assert_true(false, "Parse of JSON failed: " + err) ; + }, "Parsing submitted input"); + // and just give up + done(); + return ; + } + } + + // iterate over all of the tests for this instance + this.runTests(this.Assertions, content); + + // explicitly tell the test framework we are done + done(); + }.bind(this)); + } + }, + + // runTests - process tests + /** + * @param {object} assertions - List of assertions to process + * @param {string} content - JSON(-LD) to be evaluated + * @param {string} [testAction='continue'] - state of test processing (in parent when recursing) + * @param {integer} [level=0] - depth of recursion since assertion lists can nest + * @param {string} [compareWith='and'] - the way the results of the referenced assertions should be compared + * @returns {string} - the testAction resulting from evaluating all of the assertions + */ + runTests: function(assertions, content, testAction, level, compareWith) { + 'use strict'; + + // level + if (level === undefined) { + level = 1; + } + + // testAction + if (testAction === undefined) { + testAction = 'continue'; + } + + // compareWith + if (compareWith === undefined) { + compareWith = 'and'; + } + + var typeMap = { + 'must' : "", + 'may' : "INFORMATIONAL: ", + 'should' : "WARNING: " + }; + + + // for each assertion (in order) load the external json schema if + // one is referenced, or use the inline schema if supplied + // validate content against the referenced schema + + var theResults = [] ; + + if (assertions) { + + assertions.forEach( function(assert, num) { + + var expected = assert.hasOwnProperty('expectedResult') ? + assert.expectedResult : 'valid' ; + var message = assert.hasOwnProperty('errorMessage') ? + assert.errorMessage : "Result was not " + expected; + var type = assert.hasOwnProperty('assertionType') ? + assert.assertionType.toLowerCase() : "must" ; + if (!typeMap.hasOwnProperty(type)) { + type = "must"; + } + + // first - what is the type of the assert + if (typeof assert === "object" && !Array.isArray(assert)) { + if (assert.hasOwnProperty("compareWith") && assert.hasOwnProperty("assertions") && Array.isArray(assert.assertions) ) { + // this is a comparisonObject + var r = this.runTests(assert.assertions, content, testAction, level+1, assert.compareWith); + // r is an object that contains, among other things, an array of results from the child assertions + testAction = r.action; + + // evaluate the results against the compareWith setting + var result = true; + var data = r.results ; + var i; + + if (assert.compareWith === "or") { + result = false; + for(i = 0; i < data.length; i++) { + if (data[i]) { + result = true; + } + } + } else { + for(i = 0; i < data.length; i++) { + if (!data[i]) { + result = false; + } + } + } + + // create a test and push the result + test(function() { + var newAction = this.determineAction(assert, result) ; + // next time around we will use this action + testAction = newAction; + + var err = ";"; + + if (testAction === 'abort') { + err += "; Aborting execution of remaining assertions;"; + } else if (testAction === 'skip') { + err += "; Skipping execution of remaining assertions at level " + level + ";"; + } + + if (result === false) { + // test result was unexpected; use message + assert_true(result, message + err); + } else { + assert_true(result, err) ; + } + }.bind(this), "" + level + ":" + (num+1) + " " + assert.title); + // we are going to return out of this + return; + } + } else if (typeof assert === "object" && Array.isArray(assert)) { + // it is a nested list - recurse + var o = this.runTests(assert, content, testAction, level+1); + if (o.result && o.result === 'abort') { + // we are bailing out + testAction = 'abort'; + } + } + + if (testAction === 'abort') { + return {action: 'abort' }; + } + + var schemaName = "inline " + level + ":" + (num+1); + + if (typeof assert === "string") { + // the assertion passed in is a file name; find it in the cache + if (this._assertionCache[assert]) { + assert = this._assertionCache[assert]; + } else { + test( function() { + assert_true(false, "Reference to assertion " + assert + " at level " + level + ":" + (num+1) + " unresolved") ; + }, "Processing " + assert); + return ; + } + } + + if (assert.assertionFile) { + schemaName = "external file " + assert.assertionFile + " " + level + ":" + (num+1); + } + + var validate = null; + + try { + validate = this.ajv.compile(assert); + } + catch(err) { + test( function() { + assert_true(false, "Compilation of schema " + level + ":" + (num+1) + " failed: " + err) ; + }, "Compiling " + schemaName); + return ; + } + + if (testAction === 'continue') { + // a previous test told us to not run this test; skip it + // test(function() { }, "SKIPPED: " + assert.title); + // start an actual sub-test + var valid = validate(content) ; + + var theResult = this.determineResult(assert, valid) ; + + // remember the result + theResults.push(theResult); + + var newAction = this.determineAction(assert, theResult) ; + // next time around we will use this action + testAction = newAction; + + // only run the test if we are NOT skipping fails for some types + // or the result is expected + if ( theResult === true || !this.SkipFailures.includes(type) ) { + test(function() { + var err = ";"; + if (validate.errors !== null && !assert.hasOwnProperty("errorMessage")) { + err = "; Errors: " + this.ajv.errorsText(validate.errors) + ";" ; + } + if (testAction === 'abort') { + err += "; Aborting execution of remaining assertions;"; + } else if (testAction === 'skip') { + err += "; Skipping execution of remaining assertions at level " + level + ";"; + } + if (theResult === false) { + // test result was unexpected; use message + assert_true(theResult, typeMap[type] + message + err); + } else { + assert_true(theResult, err) ; + } + }.bind(this), "" + level + ":" + (num+1) + " " + assert.title); + } + } + }.bind(this)); + } + + return { action: testAction, results: theResults} ; + }, + + determineResult: function(schema, valid) { + 'use strict'; + var r = 'valid' ; + if (schema.hasOwnProperty('expectedResult')) { + r = schema.expectedResult; + } + + if (r === 'valid' && valid || r === 'invalid' && !valid) { + return true; + } else { + return false; + } + }, + + determineAction: function(schema, result) { + 'use strict'; + // mapping from results to actions + var mapping = { + 'failAndContinue' : 'continue', + 'failAndSkip' : 'skip', + 'failAndAbort' : 'abort', + 'passAndContinue': 'continue', + 'passAndSkip' : 'skip', + 'passAndAbort' : 'abort' + }; + + // if the result was as expected, then just keep going + if (result) { + return 'continue'; + } + + var a = 'failAndContinue'; + + if (schema.hasOwnProperty('onUnexpectedResult')) { + a = schema.onUnexpectedResult; + } + + if (mapping[a]) { + return mapping[a]; + } else { + return 'continue'; + } + }, + + // loadAssertion - load an Assertion from an external JSON file + // + // returns a promise that resolves with the contents of the assertion file + + loadAssertion: function(afile) { + 'use strict'; + if (typeof(afile) === 'string') { + var theFile = this._parseURI(afile); + // it is a file reference - load it + return new Promise(function(resolve, reject) { + this._loadFile("GET", theFile, true) + .then(function(data) { + data.assertionFile = afile; + this._assertionCache[afile] = data; + resolve(data); + }.bind(this)) + .catch(function(err) { + if (typeof err === "object") { + err.theFile = theFile; + } + reject(err); + }); + }.bind(this)); + } + else if (afile.hasOwnProperty("assertionFile")) { + // this object is referecing an external assertion + return new Promise(function(resolve, reject) { + var theFile = this._parseURI(afile.assertionFile); + this._loadFile("GET", theFile, true) + .then(function(external) { + // okay - we have an external object + Object.keys(afile).forEach(function(key) { + if (key !== 'assertionFile') { + external[key] = afile[key]; + } + }); + resolve(external); + }.bind(this)) + .catch(function(err) { + if (typeof err === "object") { + err.theFile = theFile; + } + reject(err); + }); + }.bind(this)); + } else { + // it is already a loaded assertion - just use it + return new Promise(function(resolve) { + resolve(afile); + }); + } + }, + + // loadDefinition - load a JSON Schema definition from an external JSON file + // + // returns a promise that resolves with the contents of the definition file + + loadDefinition: function(dfile) { + 'use strict'; + return new Promise(function(resolve, reject) { + this._loadFile("GET", this._parseURI(dfile), true) + .then(function(data) { + resolve(data); + }.bind(this)) + .catch(function(err) { + reject(err); + }); + }.bind(this)); + }, + + + // loadTest - load a test from an external JSON file + // + // returns a promise that resolves with the contents of the + // test + + loadTest: function(params) { + 'use strict'; + + if (params.hasOwnProperty('testFile')) { + // the test is referred to by a file name + return this._loadFile("GET", params.testFile); + } // else + return new Promise(function(resolve, reject) { + if (params.hasOwnProperty('test')) { + resolve(params.test); + } else { + reject("Must supply a 'test' or 'testFile' parameter"); + } + }); + }, + + _parseURI: function(theURI) { + 'use strict'; + // determine what the top level URI should be + if (theURI.indexOf('/') === -1) { + // no slash - it's relative to where we are + // so just use it + return this.TestDir + theURI; + } else if (theURI.indexOf('/') === 0 || theURI.indexOf('http:') === 0 || theURI.indexOf('https:') === 0) { + // it is an absolute URI so just use it + return theURI; + } else { + // it is relative and contains a slash. + // make it relative to the current test root + return this.Base + theURI; + } + }, + + /** + * return a list of all inline assertions or references + * + * @param {array} assertions list of assertions to examine + */ + + _assertionRefs: function(assertions) { + 'use strict'; + var ret = [] ; + + // when the reference is to an object that has an array of assertions in it (a conditionObject) + // then remember that one and loop over its embedded assertions + if (typeof(assertions) === "object" && !Array.isArray(assertions) && assertions.hasOwnProperty('assertions')) { + ret.push(assertions) ; + assertions = assertions.assertions; + } + if (typeof(assertions) === "object" && Array.isArray(assertions)) { + assertions.forEach( function(assert) { + // first - what is the type of the assert + if (typeof assert === "object" && Array.isArray(assert)) { + // it is a nested list - recurse + this._assertionRefs(assert).forEach( function(item) { + ret.push(item); + }.bind(this)); + } else if (typeof assert === "object") { + ret.push(assert) ; + if (assert.hasOwnProperty("assertions")) { + // there are embedded assertions; get those too + ret.concat(this._assertionRefs(assert.assertions)); + } + } else { + // it is a file name + ret.push(assert) ; + } + }.bind(this)); + } + return ret; + }, + + // _loadFile - return a promise loading a file + // + _loadFile: function(method, url, parse) { + 'use strict'; + if (parse === undefined) { + parse = true; + } + + return new Promise(function (resolve, reject) { + if (document.location.search) { + var s = document.location.search; + s = s.replace(/^\?/, ''); + if (url.indexOf('?') !== -1) { + url += "&" + s; + } else { + url += "?" + s; + } + } + var xhr = new XMLHttpRequest(); + xhr.open(method, url); + xhr.onload = function () { + if (this.status >= 200 && this.status < 300) { + var d = xhr.response; + if (parse) { + try { + d = JSON.parse(d); + resolve(d); + } + catch(err) { + reject({ status: this.status, + statusText: "Parsing of " + url + " failed: " + err } + ); + } + } else { + resolve(d); + } + } else { + reject({ + status: this.status, + statusText: xhr.statusText + }); + } + }; + xhr.onerror = function () { + reject({ + status: this.status, + statusText: xhr.statusText + }); + }; + xhr.send(); + }); + }, + +}; |