path: root/dom/canvas/test/webgl-conf/checkout/js/webgl-test-harness.js

/*
Copyright (c) 2019 The Khronos Group Inc.
Use of this source code is governed by an MIT-style license that can be
found in the LICENSE.txt file.
*/

// This is a test harness for running javascript tests in the browser.
// The only identifier exposed by this harness is WebGLTestHarnessModule.
//
// To use it make an HTML page with an iframe. Then call the harness like this
//
//    function reportResults(type, msg, success) {
//      ...
//      return true;
//    }
//
//    var fileListURL = '00_test_list.txt';
//    var testHarness = new WebGLTestHarnessModule.TestHarness(
//        iframe,
//        fileListURL,
//        reportResults,
//        options);
//
// The harness will load the fileListURL and parse it for the URLs, one URL
// per line preceded by options, see below. URLs should be on the same domain
// and at the  same folder level or below the main html file.  If any URL ends
// in .txt it will be parsed as well so you can nest .txt files. URLs inside a
// .txt file should be relative to that text file.
//
// During startup, for each page found the reportFunction will be called with
// WebGLTestHarnessModule.TestHarness.reportType.ADD_PAGE and msg will be
// the URL of the test.
//
// Each test is required to call testHarness.reportResults. This is most easily
// accomplished by storing that value on the main window with
//
//     window.webglTestHarness = testHarness
//
// and then adding these to functions to your tests.
//
//     function reportTestResultsToHarness(success, msg) {
//       if (window.parent.webglTestHarness) {
//         window.parent.webglTestHarness.reportResults(success, msg);
//       }
//     }
//
//     function notifyFinishedToHarness() {
//       if (window.parent.webglTestHarness) {
//         window.parent.webglTestHarness.notifyFinished();
//       }
//     }
//
// This way your tests will still run without the harness and you can use
// any testing framework you want.
//
// Each test should call reportTestResultsToHarness with true for success if it
// succeeded and false if it fail followed and any message it wants to
// associate with the test. If your testing framework supports checking for
// timeout you can call it with success equal to undefined in that case.
//
// To run the tests, call testHarness.runTests(options);
//
// For each test run, before the page is loaded the reportFunction will be
// called with WebGLTestHarnessModule.TestHarness.reportType.START_PAGE and msg
// will be the URL of the test. You may return false if you want the test to be
// skipped.
//
// For each test completed the reportFunction will be called with
// with WebGLTestHarnessModule.TestHarness.reportType.TEST_RESULT,
// success = true on success, false on failure, undefined on timeout
// and msg is any message the test choose to pass on.
//
// When all the tests on the page have finished your page must call
// notifyFinishedToHarness.  If notifyFinishedToHarness is not called
// the harness will assume the test timed out.
//
// When all the tests on a page have finished OR the page as timed out the
// reportFunction will be called with
// WebGLTestHarnessModule.TestHarness.reportType.FINISH_PAGE
// where success = true if the page has completed or undefined if the page timed
// out.
//
// Finally, when all the tests have completed the reportFunction will be called
// with WebGLTestHarnessModule.TestHarness.reportType.FINISHED_ALL_TESTS.
//
// Harness Options
//
// These are passed in to the TestHarness as a JavaScript object
//
// version: (required!)
//
//     Specifies a version used to filter tests. Tests marked as requiring
//     a version greater than this version will not be included.
//
//     example: new TestHarness(...., {version: "3.1.2"});
//
// minVersion:
//
//     Specifies the minimum version a test must require to be included.
//     This basically flips the filter so that only tests marked with
//     --min-version will be included if they are at this minVersion or
//     greater.
//
//     example: new TestHarness(...., {minVersion: "2.3.1"});
//
// maxVersion:
//
//     Specifies the maximum version a test must require to be included.
//     This basically flips the filter so that only tests marked with
//     --max-version will be included if they are at this maxVersion or
//     less.
//
//     example: new TestHarness(...., {maxVersion: "2.3.1"});
//
// fast:
//
//     Specifies to skip any tests marked as slow.
//
//     example: new TestHarness(..., {fast: true});
//
// Test Options:
//
// Any test URL or .txt file can be prefixed by the following options
//
// min-version:
//
//     Sets the minimum version required to include this test. A version is
//     passed into the harness options. Any test marked as requiring a
//     min-version greater than the version passed to the harness is skipped.
//     This allows you to add new tests to a suite of tests for a future
//     version of the suite without including the test in the current version.
//     If no -min-version is specified it is inheriited from the .txt file
//     including it. The default is 1.0.0
//
//     example:  --min-version 2.1.3 sometest.html
//
// max-version:
//
//     Sets the maximum version required to include this test. A version is
//     passed into the harness options. Any test marked as requiring a
//     max-version less than the version passed to the harness is skipped.
//     This allows you to test functionality that has been removed from later
//     versions of the suite.
//     If no -max-version is specified it is inherited from the .txt file
//     including it.
//
//     example:  --max-version 1.9.9 sometest.html
//
// slow:
//
//     Marks a test as slow. Slow tests can be skipped by passing fastOnly: true
//     to the TestHarness. Of course you need to pass all tests but sometimes
//     you'd like to test quickly and run only the fast subset of tests.
//
//     example:  --slow some-test-that-takes-2-mins.html
//

WebGLTestHarnessModule = function() {

/**
 * Wrapped logging function.
 */
var log = function(msg) {
  if (window.console && window.console.log) {
    window.console.log(msg);
  }
};

/**
 * Loads text from an external file. This function is synchronous.
 * @param {string} url The url of the external file.
 * @param {!function(bool, string): void} callback that is sent a bool for
 *     success and the string.
 */
var loadTextFileAsynchronous = function(url, callback) {
  log ("loading: " + url);
  var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
  var request;
  if (window.XMLHttpRequest) {
    request = new XMLHttpRequest();
    if (request.overrideMimeType) {
      request.overrideMimeType('text/plain');
    }
  } else {
    throw 'XMLHttpRequest is disabled';
  }
  try {
    request.open('GET', url, true);
    request.onreadystatechange = function() {
      if (request.readyState == 4) {
        var text = '';
        // HTTP reports success with a 200 status. The file protocol reports
        // success with zero. HTTP does not use zero as a status code (they
        // start at 100).
        // https://developer.mozilla.org/En/Using_XMLHttpRequest
        var success = request.status == 200 || request.status == 0;
        if (success) {
          text = request.responseText;
        }
        log("loaded: " + url);
        callback(success, text);
      }
    };
    request.send(null);
  } catch (e) {
    log("failed to load: " + url);
    callback(false, '');
  }
};

/**
 * @param {string} versionString WebGL version string.
 * @return {number} Integer containing the WebGL major version.
 */
var getMajorVersion = function(versionString) {
  if (!versionString) {
    return 1;
  }
  return parseInt(versionString.split(" ")[0].split(".")[0], 10);
};

/**
 * @param {string} url Base URL of the test.
 * @param {map} options Map of options to append to the URL's query string.
 * @return {string} URL that will run the test with the given WebGL version.
 */
var getURLWithOptions = function(url, options) {
  var queryArgs = 0;

  for (i in options) {
    url += queryArgs ? "&" : "?";
    url += i + "=" + options[i];
    queryArgs++;
  }

  return url;
};

/**
 * Compare version strings.
 */
var greaterThanOrEqualToVersion = function(have, want) {
  have = have.split(" ")[0].split(".");
  want = want.split(" ")[0].split(".");

  //have 1.2.3   want  1.1
  //have 1.1.1   want  1.1
  //have 1.0.9   want  1.1
  //have 1.1     want  1.1.1

  for (var ii = 0; ii < want.length; ++ii) {
    var wantNum = parseInt(want[ii]);
    var haveNum = have[ii] ? parseInt(have[ii]) : 0
    if (haveNum > wantNum) {
      return true; // 2.0.0 is greater than 1.2.3
    }
    if (haveNum < wantNum) {
      return false;
    }
  }
  return true;
};

/**
 * Reads a file, recursively adding files referenced inside.
 *
 * Each line of URL is parsed, comments starting with '#' or ';'
 * or '//' are stripped.
 *
 * arguments beginning with -- are extracted
 *
 * lines that end in .txt are recursively scanned for more files
 * other lines are added to the list of files.
 *
 * @param {string} url The url of the file to read.
 * @param {function(boolean, !Array.<string>):void} callback
 *      Callback that is called with true for success and an
 *      array of filenames.
 * @param {Object} options Optional options
 *
 * Options:
 *    version: {string} The version of the conformance test.
 *    Tests with the argument --min-version <version> will
 *    be ignored version is less then <version>
 *
 */
var getFileList = function(url, callback, options) {
  var files = [];

  var copyObject = function(obj) {
    return JSON.parse(JSON.stringify(obj));
  };

  var toCamelCase = function(str) {
    return str.replace(/-([a-z])/g, function (g) { return g[1].toUpperCase() });
  };

  var globalOptions = copyObject(options);
  globalOptions.defaultVersion = "1.0";
  globalOptions.defaultMaxVersion = null;

  var getFileListImpl = function(prefix, line, lineNum, hierarchicalOptions, callback) {
    var files = [];

    var args = line.split(/\s+/);
    var nonOptions = [];
    var useTest = true;
    var testOptions = {};
    for (var jj = 0; jj < args.length; ++jj) {
      var arg = args[jj];
      if (arg[0] == '-') {
        if (arg[1] != '-') {
          throw ("bad option at in " + url + ":" + lineNum + ": " + arg);
        }
        var option = arg.substring(2);
        switch (option) {
          // no argument options.
          case 'slow':
            testOptions[toCamelCase(option)] = true;
            break;
          // one argument options.
          case 'min-version':
          case 'max-version':
            ++jj;
            testOptions[toCamelCase(option)] = args[jj];
            break;
          default:
            throw ("bad unknown option '" + option + "' at in " + url + ":" + lineNum + ": " + arg);
        }
      } else {
        nonOptions.push(arg);
      }
    }
    var url = prefix + nonOptions.join(" ");

    if (url.substr(url.length - 4) != '.txt') {
      var minVersion = testOptions.minVersion;
      if (!minVersion) {
        minVersion = hierarchicalOptions.defaultVersion;
      }
      var maxVersion = testOptions.maxVersion;
      if (!maxVersion) {
        maxVersion = hierarchicalOptions.defaultMaxVersion;
      }
      var slow = testOptions.slow;
      if (!slow) {
        slow = hierarchicalOptions.defaultSlow;
      }

      if (globalOptions.fast && slow) {
        useTest = false;
      } else if (globalOptions.minVersion) {
        useTest = greaterThanOrEqualToVersion(minVersion, globalOptions.minVersion);
      } else if (globalOptions.maxVersion && maxVersion) {
        useTest = greaterThanOrEqualToVersion(globalOptions.maxVersion, maxVersion);
      } else {
        useTest = greaterThanOrEqualToVersion(globalOptions.version, minVersion);
        if (maxVersion) {
          useTest = useTest && greaterThanOrEqualToVersion(maxVersion, globalOptions.version);
        }
      }
    }

    if (!useTest) {
      callback(true, []);
      return;
    }

    if (url.substr(url.length - 4) == '.txt') {
      // If a version was explicity specified pass it down.
      if (testOptions.minVersion) {
        hierarchicalOptions.defaultVersion = testOptions.minVersion;
      }
      if (testOptions.maxVersion) {
        hierarchicalOptions.defaultMaxVersion = testOptions.maxVersion;
      }
      if (testOptions.slow) {
        hierarchicalOptions.defaultSlow = testOptions.slow;
      }
      loadTextFileAsynchronous(url, function() {
        return function(success, text) {
          if (!success) {
            callback(false, '');
            return;
          }
          var lines = text.split('\n');
          var prefix = '';
          var lastSlash = url.lastIndexOf('/');
          if (lastSlash >= 0) {
            prefix = url.substr(0, lastSlash + 1);
          }
          var fail = false;
          var count = 1;
          var index = 0;
          for (var ii = 0; ii < lines.length; ++ii) {
            var str = lines[ii].replace(/^\s\s*/, '').replace(/\s\s*$/, '');
            if (str.length > 4 &&
                str[0] != '#' &&
                str[0] != ";" &&
                str.substr(0, 2) != "//") {
              ++count;
              getFileListImpl(prefix, str, ii + 1, copyObject(hierarchicalOptions), function(index) {
                return function(success, new_files) {
                  //log("got files: " + new_files.length);
                  if (success) {
                    files[index] = new_files;
                  }
                  finish(success);
                };
              }(index++));
            }
          }
          finish(true);

          function finish(success) {
            if (!success) {
              fail = true;
            }
            --count;
            //log("count: " + count);
            if (!count) {
              callback(!fail, files);
            }
          }
        }
      }());
    } else {
      files.push(url);
      callback(true, files);
    }
  };

  getFileListImpl('', url, 1, globalOptions, function(success, files) {
    // flatten
    var flat = [];
    flatten(files);
    function flatten(files) {
      for (var ii = 0; ii < files.length; ++ii) {
        var value = files[ii];
        if (typeof(value) == "string") {
          flat.push(value);
        } else {
          flatten(value);
        }
      }
    }
    callback(success, flat);
  });
};

var FilterURL = (function() {
  var prefix = window.location.pathname;
  prefix = prefix.substring(0, prefix.lastIndexOf("/") + 1);
  return function(url) {
    if (url.substring(0, prefix.length) == prefix) {
      url = url.substring(prefix.length);
    }
    return url;
  };
}());

var TestFile = function(url) {
  this.url = url;
};

var Test = function(file) {
  this.file = file;
};

var TestHarness = function(iframe, filelistUrl, reportFunc, options) {
  this.window = window;
  this.iframes = iframe.length ? iframe : [iframe];
  this.reportFunc = reportFunc;
  this.timeoutDelay = 20000;
  this.files = [];
  this.allowSkip = options.allowSkip;
  this.webglVersion = getMajorVersion(options.version);
  this.dumpShaders = options.dumpShaders;
  this.quiet = options.quiet;

  var that = this;
  getFileList(filelistUrl, function() {
    return function(success, files) {
      that.addFiles_(success, files);
    };
  }(), options);

};

TestHarness.reportType = {
  ADD_PAGE: 1,
  READY: 2,
  START_PAGE: 3,
  TEST_RESULT: 4,
  FINISH_PAGE: 5,
  FINISHED_ALL_TESTS: 6
};

TestHarness.prototype.addFiles_ = function(success, files) {
  if (!success) {
    this.reportFunc(
        TestHarness.reportType.FINISHED_ALL_TESTS,
        '',
        'Unable to load tests. Are you running locally?\n' +
        'You need to run from a server or configure your\n' +
        'browser to allow access to local files (not recommended).\n\n' +
        'Note: An easy way to run from a server:\n\n' +
        '\tcd path_to_tests\n' +
        '\tpython -m SimpleHTTPServer\n\n' +
        'then point your browser to ' +
          '<a href="http://localhost:8000/webgl-conformance-tests.html">' +
          'http://localhost:8000/webgl-conformance-tests.html</a>',
        false)
    return;
  }
  log("total files: " + files.length);
  for (var ii = 0; ii < files.length; ++ii) {
    log("" + ii + ": " + files[ii]);
    this.files.push(new TestFile(files[ii]));
    this.reportFunc(TestHarness.reportType.ADD_PAGE, '', files[ii], undefined);
  }
  this.reportFunc(TestHarness.reportType.READY, '', undefined, undefined);
}

TestHarness.prototype.runTests = function(opt_options) {
  var options = opt_options || { };
  options.start = options.start || 0;
  options.count = options.count || this.files.length;

  this.idleIFrames = this.iframes.slice(0);
  this.runningTests = {};
  var testsToRun = [];
  for (var ii = 0; ii < options.count; ++ii) {
    testsToRun.push(ii + options.start);
  }
  this.numTestsRemaining = options.count;
  this.testsToRun = testsToRun;
  this.startNextTest();
};

TestHarness.prototype._bumpTimeout = function(test) {
  const newTimeoutAt = performance.now() + this.timeoutDelay;
  if (test.timeoutAt) {
    test.timeoutAt = newTimeoutAt;
    return;
  }
  test.timeoutAt = newTimeoutAt;

  const harness = this;

  function enqueueWatchdog() {
    const remaining = test.timeoutAt - performance.now();
    //console.log(`watchdog started at ${performance.now()}, ${test.timeoutAt} requested`);
    this.window.setTimeout(() => {
      if (!test.timeoutAt) return; // Timeout was cleared.
      const remainingAtCheckTime = test.timeoutAt - performance.now();
      if (performance.now() >= test.timeoutAt) {
        //console.log(`watchdog won at ${performance.now()}, ${test.timeoutAt} requested`);
        harness.timeout(test);
        return;
      }
      //console.log(`watchdog lost at ${performance.now()}, as ${test.timeoutAt} is now requested`);
      enqueueWatchdog();
    }, remaining);
  }
  enqueueWatchdog();
};

TestHarness.prototype.clearTimeout = function(test) {
  test.timeoutAt = null;
};

TestHarness.prototype.startNextTest = function() {
  if (this.numTestsRemaining == 0) {
    log("done");
    this.reportFunc(TestHarness.reportType.FINISHED_ALL_TESTS,
                    '', '', true);
  } else {
    while (this.testsToRun.length > 0 && this.idleIFrames.length > 0) {
      var testId = this.testsToRun.shift();
      var iframe = this.idleIFrames.shift();
      this.startTest(iframe, this.files[testId], this.webglVersion);
    }
  }
};

TestHarness.prototype.startTest = function(iframe, testFile, webglVersion) {
  var test = {
    iframe: iframe,
    testFile: testFile
  };
  var url = testFile.url;
  this.runningTests[url] = test;
  log("loading: " + url);
  if (this.reportFunc(TestHarness.reportType.START_PAGE, url, url, undefined)) {
    iframe.src = getURLWithOptions(url, {
      "webglVersion": webglVersion,
      "dumpShaders": this.dumpShaders,
      "quiet": this.quiet
    });
    this._bumpTimeout(test);
  } else {
    this.reportResults(url, !!this.allowSkip, "skipped", true);
    this.notifyFinished(url);
  }
};

TestHarness.prototype.getTest = function(url) {
  var test = this.runningTests[FilterURL(url)];
  if (!test) {
    throw("unknown test:" + url);
  }
  return test;
};

TestHarness.prototype.reportResults = function(url, success, msg, skipped) {
  url = FilterURL(url);
  var test = this.getTest(url);
  if (0) {
    // This is too slow to leave on for tests like
    // deqp/functional/gles3/vertexarrays/multiple_attributes.output.html
    // which has 33013505 calls to reportResults.
    log((success ? "PASS" : "FAIL") + ": " + msg);
  }
  this.reportFunc(TestHarness.reportType.TEST_RESULT, url, msg, success, skipped);
  // For each result we get, reset the timeout
  this._bumpTimeout(test);
};

TestHarness.prototype.dequeTest = function(test) {
  this.clearTimeout(test);
  this.idleIFrames.push(test.iframe);
  delete this.runningTests[test.testFile.url];
  --this.numTestsRemaining;
}

TestHarness.prototype.notifyFinished = function(url) {
  url = FilterURL(url);
  var test = this.getTest(url);
  log(url + ": finished");
  this.dequeTest(test);
  this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, url, true);
  this.startNextTest();
};

TestHarness.prototype.timeout = function(test) {
  this.dequeTest(test);
  var url = test.testFile.url;
  log(url + ": timeout");
  this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, url, undefined);
  this.startNextTest();
};

TestHarness.prototype.setTimeoutDelay = function(x) {
  this.timeoutDelay = x;
};

return {
    'TestHarness': TestHarness,
    'getMajorVersion': getMajorVersion,
    'getURLWithOptions': getURLWithOptions
  };

}();