Source code
Revision control
Copy as Markdown
Other Tools
/*
Copyright (c) 2019 The Khronos Group Inc.
Use of this source code is governed by an MIT-style license that can be
found in the LICENSE.txt file.
*/
// This is a test harness for running javascript tests in the browser.
// The only identifier exposed by this harness is WebGLTestHarnessModule.
//
// To use it make an HTML page with an iframe. Then call the harness like this
//
// function reportResults(type, msg, success) {
// ...
// return true;
// }
//
// var fileListURL = '00_test_list.txt';
// var testHarness = new WebGLTestHarnessModule.TestHarness(
// iframe,
// fileListURL,
// reportResults,
// options);
//
// The harness will load the fileListURL and parse it for the URLs, one URL
// per line preceded by options, see below. URLs should be on the same domain
// and at the same folder level or below the main html file. If any URL ends
// in .txt it will be parsed as well so you can nest .txt files. URLs inside a
// .txt file should be relative to that text file.
//
// During startup, for each page found the reportFunction will be called with
// WebGLTestHarnessModule.TestHarness.reportType.ADD_PAGE and msg will be
// the URL of the test.
//
// Each test is required to call testHarness.reportResults. This is most easily
// accomplished by storing that value on the main window with
//
// window.webglTestHarness = testHarness
//
// and then adding these to functions to your tests.
//
// function reportTestResultsToHarness(success, msg) {
// if (window.parent.webglTestHarness) {
// window.parent.webglTestHarness.reportResults(success, msg);
// }
// }
//
// function notifyFinishedToHarness() {
// if (window.parent.webglTestHarness) {
// window.parent.webglTestHarness.notifyFinished();
// }
// }
//
// This way your tests will still run without the harness and you can use
// any testing framework you want.
//
// Each test should call reportTestResultsToHarness with true for success if it
// succeeded and false if it fail followed and any message it wants to
// associate with the test. If your testing framework supports checking for
// timeout you can call it with success equal to undefined in that case.
//
// To run the tests, call testHarness.runTests(options);
//
// For each test run, before the page is loaded the reportFunction will be
// called with WebGLTestHarnessModule.TestHarness.reportType.START_PAGE and msg
// will be the URL of the test. You may return false if you want the test to be
// skipped.
//
// For each test completed the reportFunction will be called with
// with WebGLTestHarnessModule.TestHarness.reportType.TEST_RESULT,
// success = true on success, false on failure, undefined on timeout
// and msg is any message the test choose to pass on.
//
// When all the tests on the page have finished your page must call
// notifyFinishedToHarness. If notifyFinishedToHarness is not called
// the harness will assume the test timed out.
//
// When all the tests on a page have finished OR the page as timed out the
// reportFunction will be called with
// WebGLTestHarnessModule.TestHarness.reportType.FINISH_PAGE
// where success = true if the page has completed or undefined if the page timed
// out.
//
// Finally, when all the tests have completed the reportFunction will be called
// with WebGLTestHarnessModule.TestHarness.reportType.FINISHED_ALL_TESTS.
//
// Harness Options
//
// These are passed in to the TestHarness as a JavaScript object
//
// version: (required!)
//
// Specifies a version used to filter tests. Tests marked as requiring
// a version greater than this version will not be included.
//
// example: new TestHarness(...., {version: "3.1.2"});
//
// minVersion:
//
// Specifies the minimum version a test must require to be included.
// This basically flips the filter so that only tests marked with
// --min-version will be included if they are at this minVersion or
// greater.
//
// example: new TestHarness(...., {minVersion: "2.3.1"});
//
// maxVersion:
//
// Specifies the maximum version a test must require to be included.
// This basically flips the filter so that only tests marked with
// --max-version will be included if they are at this maxVersion or
// less.
//
// example: new TestHarness(...., {maxVersion: "2.3.1"});
//
// fast:
//
// Specifies to skip any tests marked as slow.
//
// example: new TestHarness(..., {fast: true});
//
// Test Options:
//
// Any test URL or .txt file can be prefixed by the following options
//
// min-version:
//
// Sets the minimum version required to include this test. A version is
// passed into the harness options. Any test marked as requiring a
// min-version greater than the version passed to the harness is skipped.
// This allows you to add new tests to a suite of tests for a future
// version of the suite without including the test in the current version.
// If no -min-version is specified it is inheriited from the .txt file
// including it. The default is 1.0.0
//
// example: --min-version 2.1.3 sometest.html
//
// max-version:
//
// Sets the maximum version required to include this test. A version is
// passed into the harness options. Any test marked as requiring a
// max-version less than the version passed to the harness is skipped.
// This allows you to test functionality that has been removed from later
// versions of the suite.
// If no -max-version is specified it is inherited from the .txt file
// including it.
//
// example: --max-version 1.9.9 sometest.html
//
// slow:
//
// Marks a test as slow. Slow tests can be skipped by passing fastOnly: true
// to the TestHarness. Of course you need to pass all tests but sometimes
// you'd like to test quickly and run only the fast subset of tests.
//
// example: --slow some-test-that-takes-2-mins.html
//
WebGLTestHarnessModule = function() {
/**
* Wrapped logging function.
*/
var log = function(msg) {
if (window.console && window.console.log) {
window.console.log(msg);
}
};
/**
* Loads text from an external file. This function is synchronous.
* @param {string} url The url of the external file.
* @param {!function(bool, string): void} callback that is sent a bool for
* success and the string.
*/
var loadTextFileAsynchronous = function(url, callback) {
log ("loading: " + url);
var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
var request;
if (window.XMLHttpRequest) {
request = new XMLHttpRequest();
if (request.overrideMimeType) {
request.overrideMimeType('text/plain');
}
} else {
throw 'XMLHttpRequest is disabled';
}
try {
request.open('GET', url, true);
request.onreadystatechange = function() {
if (request.readyState == 4) {
var text = '';
// HTTP reports success with a 200 status. The file protocol reports
// success with zero. HTTP does not use zero as a status code (they
// start at 100).
var success = request.status == 200 || request.status == 0;
if (success) {
text = request.responseText;
}
log("loaded: " + url);
callback(success, text);
}
};
request.send(null);
} catch (e) {
log("failed to load: " + url);
callback(false, '');
}
};
/**
* @param {string} versionString WebGL version string.
* @return {number} Integer containing the WebGL major version.
*/
var getMajorVersion = function(versionString) {
if (!versionString) {
return 1;
}
return parseInt(versionString.split(" ")[0].split(".")[0], 10);
};
/**
* @param {string} url Base URL of the test.
* @param {map} options Map of options to append to the URL's query string.
* @return {string} URL that will run the test with the given WebGL version.
*/
var getURLWithOptions = function(url, options) {
var queryArgs = 0;
for (i in options) {
url += queryArgs ? "&" : "?";
url += i + "=" + options[i];
queryArgs++;
}
return url;
};
/**
* Compare version strings.
*/
var greaterThanOrEqualToVersion = function(have, want) {
have = have.split(" ")[0].split(".");
want = want.split(" ")[0].split(".");
//have 1.2.3 want 1.1
//have 1.1.1 want 1.1
//have 1.0.9 want 1.1
//have 1.1 want 1.1.1
for (var ii = 0; ii < want.length; ++ii) {
var wantNum = parseInt(want[ii]);
var haveNum = have[ii] ? parseInt(have[ii]) : 0
if (haveNum > wantNum) {
return true; // 2.0.0 is greater than 1.2.3
}
if (haveNum < wantNum) {
return false;
}
}
return true;
};
/**
* Reads a file, recursively adding files referenced inside.
*
* Each line of URL is parsed, comments starting with '#' or ';'
* or '//' are stripped.
*
* arguments beginning with -- are extracted
*
* lines that end in .txt are recursively scanned for more files
* other lines are added to the list of files.
*
* @param {string} url The url of the file to read.
* @param {function(boolean, !Array.<string>):void} callback
* Callback that is called with true for success and an
* array of filenames.
* @param {Object} options Optional options
*
* Options:
* version: {string} The version of the conformance test.
* Tests with the argument --min-version <version> will
* be ignored version is less then <version>
*
*/
var getFileList = function(url, callback, options) {
var files = [];
var copyObject = function(obj) {
return JSON.parse(JSON.stringify(obj));
};
var toCamelCase = function(str) {
return str.replace(/-([a-z])/g, function (g) { return g[1].toUpperCase() });
};
var globalOptions = copyObject(options);
globalOptions.defaultVersion = "1.0";
globalOptions.defaultMaxVersion = null;
var getFileListImpl = function(prefix, line, lineNum, hierarchicalOptions, callback) {
var files = [];
var args = line.split(/\s+/);
var nonOptions = [];
var useTest = true;
var testOptions = {};
for (var jj = 0; jj < args.length; ++jj) {
var arg = args[jj];
if (arg[0] == '-') {
if (arg[1] != '-') {
throw ("bad option at in " + url + ":" + lineNum + ": " + arg);
}
var option = arg.substring(2);
switch (option) {
// no argument options.
case 'slow':
testOptions[toCamelCase(option)] = true;
break;
// one argument options.
case 'min-version':
case 'max-version':
++jj;
testOptions[toCamelCase(option)] = args[jj];
break;
default:
throw ("bad unknown option '" + option + "' at in " + url + ":" + lineNum + ": " + arg);
}
} else {
nonOptions.push(arg);
}
}
var url = prefix + nonOptions.join(" ");
if (url.substr(url.length - 4) != '.txt') {
var minVersion = testOptions.minVersion;
if (!minVersion) {
minVersion = hierarchicalOptions.defaultVersion;
}
var maxVersion = testOptions.maxVersion;
if (!maxVersion) {
maxVersion = hierarchicalOptions.defaultMaxVersion;
}
var slow = testOptions.slow;
if (!slow) {
slow = hierarchicalOptions.defaultSlow;
}
if (globalOptions.fast && slow) {
useTest = false;
} else if (globalOptions.minVersion) {
useTest = greaterThanOrEqualToVersion(minVersion, globalOptions.minVersion);
} else if (globalOptions.maxVersion && maxVersion) {
useTest = greaterThanOrEqualToVersion(globalOptions.maxVersion, maxVersion);
} else {
useTest = greaterThanOrEqualToVersion(globalOptions.version, minVersion);
if (maxVersion) {
useTest = useTest && greaterThanOrEqualToVersion(maxVersion, globalOptions.version);
}
}
}
if (!useTest) {
callback(true, []);
return;
}
if (url.substr(url.length - 4) == '.txt') {
// If a version was explicity specified pass it down.
if (testOptions.minVersion) {
hierarchicalOptions.defaultVersion = testOptions.minVersion;
}
if (testOptions.maxVersion) {
hierarchicalOptions.defaultMaxVersion = testOptions.maxVersion;
}
if (testOptions.slow) {
hierarchicalOptions.defaultSlow = testOptions.slow;
}
loadTextFileAsynchronous(url, function() {
return function(success, text) {
if (!success) {
callback(false, '');
return;
}
var lines = text.split('\n');
var prefix = '';
var lastSlash = url.lastIndexOf('/');
if (lastSlash >= 0) {
prefix = url.substr(0, lastSlash + 1);
}
var fail = false;
var count = 1;
var index = 0;
for (var ii = 0; ii < lines.length; ++ii) {
var str = lines[ii].replace(/^\s\s*/, '').replace(/\s\s*$/, '');
if (str.length > 4 &&
str[0] != '#' &&
str[0] != ";" &&
str.substr(0, 2) != "//") {
++count;
getFileListImpl(prefix, str, ii + 1, copyObject(hierarchicalOptions), function(index) {
return function(success, new_files) {
//log("got files: " + new_files.length);
if (success) {
files[index] = new_files;
}
finish(success);
};
}(index++));
}
}
finish(true);
function finish(success) {
if (!success) {
fail = true;
}
--count;
//log("count: " + count);
if (!count) {
callback(!fail, files);
}
}
}
}());
} else {
files.push(url);
callback(true, files);
}
};
getFileListImpl('', url, 1, globalOptions, function(success, files) {
// flatten
var flat = [];
flatten(files);
function flatten(files) {
for (var ii = 0; ii < files.length; ++ii) {
var value = files[ii];
if (typeof(value) == "string") {
flat.push(value);
} else {
flatten(value);
}
}
}
callback(success, flat);
});
};
var FilterURL = (function() {
var prefix = window.location.pathname;
prefix = prefix.substring(0, prefix.lastIndexOf("/") + 1);
return function(url) {
if (url.substring(0, prefix.length) == prefix) {
url = url.substring(prefix.length);
}
return url;
};
}());
var TestFile = function(url) {
this.url = url;
};
var Test = function(file) {
this.file = file;
};
var TestHarness = function(iframe, filelistUrl, reportFunc, options) {
this.window = window;
this.iframes = iframe.length ? iframe : [iframe];
this.reportFunc = reportFunc;
this.timeoutDelay = 20000;
this.files = [];
this.allowSkip = options.allowSkip;
this.webglVersion = getMajorVersion(options.version);
this.dumpShaders = options.dumpShaders;
this.quiet = options.quiet;
var that = this;
getFileList(filelistUrl, function() {
return function(success, files) {
that.addFiles_(success, files);
};
}(), options);
};
TestHarness.reportType = {
ADD_PAGE: 1,
READY: 2,
START_PAGE: 3,
TEST_RESULT: 4,
FINISH_PAGE: 5,
FINISHED_ALL_TESTS: 6
};
TestHarness.prototype.addFiles_ = function(success, files) {
if (!success) {
this.reportFunc(
TestHarness.reportType.FINISHED_ALL_TESTS,
'',
'Unable to load tests. Are you running locally?\n' +
'You need to run from a server or configure your\n' +
'browser to allow access to local files (not recommended).\n\n' +
'Note: An easy way to run from a server:\n\n' +
'\tcd path_to_tests\n' +
'\tpython -m SimpleHTTPServer\n\n' +
'then point your browser to ' +
false)
return;
}
log("total files: " + files.length);
for (var ii = 0; ii < files.length; ++ii) {
log("" + ii + ": " + files[ii]);
this.files.push(new TestFile(files[ii]));
this.reportFunc(TestHarness.reportType.ADD_PAGE, '', files[ii], undefined);
}
this.reportFunc(TestHarness.reportType.READY, '', undefined, undefined);
}
TestHarness.prototype.runTests = function(opt_options) {
var options = opt_options || { };
options.start = options.start || 0;
options.count = options.count || this.files.length;
this.idleIFrames = this.iframes.slice(0);
this.runningTests = {};
var testsToRun = [];
for (var ii = 0; ii < options.count; ++ii) {
testsToRun.push(ii + options.start);
}
this.numTestsRemaining = options.count;
this.testsToRun = testsToRun;
this.startNextTest();
};
TestHarness.prototype._bumpTimeout = function(test) {
const newTimeoutAt = performance.now() + this.timeoutDelay;
if (test.timeoutAt) {
test.timeoutAt = newTimeoutAt;
return;
}
test.timeoutAt = newTimeoutAt;
const harness = this;
function enqueueWatchdog() {
const remaining = test.timeoutAt - performance.now();
//console.log(`watchdog started at ${performance.now()}, ${test.timeoutAt} requested`);
this.window.setTimeout(() => {
if (!test.timeoutAt) return; // Timeout was cleared.
const remainingAtCheckTime = test.timeoutAt - performance.now();
if (performance.now() >= test.timeoutAt) {
//console.log(`watchdog won at ${performance.now()}, ${test.timeoutAt} requested`);
harness.timeout(test);
return;
}
//console.log(`watchdog lost at ${performance.now()}, as ${test.timeoutAt} is now requested`);
enqueueWatchdog();
}, remaining);
}
enqueueWatchdog();
};
TestHarness.prototype.clearTimeout = function(test) {
test.timeoutAt = null;
};
TestHarness.prototype.startNextTest = function() {
if (this.numTestsRemaining == 0) {
log("done");
this.reportFunc(TestHarness.reportType.FINISHED_ALL_TESTS,
'', '', true);
} else {
while (this.testsToRun.length > 0 && this.idleIFrames.length > 0) {
var testId = this.testsToRun.shift();
var iframe = this.idleIFrames.shift();
this.startTest(iframe, this.files[testId], this.webglVersion);
}
}
};
TestHarness.prototype.startTest = function(iframe, testFile, webglVersion) {
var test = {
iframe: iframe,
testFile: testFile
};
var url = testFile.url;
this.runningTests[url] = test;
log("loading: " + url);
if (this.reportFunc(TestHarness.reportType.START_PAGE, url, url, undefined)) {
iframe.src = getURLWithOptions(url, {
"webglVersion": webglVersion,
"dumpShaders": this.dumpShaders,
"quiet": this.quiet
});
this._bumpTimeout(test);
} else {
this.reportResults(url, !!this.allowSkip, "skipped", true);
this.notifyFinished(url);
}
};
TestHarness.prototype.getTest = function(url) {
var test = this.runningTests[FilterURL(url)];
if (!test) {
throw("unknown test:" + url);
}
return test;
};
TestHarness.prototype.reportResults = function(url, success, msg, skipped) {
url = FilterURL(url);
var test = this.getTest(url);
if (0) {
// This is too slow to leave on for tests like
// deqp/functional/gles3/vertexarrays/multiple_attributes.output.html
// which has 33013505 calls to reportResults.
log((success ? "PASS" : "FAIL") + ": " + msg);
}
this.reportFunc(TestHarness.reportType.TEST_RESULT, url, msg, success, skipped);
// For each result we get, reset the timeout
this._bumpTimeout(test);
};
TestHarness.prototype.dequeTest = function(test) {
this.clearTimeout(test);
this.idleIFrames.push(test.iframe);
delete this.runningTests[test.testFile.url];
--this.numTestsRemaining;
}
TestHarness.prototype.notifyFinished = function(url) {
url = FilterURL(url);
var test = this.getTest(url);
log(url + ": finished");
this.dequeTest(test);
this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, url, true);
this.startNextTest();
};
TestHarness.prototype.timeout = function(test) {
this.dequeTest(test);
var url = test.testFile.url;
log(url + ": timeout");
this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, url, undefined);
this.startNextTest();
};
TestHarness.prototype.setTimeoutDelay = function(x) {
this.timeoutDelay = x;
};
return {
'TestHarness': TestHarness,
'getMajorVersion': getMajorVersion,
'getURLWithOptions': getURLWithOptions
};
}();