Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# jit_test.py -- Python harness for JavaScript trace tests.
import os
import re
import sys
import traceback
from collections import namedtuple
from enum import Enum
if sys.platform.startswith("linux") or sys.platform.startswith("darwin"):
from .tasks_unix import run_all_tests
else:
from .tasks_win import run_all_tests
from .progressbar import NullProgressBar, ProgressBar
from .results import escape_cmdline
from .structuredlog import TestLogger
from .tempfile import TemporaryDirectory
TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__))
JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR))
TOP_SRC_DIR = os.path.dirname(os.path.dirname(JS_DIR))
TEST_DIR = os.path.join(JS_DIR, "jit-test", "tests")
LIB_DIR = os.path.join(JS_DIR, "jit-test", "lib") + os.path.sep
MODULE_DIR = os.path.join(JS_DIR, "jit-test", "modules") + os.path.sep
SHELL_XDR = "shell.xdr"
class OutputStatus(Enum):
OK = 1
SKIPPED = 2
FAILED = 3
def __bool__(self):
return self != OutputStatus.FAILED
# Backported from Python 3.1 posixpath.py
def _relpath(path, start=None):
"""Return a relative version of a path"""
if not path:
raise ValueError("no path specified")
if start is None:
start = os.curdir
start_list = os.path.abspath(start).split(os.sep)
path_list = os.path.abspath(path).split(os.sep)
# Work out how much of the filepath is shared by start and path.
i = len(os.path.commonprefix([start_list, path_list]))
rel_list = [os.pardir] * (len(start_list) - i) + path_list[i:]
if not rel_list:
return os.curdir
return os.path.join(*rel_list)
# Mapping of Python chars to their javascript string representation.
QUOTE_MAP = {
"\\": "\\\\",
"\b": "\\b",
"\f": "\\f",
"\n": "\\n",
"\r": "\\r",
"\t": "\\t",
"\v": "\\v",
}
# Quote the string S, javascript style.
def js_quote(quote, s):
result = quote
for c in s:
if c == quote:
result += "\\" + quote
elif c in QUOTE_MAP:
result += QUOTE_MAP[c]
else:
result += c
result += quote
return result
os.path.relpath = _relpath
def extend_condition(condition, value):
if condition:
condition += " || "
condition += "({})".format(value)
return condition
class JitTest:
VALGRIND_CMD = []
paths = (d for d in os.environ["PATH"].split(os.pathsep))
valgrinds = (os.path.join(d, "valgrind") for d in paths)
if any(os.path.exists(p) for p in valgrinds):
VALGRIND_CMD = [
"valgrind",
"-q",
"--smc-check=all-non-file",
"--error-exitcode=1",
"--gen-suppressions=all",
"--show-possibly-lost=no",
"--leak-check=full",
]
if os.uname()[0] == "Darwin":
VALGRIND_CMD.append("--dsymutil=yes")
del paths
del valgrinds
def __init__(self, path):
# Absolute path of the test file.
self.path = path
# Path relative to the top mozilla/ directory.
self.relpath_top = os.path.relpath(path, TOP_SRC_DIR)
# Path relative to mozilla/js/src/jit-test/tests/.
self.relpath_tests = os.path.relpath(path, TEST_DIR)
# jit flags to enable
self.jitflags = []
# True means the test is slow-running
self.slow = False
# Heavy tests will never run alongside other heavy tests
self.heavy = False
# True means that OOM is not considered a failure
self.allow_oom = False
# True means CrashAtUnhandlableOOM is not considered a failure
self.allow_unhandlable_oom = False
# True means that hitting recursion the limits is not considered a failure.
self.allow_overrecursed = False
# True means run under valgrind
self.valgrind = False
# True means force Pacific time for the test
self.tz_pacific = False
# Additional files to include, in addition to prologue.js
self.other_lib_includes = []
self.other_script_includes = []
# List of other configurations to test with.
self.test_also = []
# List of other configurations to test with all existing variants.
self.test_join = []
# Errors to expect and consider passing
self.expect_error = ""
# Exit status to expect from shell
self.expect_status = 0
# Exit status or error output.
self.expect_crash = False
self.is_module = False
# Reflect.stringify implementation to test
self.test_reflect_stringify = None
# Use self-hosted XDR instead of parsing the source stored in the binary.
self.selfhosted_xdr_path = None
self.selfhosted_xdr_mode = "off"
# Skip-if condition. We don't have a xulrunner, but we can ask the shell
# directly.
self.skip_if_cond = ""
self.skip_variant_if_cond = {}
# Expected by the test runner. Always true for jit-tests.
self.enable = True
def copy(self):
t = JitTest(self.path)
t.jitflags = self.jitflags[:]
t.slow = self.slow
t.heavy = self.heavy
t.allow_oom = self.allow_oom
t.allow_unhandlable_oom = self.allow_unhandlable_oom
t.allow_overrecursed = self.allow_overrecursed
t.valgrind = self.valgrind
t.tz_pacific = self.tz_pacific
t.other_lib_includes = self.other_lib_includes[:]
t.other_script_includes = self.other_script_includes[:]
t.test_also = self.test_also
t.test_join = self.test_join
t.expect_error = self.expect_error
t.expect_status = self.expect_status
t.expect_crash = self.expect_crash
t.test_reflect_stringify = self.test_reflect_stringify
t.selfhosted_xdr_path = self.selfhosted_xdr_path
t.selfhosted_xdr_mode = self.selfhosted_xdr_mode
t.enable = True
t.is_module = self.is_module
t.skip_if_cond = self.skip_if_cond
t.skip_variant_if_cond = self.skip_variant_if_cond
return t
def copy_and_extend_jitflags(self, variant):
t = self.copy()
t.jitflags.extend(variant)
for flags in variant:
if flags in self.skip_variant_if_cond:
t.skip_if_cond = extend_condition(
t.skip_if_cond, self.skip_variant_if_cond[flags]
)
return t
def copy_variants(self, variants):
# Append variants to be tested in addition to the current set of tests.
variants = variants + self.test_also
# For each existing variant, duplicates it for each list of options in
# test_join. This will multiply the number of variants by 2 for set of
# options.
for join_opts in self.test_join:
variants = variants + [opts + join_opts for opts in variants]
# For each list of jit flags, make a copy of the test.
return [self.copy_and_extend_jitflags(v) for v in variants]
COOKIE = b"|jit-test|"
# We would use 500019 (5k19), but quit() only accepts values up to 127, due to fuzzers
SKIPPED_EXIT_STATUS = 59
Directives = {}
@classmethod
def find_directives(cls, file_name):
meta = ""
line = open(file_name, "rb").readline()
i = line.find(cls.COOKIE)
if i != -1:
meta = ";" + line[i + len(cls.COOKIE) :].decode(errors="strict").strip("\n")
return meta
@classmethod
def from_file(cls, path, options):
test = cls(path)
# If directives.txt exists in the test's directory then it may
# contain metainformation that will be catenated with
# whatever's in the test file. The form of the directive in
# the directive file is the same as in the test file. Only
# the first line is considered, just as for the test file.
dir_meta = ""
dir_name = os.path.dirname(path)
if dir_name in cls.Directives:
dir_meta = cls.Directives[dir_name]
else:
meta_file_name = os.path.join(dir_name, "directives.txt")
if os.path.exists(meta_file_name):
dir_meta = cls.find_directives(meta_file_name)
cls.Directives[dir_name] = dir_meta
filename, file_extension = os.path.splitext(path)
meta = cls.find_directives(path)
if meta != "" or dir_meta != "":
meta = meta + dir_meta
parts = meta.split(";")
for part in parts:
part = part.strip()
if not part:
continue
name, _, value = part.partition(":")
if value:
value = value.strip()
if name == "error":
test.expect_error = value
elif name == "exitstatus":
try:
status = int(value, 0)
if status == test.SKIPPED_EXIT_STATUS:
print(
"warning: jit-tests uses {} as a sentinel"
" return value {}",
test.SKIPPED_EXIT_STATUS,
path,
)
else:
test.expect_status = status
except ValueError:
print(
"warning: couldn't parse exit status"
" {}".format(value)
)
elif name == "thread-count":
try:
test.jitflags.append(
"--thread-count={}".format(int(value, 0))
)
except ValueError:
print(
"warning: couldn't parse thread-count"
" {}".format(value)
)
elif name == "include":
test.other_lib_includes.append(value)
elif name == "local-include":
test.other_script_includes.append(value)
elif name == "skip-if":
test.skip_if_cond = extend_condition(test.skip_if_cond, value)
elif name == "skip-variant-if":
try:
[variant, condition] = value.split(",")
test.skip_variant_if_cond[variant] = extend_condition(
test.skip_if_cond, condition
)
except ValueError:
print("warning: couldn't parse skip-variant-if")
else:
print(
"{}: warning: unrecognized |jit-test| attribute"
" {}".format(path, part)
)
else:
if name == "slow":
test.slow = True
elif name == "heavy":
test.heavy = True
elif name == "allow-oom":
test.allow_oom = True
elif name == "allow-unhandlable-oom":
test.allow_unhandlable_oom = True
elif name == "allow-overrecursed":
test.allow_overrecursed = True
elif name == "valgrind":
test.valgrind = options.valgrind
elif name == "tz-pacific":
test.tz_pacific = True
elif name.startswith("test-also="):
test.test_also.append(
re.split(r"\s+", name[len("test-also=") :])
)
elif name.startswith("test-join="):
test.test_join.append(
re.split(r"\s+", name[len("test-join=") :])
)
elif name == "module":
test.is_module = True
elif name == "crash":
# Crashes are only allowed in self-test, as it is
# intended to verify that our testing infrastructure
# works, and not meant as a way to accept temporary
# failing tests. These tests should either be fixed or
# skipped.
assert (
"self-test" in path
), "{}: has an unexpected crash annotation.".format(path)
test.expect_crash = True
elif name.startswith("--"):
# // |jit-test| --ion-gvn=off; --no-sse4
test.jitflags.append(name)
elif name.startswith("-P"):
prefAndValue = name.split()
assert (
len(prefAndValue) == 2
), f"{name}: failed to parse preference"
# // |jit-test| -P pref(=value)?
test.jitflags.append("--setpref=" + prefAndValue[1])
else:
print(
"{}: warning: unrecognized |jit-test| attribute"
" {}".format(path, part)
)
if options.valgrind_all:
test.valgrind = True
if options.test_reflect_stringify is not None:
test.expect_error = ""
test.expect_status = 0
return test
def command(self, prefix, libdir, moduledir, tempdir, remote_prefix=None):
path = self.path
if remote_prefix:
path = self.path.replace(TEST_DIR, remote_prefix)
scriptdir_var = os.path.dirname(path)
if not scriptdir_var.endswith("/"):
scriptdir_var += "/"
# Note: The tempdir provided as argument is managed by the caller
# should remain alive as long as the test harness. Therefore, the XDR
# content of the self-hosted code would be accessible to all JS Shell
# instances.
self.selfhosted_xdr_path = os.path.join(tempdir, SHELL_XDR)
# Platforms where subprocess immediately invokes exec do not care
# whether we use double or single quotes. On windows and when using
# a remote device, however, we have to be careful to use the quote
# style that is the opposite of what the exec wrapper uses.
if remote_prefix:
quotechar = '"'
else:
quotechar = "'"
# Don't merge the expressions: We want separate -e arguments to avoid
exprs = [
"const platform={}".format(js_quote(quotechar, sys.platform)),
"const libdir={}".format(js_quote(quotechar, libdir)),
"const scriptdir={}".format(js_quote(quotechar, scriptdir_var)),
]
# We may have specified '-a' or '-d' twice: once via --jitflags, once
# via the "|jit-test|" line. Remove dups because they are toggles.
# Note: |dict.fromkeys(flags)| is similar to |set(flags)| but it
# preserves order.
cmd = prefix + []
cmd += list(dict.fromkeys(self.jitflags))
# Handle selfhosted XDR file.
if self.selfhosted_xdr_mode != "off":
cmd += [
"--selfhosted-xdr-path",
self.selfhosted_xdr_path,
"--selfhosted-xdr-mode",
self.selfhosted_xdr_mode,
]
for expr in exprs:
cmd += ["-e", expr]
for inc in self.other_lib_includes:
cmd += ["-f", libdir + inc]
for inc in self.other_script_includes:
cmd += ["-f", scriptdir_var + inc]
if self.skip_if_cond:
cmd += [
"-e",
"if ({}) quit({})".format(self.skip_if_cond, self.SKIPPED_EXIT_STATUS),
]
cmd += ["--module-load-path", moduledir]
if self.is_module:
cmd += ["--module", path]
elif self.test_reflect_stringify is None:
cmd += ["-f", path]
else:
cmd += ["--", self.test_reflect_stringify, "--check", path]
if self.valgrind:
cmd = self.VALGRIND_CMD + cmd
if self.allow_unhandlable_oom or self.expect_crash:
cmd += ["--suppress-minidump"]
return cmd
def get_command(self, prefix, tempdir):
"""Shim for the test runner."""
return self.command(prefix, LIB_DIR, MODULE_DIR, tempdir)
def find_tests(substring=None):
ans = []
for dirpath, dirnames, filenames in os.walk(TEST_DIR):
dirnames.sort()
filenames.sort()
if dirpath == ".":
continue
for filename in filenames:
if not filename.endswith(".js"):
continue
if filename in ("shell.js", "browser.js"):
continue
test = os.path.join(dirpath, filename)
if substring is None or substring in os.path.relpath(test, TEST_DIR):
ans.append(test)
return ans
def check_output(out, err, rc, timed_out, test, options):
# Allow skipping to compose with other expected results
if test.skip_if_cond:
if rc == test.SKIPPED_EXIT_STATUS:
return OutputStatus.SKIPPED
if timed_out:
relpath = os.path.normpath(test.relpath_tests).replace(os.sep, "/")
if relpath in options.ignore_timeouts:
return OutputStatus.OK
return OutputStatus.FAILED
if test.expect_error:
# The shell exits with code 3 on uncaught exceptions.
if rc != 3:
return OutputStatus.FAILED
return test.expect_error in err
for line in out.split("\n"):
if line.startswith("Trace stats check failed"):
return OutputStatus.FAILED
for line in err.split("\n"):
if "Assertion failed:" in line:
return OutputStatus.FAILED
if test.expect_crash:
# Python 3 on Windows interprets process exit codes as unsigned
# integers, where Python 2 used to allow signed integers. Account for
# each possibility here.
if sys.platform == "win32" and rc in (3 - 2**31, 3 + 2**31):
return OutputStatus.OK
if sys.platform != "win32" and rc == -11:
return OutputStatus.OK
# When building with ASan enabled, ASan will convert the -11 returned
# value to 1. As a work-around we look for the error output which
# includes the crash reason.
if rc == 1 and ("Hit MOZ_CRASH" in err or "Assertion failure:" in err):
return OutputStatus.OK
# When running jittests on Android, SEGV results in a return code of
# 128 + 11 = 139. Due to a bug in tinybox, we have to check for 138 as
# well.
if rc == 139 or rc == 138:
return OutputStatus.OK
# Crashing test should always crash as expected, otherwise this is an
# error. The JS shell crash() function can be used to force the test
# case to crash in unexpected configurations.
return OutputStatus.FAILED
if rc != test.expect_status:
# Allow a non-zero exit code if we want to allow OOM, but only if we
# actually got OOM.
if (
test.allow_oom
and "out of memory" in err
and "Assertion failure" not in err
and "MOZ_CRASH" not in err
):
return OutputStatus.OK
# Allow a non-zero exit code if we want to allow unhandlable OOM, but
# only if we actually got unhandlable OOM.
if test.allow_unhandlable_oom and "MOZ_CRASH([unhandlable oom]" in err:
return OutputStatus.OK
# Allow a non-zero exit code if we want to all too-much-recursion and
# the test actually over-recursed.
if (
test.allow_overrecursed
and "too much recursion" in err
and "Assertion failure" not in err
):
return OutputStatus.OK
# Allow a zero exit code if we are running under a sanitizer that
# forces the exit status.
if test.expect_status != 0 and options.unusable_error_status:
return OutputStatus.OK
return OutputStatus.FAILED
return OutputStatus.OK
def print_automation_format(ok, res, slog):
# Output test failures in a parsable format suitable for automation, eg:
# TEST-RESULT | filename.js | Failure description (code N, args "--foobar")
#
# Example:
# TEST-PASS | foo/bar/baz.js | (code 0, args "--ion-eager")
# TEST-UNEXPECTED-FAIL | foo/bar/baz.js | TypeError: or something (code -9, args "--no-ion")
# INFO exit-status : 3
# INFO timed-out : False
# INFO stdout > foo
# INFO stdout > bar
# INFO stdout > baz
# INFO stderr 2> TypeError: or something
# TEST-UNEXPECTED-FAIL | jit_test.py: Test execution interrupted by user
result = "TEST-PASS" if ok else "TEST-UNEXPECTED-FAIL"
message = "Success" if ok else res.describe_failure()
jitflags = " ".join(res.test.jitflags)
print(
'{} | {} | {} (code {}, args "{}") [{:.1f} s]'.format(
result, res.test.relpath_top, message, res.rc, jitflags, res.dt
)
)
details = {
"message": message,
"extra": {
"jitflags": jitflags,
},
}
if res.extra:
details["extra"].update(res.extra)
slog.test(res.test.relpath_tests, "PASS" if ok else "FAIL", res.dt, **details)
# For failed tests, print as much information as we have, to aid debugging.
if ok:
return
print("INFO exit-status : {}".format(res.rc))
print("INFO timed-out : {}".format(res.timed_out))
warnings = []
for line in res.out.splitlines():
if line.startswith("WARNING") and "unused DT entry" in line:
warnings.append(line)
continue
print("INFO stdout > " + line.strip())
for line in res.err.splitlines():
if line.startswith("WARNING") and "unused DT entry" in line:
warnings.append(line)
continue
print("INFO stderr 2> " + line.strip())
for line in warnings:
print("INFO (warn-stderr) 2> " + line.strip())
def print_test_summary(num_tests, failures, complete, slow_tests, doing, options):
def test_details(res):
if options.show_failed:
return escape_cmdline(res.cmd)
return " ".join(res.test.jitflags + [res.test.relpath_tests])
if failures:
if options.write_failures:
try:
out = open(options.write_failures, "w")
# Don't write duplicate entries when we are doing multiple
# failures per job.
written = set()
for res in failures:
if res.test.path not in written:
out.write(os.path.relpath(res.test.path, TEST_DIR) + "\n")
if options.write_failure_output:
out.write(res.out)
out.write(res.err)
out.write("Exit code: " + str(res.rc) + "\n")
written.add(res.test.path)
out.close()
except IOError:
sys.stderr.write(
"Exception thrown trying to write failure"
" file '{}'\n".format(options.write_failures)
)
traceback.print_exc()
sys.stderr.write("---\n")
print("FAILURES:")
for res in failures:
if not res.timed_out:
print(" " + test_details(res))
print("TIMEOUTS:")
for res in failures:
if res.timed_out:
print(" " + test_details(res))
else:
print(
"PASSED ALL"
+ (
""
if complete
else " (partial run -- interrupted by user {})".format(doing)
)
)
if options.format == "automation":
num_failures = len(failures) if failures else 0
print("Result summary:")
print("Passed: {:d}".format(num_tests - num_failures))
print("Failed: {:d}".format(num_failures))
if num_tests != 0 and options.show_slow:
threshold = options.slow_test_threshold
fraction_fast = 1 - len(slow_tests) / num_tests
print(
"{:5.2f}% of tests ran in under {}s".format(fraction_fast * 100, threshold)
)
print("Slowest tests that took longer than {}s:".format(threshold))
slow_tests.sort(key=lambda res: res.dt, reverse=True)
any = False
for i in range(min(len(slow_tests), 20)):
res = slow_tests[i]
print(" {:6.2f} {}".format(res.dt, test_details(res)))
any = True
if not any:
print("None")
return not failures
def create_progressbar(num_tests, options):
if (
not options.hide_progress
and not options.show_cmd
and ProgressBar.conservative_isatty()
):
fmt = [
{"value": "PASS", "color": "green"},
{"value": "FAIL", "color": "red"},
{"value": "TIMEOUT", "color": "blue"},
{"value": "SKIP", "color": "brightgray"},
]
return ProgressBar(num_tests, fmt)
return NullProgressBar()
def process_test_results(results, num_tests, pb, options, slog):
failures = []
timeouts = 0
skipped = 0
complete = False
output_dict = {}
doing = "before starting"
slow_tests = []
if num_tests == 0:
pb.finish(True)
complete = True
return print_test_summary(
num_tests, failures, complete, slow_tests, doing, options
)
try:
for i, res in enumerate(results):
status = check_output(
res.out, res.err, res.rc, res.timed_out, res.test, options
)
if status:
show_output = options.show_output and not options.failed_only
else:
show_output = options.show_output or not options.no_show_failed
if show_output:
pb.beginline()
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write("Exit code: {}\n".format(res.rc))
if res.test.valgrind and not show_output:
pb.beginline()
sys.stdout.write(res.err)
if options.check_output:
if res.test.path in output_dict.keys():
if output_dict[res.test.path] != res.out:
pb.message(
"FAIL - OUTPUT DIFFERS {}".format(res.test.relpath_tests)
)
else:
output_dict[res.test.path] = res.out
doing = "after {}".format(res.test.relpath_tests)
if status == OutputStatus.SKIPPED:
skipped += 1
elif status == OutputStatus.FAILED:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - {}".format(res.test.relpath_tests))
timeouts += 1
else:
pb.message("FAIL - {}".format(res.test.relpath_tests))
if options.format == "automation":
print_automation_format(status, res, slog)
n = i + 1
pb.update(
n,
{
"PASS": n - len(failures),
"FAIL": len(failures),
"TIMEOUT": timeouts,
"SKIP": skipped,
},
)
if res.dt > options.slow_test_threshold:
slow_tests.append(res)
complete = True
except KeyboardInterrupt:
print(
"TEST-UNEXPECTED-FAIL | jit_test.py"
+ " : Test execution interrupted by user"
)
pb.finish(True)
return print_test_summary(num_tests, failures, complete, slow_tests, doing, options)
def run_tests(tests, num_tests, prefix, options, remote=False):
slog = None
if options.format == "automation":
slog = TestLogger("jittests")
slog.suite_start()
if remote:
ok = run_tests_remote(tests, num_tests, prefix, options, slog)
else:
ok = run_tests_local(tests, num_tests, prefix, options, slog)
if slog:
slog.suite_end()
return ok
def run_tests_local(tests, num_tests, prefix, options, slog):
# The jstests tasks runner requires the following options. The names are
# taken from the jstests options processing code, which are frequently
# subtly different from the options jit-tests expects. As such, we wrap
# them here, as needed.
AdaptorOptions = namedtuple(
"AdaptorOptions",
[
"worker_count",
"passthrough",
"timeout",
"output_fp",
"hide_progress",
"run_skipped",
"show_cmd",
"use_xdr",
],
)
shim_options = AdaptorOptions(
options.max_jobs,
False,
options.timeout,
sys.stdout,
False,
True,
options.show_cmd,
options.use_xdr,
)
with TemporaryDirectory() as tempdir:
pb = create_progressbar(num_tests, options)
gen = run_all_tests(tests, prefix, tempdir, pb, shim_options)
ok = process_test_results(gen, num_tests, pb, options, slog)
return ok
def run_tests_remote(tests, num_tests, prefix, options, slog):
# Setup device with everything needed to run our tests.
from mozdevice import ADBError, ADBTimeoutError
from .tasks_adb_remote import get_remote_results
# Run all tests.
pb = create_progressbar(num_tests, options)
try:
gen = get_remote_results(tests, prefix, pb, options)
ok = process_test_results(gen, num_tests, pb, options, slog)
except (ADBError, ADBTimeoutError):
print("TEST-UNEXPECTED-FAIL | jit_test.py" + " : Device error during test")
raise
return ok
if __name__ == "__main__":
print("Use ../jit-test/jit_test.py to run these tests.")