Source code
Revision control
Copy as Markdown
Other Tools
Test Info:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# flake8: noqa: E501
try:
import cPickle as pickle
except ImportError:
import pickle
import json
import os
import re
from collections import defaultdict
import manifestupdate
import mozpack.path as mozpath
import mozunit
import pytest
from mozbuild.frontend.reader import BuildReader
from mozbuild.test.common import MockConfig
from moztest.resolve import (
TEST_SUITES,
BuildBackendLoader,
TestManifestLoader,
TestResolver,
)
here = os.path.abspath(os.path.dirname(__file__))
data_path = os.path.join(here, "data")
@pytest.fixture(scope="module")
def topsrcdir():
return mozpath.join(data_path, "srcdir")
@pytest.fixture(scope="module")
def create_tests(topsrcdir):
def inner(*paths, **defaults):
tests = defaultdict(list)
for path in paths:
if isinstance(path, tuple):
path, kwargs = path
else:
kwargs = {}
path = mozpath.normpath(path)
manifest_name = kwargs.get("flavor", defaults.get("flavor", "manifest"))
manifest = kwargs.pop(
"manifest",
defaults.pop(
"manifest",
mozpath.join(mozpath.dirname(path), manifest_name + ".toml"),
),
)
manifest_abspath = mozpath.join(topsrcdir, manifest)
relpath = mozpath.relpath(path, mozpath.dirname(manifest))
test = {
"name": relpath,
"path": mozpath.join(topsrcdir, path),
"relpath": relpath,
"file_relpath": path,
"flavor": "faketest",
"dir_relpath": mozpath.dirname(path),
"here": mozpath.dirname(manifest_abspath),
"manifest": manifest_abspath,
"manifest_relpath": manifest,
}
test.update(**defaults)
test.update(**kwargs)
# Normalize paths to ensure that the fixture matches reality.
for k in [
"ancestor_manifest",
"manifest",
"manifest_relpath",
"path",
"relpath",
]:
p = test.get(k)
if p:
test[k] = p.replace("/", os.path.sep)
tests[path].append(test)
# dump tests to stdout for easier debugging on failure
print("The 'create_tests' fixture returned:")
print(json.dumps(dict(tests), indent=2, sort_keys=True))
return tests
return inner
@pytest.fixture(scope="module")
def all_tests(create_tests):
return create_tests(
*[
(
"apple/test_a11y.html",
{
"expected": "pass",
"manifest": "apple/a11y.toml",
"flavor": "a11y",
},
),
(
"banana/currant/test_xpcshell_A.js",
{
"firefox-appdir": "browser",
"flavor": "xpcshell",
"head": "head_global.js head_helpers.js head_http.js",
},
),
(
"banana/currant/test_xpcshell_B.js",
{
"firefox-appdir": "browser",
"flavor": "xpcshell",
"head": "head_global.js head_helpers.js head_http.js",
},
),
(
"carrot/test_included.js",
{
"ancestor_manifest": "carrot/xpcshell-one.toml",
"manifest": "carrot/xpcshell-shared.toml",
"flavor": "xpcshell",
"stick": "one",
},
),
(
"carrot/test_included.js",
{
"ancestor_manifest": "carrot/xpcshell-two.toml",
"manifest": "carrot/xpcshell-shared.toml",
"flavor": "xpcshell",
"stick": "two",
},
),
(
"dragonfruit/elderberry/test_xpcshell_C.js",
{
"flavor": "xpcshell",
"generated-files": "head_update.js",
"head": "head_update.js",
"manifest": "dragonfruit/xpcshell.toml",
"reason": "busted",
"run-sequentially": "Launches application.",
"skip-if": "os == 'android'",
},
),
(
"dragonfruit/elderberry/test_xpcshell_C.js",
{
"flavor": "xpcshell",
"generated-files": "head_update.js",
"head": "head_update.js head2.js",
"manifest": "dragonfruit/elderberry/xpcshell_updater.toml",
"reason": "don't work",
"run-sequentially": "Launches application.",
"skip-if": "os == 'android'",
},
),
(
"fig/grape/src/TestInstrumentationA.java",
{
"flavor": "instrumentation",
"manifest": "fig/grape/instrumentation.toml",
"subsuite": "background",
},
),
(
"fig/huckleberry/src/TestInstrumentationB.java",
{
"flavor": "instrumentation",
"manifest": "fig/huckleberry/instrumentation.toml",
"subsuite": "browser",
},
),
(
"juniper/browser_chrome.js",
{
"flavor": "browser-chrome",
"manifest": "juniper/browser.toml",
"skip-if": "e10s # broken",
},
),
(
"kiwi/browser_devtools.js",
{
"flavor": "browser-chrome",
"manifest": "kiwi/browser.toml",
"subsuite": "devtools",
"tags": "devtools",
},
),
]
)
@pytest.fixture(scope="module")
def defaults(topsrcdir):
def to_abspath(relpath):
# test-defaults.pkl uses absolute paths with platform-specific path separators.
return os.path.normpath(os.path.join(topsrcdir, relpath))
return {
(to_abspath("dragonfruit/elderberry/xpcshell_updater.toml")): {
"support-files": "data/**\nxpcshell_updater.toml"
},
(
to_abspath("carrot/xpcshell-one.toml"),
to_abspath("carrot/xpcshell-shared.toml"),
): {
"head": "head_one.js",
},
(
to_abspath("carrot/xpcshell-two.toml"),
to_abspath("carrot/xpcshell-shared.toml"),
): {
"head": "head_two.js",
},
}
class WPTManifestNamespace(object):
"""Stand-in object for various WPT classes."""
def __init__(self, *args):
self.args = args
def __hash__(self):
return hash(str(self))
def __eq__(self, other):
return self.args == other.args
def __iter__(self):
yield tuple(self.args)
def fake_wpt_manifestupdate(topsrcdir, *args, **kwargs):
with open(os.path.join(topsrcdir, "wpt_manifest_data.json")) as fh:
data = json.load(fh)
items = {}
for tests_root, test_data in data.items():
kwargs = {"tests_path": os.path.join(topsrcdir, tests_root)}
for test_type, tests in test_data.items():
for test in tests:
obj = WPTManifestNamespace()
if "mozilla" in tests_root:
obj.id = "/_mozilla/" + test
else:
obj.id = "/" + test
items[WPTManifestNamespace(test_type, test, {obj})] = kwargs
return items
@pytest.fixture(params=[BuildBackendLoader, TestManifestLoader])
def resolver(request, tmpdir, monkeypatch, topsrcdir, all_tests, defaults):
topobjdir = tmpdir.mkdir("objdir").strpath
loader_cls = request.param
if loader_cls == BuildBackendLoader:
with open(os.path.join(topobjdir, "all-tests.pkl"), "wb") as fh:
pickle.dump(all_tests, fh)
with open(os.path.join(topobjdir, "test-defaults.pkl"), "wb") as fh:
pickle.dump(defaults, fh)
# The mock data already exists, so prevent BuildBackendLoader from regenerating
# the build information from the whole gecko tree...
class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader):
def backend_out_of_date(self, backend_file):
return False
loader_cls = BuildBackendLoaderNeverOutOfDate
# Patch WPT's manifestupdate.run to return tests based on the contents of
# 'data/srcdir/wpt_manifest_data.json'.
monkeypatch.setattr(manifestupdate, "run", fake_wpt_manifestupdate)
resolver = TestResolver(
topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls
)
resolver._puppeteer_loaded = True
if loader_cls == TestManifestLoader:
config = MockConfig(topsrcdir)
resolver.load_tests.reader = BuildReader(config)
return resolver
def test_load(resolver):
assert len(resolver.tests_by_path) == 9
assert len(resolver.tests_by_flavor["mochitest-plain"]) == 0
assert len(resolver.tests_by_flavor["xpcshell"]) == 4
assert len(resolver.tests_by_flavor["web-platform-tests"]) == 0
assert len(resolver.tests_by_manifest) == 9
resolver.add_wpt_manifest_data()
assert len(resolver.tests_by_path) == 11
assert len(resolver.tests_by_flavor["web-platform-tests"]) == 2
assert len(resolver.tests_by_manifest) == 11
assert "/html" in resolver.tests_by_manifest
assert "/_mozilla/html" in resolver.tests_by_manifest
def test_resolve_all(resolver):
assert len(list(resolver._resolve())) == 13
def test_resolve_filter_flavor(resolver):
assert len(list(resolver._resolve(flavor="xpcshell"))) == 6
def test_resolve_by_dir(resolver):
assert len(list(resolver._resolve(paths=["banana/currant"]))) == 2
def test_resolve_under_path(resolver):
assert len(list(resolver._resolve(under_path="banana"))) == 2
assert len(list(resolver._resolve(flavor="xpcshell", under_path="banana"))) == 2
def test_resolve_multiple_paths(resolver):
result = list(resolver.resolve_tests(paths=["banana", "dragonfruit"]))
assert len(result) == 4
def test_resolve_support_files(resolver):
expected_support_files = "data/**\nxpcshell_updater.toml"
tests = list(resolver.resolve_tests(paths=["dragonfruit"]))
assert len(tests) == 2
for test in tests:
if test["manifest"].endswith("xpcshell_updater.toml"):
assert test["support-files"] == expected_support_files
else:
assert "support-files" not in test
def test_resolve_path_prefix(resolver):
tests = list(resolver._resolve(paths=["juniper"]))
assert len(tests) == 1
# relative manifest
tests = list(resolver._resolve(paths=["apple/a11y.toml"]))
assert len(tests) == 1
assert tests[0]["name"] == "test_a11y.html"
# absolute manifest
tests = list(
resolver._resolve(paths=[os.path.join(resolver.topsrcdir, "apple/a11y.toml")])
)
assert len(tests) == 1
assert tests[0]["name"] == "test_a11y.html"
def test_cwd_children_only(resolver):
"""If cwd is defined, only resolve tests under the specified cwd."""
# Pretend we're under '/services' and ask for 'common'. This should
# pick up all tests from '/services/common'
tests = list(
resolver.resolve_tests(
paths=["currant"], cwd=os.path.join(resolver.topsrcdir, "banana")
)
)
assert len(tests) == 2
# Tests should be rewritten to objdir.
for t in tests:
assert t["here"] == mozpath.join(
resolver.topobjdir, "_tests/xpcshell/banana/currant"
)
def test_various_cwd(resolver):
"""Test various cwd conditions are all equal."""
expected = list(resolver.resolve_tests(paths=["banana"]))
actual = list(resolver.resolve_tests(paths=["banana"], cwd="/"))
assert actual == expected
actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topsrcdir))
assert actual == expected
actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topobjdir))
assert actual == expected
def test_subsuites(resolver):
"""Test filtering by subsuite."""
tests = list(resolver.resolve_tests(paths=["fig"]))
assert len(tests) == 2
tests = list(resolver.resolve_tests(paths=["fig"], subsuite="browser"))
assert len(tests) == 1
assert tests[0]["name"] == "src/TestInstrumentationB.java"
tests = list(resolver.resolve_tests(paths=["fig"], subsuite="background"))
assert len(tests) == 1
assert tests[0]["name"] == "src/TestInstrumentationA.java"
# Resolve tests *without* a subsuite.
tests = list(resolver.resolve_tests(flavor="browser-chrome", subsuite="undefined"))
assert len(tests) == 1
assert tests[0]["name"] == "browser_chrome.js"
def test_wildcard_patterns(resolver):
"""Test matching paths by wildcard."""
tests = list(resolver.resolve_tests(paths=["fig/**"]))
assert len(tests) == 2
for t in tests:
assert t["file_relpath"].startswith("fig")
tests = list(resolver.resolve_tests(paths=["**/**.js", "apple/**"]))
assert len(tests) == 9
for t in tests:
path = t["file_relpath"]
assert path.startswith("apple") or path.endswith(".js")
def test_resolve_metadata(resolver):
"""Test finding metadata from outgoing files."""
suites, tests = resolver.resolve_metadata(["bc"])
assert suites == {"mochitest-browser-chrome"}
assert tests == []
suites, tests = resolver.resolve_metadata(
["mochitest-a11y", "/browser", "xpcshell"]
)
assert suites == {"mochitest-a11y", "xpcshell"}
assert sorted(t["file_relpath"] for t in tests) == [
"juniper/browser_chrome.js",
"kiwi/browser_devtools.js",
]
def test_ancestor_manifest_defaults(resolver, topsrcdir, defaults):
"""Test that defaults from ancestor manifests are found."""
tests = list(resolver._resolve(paths=["carrot/test_included.js"]))
assert len(tests) == 2
if tests[0]["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.toml"):
[testOne, testTwo] = tests
else:
[testTwo, testOne] = tests
assert testOne["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.toml")
assert testOne["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.toml")
assert testOne["head"] == "head_one.js"
assert testOne["stick"] == "one"
assert testTwo["ancestor_manifest"] == os.path.join("carrot", "xpcshell-two.toml")
assert testTwo["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.toml")
assert testTwo["head"] == "head_two.js"
assert testTwo["stick"] == "two"
def test_task_regexes():
"""Test the task_regexes defined in TEST_SUITES."""
task_labels = [
"test-linux64/opt-marionette",
"test-linux64/opt-mochitest-plain",
"test-linux64/debug-mochitest-plain-e10s",
"test-linux64/opt-mochitest-a11y",
"test-linux64/opt-mochitest-browser",
"test-linux64/opt-mochitest-browser-chrome",
"test-linux64/opt-mochitest-browser-chrome-e10s",
"test-linux64/opt-mochitest-browser-chrome-e10s-11",
"test-linux64/opt-mochitest-chrome",
"test-linux64/opt-mochitest-devtools",
"test-linux64/opt-mochitest-devtools-chrome",
"test-linux64/opt-mochitest-gpu",
"test-linux64/opt-mochitest-gpu-e10s",
"test-linux64/opt-mochitest-media-e10s-1",
"test-linux64/opt-mochitest-media-e10s-11",
"test-linux64/opt-mochitest-browser-screenshots-1",
"test-linux64/opt-mochitest-browser-screenshots-e10s-1",
"test-linux64/opt-reftest",
"test-linux64/opt-geckoview-reftest",
"test-linux64/debug-reftest-e10s-1",
"test-linux64/debug-reftest-e10s-11",
"test-linux64/opt-robocop",
"test-linux64/opt-robocop-1",
"test-linux64/opt-robocop-e10s",
"test-linux64/opt-robocop-e10s-1",
"test-linux64/opt-robocop-e10s-11",
"test-linux64/opt-web-platform-tests-e10s-1",
"test-linux64/opt-web-platform-tests-reftest-e10s-1",
"test-linux64/opt-web-platform-tests-wdspec-e10s-1",
"test-linux64/opt-web-platform-tests-1",
"test-linux64/opt-web-platform-test-e10s-1",
"test-linux64/opt-xpcshell",
"test-linux64/opt-xpcshell-1",
"test-linux64/opt-xpcshell-2",
]
test_cases = {
"mochitest-browser-chrome": [
"test-linux64/opt-mochitest-browser-chrome",
"test-linux64/opt-mochitest-browser-chrome-e10s",
],
"mochitest-chrome": [
"test-linux64/opt-mochitest-chrome",
],
"mochitest-devtools-chrome": [
"test-linux64/opt-mochitest-devtools-chrome",
],
"mochitest-media": [
"test-linux64/opt-mochitest-media-e10s-1",
],
"mochitest-plain": [
"test-linux64/opt-mochitest-plain",
"test-linux64/debug-mochitest-plain-e10s",
],
"mochitest-plain-gpu": [
"test-linux64/opt-mochitest-gpu",
"test-linux64/opt-mochitest-gpu-e10s",
],
"mochitest-browser-screenshots": [
"test-linux64/opt-mochitest-browser-screenshots-1",
"test-linux64/opt-mochitest-browser-screenshots-e10s-1",
],
"reftest": [
"test-linux64/opt-reftest",
"test-linux64/opt-geckoview-reftest",
"test-linux64/debug-reftest-e10s-1",
],
"robocop": [
"test-linux64/opt-robocop",
"test-linux64/opt-robocop-1",
"test-linux64/opt-robocop-e10s",
"test-linux64/opt-robocop-e10s-1",
],
"web-platform-tests": [
"test-linux64/opt-web-platform-tests-e10s-1",
"test-linux64/opt-web-platform-tests-1",
],
"web-platform-tests-reftest": [
"test-linux64/opt-web-platform-tests-reftest-e10s-1",
],
"web-platform-tests-wdspec": [
"test-linux64/opt-web-platform-tests-wdspec-e10s-1",
],
"xpcshell": [
"test-linux64/opt-xpcshell",
"test-linux64/opt-xpcshell-1",
],
}
regexes = []
def match_task(task):
return any(re.search(pattern, task) for pattern in regexes)
for suite, expected in sorted(test_cases.items()):
print(suite)
regexes = TEST_SUITES[suite]["task_regex"]
assert set(filter(match_task, task_labels)) == set(expected)
if __name__ == "__main__":
mozunit.main()