Source code
Revision control
Copy as Markdown
Other Tools
import os
import sys
import threading
import traceback
from unittest import mock
from unittest.mock import patch
import mozunit
import pytest
from mozprofile import BaseProfile
# need this so the raptor unit tests can find output & filter classes
here = os.path.abspath(os.path.dirname(__file__))
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
sys.path.insert(0, raptor_dir)
from browsertime import BrowsertimeAndroid, BrowsertimeDesktop
DEFAULT_TIMEOUT = 125
class TestBrowserThread(threading.Thread):
def __init__(self, raptor_instance, tests, names):
super(TestBrowserThread, self).__init__()
self.raptor_instance = raptor_instance
self.tests = tests
self.names = names
self.exc = None
def print_error(self):
if self.exc is None:
return
type, value, tb = self.exc
traceback.print_exception(type, value, tb, None, sys.stderr)
def run(self):
try:
self.raptor_instance.run_tests(self.tests, self.names)
except BaseException:
self.exc = sys.exc_info()
# Perftest tests
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
@pytest.mark.parametrize(
"perftest_class, app_name",
[
[BrowsertimeDesktop, "firefox"],
[BrowsertimeAndroid, "geckoview"],
],
)
def test_build_profile(
mock_info, mock_critical, options, perftest_class, app_name, get_prefs
):
options["app"] = app_name
# We need to do the mock ourselves because of how the perftest_class
# is being defined
original_get = perftest_class.get_browser_meta
perftest_class.get_browser_meta = mock.MagicMock()
perftest_class.get_browser_meta.return_value = (app_name, "100")
perftest_instance = perftest_class(**options)
perftest_class.get_browser_meta = original_get
assert isinstance(perftest_instance.profile, BaseProfile)
if app_name != "firefox":
return
# These prefs are set in mozprofile
firefox_prefs = [
'user_pref("app.update.checkInstallTime", false);',
'user_pref("app.update.disabledForTesting", true);',
'user_pref("'
'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);',
]
# This pref is set in raptor
raptor_pref = 'user_pref("security.enable_java", false);'
prefs_file = os.path.join(perftest_instance.profile.profile, "user.js")
with open(prefs_file, "r") as fh:
prefs = fh.read()
for firefox_pref in firefox_prefs:
assert firefox_pref in prefs
assert raptor_pref in prefs
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
def test_perftest_host_ip(
mock_info, mock_critical, ConcretePerftest, options, get_prefs
):
os.environ["HOST_IP"] = "some_dummy_host_ip"
options["host"] = "HOST_IP"
perftest = ConcretePerftest(**options)
assert perftest.config["host"] == os.environ["HOST_IP"]
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
@pytest.mark.parametrize(
"app_name, expected_e10s_flag",
[["firefox", True], ["geckoview", True]],
)
def test_e10s_enabling(
mock_info, mock_critical, ConcretePerftest, options, app_name, expected_e10s_flag
):
options["app"] = app_name
perftest = ConcretePerftest(profile_class="firefox", **options)
assert perftest.config["e10s"] == expected_e10s_flag
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
def test_profile_was_provided_locally(
mock_info, mock_critical, ConcretePerftest, options
):
perftest = ConcretePerftest(**options)
assert os.path.isdir(perftest.config["local_profile_dir"])
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
@pytest.mark.parametrize(
"profile_class, app, expected_profile",
[
["firefox", "firefox", "firefox"],
[None, "firefox", "firefox"],
["firefox", None, "firefox"],
],
)
def test_profile_class_assignation(
mock_info,
mock_critical,
ConcretePerftest,
options,
profile_class,
app,
expected_profile,
):
options["app"] = app
perftest = ConcretePerftest(profile_class=profile_class, **options)
assert perftest.profile_class == expected_profile
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
def test_raptor_venv(mock_info, mock_critical, ConcretePerftest, options):
perftest = ConcretePerftest(**options)
assert perftest.raptor_venv.endswith("raptor-venv")
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
@mock.patch("perftest.Perftest.build_browser_profile", new=mock.MagicMock())
@pytest.mark.parametrize(
"app,"
"run_local,"
"debug_mode,"
"conditioned_profile,"
"post_startup_delay,"
"expected_post_startup_delay,"
"expected_debug_mode",
[
["firefox", True, True, None, 1234, 1234, True],
["firefox", True, True, None, None, 3000, True],
["firefox", True, False, None, None, 30000, False],
["firefox", True, False, "settled", None, 1000, False],
["fenix", True, False, None, None, 20000, False],
["fenix", True, False, "settled", None, 1000, False],
["firefox", False, False, None, 1234, 1234, False],
["firefox", False, False, None, 12345, 12345, False],
["firefox", True, False, None, 1234, 1234, False],
["firefox", True, False, None, 12345, 12345, False],
["firefox", False, True, None, 1234, 1234, False],
["firefox", False, True, None, 12345, 12345, False],
],
)
def test_post_startup_delay(
mock_info,
mock_critical,
ConcretePerftest,
options,
app,
run_local,
debug_mode,
conditioned_profile,
post_startup_delay,
expected_post_startup_delay,
expected_debug_mode,
):
options["app"] = app
perftest = ConcretePerftest(
run_local=run_local,
debug_mode=debug_mode,
post_startup_delay=post_startup_delay,
conditioned_profile=conditioned_profile,
**options
)
assert perftest.post_startup_delay == expected_post_startup_delay
assert perftest.debug_mode == expected_debug_mode
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
@pytest.mark.parametrize(
"alert, expected_alert", [["test_to_alert_on", "test_to_alert_on"], [None, None]]
)
def test_perftest_run_test_setup(
mock_info,
mock_critical,
ConcretePerftest,
options,
mock_test,
alert,
expected_alert,
):
perftest = ConcretePerftest(**options)
mock_test["alert_on"] = alert
perftest.run_test_setup(mock_test)
assert perftest.config["subtest_alert_on"] == expected_alert
# Browsertime tests
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
def test_cmd_arguments(
mock_info, mock_critical, ConcreteBrowsertime, browsertime_options, mock_test
):
expected_cmd = {
browsertime_options["browsertime_node"],
browsertime_options["browsertime_browsertimejs"],
"--firefox.geckodriverPath",
browsertime_options["browsertime_geckodriver"],
"--browsertime.page_cycles",
"1",
"--browsertime.url",
mock_test["test_url"],
"--browsertime.secondary_url",
mock_test["secondary_url"],
"--browsertime.page_cycle_delay",
"1000",
"--browsertime.post_startup_delay",
str(DEFAULT_TIMEOUT),
"--firefox.profileTemplate",
"--skipHar",
"--video",
"true",
"--visualMetrics",
"false",
"--timeouts.pageLoad",
str(DEFAULT_TIMEOUT),
"--timeouts.script",
str(DEFAULT_TIMEOUT),
"--resultDir",
"--iterations",
"1",
}
if browsertime_options.get("app") in ["chrome", "chrome-m"]:
expected_cmd.add(
"--chrome.chromedriverPath", browsertime_options["browsertime_chromedriver"]
)
browsertime = ConcreteBrowsertime(
post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
)
browsertime.run_test_setup(mock_test)
cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)
assert expected_cmd.issubset(set(cmd))
def extract_arg_value(cmd, arg):
param_index = cmd.index(arg) + 1
return cmd[param_index]
@pytest.mark.parametrize(
"arg_to_test, expected, test_patch, options_patch",
[
["--iterations", "1", {}, {"browser_cycles": None}],
["--iterations", "123", {"browser_cycles": 123}, {}],
["--video", "false", {}, {"browsertime_video": None}],
["--video", "true", {}, {"browsertime_video": "dummy_value"}],
["--timeouts.script", str(DEFAULT_TIMEOUT), {}, {"page_cycles": None}],
["--timeouts.script", str(123 * DEFAULT_TIMEOUT), {"page_cycles": 123}, {}],
["--browsertime.page_cycles", "1", {}, {"page_cycles": None}],
["--browsertime.page_cycles", "123", {"page_cycles": 123}, {}],
],
)
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
def test_browsertime_arguments(
mock_info,
mock_critical,
ConcreteBrowsertime,
browsertime_options,
mock_test,
arg_to_test,
expected,
test_patch,
options_patch,
):
mock_test.update(test_patch)
browsertime_options.update(options_patch)
browsertime = ConcreteBrowsertime(
post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
)
browsertime.run_test_setup(mock_test)
cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)
param_value = extract_arg_value(cmd, arg_to_test)
assert expected == param_value
@pytest.mark.parametrize(
"timeout, expected_timeout, test_patch, options_patch",
[
[0, 80, {}, {}],
[0, 80, {}, {"gecko_profile": False}],
[1000, 381, {}, {"gecko_profile": True}],
],
)
@patch("logger.logger.RaptorLogger.info")
@patch("logger.logger.RaptorLogger.critical")
def test_compute_process_timeout(
mock_info,
mock_critical,
ConcreteBrowsertime,
browsertime_options,
mock_test,
timeout,
expected_timeout,
test_patch,
options_patch,
):
mock_test.update(test_patch)
browsertime_options.update(options_patch)
browsertime = ConcreteBrowsertime(
post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
)
bt_timeout = browsertime._compute_process_timeout(mock_test, timeout, [])
assert bt_timeout == expected_timeout
if __name__ == "__main__":
mozunit.main()