Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
import importlib
import inspect
import pathlib
BUILTIN_COMPARATORS = {}
class ComparatorNotFound(Exception):
"""Raised when we can't find the specified comparator.
Triggered when either the comparator name is incorrect for a builtin one,
or when a path to a specified comparator cannot be found.
"""
pass
class GithubRequestFailure(Exception):
"""Raised when we hit a failure during PR link parsing."""
pass
class BadComparatorArgs(Exception):
"""Raised when the args given to the comparator are incorrect."""
pass
def comparator(comparator_klass):
BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass
return comparator_klass
@comparator
class BasePerfComparator:
def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args):
"""Initialize the standard/default settings for Comparators.
:param vcs object: Used for updating the local repo.
:param compare_commit str: The base revision found for the local repo.
:param current_revision_ref str: The current revision of the local repo.
:param comparator_args list: List of comparator args in the format NAME=VALUE.
"""
self.vcs = vcs
self.compare_commit = compare_commit
self.current_revision_ref = current_revision_ref
self.comparator_args = comparator_args
# Used to ensure that the local repo gets cleaned up appropriately on failures
self._updated = False
def setup_base_revision(self, extra_args):
"""Setup the base try run/revision.
In this case, we update to the repo to the base revision and
push that to try. The extra_args can be used to set additional
arguments for Raptor (not available for other harnesses).
:param extra_args list: A list of extra arguments to pass to the try tasks.
"""
self.vcs.update(self.compare_commit)
self._updated = True
def teardown_base_revision(self):
"""Teardown the setup for the base revision."""
if self._updated:
self.vcs.update(self.current_revision_ref)
self._updated = False
def setup_new_revision(self, extra_args):
"""Setup the new try run/revision.
Note that the extra_args are reset between the base, and new revision runs.
:param extra_args list: A list of extra arguments to pass to the try tasks.
"""
pass
def teardown_new_revision(self):
"""Teardown the new run/revision setup."""
pass
def teardown(self):
"""Teardown for failures.
This method can be used for ensuring that the repo is cleaned up
when a failure is hit at any point in the process of doing the
new/base revision setups, or the pushes to try.
"""
self.teardown_base_revision()
def get_github_pull_request_info(link):
"""Returns information about a PR link.
This method accepts a Github link in either of these formats:
and returns the Github link, branch, and revision of the commit.
"""
from urllib.parse import urlparse
import requests
# Parse the url, and get all the necessary info
parsed_url = urlparse(link)
path_parts = parsed_url.path.strip("/").split("/")
owner, repo = path_parts[0], path_parts[1]
pr_number = path_parts[-1]
if "/pull/" not in parsed_url.path:
raise GithubRequestFailure(
f"Link for Github PR is invalid (missing /pull/): {link}"
)
# Get the commit being targeted in the PR
pr_commit = None
if "/commits/" in parsed_url.path:
pr_commit = path_parts[-1]
pr_number = path_parts[-3]
# Make the request, and get the PR info, otherwise,
# raise an exception if the response code is not 200
response = requests.get(api_url)
if response.status_code == 200:
link_info = response.json()
return (
link_info["head"]["repo"]["html_url"],
pr_commit if pr_commit else link_info["head"]["sha"],
link_info["head"]["ref"],
)
raise GithubRequestFailure(
f"The following url returned a non-200 status code: {api_url}"
)
@comparator
class BenchmarkComparator(BasePerfComparator):
def _get_benchmark_info(self, arg_prefix):
# Get the flag from the comparator args
benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None}
for arg in self.comparator_args:
if arg.startswith(arg_prefix):
_, settings = arg.split(arg_prefix)
setting, val = settings.split("=")
if setting not in benchmark_info:
raise BadComparatorArgs(
f"Unknown argument provided `{setting}`. Only the following "
f"are available (prefixed with `{arg_prefix}`): "
f"{list(benchmark_info.keys())}"
)
benchmark_info[setting] = val
# Parse the link for any required information
if benchmark_info.get("link", None) is not None:
(
benchmark_info["repo"],
benchmark_info["revision"],
benchmark_info["branch"],
) = get_github_pull_request_info(benchmark_info["link"])
return benchmark_info
def _setup_benchmark_args(self, extra_args, benchmark_info):
# Setup the arguments for Raptor
extra_args.append(f"benchmark-repository={benchmark_info['repo']}")
extra_args.append(f"benchmark-revision={benchmark_info['revision']}")
if benchmark_info.get("branch", None):
extra_args.append(f"benchmark-branch={benchmark_info['branch']}")
def setup_base_revision(self, extra_args):
"""Sets up the options for a base benchmark revision run.
Checks for a `base-link` in the
command and adds the appropriate commands to the extra_args
which will be added to the PERF_FLAGS environment variable.
If that isn't provided, then you must provide the repo, branch,
and revision directly through these (branch is optional):
base-branch=main
base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365
Otherwise, we'll use the default mach try perf
base behaviour.
TODO: Get the information automatically from a commit link. Github
API doesn't provide the branch name from a link like that.
"""
base_info = self._get_benchmark_info("base-")
# If no options were provided, use the default BasePerfComparator behaviour
if not any(v is not None for v in base_info.values()):
raise BadComparatorArgs(
f"Could not find the correct base-revision arguments in: {self.comparator_args}"
)
self._setup_benchmark_args(extra_args, base_info)
def setup_new_revision(self, extra_args):
"""Sets up the options for a new benchmark revision run.
Same as `setup_base_revision`, except it uses
`new-` as the prefix instead of `base-`.
"""
new_info = self._get_benchmark_info("new-")
# If no options were provided, use the default BasePerfComparator behaviour
if not any(v is not None for v in new_info.values()):
raise BadComparatorArgs(
f"Could not find the correct new-revision arguments in: {self.comparator_args}"
)
self._setup_benchmark_args(extra_args, new_info)
def get_comparator(comparator):
if comparator in BUILTIN_COMPARATORS:
return BUILTIN_COMPARATORS[comparator]
file = pathlib.Path(comparator)
if not file.exists():
raise ComparatorNotFound(
f"Expected either a path to a file containing a comparator, or a "
f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}"
)
# Importing a source file directly
spec = importlib.util.spec_from_file_location(name=file.name, location=comparator)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
members = inspect.getmembers(
module,
lambda c: inspect.isclass(c)
and issubclass(c, BasePerfComparator)
and c != BasePerfComparator,
)
if not members:
raise ComparatorNotFound(
f"The path {comparator} was found but it was not a valid comparator. "
f"Ensure it is a subclass of BasePerfComparator and optionally contains the "
f"following methods: "
f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}"
)
return members[0][-1]