Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
import glob
import os
import re
import subprocess
import sys
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))
# import the guts
import mozharness
from mozharness.base.log import DEBUG, ERROR, FATAL
from mozharness.base.transfer import TransferMixin
from mozharness.base.vcs.vcsbase import VCSScript
from mozharness.mozilla.tooltool import TooltoolMixin
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
"external_tools",
)
class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin):
all_actions = [
"clobber",
"get-tooltool",
"checkout-sources",
"build",
"test",
"package",
"dump-symbols",
]
default_actions = [
"get-tooltool",
"checkout-sources",
"build",
"package",
"dump-symbols",
]
config_options = [
[
["--repo"],
{
"dest": "repo",
"help": "OpenH264 repository to use",
},
],
[
["--rev"],
{"dest": "revision", "help": "revision to checkout", "default": "master"},
],
[
["--debug"],
{
"dest": "debug_build",
"action": "store_true",
"help": "Do a debug build",
},
],
[
["--arch"],
{
"dest": "arch",
"help": "Arch type to use (x64, x86, arm, or aarch64)",
},
],
[
["--os"],
{
"dest": "operating_system",
"help": "Specify the operating system to build for",
},
],
[
["--branch"],
{
"dest": "branch",
"help": "dummy option",
},
],
[
["--build-pool"],
{
"dest": "build_pool",
"help": "dummy option",
},
],
]
def __init__(
self,
require_config_file=False,
config={},
all_actions=all_actions,
default_actions=default_actions,
):
# Default configuration
default_config = {
"debug_build": False,
"upload_ssh_key": "~/.ssh/ffxbld_rsa",
"upload_ssh_user": "ffxbld",
"upload_ssh_host": "upload.ffxbld.productdelivery.prod.mozaws.net",
"upload_path_base": "/tmp/openh264",
}
default_config.update(config)
VCSScript.__init__(
self,
config_options=self.config_options,
require_config_file=require_config_file,
config=default_config,
all_actions=all_actions,
default_actions=default_actions,
)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
dirs = super(OpenH264Build, self).query_abs_dirs()
dirs["abs_upload_dir"] = os.path.join(dirs["abs_work_dir"], "upload")
self.abs_dirs = dirs
return self.abs_dirs
def get_tooltool(self):
c = self.config
if not c.get("tooltool_manifest_file"):
self.info("Skipping tooltool fetching since no tooltool manifest")
return
dirs = self.query_abs_dirs()
self.mkdir_p(dirs["abs_work_dir"])
manifest = os.path.join(
dirs["abs_src_dir"],
"testing",
"mozharness",
"configs",
"openh264",
"tooltool-manifests",
c["tooltool_manifest_file"],
)
self.info("Getting tooltool files from manifest (%s)" % manifest)
try:
self.tooltool_fetch(
manifest=manifest,
output_dir=os.path.join(dirs["abs_work_dir"]),
cache=c.get("tooltool_cache"),
)
except KeyError:
self.error("missing a required key.")
def query_package_name(self):
if self.config["arch"] in ("x64", "aarch64"):
bits = "64"
else:
bits = "32"
version = self.config["revision"]
if sys.platform in ("linux2", "linux"):
if self.config.get("operating_system") == "android":
return "openh264-android-{arch}-{version}.zip".format(
version=version, arch=self.config["arch"]
)
elif self.config.get("operating_system") == "darwin":
suffix = ""
if self.config["arch"] != "x64":
suffix = "-" + self.config["arch"]
return "openh264-macosx{bits}{suffix}-{version}.zip".format(
version=version, bits=bits, suffix=suffix
)
elif self.config["arch"] == "aarch64":
return "openh264-linux64-aarch64-{version}.zip".format(version=version)
else:
return "openh264-linux{bits}-{version}.zip".format(
version=version, bits=bits
)
elif sys.platform == "win32":
if self.config["arch"] == "aarch64":
return "openh264-win64-aarch64-{version}.zip".format(version=version)
else:
return "openh264-win{bits}-{version}.zip".format(
version=version, bits=bits
)
self.fatal("can't determine platform")
def query_make_params(self):
retval = []
if self.config["debug_build"]:
retval.append("BUILDTYPE=Debug")
if self.config["arch"] in ("x64", "aarch64"):
retval.append("ENABLE64BIT=Yes")
else:
retval.append("ENABLE64BIT=No")
if self.config["arch"] == "x86":
retval.append("ARCH=x86")
elif self.config["arch"] == "x64":
retval.append("ARCH=x86_64")
elif self.config["arch"] == "aarch64":
retval.append("ARCH=arm64")
else:
self.fatal("Unknown arch: {}".format(self.config["arch"]))
if "operating_system" in self.config:
retval.append("OS=%s" % self.config["operating_system"])
if self.config["operating_system"] == "android":
retval.append("TARGET=invalid")
retval.append("NDKLEVEL=%s" % self.config["min_sdk"])
retval.append("NDKROOT=%s/android-ndk" % os.environ["MOZ_FETCHES_DIR"])
retval.append("NDK_TOOLCHAIN_VERSION=clang")
if self.config["operating_system"] == "darwin":
retval.append("OS=darwin")
if self._is_windows():
retval.append("OS=msvc")
retval.append("CC=clang-cl")
retval.append("CXX=clang-cl")
if self.config["arch"] == "aarch64":
retval.append("CXX_LINK_O=-nologo --target=aarch64-windows-msvc -Fe$@")
else:
retval.append("CC=clang")
retval.append("CXX=clang++")
return retval
def query_upload_ssh_key(self):
return self.config["upload_ssh_key"]
def query_upload_ssh_host(self):
return self.config["upload_ssh_host"]
def query_upload_ssh_user(self):
return self.config["upload_ssh_user"]
def query_upload_ssh_path(self):
return "%s/%s" % (self.config["upload_path_base"], self.config["revision"])
def run_make(self, target, capture_output=False):
make = (
f"{os.environ['MOZ_FETCHES_DIR']}/mozmake/mozmake"
if sys.platform == "win32"
else "make"
)
cmd = [make, target] + self.query_make_params()
dirs = self.query_abs_dirs()
repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
env = None
if self.config.get("partial_env"):
env = self.query_env(self.config["partial_env"])
kwargs = dict(cwd=repo_dir, env=env)
if capture_output:
return self.get_output_from_command(cmd, **kwargs)
else:
return self.run_command(cmd, **kwargs)
def _git_checkout(self, repo, repo_dir, rev):
try:
subprocess.run(["git", "clone", "-q", "--no-checkout", repo, repo_dir])
subprocess.run(["git", "checkout", "-q", "-f", f"{rev}^0"], cwd=repo_dir)
except Exception:
self.rmtree(repo_dir)
raise
return True
def checkout_sources(self):
repo = self.config["repo"]
rev = self.config["revision"]
dirs = self.query_abs_dirs()
repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
if self._is_windows():
# We don't have git on our windows builders, so download a zip
# package instead.
path = repo.replace(".git", "/archive/") + rev + ".zip"
self.download_file(path)
self.unzip(rev + ".zip", dirs["abs_work_dir"])
self.move(
os.path.join(dirs["abs_work_dir"], "openh264-" + rev),
os.path.join(dirs["abs_work_dir"], "openh264"),
)
# Retrieve in-tree version of gmp-api
self.copytree(
os.path.join(dirs["abs_src_dir"], "dom", "media", "gmp", "gmp-api"),
os.path.join(repo_dir, "gmp-api"),
)
# We need gas-preprocessor.pl for arm64 builds
if self.config["arch"] == "aarch64":
openh264_dir = os.path.join(dirs["abs_work_dir"], "openh264")
self.download_file(
(
"gas-preprocessor/c2bc63c96678d9739509e58"
"7aa30c94bdc0e636d/gas-preprocessor.pl"
),
parent_dir=openh264_dir,
)
self.chmod(os.path.join(openh264_dir, "gas-preprocessor.pl"), 744)
# gas-preprocessor.pl expects cpp to exist
# os.symlink is not available on Windows until we switch to
# Python 3.
os.system(
"ln -s %s %s"
% (
os.path.join(
os.environ["MOZ_FETCHES_DIR"], "clang", "bin", "clang.exe"
),
os.path.join(openh264_dir, "cpp"),
)
)
return 0
self.retry(
self._git_checkout,
error_level=FATAL,
error_message="Automation Error: couldn't clone repo",
args=(repo, repo_dir, rev),
)
# Checkout gmp-api
# TODO: Nothing here updates it yet, or enforces versions!
if not os.path.exists(os.path.join(repo_dir, "gmp-api")):
retval = self.run_make("gmp-bootstrap")
if retval != 0:
self.fatal("couldn't bootstrap gmp")
else:
self.info("skipping gmp bootstrap - we have it locally")
# Checkout gtest
# TODO: Requires svn!
if not os.path.exists(os.path.join(repo_dir, "gtest")):
retval = self.run_make("gtest-bootstrap")
if retval != 0:
self.fatal("couldn't bootstrap gtest")
else:
self.info("skipping gtest bootstrap - we have it locally")
return retval
def build(self):
retval = self.run_make("plugin")
if retval != 0:
self.fatal("couldn't build plugin")
def package(self):
dirs = self.query_abs_dirs()
srcdir = os.path.join(dirs["abs_work_dir"], "openh264")
package_name = self.query_package_name()
package_file = os.path.join(dirs["abs_work_dir"], package_name)
if os.path.exists(package_file):
os.unlink(package_file)
to_package = []
for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")):
if not re.search(
r"(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)", f
):
# Don't package unnecessary zip bloat
# Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1
self.log("Skipping packaging of {package}".format(package=f))
continue
to_package.append(os.path.basename(f))
self.log("Packaging files %s" % to_package)
cmd = ["zip", package_file] + to_package
retval = self.run_command(cmd, cwd=srcdir)
if retval != 0:
self.fatal("couldn't make package")
self.copy_to_upload_dir(
package_file, dest=os.path.join(srcdir, "artifacts", package_name)
)
# Taskcluster expects this path to exist, but we don't use it
# because our builds are private.
path = os.path.join(
self.query_abs_dirs()["abs_work_dir"], "..", "public", "build"
)
self.mkdir_p(path)
def dump_symbols(self):
dirs = self.query_abs_dirs()
c = self.config
srcdir = os.path.join(dirs["abs_work_dir"], "openh264")
package_name = self.run_make("echo-plugin-name", capture_output=True)
if not package_name:
self.fatal("failure running make")
zip_package_name = self.query_package_name()
if not zip_package_name[-4:] == ".zip":
self.fatal("Unexpected zip_package_name")
symbol_package_name = "{base}.symbols.zip".format(base=zip_package_name[:-4])
symbol_zip_path = os.path.join(srcdir, "artifacts", symbol_package_name)
repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
env = None
if self.config.get("partial_env"):
env = self.query_env(self.config["partial_env"])
kwargs = dict(cwd=repo_dir, env=env)
dump_syms = os.path.join(dirs["abs_work_dir"], c["dump_syms_binary"])
self.chmod(dump_syms, 0o755)
python = self.query_exe("python3")
cmd = [
python,
os.path.join(external_tools_path, "packagesymbols.py"),
"--symbol-zip",
symbol_zip_path,
dump_syms,
os.path.join(srcdir, package_name),
]
self.run_command(cmd, **kwargs)
def test(self):
retval = self.run_make("test")
if retval != 0:
self.fatal("test failures")
def copy_to_upload_dir(
self,
target,
dest=None,
log_level=DEBUG,
error_level=ERROR,
compress=False,
upload_dir=None,
):
"""Copy target file to upload_dir/dest.
Potentially update a manifest in the future if we go that route.
Currently only copies a single file; would be nice to allow for
recursive copying; that would probably done by creating a helper
_copy_file_to_upload_dir().
"""
dest_filename_given = dest is not None
if upload_dir is None:
upload_dir = self.query_abs_dirs()["abs_upload_dir"]
if dest is None:
dest = os.path.basename(target)
if dest.endswith("/"):
dest_file = os.path.basename(target)
dest_dir = os.path.join(upload_dir, dest)
dest_filename_given = False
else:
dest_file = os.path.basename(dest)
dest_dir = os.path.join(upload_dir, os.path.dirname(dest))
if compress and not dest_filename_given:
dest_file += ".gz"
dest = os.path.join(dest_dir, dest_file)
if not os.path.exists(target):
self.log("%s doesn't exist!" % target, level=error_level)
return None
self.mkdir_p(dest_dir)
self.copyfile(target, dest, log_level=log_level, compress=compress)
if os.path.exists(dest):
return dest
else:
self.log("%s doesn't exist after copy!" % dest, level=error_level)
return None
# main {{{1
if __name__ == "__main__":
myScript = OpenH264Build()
myScript.run_and_exit()