Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""
import logging
from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
from taskgraph.util.treeherder import add_suffix
from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level
from gecko_taskgraph.util.workertypes import worker_type_implementation
logger = logging.getLogger(__name__)
transforms = TransformSequence()
@transforms.add
def set_defaults(config, jobs):
"""Set defaults, including those that differ per worker implementation"""
for job in jobs:
job["treeherder"].setdefault("kind", "build")
job["treeherder"].setdefault("tier", 1)
_, worker_os = worker_type_implementation(
config.graph_config, config.params, job["worker-type"]
)
worker = job.setdefault("worker", {})
worker.setdefault("env", {})
worker["chain-of-trust"] = True
yield job
@transforms.add
def stub_installer(config, jobs):
for job in jobs:
resolve_keyed_by(
job,
"stub-installer",
item_name=job["name"],
project=config.params["project"],
**{
"release-type": config.params["release_type"],
},
)
job.setdefault("attributes", {})
if job.get("stub-installer"):
job["attributes"]["stub-installer"] = job["stub-installer"]
job["worker"]["env"].update({"USE_STUB_INSTALLER": "1"})
if "stub-installer" in job:
del job["stub-installer"]
yield job
@transforms.add
def resolve_shipping_product(config, jobs):
for job in jobs:
resolve_keyed_by(
job,
"shipping-product",
item_name=job["name"],
**{
"release-type": config.params["release_type"],
},
)
yield job
@transforms.add
def update_channel(config, jobs):
keys = [
"run.update-channel",
"run.mar-channel-id",
"run.accepted-mar-channel-ids",
]
for job in jobs:
job["worker"].setdefault("env", {})
for key in keys:
resolve_keyed_by(
job,
key,
item_name=job["name"],
**{
"project": config.params["project"],
"release-type": config.params["release_type"],
},
)
update_channel = job["run"].pop("update-channel", None)
if update_channel:
job["run"].setdefault("extra-config", {})["update_channel"] = update_channel
job["attributes"]["update-channel"] = update_channel
mar_channel_id = job["run"].pop("mar-channel-id", None)
if mar_channel_id:
job["attributes"]["mar-channel-id"] = mar_channel_id
job["worker"]["env"]["MAR_CHANNEL_ID"] = mar_channel_id
accepted_mar_channel_ids = job["run"].pop("accepted-mar-channel-ids", None)
if accepted_mar_channel_ids:
job["attributes"]["accepted-mar-channel-ids"] = accepted_mar_channel_ids
job["worker"]["env"]["ACCEPTED_MAR_CHANNEL_IDS"] = accepted_mar_channel_ids
yield job
@transforms.add
def mozconfig(config, jobs):
for job in jobs:
resolve_keyed_by(
job,
"run.mozconfig-variant",
item_name=job["name"],
**{
"release-type": config.params["release_type"],
},
)
mozconfig_variant = job["run"].pop("mozconfig-variant", None)
if mozconfig_variant:
job["run"].setdefault("extra-config", {})[
"mozconfig_variant"
] = mozconfig_variant
yield job
@transforms.add
def use_artifact(config, jobs):
if is_try(config.params):
use_artifact = config.params["try_task_config"].get(
"use-artifact-builds", False
)
else:
use_artifact = False
for job in jobs:
if (
config.kind == "build"
and use_artifact
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
# If tests aren't packaged, then we are not able to rebuild all the packages
and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1"
# Android shippable artifact builds are not supported
and not (
"android" in job["name"] and job["attributes"].get("shippable", False)
)
):
job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a")
job["worker"]["env"]["USE_ARTIFACT"] = "1"
job["attributes"]["artifact-build"] = True
yield job
@transforms.add
def use_profile_data(config, jobs):
for job in jobs:
use_pgo = job.pop("use-pgo", False)
disable_pgo = config.params["try_task_config"].get("disable-pgo", False)
artifact_build = job["attributes"].get("artifact-build")
if not use_pgo or disable_pgo or artifact_build:
yield job
continue
# If use_pgo is True, the task uses the generate-profile task of the
# same name. Otherwise a task can specify a specific generate-profile
# task to use in the use_pgo field.
if use_pgo is True:
name = job["name"]
else:
name = use_pgo
dependencies = f"generate-profile-{name}"
job.setdefault("dependencies", {})["generate-profile"] = dependencies
job.setdefault("fetches", {})["generate-profile"] = ["profdata.tar.xz"]
job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
_, worker_os = worker_type_implementation(
config.graph_config, config.params, job["worker-type"]
)
if worker_os == "linux":
# LTO linkage needs more open files than the default from run-task.
job["worker"]["env"].update({"MOZ_LIMIT_NOFILE": "8192"})
if job.get("use-sccache"):
raise Exception(
"use-sccache is incompatible with use-pgo in {}".format(job["name"])
)
yield job
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(
job,
"use-sccache",
item_name=job["name"],
**{"release-level": release_level(config.params["project"])},
)
yield job
@transforms.add
def enable_full_crashsymbols(config, jobs):
"""Enable full crashsymbols on jobs with
'enable-full-crashsymbols' set to True and on release branches, or
on try"""
branches = RELEASE_PROJECTS | {
"toolchains",
"try",
}
for job in jobs:
enable_full_crashsymbols = job["attributes"].get("enable-full-crashsymbols")
if enable_full_crashsymbols and config.params["project"] in branches:
logger.debug("Enabling full symbol generation for %s", job["name"])
job["worker"]["env"]["MOZ_ENABLE_FULL_SYMBOLS"] = "1"
else:
logger.debug("Disabling full symbol generation for %s", job["name"])
job["attributes"].pop("enable-full-crashsymbols", None)
yield job
@transforms.add
def set_expiry(config, jobs):
for job in jobs:
attributes = job["attributes"]
if (
"shippable" in attributes
and attributes["shippable"]
and config.kind
in {
"build",
}
):
expiration_policy = "long"
else:
expiration_policy = "medium"
job["expiration-policy"] = expiration_policy
yield job