Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
"""
Transform the partials task into an actual task description.
"""
import logging
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.dependencies import get_primary_dependency
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from gecko_taskgraph.util.attributes import (
copy_attributes_from_dependent_job,
release_level,
)
from gecko_taskgraph.util.partials import get_builds
from gecko_taskgraph.util.platforms import architecture
logger = logging.getLogger(__name__)
transforms = TransformSequence()
def _generate_task_output_files(job, filenames, locale=None):
locale_output_path = f"{locale}/" if locale else ""
artifact_prefix = get_artifact_prefix(job)
data = list()
for filename in filenames:
data.append(
{
"type": "file",
"path": f"/home/worker/artifacts/{filename}",
"name": f"{artifact_prefix}/{locale_output_path}{filename}",
}
)
data.append(
{
"type": "file",
"path": "/home/worker/artifacts/manifest.json",
"name": f"{artifact_prefix}/{locale_output_path}manifest.json",
}
)
return data
def identify_desired_signing_keys(project, product):
if project in ["mozilla-central", "comm-central", "larch", "pine"]:
return "nightly"
if project == "mozilla-beta":
if product == "devedition":
return "nightly"
return "release"
if (
project in ["mozilla-release", "comm-release", "comm-beta"]
or project.startswith("mozilla-esr")
or project.startswith("comm-esr")
):
return "release"
return "dep1"
@transforms.add
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get("release_history"):
return
for job in jobs:
dep_job = get_primary_dependency(config, job)
assert dep_job
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault("symbol", "p(N)")
label = job.get("label", f"partials-{dep_job.label}")
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get("locale")
if locale:
attributes["locale"] = locale
treeherder["symbol"] = f"p({locale})"
attributes["shipping_phase"] = job["shipping-phase"]
build_locale = locale or "en-US"
build_platform = attributes["build_platform"]
builds = get_builds(
config.params["release_history"], build_platform, build_locale
)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {"funsize": {"partials": list()}}
update_number = 1
locale_suffix = ""
if locale:
locale_suffix = f"{locale}/"
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
"locale": build_locale,
"from_mar": builds[build]["mar_url"],
"to_mar": {"artifact-reference": artifact_path},
"branch": config.params["project"],
"update_number": update_number,
"dest_mar": build,
}
if "product" in builds[build]:
partial_info["product"] = builds[build]["product"]
if "previousVersion" in builds[build]:
partial_info["previousVersion"] = builds[build]["previousVersion"]
if "previousBuildNumber" in builds[build]:
partial_info["previousBuildNumber"] = builds[build][
"previousBuildNumber"
]
extra["funsize"]["partials"].append(partial_info)
update_number += 1
level = config.params["level"]
worker = {
"artifacts": _generate_task_output_files(dep_job, builds.keys(), locale),
"implementation": "docker-worker",
"docker-image": {"in-tree": "funsize-update-generator"},
"os": "linux",
"max-run-time": 3600 if "asan" in dep_job.label else 1800,
"chain-of-trust": True,
"taskcluster-proxy": True,
"env": {
"SIGNING_CERT": identify_desired_signing_keys(
config.params["project"], config.params["release_product"]
),
"EXTRA_PARAMS": f"--arch={architecture(build_platform)}",
"MAR_CHANNEL_ID": attributes["mar-channel-id"],
},
}
if release_level(config.params["project"]) == "staging":
worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true"
task = {
"label": label,
"description": f"{dep_job.description} Partials",
"worker-type": "b-linux-gcp",
"dependencies": dependencies,
"scopes": [],
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
"worker": worker,
}
if int(level) == 3 and any(
[build_platform.startswith(prefix) for prefix in ["linux", "win"]]
):
task["scopes"].append(
"auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/"
)
yield task