.taskcluster.yml - mozsearch

# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# yamllint disable rule:line-length

# This file is rendered via JSON-e by

# - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py

#   {

#     tasks_for: 'hg-push',

#     push: {owner, comment, pushlog_id, pushdate},

#     repository: {url, project, level},

#     now,

#     as_slugid: // function

#     ownTaskId: // taskId of the task that will be created

#   }

# - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py

#   {

#     tasks_for: 'cron',

#     push: {revision, pushlog_id, pushdate, owner}

#     repository: {url, project, level},

#     cron: {task_id, job_name, job_symbol, quoted_args},

#     now,

#     ownTaskId: // taskId of the task that will be created

#   }

# - action tasks - See:

#   * taskcluster/gecko_taskgraph/actions/registry.py,

#   * https://docs.taskcluster.net/docs/manual/using/actions/spec

#   * ci-admin:ciadmin/generate/in_tree_actions.py

#   The registry generates the hookPayload that appears in actions.json, and

#   contains data from the decision task as well as JSON-e code to combine that

#   with data supplied as part of the action spec.  When the hook is fired, the

#   hookPayload is rendered with JSON-e to produce a payload for the hook task

#   template.

#   The ci-admin code wraps the content of this file (.taskcluster.yml) with a

#   JSON-e $let statement that produces the context described below, and

#   installs that as the hook task template.

#   {

#     tasks_for: 'action',

#     push: {owner, pushlog_id, revision, base_revision},

#     repository: {url, project, level},

#     input,

#     taskId,      // targeted taskId

#     taskGroupId, // targeted taskGroupId

#     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}

#     ownTaskId:   // taskId of the task that will be created

#     clientId:    // clientId that triggered this hook

#   }

---

version: 1

tasks:

    # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering

    # takes place.

    - $if: 'tasks_for in ["hg-push", "action", "cron"]'

      then:

          $let:

              # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..

              ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}

              # ensure there's no trailing `/` on the repo URL

              repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}

              # expire try-comm-central earlier than other branches

              expires:

                  $if: 'repository.project == "try-comm-central"'

                  then: {$fromNow: '28 days'}

                  else: {$fromNow: '1 year'}

              trustDomain: comm

              treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'

in:

              taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}

              taskGroupId:

                  $if: 'tasks_for == "action"'

                  then:

                      '${action.taskGroupId}'

                  else:

                      '${ownTaskId}'  # same as taskId; this is how automation identifies a decision task

              schedulerId: '${trustDomain}-level-${repository.level}'

              created: {$fromNow: ''}

              deadline: {$fromNow: '1 day'}

              expires: {$eval: 'expires'}

              metadata:

                  $merge:

                      - owner: "${ownerEmail}"

                        source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"

                      - $if: 'tasks_for == "hg-push"'

                        then:

                            name: "Thunderbird Decision Task"

                            description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'

                        else:

                            $if: 'tasks_for == "action"'

                            then:

                                name: "Action: ${action.title}"

                                description: |

                                    ${action.description}

                                    ${treeherder_link}

                                    Action triggered by clientID `${clientId}`

                            else:

                                name: "Decision Task for cron job ${cron.job_name}"

                                description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'

              provisionerId: "${trustDomain}-${repository.level}"

              workerType: "decision-gcp"

              tags:

                  $if: 'tasks_for == "hg-push"'

                  then:

                      createdForUser: "${ownerEmail}"

                      kind: decision-task

                  else:

                      $if: 'tasks_for == "action"'

                      then:

                          createdForUser: '${ownerEmail}'

                          kind: 'action-callback'

                      else:

                          $if: 'tasks_for == "cron"'

                          then:

                              kind: cron-task

              routes:

                  $flattenDeep:

                      - "tc-treeherder.v2.${repository.project}.${push.revision}"

                      - $if: 'tasks_for == "hg-push"'

                        then:

                            - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"

                            - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"

                            - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"

                            - "notify.email.${ownerEmail}.on-failed"

                            - "notify.email.${ownerEmail}.on-exception"

                            # Send a notification email if the push comes from try

                            - $if: 'repository.project == "try-comm-central"'

                              then:

                                  "notify.email.${ownerEmail}.on-completed"

                        else:

                            $if: 'tasks_for == "action"'

                            then:

                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"

                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"

                            else:  # cron

                                - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"

                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"

                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"

                                # list each cron task on this revision, so actions can find them

                                - 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'

                                # Notify tb-builds by email if a nightly hook fails

                                - $if: 'repository.project != "try-comm-central"'

                                  then:

                                      - "notify.email.tb-builds@thunderbird.net.on-failed"

                                      - "notify.email.tb-builds@thunderbird.net.on-exception"

              scopes:

                  $if: 'tasks_for == "hg-push"'

                  then:

                      - 'assume:repo:${repoUrl[8:]}:branch:default'

                      - 'queue:route:notify.email.${ownerEmail}.*'

                      - 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'

                      - 'index:insert-task:${trustDomain}.v2.${repository.project}.*'

                  else:

                      $if: 'tasks_for == "action"'

                      then:

                          # when all actions are hooks, we can calculate this directly rather than using a variable

                          - '${action.repo_scope}'

                      else:

                          - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

              dependencies: []

              requires: all-completed

              priority:

                  # Most times, there is plenty of worker capacity so everything runs

                  # quickly, but sometimes a storm of action tasks lands.  Then we

                  # want, from highest to lowest:

                  # - cron tasks (time-sensitive) (low)

                  # - action tasks (avoid interfering with the other two) (very-low)

                  # - decision tasks (minimize user-visible delay) (lowest)

                  # SCM levels all use different workerTypes, so there is no need for priority

                  # between levels; "low" is the highest priority available at all levels, and

                  # nothing runs at any higher priority on these workerTypes.

                  $if: "tasks_for == 'cron'"

                  then: low

                  else:

                      $if: "tasks_for == 'action'"

                      then: very-low

                      else: lowest  # tasks_for == 'hg-push'

              retries:

                  $if: "tasks_for == 'hg-push'"

                  then: 0

                  else: 5

              payload:

                  env:

                      # run-task uses these to check out the source; the inputs

                      # to `mach taskgraph decision` are all on the command line.

                      $merge:

                          - COMM_BASE_REPOSITORY: 'https://hg.mozilla.org/comm-unified'

                            COMM_BASE_REV: '${push.base_revision}'

                            COMM_HEAD_REPOSITORY: '${repoUrl}'

                            COMM_HEAD_REF: '${push.revision}'

                            COMM_HEAD_REV: '${push.revision}'

                            HG_STORE_PATH: /builds/worker/checkouts/hg-store

                            TASKCLUSTER_CACHES: /builds/worker/checkouts

                            MOZ_AUTOMATION: '1'

                            # mach generates pyc files when reading `mach_commands.py`

                            # This causes cached_task digest generation to be random for

                            # some tasks. Disable bytecode generation to work around that.

                            PYTHONDONTWRITEBYTECODE: '1'

                          - $if: 'tasks_for == "action"'

                            then:

                                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task

                                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)

                                ACTION_INPUT: {$json: {$eval: 'input'}}

                                ACTION_CALLBACK: '${action.cb_name}'

                  cache:

                      "${trustDomain}-level-${repository.level}-checkouts-sparse-v4": /builds/worker/checkouts

                  features:

                      taskclusterProxy: true

                      chainOfTrust: true

                  # Note: This task is built server side without the context or tooling that

                  # exist in tree so we must hard code the hash

                  image: 'mozillareleases/gecko_decision:4.1.0@sha256:ea73900129328095da4fb033697690d538a08d935298fef4d5fa4aae80273bce'

                  maxRunTime: 1800

                  command:

                      - /builds/worker/bin/comm-task-env

                      - /builds/worker/bin/run-task

                      - '--gecko-checkout=/builds/worker/checkouts/gecko'

                      - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'

                      - '--comm-checkout=/builds/worker/checkouts/gecko/comm'

                      - '--'

                      - bash

                      - -cx

                      - $let:

                            extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}

in:

                            $if: 'tasks_for == "action"'

                            then: >

                                cd /builds/worker/checkouts/gecko &&

                                ln -s /builds/worker/artifacts artifacts &&

                                ./mach --log-no-times taskgraph action-callback

                                --root=comm/taskcluster

                            else: >

                                cd /builds/worker/checkouts/gecko &&

                                ln -s /builds/worker/artifacts artifacts &&

                                ./mach --log-no-times taskgraph decision

                                --root=comm/taskcluster

                                --pushlog-id='${push.pushlog_id}'

                                --pushdate='${push.pushdate}'

                                --project='${repository.project}'

                                --owner='${ownerEmail}'

                                --level='${repository.level}'

                                --tasks-for='${tasks_for}'

                                --repository-type=hg

                                --base-repository="$GECKO_BASE_REPOSITORY"

                                --head-repository="$GECKO_HEAD_REPOSITORY"

                                --head-ref="$GECKO_HEAD_REF"

                                --head-rev="$GECKO_HEAD_REV"

                                --try-task-config-file=comm/try_task_config.json

                                ${extraArgs}

                  artifacts:

                      'public':

                          type: 'directory'

                          path: '/builds/worker/artifacts'

                          expires: {$eval: expires}

                      'public/docker-contexts':

                          type: 'directory'

                          path: '/builds/worker/checkouts/gecko/docker-contexts'

                          # This needs to be at least the deadline of the

                          # decision task + the docker-image task deadlines.

                          # It is set to a week to allow for some time for

                          # debugging, but they are not useful long-term.

                          expires: {$fromNow: '7 day'}

              extra:

                  $merge:

                      - treeherder:

                            $merge:

                                - machine:

                                      platform: gecko-decision

                                - $if: 'tasks_for == "hg-push"'

                                  then:

                                      symbol: D

                                  else:

                                      $if: 'tasks_for == "action"'

                                      then:

                                          groupName: 'action-callback'

                                          groupSymbol: AC

                                          symbol: "${action.symbol}"

                                      else:

                                          groupSymbol: cron

                                          symbol: "${cron.job_symbol}"

                      - $if: 'tasks_for == "action"'

                        then:

                            parent: '${action.taskGroupId}'

                            action:

                                name: '${action.name}'

                                context:

                                    taskGroupId: '${action.taskGroupId}'

                                    taskId: {$eval: 'taskId'}

                                    input: {$eval: 'input'}

                                    clientId: {$eval: 'clientId'}

                      - $if: 'tasks_for == "cron"'

                        then:

                            cron: {$json: {$eval: 'cron'}}

                      - tasks_for: '${tasks_for}'

                      # Email for all pushes should link to treeherder

                      - $if: 'tasks_for == "hg-push"'

                        then:

                            notify:

                                email:

                                    $merge:

                                        - link:

                                              text: "Treeherder Jobs"

                                              href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"

                                        # Email for try pushes should thank you for your revision

                                        - $if: 'repository.project == "try-comm-central"'

                                          then:

                                              subject: "try-comm-central submission - ${push.revision}"

                                              content: "Your try-comm-central push has been submitted. It's the best! Use the link to view the status of your jobs."

Revision control

Copy as Markdown

Other Tools