Revision control
Copy as Markdown
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
---
version: 2.1
###########################################################################
# DEFINITIONS
definitions:
- release_filters: &release-filters
branches:
ignore: /.*/
tags:
only: /^v.*/
- ci_filters: &ci-filters
branches:
ignore: release
##########################################################################
# COMMANDS
commands:
skip-if-doc-only:
steps:
- run:
name: Check doc only skip condition
command: |
if git log -1 "$CIRCLE_SHA1" | grep -q '\[doc only\]'; then
echo "Skipping this step. Last commit was tagged to not require tests."
circleci-agent step halt
fi
setup-rust-toolchain:
parameters:
rust-version:
type: string
default: "stable"
steps:
- run:
name: Turn on RUST_BACKTRACE and RUST_LOG for any job that installs rustc
command: |
echo "export RUST_BACKTRACE=1" >> $BASH_ENV
echo "export RUST_LOG=glean_core=debug" >> $BASH_ENV
echo "export CARGO_INCREMENTAL=0" >> $BASH_ENV
- run:
name: Setup Rust toolchain
command: |
rustup install <<parameters.rust-version>>
rustup default <<parameters.rust-version>>
rustc --version
test-rust:
parameters:
rust-version:
type: string
default: "stable"
steps:
- checkout
- skip-if-doc-only
- setup-rust-toolchain:
rust-version: <<parameters.rust-version>>
- run:
name: Install required Python dependencies
command: |
sudo apt update
sudo apt install --yes --no-install-recommends \
python3-pip \
python3.10-venv
- run:
name: Install nextest
command: |
NEXTEST_SHA256=bde8d231e435099b068e654c591224defe686c784b4920682ee12784b6bfcf9e
NEXTEST=cargo-nextest
echo "${NEXTEST_SHA256} *${NEXTEST}.tar.gz" | shasum -a 256 -c -
tar -xf "${NEXTEST}.tar.gz"
cp ./${NEXTEST} ~/.cargo/bin/
- run:
name: Install circleci-junit-fix
command: |
JUNIT_FIX_SHA256=e8dd9012c841d9c03037783db676aa43a2ac1900346ccf4a61bf9956acfc6032
JUNIT_FIX=circleci-junit-fix
JUNIT_FIX_VERSION=v0.2.0
curl -sfSL --retry 5 -o "${JUNIT_FIX}.tar.gz" "https://github.com/conradludgate/circleci-junit-fix/releases/download/${JUNIT_FIX_VERSION}/${JUNIT_FIX}-${JUNIT_FIX_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
echo "${JUNIT_FIX_SHA256} *${JUNIT_FIX}.tar.gz" | shasum -a 256 -c -
tar -xf "${JUNIT_FIX}.tar.gz"
cp ./${JUNIT_FIX} ~/.cargo/bin/
- run:
name: Test
command: |
cargo nextest run --workspace --profile ci
- run:
name: Fix junit output
command: |
mkdir -p ~/test-results/
cat target/nextest/ci/junit.xml | circleci-junit-fix > ~/test-results/junit.xml
when: always
- store_test_results:
path: ~/test-results
- run:
name: Run Rust sample
command: |
cargo run -p sample
- run:
name: Run Rust RLB test
command: |
glean-core/rlb/tests/test-shutdown-blocking.sh
- run:
name: Run Rust RLB crash test
command: |
glean-core/rlb/tests/test-thread-crashing.sh
- run:
name: Run Rust RLB delayed ping data test
command: |
glean-core/rlb/tests/test-delayed-ping-data.sh
- run:
name: Run Rust RLB flush test
command: |
glean-core/rlb/tests/test-ping-lifetime-flush.sh
- run:
name: Run Rust RLB enabled-pings test
command: |
glean-core/rlb/tests/test-enabled-pings.sh
- run:
name: Run Rust RLB pending-gets-removed test
command: |
glean-core/rlb/tests/test-pending-gets-removed.sh
install-rustup:
steps:
- run:
name: Installing rustup
- run:
name: Setup custom environment variables
command: |
echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV
install-android-ndk:
steps:
- run:
name: Install missing Android SDK & NDK
command: |
sdkmanager \
"build-tools;35.0.0" \
"ndk;28.1.13356709"
android-setup:
steps:
- install-rustup
- setup-rust-toolchain:
rust-version: stable
- install-android-ndk
- run:
name: Restrict to Linux builds only
command: |
echo "rust.targets=linux-x86-64" > local.properties
test-python:
steps:
- install-rustup
- setup-rust-toolchain
- run:
name: Remove coredump file restriction
command: |
# tell the operating system to remove the file size limit on core dump files
ulimit -c unlimited
- run:
name: Python tests
command: |
export PYTEST_ARGS=-v
make test-python
- run:
name: Detect and gather coredump files
when: always
command: |
mkdir -p ~/coredumps
# Try to copy the core file(s). Don't fail if they don't exist.
cp core.* ~/coredumps || true
- store_artifacts:
path: ~/coredumps
destination: coredumps
build-windows-x86_64-wheel:
steps:
- install-rustup
- setup-rust-toolchain
- install-mingw
- run:
name: Install Python development tools for host
command:
make setup-python
- run:
name: Build Windows wheel
command: |
make build-python-wheel GLEAN_BUILD_TARGET=x86_64-pc-windows-gnu
build-windows-i686-wheel:
steps:
- install-rustup
- setup-rust-toolchain
- install-mingw
- run:
name: Install Python development tools for host
command:
make setup-python
- run:
name: Build Windows wheel
command: |
make build-python-wheel GLEAN_BUILD_TARGET=i686-pc-windows-gnu
install-python-windows-deps:
steps:
- run:
name: Set up dependencies for Python for Windows
command: |
echo "export WINEDEBUG=-all" >> $BASH_ENV
$WINPYTHON get-pip.py
echo "import site" >> winpython/python39._pth
echo "import sys; sys.path.insert(0, '')" >> winpython/sitecustomize.py
# The Windows-Python-installed-inside-Wine thing can't actually build wheels,
# so just install all of the wheels that were created as part of creating the
# environment on the host system before attempting to install everything in
# requirements_dev.txt
find ~/.cache/pip -name "*.whl" -exec $WINPYTHON -m pip install {} \;
$WINPYTHON -m pip install -r glean-core/python/requirements_dev.txt --no-warn-script-location
$WINPYTHON -m pip install target/wheels/*.whl --no-warn-script-location
install-mingw:
steps:
- run:
name: Install mingw
command: |
sudo apt update
# This package contains tools for both i686 and x86_64 targets
sudo apt install -y gcc-mingw-w64
- run:
name: Add mingw target
command: |
rustup target add x86_64-pc-windows-gnu
rustup target add i686-pc-windows-gnu
# Set the linker to use for Rust/mingw
echo '[target.x86_64-pc-windows-gnu]' >> ~/.cargo/config.toml
echo 'linker = "/usr/bin/x86_64-w64-mingw32-gcc"' >> ~/.cargo/config.toml
echo '[target.i686-pc-windows-gnu]' >> ~/.cargo/config.toml
echo 'linker = "/usr/bin/i686-w64-mingw32-gcc"' >> ~/.cargo/config.toml
install-ghr-darwin:
steps:
- run:
name: Get ghr release tool
command: |
GHR_VERSION=v0.16.2
GHR=ghr_${GHR_VERSION}_darwin_arm64
GHR_SHA256=f762742b3a1a6a10c87ca1bfcc043e9e535ce4d1defa02e4e9d5a4875c1116ba
echo "${GHR_SHA256} *${GHR}.zip" | shasum -a 256 -c -
unzip "${GHR}.zip"
cp ./${GHR}/ghr ghr
install-ghr-linux:
steps:
- run:
name: Get ghr release tool
command: |
GHR_VERSION=v0.16.2
GHR=ghr_${GHR_VERSION}_linux_amd64
GHR_SHA256=084ed9819dff71ea77f77a3071a643b6d1cbe5d2ab57bb5f56bb23de17189cd0
curl -sfSL --retry 5 -O "https://github.com/tcnksm/ghr/releases/download/${GHR_VERSION}/${GHR}.tar.gz"
echo "${GHR_SHA256} *${GHR}.tar.gz" | sha256sum -c -
tar -xf "${GHR}.tar.gz"
cp ./${GHR}/ghr ghr
jobs:
###########################################################################
# Project-level
License check:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- run:
name: Install cargo-deny
command: |
DENY_VERSION=0.14.20
DENY="cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl"
DENY_SHA256=1c9f8cfc23647346f1aa7ba0ed3167191f3198aba3dc5a957fda6f85a82fc424
curl -sfSL --retry 5 -O "https://github.com/EmbarkStudios/cargo-deny/releases/download/${DENY_VERSION}/${DENY}.tar.gz"
echo "${DENY_SHA256} *${DENY}.tar.gz" | shasum -a 256 -c -
tar -xvf "${DENY}.tar.gz"
mv "${DENY}/cargo-deny" ~/.cargo/bin/cargo-deny
chmod +x ~/.cargo/bin/cargo-deny
- run:
name: Run license check
command: cargo deny check licenses
- run:
name: Run dependency ban check
command: cargo deny check bans
Check vendored schema:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- run:
name: Check vendored schema for upstream updates
command: |
bin/update-schema.sh HEAD
if ! git diff --exit-code HEAD -- glean-core/preview/tests/glean.1.schema.json; then
echo "===================================="
echo "Latest schema from upstream changed."
echo "Please regenerate the file using:"
echo " bin/update-schema.sh latest"
echo "Commit the modified files and push."
echo "===================================="
exit 1
fi
Lint YAML with yamllint:
docker:
- image: cimg/python:3.8
steps:
- checkout
- run: pip install yamllint
- run: make lint-yaml
###########################################################################
# Rust / C / FFI
Check Rust formatting:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- run: rustup component add rustfmt
- run: rustfmt --version
- run: cargo fmt -- --check
Lint Rust with clippy:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- run: rustup component add clippy
- run: cargo clippy --version
- run:
name: Install required Python dependencies
command: |
sudo apt update
sudo apt install --yes --no-install-recommends \
python3.10-venv
- run:
name: Clippy
command: make lint-rust
Rust tests - stable:
docker:
- image: cimg/rust:1.82
resource_class: "medium+"
steps:
- test-rust
# RLB Windows compile test
- install-mingw
- run:
name: Check RLB on Windows x86_64
command: |
cargo check -p glean --examples --tests --target x86_64-pc-windows-gnu
- run:
name: Check RLB on Windows i686
command: |
cargo check -p glean --examples --tests --target i686-pc-windows-gnu
Rust tests - beta:
docker:
- image: cimg/rust:1.82
steps:
- test-rust:
rust-version: "beta"
Rust tests - minimum version:
docker:
- image: cimg/rust:1.82
resource_class: "medium+"
steps:
- test-rust:
rust-version: "1.82.0"
Generate Rust documentation:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- run:
name: Version information
command: rustc --version; cargo --version; rustup --version
- run:
name: Install mdbook-dtmo
command: |
MDBOOK_VERSION=0.15.2
MDBOOK="mdbook-dtmo-${MDBOOK_VERSION}-x86_64-unknown-linux-musl.tar.gz"
MDBOOK_SHA256=87f5cb874faadc745f033b646d358669b172edf19d2ca0a4e291a9c627e52e13
curl -sfSL --retry 5 -O "https://github.com/badboy/mdbook-dtmo/releases/download/${MDBOOK_VERSION}/${MDBOOK}"
echo "${MDBOOK_SHA256} *${MDBOOK}" | shasum -a 256 -c -
tar -xvf "${MDBOOK}"
# We rename it to mdbook here, so other tools keep working as expected
mv mdbook-dtmo ~/.cargo/bin/mdbook
mdbook --version
- run:
name: Build Rust documentation
command: bin/build-rust-docs.sh
- persist_to_workspace:
root: build/
paths:
- docs/book
- docs/dev
- docs/docs
- docs/shared
- docs/index.html
Publish Rust crates:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- run:
name: Publish Cargo Package
command: |
# Login to crates.io so the following commands work
cargo login "$CRATES_IO_TOKEN"
# Publish all crates from CI.
# The token is set in CircleCI settings.
pushd glean-core
cargo publish --verbose
pushd rlb
cargo publish --verbose
###########################################################################
# Android / Kotlin / Java
Lint Android with ktlint and detekt:
docker:
- image: cimg/android:2025.04.1-browsers
steps:
- checkout
- android-setup
- run: ./gradlew --no-daemon lint
- run: ./gradlew --no-daemon ktlint
- run: ./gradlew --no-daemon detekt
Android tests:
docker:
- image: cimg/android:2025.04.1-browsers
steps:
- checkout
- skip-if-doc-only
- android-setup
- run:
name: Remove coredump file restriction
command: |
# tell the operating system to remove the file size limit on core dump files
ulimit -c unlimited
- run:
name: Android tests
command: ./gradlew --no-daemon :glean:testDebugUnitTest
environment:
GRADLE_OPTS: -Xmx2048m
TARGET_CFLAGS: -DNDEBUG
- run:
name: Save test results
command: |
mkdir -p ~/test-results/junit/
mkdir -p ~/test-results/tests/
cp -a glean-core/android/build/reports/tests ~/test-results/
find glean-core/android/build -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/junit/ \;
when: always
- store_artifacts:
path: ~/test-results/tests
destination: test-results
- run:
name: Detect and gather coredump files
command: |
mkdir -p ~/coredumps
# Try to copy the core file from a default location. Don't fail if it doesn't exist.
cp -a glean-core/android/core ~/coredumps || true
# The JVM/Gradle might also produce a log file named like "hs_err_pid3247.log", let's copy that as well
find . -type f -name "hs_*.log" | xargs -I% cp -a % ~/coredumps/%
- store_artifacts:
path: ~/coredumps
destination: coredumps
- store_test_results:
path: ~/test-results
- run:
name: Build Android Sample App
command: |
./gradlew :glean-sample-app:assembleAndroidTest
environment:
GRADLE_OPTS: -Xmx2048m
TARGET_CFLAGS: -DNDEBUG
###########################################################################
# Swift / iOS / macOS
Check Swift formatting:
macos:
xcode: "15.1"
resource_class: "macos.m1.medium.gen1"
steps:
- checkout
- run:
name: Install lint tools
command: |
export HOMEBREW_NO_AUTO_UPDATE=1
export HOMEBREW_NO_INSTALL_CLEANUP=1
brew install swiftlint
- run:
name: Run swiftlint
command: |
swiftlint version
swiftlint --strict
iOS build and test:
macos:
xcode: "15.1"
resource_class: "macos.m1.medium.gen1"
steps:
- checkout
- run:
name: Show Ruby environment
command: |
ruby --version
gem env
- install-rustup
- setup-rust-toolchain
- restore_cache:
name: Restore rubygems cache
key: swift-docs-gems-v20
- run:
name: Install jazzy
command: gem install --no-document jazzy
- save_cache:
name: Save rubygems cache
# NEEDS TO CHANGE WHEN JAZZY OR RUBY IS UPDATED
key: swift-docs-gems-v20
paths:
- ~/.rbenv/versions/3.1.4/lib/ruby/gems/3.1.0
- run:
name: Setup build environment
command: |
set -x
rustup target add aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-ios
# For some reason everything works fine if we use the host clang,
# not the Xcode-bundled clang.
echo '[target.aarch64-apple-darwin]' >> ~/.cargo/config.toml
echo 'linker = "/usr/bin/cc"' >> ~/.cargo/config.toml
# List available devices -- allows us to see what's there
DEVICES=$(xcrun xctrace list devices 2>&1)
echo "$DEVICES"
# Pick a device and start it
UUID=$(echo "$DEVICES" | grep --max-count=1 'iPhone 15 Simulator (17' | awk -F'[()]' '{print $4}')
xcrun simctl boot "$UUID"
# Store build type for use in cache key
if [ -z "${CIRCLE_TAG}" ]; then
echo "debug" > buildtype.txt
else
echo "release" > buildtype.txt
fi
- restore_cache:
keys:
- v2-cargo-cache-{{arch}}-{{checksum "buildtype.txt"}}-{{checksum "Cargo.lock"}}
- run:
name: Run iOS build
command: bash bin/run-ios-build.sh
- save_cache:
paths:
- /Users/distiller/.cargo/registry
- target
key: v2-cargo-cache-{{arch}}-{{checksum "buildtype.txt"}}-{{checksum "Cargo.lock"}}
- run:
name: Run iOS tests
command: |
if git log -1 "$CIRCLE_SHA1" | grep -q '\[doc only\]'; then
echo "Skipping this step. Last commit was tagged to not require tests."
else
bash bin/run-ios-tests.sh
fi
- run:
name: Generate Swift documentation
command: |
# Skip doc generation for pull requests.
if [ "$CIRCLE_BRANCH" = "main" ]; then
bash bin/build-swift-docs.sh
else
mkdir -p build/docs/swift
fi
- store_artifacts:
path: raw_xcodebuild.log
destination: raw_xcodebuild.log
- store_artifacts:
path: raw_xcodetest.log
destination: raw_xcodetest.log
- persist_to_workspace:
root: build/
paths: docs/swift
- run:
name: Build XCFramework archive
no_output_timeout: 20m
command: |
if [ -z "${CIRCLE_TAG}" ]; then
# No need to build the framework archive unless we build for a tagged release.
circleci-agent step halt
else
bash bin/build-xcframework.sh
fi
- persist_to_workspace:
root: .
paths:
- Glean.xcframework.zip
iOS integration test:
macos:
xcode: "15.1"
resource_class: "macos.m1.medium.gen1"
steps:
- checkout
- skip-if-doc-only
- install-rustup
- setup-rust-toolchain
- run:
name: Setup build environment
command: |
set -x
rustup target add aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-ios
# For some reason everything works fine if we use the host clang,
# not the Xcode-bundled clang.
echo '[target.aarch64-apple-darwin]' >> ~/.cargo/config.toml
echo 'linker = "/usr/bin/cc"' >> ~/.cargo/config.toml
# List available devices -- allows us to see what's there
DEVICES=$(xcrun xctrace list devices 2>&1)
echo "$DEVICES"
# Pick a device and start it
UUID=$(echo "$DEVICES" | grep --max-count=1 'iPhone 15 Simulator (17' | awk -F'[()]' '{print $4}')
xcrun simctl boot "$UUID"
- run:
name: Build XCFramework archive
command: |
bash bin/build-xcframework.sh
- run:
name: Build sample app
command: |
bash bin/run-ios-sample-app-build.sh
- store_artifacts:
path: raw_sample_xcodebuild.log
destination: raw_sample_xcodebuild.log
- run:
name: Run sample app tests
command: |
bash bin/run-ios-sample-app-test.sh
- store_artifacts:
path: raw_sample_xcodetest.log
destination: raw_sample_xcodetest.log
iOS Framework release:
macos:
xcode: "15.1"
resource_class: "macos.m1.medium.gen1"
steps:
- checkout
- attach_workspace:
at: .
- install-ghr-darwin
- run:
name: Release framework archive on GitHub
command: |
./ghr -replace "${CIRCLE_TAG}" Glean.xcframework.zip
glean-swift release:
docker:
- image: cimg/rust:1.82
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Release glean-swift
command: |
git config --global user.email "jrediger@mozilla.com"
git config --global user.name "Glean automated release"
./bin/publish-glean-swift.sh "${CIRCLE_TAG}"
###########################################################################
# Python
Lint Python:
docker:
- image: cimg/python:3.10
steps:
- checkout
- install-rustup
- run:
name: Run uniffi-bindgen
command: make bindgen-python
- run:
name: Python lints
command: make lint-python
Python 3_8 tests:
docker:
- image: cimg/python:3.8
steps:
- checkout
- test-python
Python 3_9 tests:
docker:
- image: cimg/python:3.9
steps:
- checkout
- test-python
Python 3_9 on Alpine tests:
docker:
- image: python:3.9-alpine
shell: /bin/sh -leo pipefail
environment:
- BASH_ENV: /etc/profile
steps:
- run:
name: Install dependencies
command: |
apk add curl git gcc musl-dev libffi-dev openssh-client make openssl-dev
- checkout
- test-python
Python 3_10 tests:
docker:
- image: cimg/python:3.10
steps:
- checkout
- test-python
- persist_to_workspace:
root: .
paths: .venv3.10
Python 3_11 tests:
docker:
- image: cimg/python:3.11
steps:
- checkout
- test-python
Python 3_12 tests:
docker:
- image: cimg/python:3.12
steps:
- checkout
- test-python
Python Windows x86_64 tests:
docker:
- image: cimg/python:3.9
steps:
- checkout
- skip-if-doc-only
- build-windows-x86_64-wheel
- run:
name: Install Wine
command: |
sudo apt install wine64
- run:
name: Install Python for Windows
command: |
mkdir winpython
unzip python-3.9.13-embed-amd64.zip -d winpython
rm python-3.9.13-embed-amd64.zip
echo "export WINPYTHON=\"wine64-stable winpython/python.exe\"" >> $BASH_ENV
- install-python-windows-deps
- run:
name: Build bcryptprimitives.dll shim
command: |
rustc tools/patches/bcryptprimitives.rs -Copt-level=3 -Clto=fat --out-dir wine_shims --target x86_64-pc-windows-gnu
# This preloads our bcryptprimitives shim.
shimpath='Z:/home/circleci/project/wine_shims/bcryptprimitives.dll'
echo "import ctypes; ctypes.cdll.LoadLibrary('$shimpath')" >> winpython/sitecustomize.py
- run:
name: Run tests
command: |
$WINPYTHON -m pytest -s glean-core/python/tests
Python Windows i686 tests:
docker:
- image: cimg/python:3.9
steps:
- checkout
- skip-if-doc-only
- build-windows-i686-wheel
- run:
name: Install Wine
command: |
sudo dpkg --add-architecture i386
sudo apt update
sudo apt install wine32
- run:
name: Install Python for Windows
command: |
mkdir winpython
unzip python-3.9.13-embed-win32.zip -d winpython
rm python-3.9.13-embed-win32.zip
echo "export WINPYTHON=\"wine winpython/python.exe\"" >> $BASH_ENV
- install-python-windows-deps
- run:
name: Build bcryptprimitives.dll shim
command: |
rustc tools/patches/bcryptprimitives.rs -Copt-level=3 -Clto=fat --out-dir wine_shims --target i686-pc-windows-gnu
# This preloads our bcryptprimitives shim.
shimpath='Z:/home/circleci/project/wine_shims/bcryptprimitives.dll'
echo "import ctypes; ctypes.cdll.LoadLibrary('$shimpath')" >> winpython/sitecustomize.py
- run:
name: Run tests
command: |
$WINPYTHON -m pytest -s glean-core/python/tests
Generate Python documentation:
docker:
- image: cimg/python:3.10
steps:
- install-rustup
- setup-rust-toolchain
- checkout
- attach_workspace:
at: .
- run:
name: Generate Python docs
command: |
make docs-python
- persist_to_workspace:
root: build/
paths: docs/python
pypi-source-release:
docker:
- image: cimg/python:3.8
steps:
- install-rustup
- setup-rust-toolchain
- checkout
- run:
name: Setup default Python version
command: |
echo 'export PATH=/opt/python/cp38-cp38/bin:$PATH' >> $BASH_ENV
- run:
name: Setup Python env
command: |
make setup-python
- run:
name: Build Python source distribution
command: |
make build-python-sdist
- run:
name: Upload Linux source distribution
command: |
# Requires that the TWINE_USERNAME and TWINE_PASSWORD environment
# variables are configured in CircleCI's environment variables.
.venv3.8/bin/python3 -m twine upload target/wheels/*
pypi-linux-release:
docker:
# The official docker image for building manylinux2010 wheels
- image: quay.io/pypa/manylinux2014_x86_64
steps:
# manylinux2014 doesn't have ssh installed.
- run:
name: Install missing tools
command: yum install -y openssh-clients
- install-rustup
- setup-rust-toolchain
- checkout
- run:
name: Setup default Python version
command: |
echo 'export PATH=/opt/python/cp38-cp38/bin:$PATH' >> $BASH_ENV
- run:
name: Setup Python env
command: |
make setup-python
- run:
name: Build Python package
command: |
make build-python-wheel
- run:
name: Upload Linux wheel
command: |
# Requires that the TWINE_USERNAME and TWINE_PASSWORD environment
# variables are configured in CircleCI's environment variables.
.venv3.8/bin/python3 -m twine upload target/wheels/*
- install-ghr-linux
- run:
name: Publish to Github
command: |
# Upload to GitHub
./ghr -replace ${CIRCLE_TAG} target/wheels
pypi-macos-release:
macos:
xcode: "15.1"
resource_class: "macos.m1.medium.gen1"
steps:
- install-rustup
- setup-rust-toolchain
- checkout
- run:
name: Install Python development tools for host
command: |
rustup target add x86_64-apple-darwin
make setup-python
- run:
name: Build macOS x86_64 wheel
command: |
make build-python-wheel GLEAN_BUILD_TARGET=x86_64-apple-darwin
- run:
name: Build macOS aarch64 wheel
command: |
make build-python-wheel GLEAN_BUILD_TARGET=aarch64-apple-darwin
- run:
name: Build macOS universal2 wheel
command: |
make build-python-wheel GLEAN_BUILD_TARGET="universal2-apple-darwin"
- run:
name: Upload wheels to PyPI
command: |
# Requires that the TWINE_USERNAME and TWINE_PASSWORD environment
# variables are configured in CircleCI's environment variables.
.venv3.11/bin/python3 -m twine upload target/wheels/*
- install-ghr-darwin
- run:
name: Publish to Github
command: |
# Upload to GitHub
./ghr -replace ${CIRCLE_TAG} target/wheels
pypi-windows-x86_64-release:
docker:
- image: cimg/python:3.8
steps:
- checkout
- build-windows-x86_64-wheel
- run:
name: Upload to PyPI
command: |
# Requires that the TWINE_USERNAME and TWINE_PASSWORD environment
# variables are configured in CircleCI's environment variables.
.venv3.8/bin/python3 -m twine upload target/wheels/*
- install-ghr-linux
- run:
name: Publish to Github
command: |
# Upload to GitHub
./ghr -replace ${CIRCLE_TAG} target/wheels
pypi-windows-i686-release:
docker:
- image: cimg/python:3.8
steps:
- checkout
- build-windows-i686-wheel
- run:
name: Upload to PyPI
command: |
# Requires that the TWINE_USERNAME and TWINE_PASSWORD environment
# variables are configured in CircleCI's environment variables.
.venv3.8/bin/python3 -m twine upload target/wheels/*
- install-ghr-linux
- run:
name: Publish to Github
command: |
# Upload to GitHub
./ghr -replace ${CIRCLE_TAG} target/wheels
###########################################################################
# Docs
Docs internal metrics check:
docker:
- image: cimg/python:3.10
steps:
- checkout
- run:
name: Internal metrics docs consistency check
command: |
make docs-metrics
if ! git diff --exit-code HEAD -- docs/user/collected-metrics/metrics.md; then
echo "=================================================="
echo "metrics.md is different from what's stored in git."
echo "Please regenerate the file using:"
echo " make docs-metrics"
echo "Commit the modified file and push."
echo "=================================================="
exit 1
fi
docs-linkcheck:
docker:
- image: cimg/node:22.6
steps:
- checkout
- run:
name: Install linkchecker
command: npm install link-checker
- attach_workspace:
at: build/
- run:
name: Check internal documentation links
command: |
make linkcheck-raw
docs-spellcheck:
docker:
- image: cimg/base:2022.03
steps:
- checkout
- run:
name: Upgrade packages
command: sudo apt update
- run:
name: Install aspell
command: sudo apt install aspell aspell-en
- run:
name: Check documentation spelling
command: bin/spellcheck.sh list
# via https://circleci.com/blog/deploying-documentation-to-github-pages-with-continuous-integration/
docs-deploy:
docker:
- image: cimg/node:22.6
steps:
- checkout
- attach_workspace:
at: build/
- run:
name: Disable jekyll builds
command: touch build/docs/.nojekyll
- run:
name: Show contents
command: ls -R
# Needed for write access to the GitHub repository;
- add_ssh_keys:
fingerprints:
- "52:a9:c6:64:9e:df:60:70:73:da:81:02:71:9e:00:1b"
# The gh-pages npm package can be used to push a directory to a git branch;
- run:
name: Deploy docs to gh-pages branch
command: |
git config user.email "jrediger@mozilla.com"
git config user.name "CircleCI docs-deploy job"
npm install gh-pages@6.1.1
npx gh-pages --dotfiles --message "[skip ci] Updates" --dist build/docs
###########################################################################
# Workflows
workflows:
version: 2
lint:
jobs:
- Lint YAML with yamllint:
filters: *ci-filters
- License check:
filters: *ci-filters
- Lint Rust with clippy:
filters: *ci-filters
- Docs internal metrics check:
filters: *ci-filters
- Lint Android with ktlint and detekt:
filters: *ci-filters
- Lint Python:
filters: *ci-filters
- Check vendored schema:
filters: *ci-filters
- Check Rust formatting:
filters: *ci-filters
- Check Swift formatting:
filters: *ci-filters
ci:
jobs:
- Rust tests - stable:
filters: *ci-filters
- Rust tests - minimum version:
filters: *ci-filters
- Android tests:
filters: *ci-filters
# iOS jobs run only on main by default, see below for manual-approved jobs
- iOS build and test:
filters:
branches:
only: main
- iOS integration test:
filters:
branches:
only: main
- Python 3_8 tests:
filters: *ci-filters
- Python 3_9 tests:
filters: *ci-filters
- Python 3_9 on Alpine tests:
filters: *ci-filters
- Python 3_10 tests:
filters: *ci-filters
- Python 3_11 tests:
filters: *ci-filters
- Python 3_12 tests:
filters: *ci-filters
- Python Windows x86_64 tests:
filters: *ci-filters
- Python Windows i686 tests:
filters: *ci-filters
- Generate Rust documentation:
requires:
- docs-spellcheck
filters: *ci-filters
- Generate Python documentation:
requires:
- Python 3_10 tests
filters: *ci-filters
- docs-linkcheck:
requires:
- Generate Rust documentation
- Generate Python documentation
filters: *ci-filters
- docs-spellcheck:
filters: *ci-filters
- docs-deploy:
requires:
- docs-linkcheck
- iOS build and test
filters:
branches:
only: main
# iOS jobs require manual approval on PRs
iOS:
jobs:
- hold:
type: approval
filters:
branches:
ignore:
- main
- release
- iOS build and test:
requires:
- hold
filters:
branches:
ignore:
- main
- release
- iOS integration test:
requires:
- hold
filters:
branches:
ignore:
- main
- release
release:
jobs:
- Python 3_8 tests:
filters: *release-filters
- pypi-source-release:
requires:
- Python 3_8 tests
filters: *release-filters
- pypi-linux-release:
requires:
- Python 3_8 tests
filters: *release-filters
- pypi-macos-release:
requires:
- Python 3_8 tests
filters: *release-filters
- pypi-windows-i686-release:
requires:
- Python 3_8 tests
filters: *release-filters
- pypi-windows-x86_64-release:
requires:
- Python 3_8 tests
filters: *release-filters
- iOS build and test:
filters: *release-filters
- iOS Framework release:
requires:
- iOS build and test
filters: *release-filters
- glean-swift release:
requires:
- iOS Framework release
filters: *release-filters
- Publish Rust crates:
filters: *release-filters