CI: Re-write run_task.sh

Recently we re-wrote CI to increase VM level parallelism, in hindsite
this has proved to be not that great because:

- It resulted in approx 180 jobs
- We are on free tier so only get 20 jobs (VMs) at a time so its slow to run
- The UI is annoying to dig through the long job list to find failures

Have another go at organising the jobs with the main aim of shortening
total run time and making it easier to quickly see fails.

Re-write the `run_task.sh` script, notable moving manifest handling
to the workflow. Also don't bother testing with beta toolchain.

WASM Note

Removes the `cdylib` and `rlib` from the manifest patching during wasm
build - I do not know the following:

- Why this breaks on this PR but not on other PRs
- Why I can't get wasm test to run locally on master but PRs are passing
- What the `cdylib` and `rlib` were meant to be doing

This is the docs from: https://doc.rust-lang.org/reference/linkage.html

* --crate-type=cdylib, #![crate_type = "cdylib"] - A dynamic system
library will be produced. This is used when compiling a dynamic library
to be loaded from another language. This output type will create *.so
files on Linux, *.dylib files on macOS, and *.dll files on Windows.

* --crate-type=rlib, #![crate_type = "rlib"] - A "Rust library" file
will be produced. This is used as an intermediate artifact and can be
thought of as a "static Rust library". These rlib files, unlike
staticlib files, are interpreted by the compiler in future linkage. This
essentially means that rustc will look for metadata in rlib files like
it looks for metadata in dynamic libraries. This form of output is used
to produce statically linked executables as well as staticlib outputs.
This commit is contained in:
Tobin C. Harding 2024-03-27 09:10:45 +11:00
parent a9cdcb4392
commit 26b9782d8b
No known key found for this signature in database
GPG Key ID: 40BF9E4C269D6607
9 changed files with 299 additions and 194 deletions

View File

@ -11,8 +11,6 @@ jobs:
Prepare:
runs-on: ubuntu-latest
outputs:
crates: ${{ steps.get_matrix.outputs.crates }}
deps: ${{ steps.get_matrix.outputs.deps }}
nightly_version: ${{ steps.read_toolchain.outputs.nightly_version }}
steps:
- name: Checkout Crate
@ -20,220 +18,261 @@ jobs:
- name: Read nightly version
id: read_toolchain
run: echo "nightly_version=$(cat nightly-version)" >> $GITHUB_OUTPUT
- name: Prepare tests
id: get_matrix
run: contrib/get_matrix.sh
Stable:
needs: Prepare
Stable: # 2 jobs, one per manifest.
name: Test - stable toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test, docs, feature_matrix, dup_deps]
dep: [minimal, recent]
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
# https://github.com/dtolnay/rust-toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
- name: Set dependencies
- name: "Set dependencies"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
- name: "Run test script"
run: ./contrib/run_task.sh stable
Beta:
needs: Prepare
name: Test - beta toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Checkout Toolchain
uses: dtolnay/rust-toolchain@beta
- name: Set dependencies
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Nightly:
needs: Prepare
Nightly: # 2 jobs, one per manifest.
name: Test - nightly toolchain
needs: Prepare
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test, lint, bench, docsrs]
dep: [minimal, recent]
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@v1
with:
toolchain: ${{ needs.Prepare.outputs.nightly_version }}
- name: Install clippy
run: rustup component add clippy
- name: Set dependencies
- name: "Set dependencies"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
- name: "Run test script"
run: ./contrib/run_task.sh nightly
MSRV:
needs: Prepare
MSRV: # 2 jobs, one per manifest.
name: Test - 1.56.1 toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test, feature_matrix]
dep: [minimal, recent]
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.56.1"
- name: Set dependencies
- name: "Copy lock file"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
- name: "Run test script"
run: ./contrib/run_task.sh msrv
Lint:
name: Lint - nightly toolchain
needs: Prepare
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
dep: [recent]
steps:
- name: "Checkout repo"
uses: actions/checkout@v4
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@v1
with:
toolchain: ${{ needs.Prepare.outputs.nightly_version }}
- name: Install clippy
run: rustup component add clippy
- name: "Set dependencies"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: "Run test script"
run: ./contrib/run_task.sh lint
Docs:
name: Docs - stable toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
dep: [recent]
steps:
- name: "Checkout repo"
uses: actions/checkout@v4
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
- name: "Copy lock file"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: "Run test script"
run: ./contrib/run_task.sh docs
Docsrs:
name: Docs - nightly toolchain
needs: Prepare
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
dep: [recent]
steps:
- name: "Checkout repo"
uses: actions/checkout@v4
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@v1
with:
toolchain: ${{ needs.Prepare.outputs.nightly_version }}
- name: "Set dependencies"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: "Run test script"
run: ./contrib/run_task.sh docsrs
Bench:
name: Bench - nightly toolchain
needs: Prepare
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
dep: [recent]
steps:
- name: "Checkout repo"
uses: actions/checkout@v4
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@v1
with:
toolchain: ${{ needs.Prepare.outputs.nightly_version }}
- name: "Copy lock file"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: "Run test script"
run: ./contrib/run_task.sh bench
Arch32bit:
name: Test 32-bit version
runs-on: ubuntu-latest
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
- name: Add architecture i386
- name: "Add architecture i386"
run: sudo dpkg --add-architecture i386
- name: Install i686 gcc
- name: "Install i686 gcc"
run: sudo apt-get update -y && sudo apt-get install -y gcc-multilib
- name: Install target
- name: "Install target"
run: rustup target add i686-unknown-linux-gnu
- name: Run test on i686
- name: "Run test on i686"
run: cargo test --target i686-unknown-linux-gnu
Cross:
name: Cross test
name: Cross test - stable toolchain
if: ${{ !github.event.act }}
runs-on: ubuntu-latest
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
- name: Install target
- name: "Install target"
run: rustup target add s390x-unknown-linux-gnu
- name: install cross
- name: "Install cross"
run: cargo install cross --locked
- name: run cross test
- name: "Run cross test"
run: cross test --target s390x-unknown-linux-gnu
Embedded:
name: Embedded - nightly toolchain
needs: Prepare
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C link-arg=-Tlink.x"
CARGO_TARGET_THUMBV7M_NONE_EABI_RUNNER: "qemu-system-arm -cpu cortex-m3 -machine mps2-an385 -nographic -semihosting-config enable=on,target=native -kernel"
steps:
- name: Checkout
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Set up QEMU
- name: "Set up QEMU"
run: sudo apt update && sudo apt install -y qemu-system-arm gcc-arm-none-eabi
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@v1
with:
toolchain: ${{ needs.Prepare.outputs.nightly_version }}
targets: thumbv7m-none-eabi
- name: Install src
- name: "Install rust-src"
run: rustup component add rust-src
- name: Run bitcoin/embedded
- name: "Run bitcoin/embedded"
run: cd bitcoin/embedded && cargo run --target thumbv7m-none-eabi
- name: Run hashes/embedded no alloc
- name: "Run hashes/embedded no alloc"
run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi
- name: Run hashes/embedded with alloc
- name: "Run hashes/embedded with alloc"
run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi --features=alloc
ASAN:
ASAN: # hashes crate only.
name: ASAN - nightly toolchain
needs: Prepare
name: Address sanitizer # hashes crate only.
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [hashes]
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [asan]
dep: [recent]
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@v1
with:
toolchain: ${{ needs.Prepare.outputs.nightly_version }}
- name: Install src
run: rustup component add rust-src
- name: Running address sanitizer
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
- name: "Copy lock file"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: "Run test script"
run: ./contrib/run_task.sh asan
WASM:
needs: Prepare
name: WebAssembly Build # hashes crate only.
WASM: # hashes crate only.
name: WASM - stable toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
# Note we do not use the recent lock file for wasm testing.
steps:
- name: "Checkout repo"
uses: actions/checkout@v4
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
- name: "Run test script"
run: ./contrib/run_task.sh wasm
Schemars: # hashes crate only.
name: Schemars - stable toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [hashes]
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [wasm]
dep: [recent]
steps:
- name: Checkout Crate
- name: "Checkout repo"
uses: actions/checkout@v4
- name: Checkout Toolchain
- name: "Select toolchain"
uses: dtolnay/rust-toolchain@stable
- name: Running WASM build
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Schemars:
needs: Prepare
name: Schemars
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [hashes]
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [schemars]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Checkout Toolchain
uses: dtolnay/rust-toolchain@stable
- name: Running schemars test
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
- name: "Copy lock file"
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: "Run test script"
run: ./contrib/run_task.sh schemars
Kani:
name: Kani codegen - stable toolchain
runs-on: ubuntu-20.04
steps:
- name: 'Checkout your code.'
- name: "Checkout repo"
uses: actions/checkout@v4
- name: 'Kani build proofs'
- name: "Build Kani proofs"
uses: model-checking/kani-github-action@v1.1
with:
args: '--only-codegen'
args: "--only-codegen"

View File

@ -6,5 +6,5 @@ FEATURES_WITH_STD=""
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD=""
# Run and lint these examples.
# Run these examples.
EXAMPLES=""

View File

@ -6,5 +6,5 @@ FEATURES_WITH_STD="rand-std serde secp-recovery bitcoinconsensus-std base64 orde
# Test all these features without "std" or "alloc" enabled.
FEATURES_WITHOUT_STD="rand serde secp-recovery bitcoinconsensus base64 ordered"
# Run and lint these examples.
# Run these examples.
EXAMPLES="ecdsa-psbt:std,bitcoinconsensus sign-tx-segwit-v0:rand-std sign-tx-taproot:rand-std taproot-psbt:bitcoinconsensus,rand-std sighash:std"

View File

@ -1,13 +1,52 @@
#!/bin/env bash
#!/usr/bin/env bash
#
# We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier).
#
# ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
#
# The minimal/recent manifests are handled by CI (rust.yaml).
#
# Jobs (shell functions) that get run twice, once for each manifest:
#
# 1+2 stable
# 3+4 nightly
# 5+6 msrv
#
# Jobs (shell functions) that get run once:
#
# 7 lint
# 8 docs
# 9 docsrs
# 10 bench
# 11 asan
# 12 wasm
# 13 schemars
#
# Jobs run directly by rust.yml workflow:
#
# 0 Prepare
#
# 14 Arch32bit
# 15 Cross
# 16 Embedded
# 17 Kani
#
# Jobs run directly from other workflows:
#
# 18 Coveralls - run by coveralls.yml
# 19 release - run by release.yml
# 20 labeler - run by manage-pr.yml
set -euox pipefail
REPO_DIR=$(git rev-parse --show-toplevel)
CRATES=("bitcoin" "hashes" "internals" "io" "units" "base58")
# Make all cargo invocations verbose.
export CARGO_TERM_VERBOSE=true
main() {
crate="$1"
task="$2"
local task="$1"
check_required_commands
@ -17,32 +56,25 @@ main() {
locale
env
cd "$crate"
# Building the fuzz crate is more-or-less just a sanity check.
if [ "$crate" == "fuzz" ]
then
cargo --locked build
exit 0
fi
# Every crate must define EXAMPLES.
. contrib/test_vars.sh || exit 1
case $task in
test)
do_test
# 2 jobs each for these (one for each lock file).
stable)
# Test, run examples, do feature matrix.
# crate/contrib/test_vars.sh is sourced in this function.
build_and_test
;;
feature_matrix)
do_feature_matrix
nightly)
build_and_test
;;
msrv)
build_and_test
;;
# 1 job each for these.
lint)
do_lint
;;
dup_deps)
do_dup_deps
;;
@ -54,27 +86,51 @@ main() {
build_docs_with_nightly_toolchain
;;
wasm)
do_wasm
;;
asan)
do_asan
;;
bench)
do_bench
;;
wasm)
# hashes crate only.
do_wasm
;;
asan)
# hashes crate only - hashes/contrib/test_vars.sh is sourced in this function.
do_asan
;;
schemars)
# hashes crate only.
do_schemars
;;
*)
err "Error: unknown task $task"
;;
esac
}
# Build and test for each crate, done with each toolchain.
build_and_test() {
# Building the fuzz crate is more-or-less just a sanity check.
pushd "$REPO_DIR/fuzz" > /dev/null
cargo --locked build
popd > /dev/null
for crate in "${CRATES[@]}"; do
pushd "$REPO_DIR/$crate" > /dev/null
# Set crate specific variables.
. contrib/test_vars.sh || exit 1
do_test
do_feature_matrix
popd > /dev/null
done
}
do_test() {
# Use the current (recent/minimal) lock file.
local cargo="cargo --locked"
@ -83,7 +139,7 @@ do_test() {
$cargo build
$cargo test
for example in $EXAMPLES; do
for example in $EXAMPLES; do # EXAMPLES is set in contrib/test_vars.sh
name="$(echo "$example" | cut -d ':' -f 1)"
features="$(echo "$example" | cut -d ':' -f 2)"
$cargo run --example "$name" --features="$features"
@ -103,7 +159,8 @@ do_feature_matrix() {
$cargo build --no-default-features
$cargo test --no-default-features
# All crates have a "std" feature.
# All crates have a "std" feature and FEATURES_WITH_STD is set in
# contrib/test_vars.sh
loop_features "std" "$FEATURES_WITH_STD"
# All but `bitcoin` crate have an "alloc" feature, this tests it
@ -146,20 +203,15 @@ loop_features() {
fi
}
# Lint the workspace then the individual crate examples.
# Lint the workspace.
do_lint() {
need_nightly
# Use the current (recent/minimal) lock file.
local cargo="cargo --locked"
$cargo clippy --workspace -- -D warnings
for example in $EXAMPLES; do
name=$(echo "$example" | cut -d ':' -f 1)
features=$(echo "$example" | cut -d ':' -f 2)
$cargo clippy --example "$name" --features="$features" -- -D warnings
done
# Lint various feature combinations to try and catch mistakes in feature gating.
$cargo clippy --workspace --all-targets --keep-going -- -D warnings
$cargo clippy --workspace --all-targets --all-features --keep-going -- -D warnings
$cargo clippy --workspace --all-targets --no-default-features --keep-going -- -D warnings
}
# We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies
@ -173,7 +225,7 @@ do_dup_deps() {
# Only show the actual duplicated deps, not their reverse tree, then
# whitelist the 'syn' crate which is duplicated but it's not our fault.
#
# Temporarily allow 2 versions of `hashes` and `hex` while we upgrade.
# Temporarily allow 2 versions of `hashes`, `internals`, and `hex` while we upgrade.
cargo tree --target=all --all-features --duplicates \
| grep '^[0-9A-Za-z]' \
| grep -v 'syn' \
@ -193,7 +245,6 @@ do_dup_deps() {
build_docs_with_nightly_toolchain() {
need_nightly
local cargo="cargo --locked"
RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features
}
@ -201,20 +252,45 @@ build_docs_with_nightly_toolchain() {
# above this checks that we feature guarded docs imports correctly.
build_docs_with_stable_toolchain() {
local cargo="cargo +stable --locked"
RUSTDOCFLAGS="-D warnings" $cargo doc --all-features
}
# Bench only works with a non-stable toolchain (nightly, beta).
do_bench() {
for crate in bitcoin hashes; do
pushd "$REPO_DIR/$crate" > /dev/null
RUSTFLAGS='--cfg=bench' cargo bench
popd > /dev/null
done
}
# This is only relevant for hashes.
do_schemars() {
pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null
cargo test
popd > /dev/null
}
# Note we do not use the recent lock file or `--locked` when running the wasm tests.
do_wasm() {
pushd "$REPO_DIR/hashes" > /dev/null
clang --version &&
CARGO_TARGET_DIR=wasm cargo install --force wasm-pack &&
printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml &&
printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml &&
CC=clang-9 wasm-pack build &&
CC=clang-9 wasm-pack test --node;
popd > /dev/null
}
do_asan() {
pushd "$REPO_DIR/hashes" > /dev/null
# Set ASAN_FEATURES
. contrib/test_vars.sh || exit 1
cargo clean
CC='clang -fsanitize=address -fno-omit-frame-pointer' \
RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \
@ -226,17 +302,8 @@ do_asan() {
# CC='clang -fsanitize=memory -fno-omit-frame-pointer' \
# RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \
# cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
}
# Bench only works with a non-stable toolchain (nightly, beta).
do_bench() {
RUSTFLAGS='--cfg=bench' cargo bench
}
# This is only relevant for hashes.
do_schemars() {
cd "extended_tests/schemars" > /dev/null
cargo test
popd > /dev/null
}
# Check all the commands we use are present in the current environment.

View File

@ -9,5 +9,5 @@ FEATURES_WITHOUT_STD="alloc serde small-hash"
# Run address sanitizer with these features.
ASAN_FEATURES="std io serde"
# Run and lint these examples.
# Run these examples.
EXAMPLES=""

View File

@ -1027,10 +1027,9 @@ mod tests {
#[cfg(target_arch = "wasm32")]
mod wasm_tests {
extern crate wasm_bindgen_test;
use self::wasm_bindgen_test::*;
use super::*;
#[wasm_bindgen_test]
#[test]
#[wasm_bindgen_test::wasm_bindgen_test]
fn sha256_tests() {
test();
midstate();

View File

@ -6,5 +6,5 @@ FEATURES_WITH_STD="serde"
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc serde"
# Run and lint these examples.
# Run these examples.
EXAMPLES=""

View File

@ -6,5 +6,5 @@ FEATURES_WITH_STD=""
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc"
# Run and lint these examples.
# Run these examples.
EXAMPLES=""

View File

@ -6,5 +6,5 @@ FEATURES_WITH_STD="serde"
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc serde"
# Run and lint these examples.
# Run these examples.
EXAMPLES=""