CI: Epic overhaul

Re-write the whole CI pipeline.

Co-developed-by: Martin Habovstiak <martin.habovstiak@gmail.com>
This commit is contained in:
Martin Habovstiak 2024-01-15 15:09:48 +01:00 committed by Tobin C. Harding
parent 242aa676b3
commit 5c15ed5441
No known key found for this signature in database
GPG Key ID: 40BF9E4C269D6607
14 changed files with 395 additions and 228 deletions

View File

@ -8,45 +8,69 @@ on:
name: Continuous integration
jobs:
Prepare:
runs-on: ubuntu-latest
outputs:
crates: ${{ steps.get_matrix.outputs.crates }}
deps: ${{ steps.get_matrix.outputs.deps }}
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Prepare tests
id: get_matrix
run: contrib/get_matrix.sh
Stable:
needs: Prepare
name: Test - stable toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test, docs, feature_matrix, dup_deps]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Checkout Toolchain
# https://github.com/dtolnay/rust-toolchain
uses: dtolnay/rust-toolchain@stable
- name: Set dependencies
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
env:
AS_DEPENDENCY: false
DO_DOCS: true
DO_FEATURE_MATRIX: true
DO_SCHEMARS_TESTS: true # Currently only used in hashes crate.
run: ./contrib/test.sh
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Beta:
needs: Prepare
name: Test - beta toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Checkout Toolchain
uses: dtolnay/rust-toolchain@beta
- name: Set dependencies
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
env:
AS_DEPENDENCY: false
run: ./contrib/test.sh
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Nightly:
needs: Prepare
name: Test - nightly toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test, lint, bench, docsrs]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
@ -54,20 +78,21 @@ jobs:
uses: dtolnay/rust-toolchain@nightly
- name: Install clippy
run: rustup component add clippy
- name: Set dependencies
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
env:
DO_LINT: true
DO_FMT: false
DO_BENCH: true
AS_DEPENDENCY: false
DO_DOCSRS: true
run: ./contrib/test.sh
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
MSRV:
needs: Prepare
name: Test - 1.56.1 toolchain
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: ${{ fromJson(needs.Prepare.outputs.crates) }}
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [test, feature_matrix]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
@ -75,10 +100,10 @@ jobs:
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.56.1"
- name: Set dependencies
run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock
- name: Running test script
env:
DO_FEATURE_MATRIX: true
run: ./contrib/test.sh
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Arch32bit:
name: Test 32-bit version
@ -137,8 +162,15 @@ jobs:
run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi --features=alloc
ASAN:
needs: Prepare
name: Address sanitizer # hashes crate only.
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [hashes]
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [asan]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
@ -147,22 +179,43 @@ jobs:
- name: Install src
run: rustup component add rust-src
- name: Running address sanitizer
env:
DO_ASAN: true
run: cd hashes && ./contrib/test.sh
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
WASM:
needs: Prepare
name: WebAssembly Build # hashes crate only.
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [hashes]
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [wasm]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Checkout Toolchain
uses: dtolnay/rust-toolchain@stable
- name: Running WASM build
env:
DO_WASM: true
run: cd hashes && ./contrib/test.sh
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Schemars:
needs: Prepare
name: Schemars
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [hashes]
dep: ${{ fromJson(needs.Prepare.outputs.deps) }}
task: [schemars]
steps:
- name: Checkout Crate
uses: actions/checkout@v4
- name: Checkout Toolchain
uses: dtolnay/rust-toolchain@stable
- name: Running schemars test
run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }}
Kani:
runs-on: ubuntu-20.04

6
bitcoin/contrib/extra_tests.sh Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -ex
cargo run --locked --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd
cargo run --locked --no-default-features --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd

View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Test all these features with "std" enabled.
FEATURES_WITH_STD="rand-std serde secp-recovery bitcoinconsensus-std base64 ordered"
# Test all these features without "std" or "alloc" enabled.
FEATURES_WITHOUT_STD="rand serde secp-recovery bitcoinconsensus base64 ordered"
# Run and lint these examples.
EXAMPLES="ecdsa-psbt:std,bitcoinconsensus sign-tx-segwit-v0:rand-std sign-tx-taproot:rand-std taproot-psbt:bitcoinconsensus,rand-std"

12
contrib/get_matrix.sh Executable file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
. contrib/test_vars.sh
crates="`cargo metadata --no-deps --format-version 1 | jq -c '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/"))'`"
deps="`echo -n $DEPS | jq -R -c 'split(" ")'`"
# debug
echo "$crates"
echo "$deps"
echo "crates=$crates" >> $GITHUB_OUTPUT
echo "deps=$deps" >> $GITHUB_OUTPUT

229
contrib/run_task.sh Executable file
View File

@ -0,0 +1,229 @@
#!/bin/env bash
set -ex
# Make all cargo invocations verbose.
export CARGO_TERM_VERBOSE=true
cargo --version
rustc --version
main() {
crate="$1"
task="$2"
cd "$crate"
# Building the fuzz crate is more-or-less just a sanity check.
if [ "$crate" == "fuzz" ]
then
cargo --locked build
exit 0
fi
# Every crate must define EXAMPLES.
. contrib/test_vars.sh || exit 1
case $task in
test)
do_test
;;
feature_matrix)
do_feature_matrix
;;
lint)
do_lint
;;
dup_deps)
do_dup_deps
;;
docs)
build_docs_with_stable_toolchain
;;
docsrs)
build_docs_with_nightly_toolchain
;;
wasm)
do_wasm
;;
asan)
do_asan
;;
bench)
do_bench
;;
schemars)
do_schemars
;;
*)
echo "Error: unknown task $task" >&2
exit 1
;;
esac
}
do_test() {
# Use the current (recent/minimal) lock file.
local cargo="cargo --locked"
# Defaults / sanity checks
$cargo build
$cargo test
for example in $EXAMPLES; do
name="$(echo "$example" | cut -d ':' -f 1)"
features="$(echo "$example" | cut -d ':' -f 2)"
$cargo run --example "$name" --features="$features"
done
if [ -e ./contrib/extra_tests.sh ];
then
./contrib/extra_tests.sh
fi
}
# Each crate defines its own feature matrix test so feature combinations
# can be better controlled.
do_feature_matrix() {
local cargo="cargo --locked"
$cargo build --no-default-features
$cargo test --no-default-features
# All crates have a "std" feature.
loop_features "std" "$FEATURES_WITH_STD"
# All but `bitcoin` crate have an "alloc" feature, this tests it
# along with any other features that should work with "std".
if [ -n "$FEATURES_WITHOUT_STD" ]
then
loop_features "" "$FEATURES_WITHOUT_STD"
fi
}
# Build with each feature as well as all combinations of two features.
#
# Usage: loop_features "std" "this-feature that-feature other"
loop_features() {
local use="$1"
local features="$2"
local cargo="cargo --locked"
# All the provided features including $use
$cargo build --no-default-features --features="$use $features"
$cargo test --no-default-features --features="$use $features"
read -r -a array <<< "$features"
local len="${#array[@]}"
if (( len > 1 )); then
for ((i = 0 ; i < len ; i++ ));
do
$cargo build --features="$use ${array[i]}"
$cargo test --features="$use ${array[i]}"
if (( i < len - 1 )); then
for ((j = i + 1 ; j < len ; j++ ));
do
$cargo build --features="$use ${array[i]} ${array[j]}"
$cargo test --features="$use ${array[i]} ${array[j]}"
done
fi
done
fi
}
# Lint the workspace then the individual crate examples.
do_lint() {
# Use the current (recent/minimal) lock file.
local cargo="cargo +nightly --locked"
$cargo clippy --workspace -- -D warnings
for example in $EXAMPLES; do
name=$(echo "$example" | cut -d ':' -f 1)
features=$(echo "$example" | cut -d ':' -f 2)
$cargo clippy --example "$name" --features="$features" -- -D warnings
done
}
# We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies
# in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp).
do_dup_deps() {
duplicate_dependencies=$(
# Only show the actual duplicated deps, not their reverse tree, then
# whitelist the 'syn' crate which is duplicated but it's not our fault.
cargo tree --target=all --all-features --duplicates \
| grep '^[0-9A-Za-z]' \
| grep -v 'syn' \
| wc -l
)
if [ "$duplicate_dependencies" -ne 0 ]; then
echo "Dependency tree is broken, contains duplicates"
cargo tree --target=all --all-features --duplicates
exit 1
fi
}
# Build the docs with a nightly toolchain, in unison with the function
# below this checks that we feature guarded docs imports correctly.
build_docs_with_nightly_toolchain() {
local cargo="cargo +nightly --locked"
RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features
}
# Build the docs with a stable toolchain, in unison with the function
# above this checks that we feature guarded docs imports correctly.
build_docs_with_stable_toolchain() {
local cargo="cargo +stable --locked"
RUSTDOCFLAGS="-D warnings" $cargo doc --all-features
}
do_wasm() {
clang --version &&
CARGO_TARGET_DIR=wasm cargo install --force wasm-pack &&
printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml &&
printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml &&
CC=clang-9 wasm-pack build &&
CC=clang-9 wasm-pack test --node;
}
do_asan() {
cargo clean
CC='clang -fsanitize=address -fno-omit-frame-pointer' \
RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \
ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \
cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
cargo clean
CC='clang -fsanitize=memory -fno-omit-frame-pointer' \
RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \
cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
}
# Bench only works with a non-stable toolchain (nightly, beta).
do_bench() {
RUSTFLAGS='--cfg=bench' cargo bench
}
# This is only relevant for hashes.
do_schemars() {
cd "extended_tests/schemars" > /dev/null
cargo test
}
#
# Main script
#
main "$@"
exit 0

4
contrib/test_vars.sh Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
CRATES="`cargo metadata --no-deps --format-version 1 | jq -j -r '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/")) | join(" ")'`"
DEPS="recent minimal"

11
hashes/contrib/extra_tests.sh Executable file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -ex
REPO_DIR=$(git rev-parse --show-toplevel)
if [ "$DO_SCHEMARS_TESTS" = true ]; then
pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null
cargo test
popd > /dev/null
fi

View File

@ -1,124 +0,0 @@
#!/usr/bin/env bash
set -ex
FEATURES="serde std io alloc"
cargo --version
rustc --version
# Work out if we are using a nightly toolchain.
NIGHTLY=false
if cargo --version | grep nightly >/dev/null; then
NIGHTLY=true
fi
# Pin dependencies as required if we are using MSRV toolchain.
if cargo --version | grep "1\.48"; then
# 1.0.157 uses syn 2.0 which requires edition 2021
cargo update -p serde_json --precise 1.0.99
cargo update -p serde --precise 1.0.156
fi
# Make all cargo invocations verbose
export CARGO_TERM_VERBOSE=true
# Defaults / sanity checks
cargo build
cargo test
if [ "$DO_LINT" = true ]
then
cargo clippy --locked --all-features --all-targets -- -D warnings
fi
if [ "$DO_FEATURE_MATRIX" = true ]; then
cargo build --locked --no-default-features
cargo test --locked --no-default-features
# All features
cargo build --locked --no-default-features --features="$FEATURES"
cargo test --locked --no-default-features --features="$FEATURES"
# Single features
for feature in ${FEATURES}
do
cargo build --locked --no-default-features --features="$feature"
cargo test --locked --no-default-features --features="$feature"
# All combos of two features
for featuretwo in ${FEATURES}; do
cargo build --locked --no-default-features --features="$feature $featuretwo"
cargo test --locked --no-default-features --features="$feature $featuretwo"
done
done
# Other combos
cargo test --locked --no-default-features --features="std,schemars"
fi
REPO_DIR=$(git rev-parse --show-toplevel)
if [ "$DO_SCHEMARS_TESTS" = true ]; then
pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null
cargo test
popd > /dev/null
fi
# Build the docs if told to (this only works with the nightly toolchain)
if [ "$DO_DOCSRS" = true ]; then
RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features
fi
# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command
# above this checks that we feature guarded docs imports correctly.
if [ "$DO_DOCS" = true ]; then
RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features
fi
# Webassembly stuff
if [ "$DO_WASM" = true ]; then
clang --version &&
CARGO_TARGET_DIR=wasm cargo install --force wasm-pack &&
printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml &&
printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml &&
CC=clang-9 wasm-pack build &&
CC=clang-9 wasm-pack test --node;
fi
# Address Sanitizer
if [ "$DO_ASAN" = true ]; then
cargo clean
CC='clang -fsanitize=address -fno-omit-frame-pointer' \
RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \
ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \
cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
cargo clean
CC='clang -fsanitize=memory -fno-omit-frame-pointer' \
RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \
cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
fi
# Run formatter if told to.
if [ "$DO_FMT" = true ]; then
if [ "$NIGHTLY" = false ]; then
echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)"
exit 1
fi
rustup component add rustfmt
cargo fmt --check
fi
# Bench if told to, only works with non-stable toolchain (nightly, beta).
if [ "$DO_BENCH" = true ]
then
if [ "$NIGHTLY" = false ]
then
if [ -n "$RUSTUP_TOOLCHAIN" ]
then
echo "RUSTUP_TOOLCHAIN is set to a non-nightly toolchain but DO_BENCH requires a nightly toolchain"
else
echo "DO_BENCH requires a nightly toolchain"
fi
exit 1
fi
RUSTFLAGS='--cfg=bench' cargo bench
fi

View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Test all these features with "std" enabled.
FEATURES_WITH_STD="io serde small-hash schemars"
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc serde small-hash"
# Run address sanitizer with these features.
ASAN_FEATURES="std io serde"
# Run and lint these examples.
EXAMPLES=""

View File

@ -1,2 +1,4 @@
#!/usr/bin/env bash
export RUSTFLAGS="-C link-arg=-Tlink.x"
export CARGO_TARGET_THUMBV7M_NONE_EABI_RUNNER="qemu-system-arm -cpu cortex-m3 -machine mps2-an385 -nographic -semihosting-config enable=on,target=native -kernel"

View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Test all these features with "std" enabled.
FEATURES_WITH_STD="serde"
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc serde"
# Run and lint these examples.
EXAMPLES=""

10
io/contrib/test_vars.sh Normal file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Test all these features with "std" enabled.
FEATURES_WITH_STD=""
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc"
# Run and lint these examples.
EXAMPLES=""

View File

@ -1,79 +0,0 @@
#!/usr/bin/env bash
set -ex
FEATURES="std alloc serde"
cargo --version
rustc --version
# Work out if we are using a nightly toolchain.
NIGHTLY=false
if cargo --version | grep nightly >/dev/null; then
NIGHTLY=true
fi
# Make all cargo invocations verbose
export CARGO_TERM_VERBOSE=true
# Defaults / sanity checks
cargo build
cargo test
if [ "$DO_LINT" = true ]
then
cargo clippy --locked --all-features --all-targets -- -D warnings
fi
if [ "$DO_FEATURE_MATRIX" = true ]; then
# No features
cargo build --locked --no-default-features
cargo test --locked --no-default-features
# Default features (this is std and alloc)
cargo build --locked
cargo test --locked
# All features
cargo build --locked --no-default-features --all-features
cargo test --locked --no-default-features --all-features
fi
REPO_DIR=$(git rev-parse --show-toplevel)
# Build the docs if told to (this only works with the nightly toolchain)
if [ "$DO_DOCSRS" = true ]; then
RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features
fi
# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command
# above this checks that we feature guarded docs imports correctly.
if [ "$DO_DOCS" = true ]; then
RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features
fi
# Run formatter if told to.
if [ "$DO_FMT" = true ]; then
if [ "$NIGHTLY" = false ]; then
echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)"
exit 1
fi
rustup component add rustfmt
cargo fmt --check
fi
# Bench if told to, only works with non-stable toolchain (nightly, beta).
if [ "$DO_BENCH" = true ]
then
if [ "$NIGHTLY" = false ]
then
if [ -n "$RUSTUP_TOOLCHAIN" ]
then
echo "RUSTUP_TOOLCHAIN is set to a non-nightly toolchain but DO_BENCH requires a nightly toolchain"
else
echo "DO_BENCH requires a nightly toolchain"
fi
exit 1
fi
RUSTFLAGS='--cfg=bench' cargo bench
fi

View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Test all these features with "std" enabled.
FEATURES_WITH_STD="serde"
# Test all these features without "std" enabled.
FEATURES_WITHOUT_STD="alloc serde"
# Run and lint these examples.
EXAMPLES=""