diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..b8896a950 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,34 @@ +# rust-bitcoin workflow notes + +We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). + +ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration + +The minimal/recent lock files are handled by CI (`rust.yml`). + +## Jobs + +Run from `rust.yml` unless stated otherwise. Total 21 jobs but +`Prepare` is quick and must be run first anyway. + +0. `Prepare` +1. `Stable - minimal` +2. `Stable - recent` +3. `Nightly - minimal` +4. `Nightly - recent` +5. `MSRV - minimal` +6. `MSRV - recent` +7. `Lint` +8. `Docs` +9. `Docsrs` +10. `Bench` +11. `ASAN` +12. `WASM` +13. `schemars` +14. `Arch32bit` +15. `Cross` +16. `Embedded` +17. `Kani` +18. `Coveralls` - run by `coveralls.yml` +19. `release` - run by `release.yml` +20. `labeler` - run by `manage-pr.yml` diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index da20ed1d4..7f778e8fa 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,4 +1,5 @@ -on: +--- # rust-bitcoin CI: If you edit this file please update README.md +on: # yamllint disable-line rule:truthy push: branches: - master @@ -29,12 +30,18 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh stable + run: ./maintainer-tools/ci/run_task.sh stable Nightly: # 2 jobs, one per manifest. name: Test - nightly toolchain @@ -47,6 +54,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -54,7 +67,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh nightly + run: ./maintainer-tools/ci/run_task.sh nightly MSRV: # 2 jobs, one per manifest. name: Test - 1.56.1 toolchain @@ -66,14 +79,20 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable with: toolchain: "1.56.1" - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh msrv + run: ./maintainer-tools/ci/run_task.sh msrv Lint: name: Lint - nightly toolchain @@ -86,6 +105,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -95,7 +120,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh lint + run: ./maintainer-tools/ci/run_task.sh lint Docs: name: Docs - stable toolchain @@ -107,12 +132,18 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh docs + run: ./maintainer-tools/ci/run_task.sh docs Docsrs: name: Docs - nightly toolchain @@ -125,6 +156,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -132,7 +169,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh docsrs + run: ./maintainer-tools/ci/run_task.sh docsrs Bench: name: Bench - nightly toolchain @@ -145,14 +182,20 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: toolchain: ${{ needs.Prepare.outputs.nightly_version }} - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh bench + run: ./maintainer-tools/ci/run_task.sh bench Arch32bit: name: Test 32-bit version @@ -213,7 +256,7 @@ jobs: - name: "Run hashes/embedded with alloc" run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi --features=alloc - ASAN: # hashes crate only. + ASAN: # hashes crate only. name: ASAN - nightly toolchain needs: Prepare runs-on: ubuntu-latest @@ -230,12 +273,12 @@ jobs: toolchain: ${{ needs.Prepare.outputs.nightly_version }} - name: Install src run: rustup component add rust-src - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: "Run test script" - run: ./contrib/run_task.sh asan + - name: "Run sanitizer script" + run: cd ./hashes && ./contrib/sanitizer.sh - WASM: # hashes crate only. + WASM: # hashes crate only. name: WASM - stable toolchain runs-on: ubuntu-latest strategy: @@ -246,25 +289,22 @@ jobs: uses: actions/checkout@v4 - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Run test script" - run: ./contrib/run_task.sh wasm + - name: "Run wasm script" + run: cd hashes && ./contrib/wasm.sh - Schemars: # hashes crate only. + Schemars: # hashes crate only. name: Schemars - stable toolchain runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - dep: [recent] + # Note we do not use the recent lock file for schemars testing. steps: - name: "Checkout repo" uses: actions/checkout@v4 - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Copy lock file" - run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: "Run test script" - run: ./contrib/run_task.sh schemars + - name: "Run schemars test" + run: cd hashes/extended_tests/schemars && cargo test Kani: name: Kani codegen - stable toolchain diff --git a/contrib/crates.sh b/contrib/crates.sh new file mode 100755 index 000000000..4686b3b06 --- /dev/null +++ b/contrib/crates.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Crates in this workspace to test (note "fuzz" is only built not tested). +CRATES=("base58" "bitcoin" "fuzz" "hashes" "internals" "io" "units") diff --git a/contrib/get_matrix.sh b/contrib/get_matrix.sh deleted file mode 100755 index d72a865bc..000000000 --- a/contrib/get_matrix.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -. contrib/test_vars.sh - -crates="`cargo metadata --no-deps --format-version 1 | jq -c '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/"))'`" -deps="`echo -n $DEPS | jq -R -c 'split(" ")'`" -# debug -echo "$crates" -echo "$deps" - -echo "crates=$crates" >> $GITHUB_OUTPUT -echo "deps=$deps" >> $GITHUB_OUTPUT diff --git a/contrib/run_task.sh b/contrib/run_task.sh deleted file mode 100755 index 424dd902a..000000000 --- a/contrib/run_task.sh +++ /dev/null @@ -1,341 +0,0 @@ -#!/usr/bin/env bash -# -# We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). -# -# ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration -# -# The minimal/recent manifests are handled by CI (rust.yaml). -# -# Jobs (shell functions) that get run twice, once for each manifest: -# -# 1+2 stable -# 3+4 nightly -# 5+6 msrv -# -# Jobs (shell functions) that get run once: -# -# 7 lint -# 8 docs -# 9 docsrs -# 10 bench -# 11 asan -# 12 wasm -# 13 schemars -# -# Jobs run directly by rust.yml workflow: -# -# 0 Prepare -# -# 14 Arch32bit -# 15 Cross -# 16 Embedded -# 17 Kani -# -# Jobs run directly from other workflows: -# -# 18 Coveralls - run by coveralls.yml -# 19 release - run by release.yml -# 20 labeler - run by manage-pr.yml - -set -euox pipefail - -REPO_DIR=$(git rev-parse --show-toplevel) -CRATES=("bitcoin" "hashes" "internals" "io" "units" "base58") - -# Make all cargo invocations verbose. -export CARGO_TERM_VERBOSE=true - -main() { - local task="$1" - - check_required_commands - - cargo --version - rustc --version - /usr/bin/env bash --version - locale - env - - case $task in - # 2 jobs each for these (one for each lock file). - stable) - # Test, run examples, do feature matrix. - # crate/contrib/test_vars.sh is sourced in this function. - build_and_test - ;; - - nightly) - build_and_test - ;; - - msrv) - build_and_test - ;; - - # 1 job each for these. - lint) - do_lint - do_dup_deps - ;; - - docs) - build_docs_with_stable_toolchain - ;; - - docsrs) - build_docs_with_nightly_toolchain - ;; - - bench) - do_bench - ;; - - wasm) - # hashes crate only. - do_wasm - ;; - - asan) - # hashes crate only - hashes/contrib/test_vars.sh is sourced in this function. - do_asan - ;; - - schemars) - # hashes crate only. - do_schemars - ;; - - *) - err "Error: unknown task $task" - ;; - esac -} - -# Build and test for each crate, done with each toolchain. -build_and_test() { - # Building the fuzz crate is more-or-less just a sanity check. - pushd "$REPO_DIR/fuzz" > /dev/null - cargo --locked build - popd > /dev/null - - for crate in "${CRATES[@]}"; do - pushd "$REPO_DIR/$crate" > /dev/null - - # Set crate specific variables. - . contrib/test_vars.sh || exit 1 - - do_test - do_feature_matrix - - popd > /dev/null - done -} - -do_test() { - # Use the current (recent/minimal) lock file. - local cargo="cargo --locked" - - # Defaults / sanity checks - $cargo build - $cargo test - - for example in $EXAMPLES; do # EXAMPLES is set in contrib/test_vars.sh - name="$(echo "$example" | cut -d ':' -f 1)" - features="$(echo "$example" | cut -d ':' -f 2)" - $cargo run --example "$name" --features="$features" - done - - if [ -e ./contrib/extra_tests.sh ]; - then - ./contrib/extra_tests.sh - fi -} - -# Each crate defines its own feature matrix test so feature combinations -# can be better controlled. -do_feature_matrix() { - local cargo="cargo --locked" - - $cargo build --no-default-features - $cargo test --no-default-features - - # All crates have a "std" feature and FEATURES_WITH_STD is set in - # contrib/test_vars.sh - loop_features "std" "$FEATURES_WITH_STD" - - # All but `bitcoin` crate have an "alloc" feature, this tests it - # along with any other features that should work with "std". - if [ -n "$FEATURES_WITHOUT_STD" ] - then - loop_features "" "$FEATURES_WITHOUT_STD" - fi -} - -# Build with each feature as well as all combinations of two features. -# -# Usage: loop_features "std" "this-feature that-feature other" -loop_features() { - local use="$1" - local features="$2" - local cargo="cargo --locked" - - # All the provided features including $use - $cargo build --no-default-features --features="$use $features" - $cargo test --no-default-features --features="$use $features" - - read -r -a array <<< "$features" - local len="${#array[@]}" - - if (( len > 1 )); then - for ((i = 0 ; i < len ; i++ )); - do - $cargo build --features="$use ${array[i]}" - $cargo test --features="$use ${array[i]}" - - if (( i < len - 1 )); then - for ((j = i + 1 ; j < len ; j++ )); - do - $cargo build --features="$use ${array[i]} ${array[j]}" - $cargo test --features="$use ${array[i]} ${array[j]}" - done - fi - done - fi -} - -# Lint the workspace. -do_lint() { - need_nightly - local cargo="cargo --locked" - - # Lint various feature combinations to try and catch mistakes in feature gating. - $cargo clippy --workspace --all-targets --keep-going -- -D warnings - $cargo clippy --workspace --all-targets --all-features --keep-going -- -D warnings - $cargo clippy --workspace --all-targets --no-default-features --keep-going -- -D warnings -} - -# We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies -# in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp). -do_dup_deps() { - # We can't use pipefail because these grep statements fail by design when there is no duplicate, - # the shell therefore won't pick up mistakes in your pipe - you are on your own. - set +o pipefail - - duplicate_dependencies=$( - # Only show the actual duplicated deps, not their reverse tree, then - # whitelist the 'syn' crate which is duplicated but it's not our fault. - # - # Temporarily allow 2 versions of `hashes`, `internals`, and `hex` while we upgrade. - cargo tree --target=all --all-features --duplicates \ - | grep '^[0-9A-Za-z]' \ - | grep -v 'syn' \ - | wc -l - ) - if [ "$duplicate_dependencies" -ne 0 ]; then - echo "Dependency tree is broken, contains duplicates" - cargo tree --target=all --all-features --duplicates - exit 1 - fi - - set -o pipefail -} - -# Build the docs with a nightly toolchain, in unison with the function -# below this checks that we feature guarded docs imports correctly. -build_docs_with_nightly_toolchain() { - need_nightly - local cargo="cargo --locked" - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features -} - -# Build the docs with a stable toolchain, in unison with the function -# above this checks that we feature guarded docs imports correctly. -build_docs_with_stable_toolchain() { - local cargo="cargo +stable --locked" - RUSTDOCFLAGS="-D warnings" $cargo doc --all-features -} - -# Bench only works with a non-stable toolchain (nightly, beta). -do_bench() { - for crate in bitcoin hashes; do - pushd "$REPO_DIR/$crate" > /dev/null - RUSTFLAGS='--cfg=bench' cargo bench - popd > /dev/null - done -} - -# This is only relevant for hashes. -do_schemars() { - pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null - cargo test - popd > /dev/null -} - -# Note we do not use the recent lock file or `--locked` when running the wasm tests. -do_wasm() { - pushd "$REPO_DIR/hashes" > /dev/null - - clang --version && - CARGO_TARGET_DIR=wasm cargo install --force wasm-pack && - printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml && - printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml && - CC=clang-9 wasm-pack build && - CC=clang-9 wasm-pack test --node; - - popd > /dev/null -} - -do_asan() { - pushd "$REPO_DIR/hashes" > /dev/null - - # Set ASAN_FEATURES - . contrib/test_vars.sh || exit 1 - - cargo clean - CC='clang -fsanitize=address -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ - cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - # There is currently a bug in the MemorySanitizer (MSAN) - disable the job for now. - # - # cargo clean - # CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ - # RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ - # cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - - popd > /dev/null -} - -# Check all the commands we use are present in the current environment. -check_required_commands() { - need_cmd cargo - need_cmd rustc - need_cmd jq - need_cmd cut - need_cmd grep - need_cmd wc -} - -need_cmd() { - if ! command -v "$1" > /dev/null 2>&1 - then err "need '$1' (command not found)" - fi -} - -need_nightly() { - cargo_ver=$(cargo --version) - if echo "$cargo_ver" | grep -q -v nightly; then - err "Need a nightly compiler; have $(cargo --version)" - fi -} - -err() { - echo "$1" >&2 - exit 1 -} - -# -# Main script -# -main "$@" -exit 0 diff --git a/contrib/test_vars.sh b/contrib/test_vars.sh index 3f96f89e7..eb2b25119 100644 --- a/contrib/test_vars.sh +++ b/contrib/test_vars.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +# +# Used by labeler.yaml +# +# Not to be confused with the per crate `test_vars.sh` used by +# `rust-bitcoin-maintainer-tools-run_task.sh`. CRATES="`cargo metadata --no-deps --format-version 1 | jq -j -r '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/")) | join(" ")'`" DEPS="recent minimal" diff --git a/hashes/contrib/sanitizer.sh b/hashes/contrib/sanitizer.sh new file mode 100755 index 000000000..5a1e56c2f --- /dev/null +++ b/hashes/contrib/sanitizer.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Run the Address/Memory Sanitizer tests. + +set -euox pipefail + +# Run the sanitizer with these features. +FEATURES="std io serde" + +cargo clean +CC='clang -fsanitize=address -fno-omit-frame-pointer' \ + RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ + ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ + cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu + +# There is currently a bug in the MemorySanitizer (MSAN) - disable the job for now. +# +# cargo clean +# CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ + # RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ + # cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu diff --git a/hashes/contrib/test_vars.sh b/hashes/contrib/test_vars.sh index 5e89c21e7..d4073ddbe 100644 --- a/hashes/contrib/test_vars.sh +++ b/hashes/contrib/test_vars.sh @@ -6,8 +6,5 @@ FEATURES_WITH_STD="io serde small-hash schemars" # Test all these features without "std" enabled. FEATURES_WITHOUT_STD="alloc serde small-hash" -# Run address sanitizer with these features. -ASAN_FEATURES="std io serde" - # Run these examples. EXAMPLES="" diff --git a/hashes/contrib/wasm.sh b/hashes/contrib/wasm.sh new file mode 100755 index 000000000..6139e3078 --- /dev/null +++ b/hashes/contrib/wasm.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Run the WASM tests. + +set -euox pipefail + +clang --version && + CARGO_TARGET_DIR=wasm cargo install --force wasm-pack && + printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml && + printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml && + CC=clang-9 wasm-pack build && + CC=clang-9 wasm-pack test --node;