From 8d7117bb0eeb740946e979a25868b7028842eff8 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Fri, 3 May 2024 12:03:36 +1000 Subject: [PATCH 1/9] CI: Use original name I've been a bit confused lately about pinning and the locks, at some stage recently I changed 'Set dependencies' to 'Copy lock file'. Its no biggy but the original that Kix wrote was correct and descriptive - it was me who was confused. Revert it back to the original. --- .github/workflows/rust.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index da20ed1d4..33df96b84 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -70,7 +70,7 @@ jobs: uses: dtolnay/rust-toolchain@stable with: toolchain: "1.56.1" - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" run: ./contrib/run_task.sh msrv @@ -109,7 +109,7 @@ jobs: uses: actions/checkout@v4 - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" run: ./contrib/run_task.sh docs @@ -149,7 +149,7 @@ jobs: uses: dtolnay/rust-toolchain@v1 with: toolchain: ${{ needs.Prepare.outputs.nightly_version }} - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" run: ./contrib/run_task.sh bench @@ -230,7 +230,7 @@ jobs: toolchain: ${{ needs.Prepare.outputs.nightly_version }} - name: Install src run: rustup component add rust-src - - name: "Copy lock file" + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" run: ./contrib/run_task.sh asan From 1fb12e1917cc90ab00e9a3754ffb3f636c0561f4 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 12:51:33 +1000 Subject: [PATCH 2/9] CI: Remove recent from schemars job We do not used `cargo --locked` in the schemars job so copying the recent lock file is deceiving. --- .github/workflows/rust.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 33df96b84..120a0cda2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -254,15 +254,12 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - dep: [recent] + # Note we do not use the recent lock file for schemars testing. steps: - name: "Checkout repo" uses: actions/checkout@v4 - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Copy lock file" - run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" run: ./contrib/run_task.sh schemars From cc14edf63f6aa27808073820c2171a99a22ce514 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 12:54:53 +1000 Subject: [PATCH 3/9] CI: Run the schemars job directly using cargo The `Schemars` job calls the `run_task.sh` script only to do a single invocation of `cargo test` - just call cargo directly. --- .github/workflows/rust.yml | 4 ++-- contrib/run_task.sh | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 120a0cda2..ddcc6ee61 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -260,8 +260,8 @@ jobs: uses: actions/checkout@v4 - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Run test script" - run: ./contrib/run_task.sh schemars + - name: "Run schemars test" + run: cd hashes/extended_tests/schemars && cargo test Kani: name: Kani codegen - stable toolchain diff --git a/contrib/run_task.sh b/contrib/run_task.sh index 424dd902a..f74dacf83 100755 --- a/contrib/run_task.sh +++ b/contrib/run_task.sh @@ -100,11 +100,6 @@ main() { do_asan ;; - schemars) - # hashes crate only. - do_schemars - ;; - *) err "Error: unknown task $task" ;; @@ -264,13 +259,6 @@ do_bench() { done } -# This is only relevant for hashes. -do_schemars() { - pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null - cargo test - popd > /dev/null -} - # Note we do not use the recent lock file or `--locked` when running the wasm tests. do_wasm() { pushd "$REPO_DIR/hashes" > /dev/null From 34072579363973265d7e718eb482ae8d46465914 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 12:56:49 +1000 Subject: [PATCH 4/9] CI: Add WASM script In preparation for using the `run_task` script from maintainer tools we want to have all the things that are particular to `rust-bitcoin` out of the current `run_task` script. The wasm test is specific to `hashes`. Add a script in `hashes/contrib` and call it from the wasm job. No test coverage change. --- .github/workflows/rust.yml | 4 ++-- contrib/run_task.sh | 19 ------------------- hashes/contrib/wasm.sh | 12 ++++++++++++ 3 files changed, 14 insertions(+), 21 deletions(-) create mode 100755 hashes/contrib/wasm.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ddcc6ee61..8448d5571 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -246,8 +246,8 @@ jobs: uses: actions/checkout@v4 - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: "Run test script" - run: ./contrib/run_task.sh wasm + - name: "Run wasm script" + run: cd hashes && ./contrib/wasm.sh Schemars: # hashes crate only. name: Schemars - stable toolchain diff --git a/contrib/run_task.sh b/contrib/run_task.sh index f74dacf83..2bb10dddc 100755 --- a/contrib/run_task.sh +++ b/contrib/run_task.sh @@ -90,11 +90,6 @@ main() { do_bench ;; - wasm) - # hashes crate only. - do_wasm - ;; - asan) # hashes crate only - hashes/contrib/test_vars.sh is sourced in this function. do_asan @@ -259,20 +254,6 @@ do_bench() { done } -# Note we do not use the recent lock file or `--locked` when running the wasm tests. -do_wasm() { - pushd "$REPO_DIR/hashes" > /dev/null - - clang --version && - CARGO_TARGET_DIR=wasm cargo install --force wasm-pack && - printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml && - printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml && - CC=clang-9 wasm-pack build && - CC=clang-9 wasm-pack test --node; - - popd > /dev/null -} - do_asan() { pushd "$REPO_DIR/hashes" > /dev/null diff --git a/hashes/contrib/wasm.sh b/hashes/contrib/wasm.sh new file mode 100755 index 000000000..6139e3078 --- /dev/null +++ b/hashes/contrib/wasm.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Run the WASM tests. + +set -euox pipefail + +clang --version && + CARGO_TARGET_DIR=wasm cargo install --force wasm-pack && + printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml && + printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml && + CC=clang-9 wasm-pack build && + CC=clang-9 wasm-pack test --node; From 44cb2255d3761df825ca4f130bafaa25119a1b08 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 12:59:55 +1000 Subject: [PATCH 5/9] CI: Add sanitizer script As we did for the wasm job. In preparation for using the `run_task` script from maintainer tools we want to have all the things that are particular to `rust-bitcoin` out of the current `run_task` script. The address/memory sanitizer test is specific to `hashes`. Add a script in `hashes/contrib` and call it from the ASAN job. No test coverage change. --- .github/workflows/rust.yml | 4 ++-- contrib/run_task.sh | 26 -------------------------- hashes/contrib/sanitizer.sh | 21 +++++++++++++++++++++ hashes/contrib/test_vars.sh | 3 --- 4 files changed, 23 insertions(+), 31 deletions(-) create mode 100755 hashes/contrib/sanitizer.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8448d5571..489dd211e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -232,8 +232,8 @@ jobs: run: rustup component add rust-src - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: "Run test script" - run: ./contrib/run_task.sh asan + - name: "Run sanitizer script" + run: cd ./hashes && ./contrib/sanitizer.sh WASM: # hashes crate only. name: WASM - stable toolchain diff --git a/contrib/run_task.sh b/contrib/run_task.sh index 2bb10dddc..1c7c54d1c 100755 --- a/contrib/run_task.sh +++ b/contrib/run_task.sh @@ -90,11 +90,6 @@ main() { do_bench ;; - asan) - # hashes crate only - hashes/contrib/test_vars.sh is sourced in this function. - do_asan - ;; - *) err "Error: unknown task $task" ;; @@ -254,27 +249,6 @@ do_bench() { done } -do_asan() { - pushd "$REPO_DIR/hashes" > /dev/null - - # Set ASAN_FEATURES - . contrib/test_vars.sh || exit 1 - - cargo clean - CC='clang -fsanitize=address -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ - cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - # There is currently a bug in the MemorySanitizer (MSAN) - disable the job for now. - # - # cargo clean - # CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ - # RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ - # cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - - popd > /dev/null -} - # Check all the commands we use are present in the current environment. check_required_commands() { need_cmd cargo diff --git a/hashes/contrib/sanitizer.sh b/hashes/contrib/sanitizer.sh new file mode 100755 index 000000000..5a1e56c2f --- /dev/null +++ b/hashes/contrib/sanitizer.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Run the Address/Memory Sanitizer tests. + +set -euox pipefail + +# Run the sanitizer with these features. +FEATURES="std io serde" + +cargo clean +CC='clang -fsanitize=address -fno-omit-frame-pointer' \ + RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ + ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ + cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu + +# There is currently a bug in the MemorySanitizer (MSAN) - disable the job for now. +# +# cargo clean +# CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ + # RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ + # cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu diff --git a/hashes/contrib/test_vars.sh b/hashes/contrib/test_vars.sh index 5e89c21e7..d4073ddbe 100644 --- a/hashes/contrib/test_vars.sh +++ b/hashes/contrib/test_vars.sh @@ -6,8 +6,5 @@ FEATURES_WITH_STD="io serde small-hash schemars" # Test all these features without "std" enabled. FEATURES_WITHOUT_STD="alloc serde small-hash" -# Run address sanitizer with these features. -ASAN_FEATURES="std io serde" - # Run these examples. EXAMPLES="" From 0c0e88165e4e8ffa88497feb10ccbcf878950d63 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 13:01:41 +1000 Subject: [PATCH 6/9] CI: Add README file In preparation for using the `run_task` file from maintainer tools pull the docs out and throw them in a readme file in the workflow directory. --- .github/workflows/README.md | 34 ++++++++++++++++++++++++++++++++++ contrib/run_task.sh | 37 ------------------------------------- 2 files changed, 34 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..b8896a950 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,34 @@ +# rust-bitcoin workflow notes + +We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). + +ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration + +The minimal/recent lock files are handled by CI (`rust.yml`). + +## Jobs + +Run from `rust.yml` unless stated otherwise. Total 21 jobs but +`Prepare` is quick and must be run first anyway. + +0. `Prepare` +1. `Stable - minimal` +2. `Stable - recent` +3. `Nightly - minimal` +4. `Nightly - recent` +5. `MSRV - minimal` +6. `MSRV - recent` +7. `Lint` +8. `Docs` +9. `Docsrs` +10. `Bench` +11. `ASAN` +12. `WASM` +13. `schemars` +14. `Arch32bit` +15. `Cross` +16. `Embedded` +17. `Kani` +18. `Coveralls` - run by `coveralls.yml` +19. `release` - run by `release.yml` +20. `labeler` - run by `manage-pr.yml` diff --git a/contrib/run_task.sh b/contrib/run_task.sh index 1c7c54d1c..ff647eec6 100755 --- a/contrib/run_task.sh +++ b/contrib/run_task.sh @@ -1,41 +1,4 @@ #!/usr/bin/env bash -# -# We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). -# -# ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration -# -# The minimal/recent manifests are handled by CI (rust.yaml). -# -# Jobs (shell functions) that get run twice, once for each manifest: -# -# 1+2 stable -# 3+4 nightly -# 5+6 msrv -# -# Jobs (shell functions) that get run once: -# -# 7 lint -# 8 docs -# 9 docsrs -# 10 bench -# 11 asan -# 12 wasm -# 13 schemars -# -# Jobs run directly by rust.yml workflow: -# -# 0 Prepare -# -# 14 Arch32bit -# 15 Cross -# 16 Embedded -# 17 Kani -# -# Jobs run directly from other workflows: -# -# 18 Coveralls - run by coveralls.yml -# 19 release - run by release.yml -# 20 labeler - run by manage-pr.yml set -euox pipefail From 62ba10503a7528fb2b34f12d5450d7f5e2dc09d4 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Fri, 3 May 2024 11:41:19 +1000 Subject: [PATCH 7/9] CI: Use correct spacing `yamllint` emits a few warnings of form: warning too few spaces before comment (comments) Use "correct" spacing. --- .github/workflows/rust.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 489dd211e..5855ddacb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -213,7 +213,7 @@ jobs: - name: "Run hashes/embedded with alloc" run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi --features=alloc - ASAN: # hashes crate only. + ASAN: # hashes crate only. name: ASAN - nightly toolchain needs: Prepare runs-on: ubuntu-latest @@ -235,7 +235,7 @@ jobs: - name: "Run sanitizer script" run: cd ./hashes && ./contrib/sanitizer.sh - WASM: # hashes crate only. + WASM: # hashes crate only. name: WASM - stable toolchain runs-on: ubuntu-latest strategy: @@ -249,7 +249,7 @@ jobs: - name: "Run wasm script" run: cd hashes && ./contrib/wasm.sh - Schemars: # hashes crate only. + Schemars: # hashes crate only. name: Schemars - stable toolchain runs-on: ubuntu-latest strategy: From c5af52847b80836f4a94bcf73983324fea1b4435 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Fri, 3 May 2024 11:42:42 +1000 Subject: [PATCH 8/9] CI: Add docs and document start Add a document start and comment to help try to stop the readme from going stale. This removes a `yamllint` warning. While we are clearing lint warnings disable the one causing warning truthy value should be one of [false, true] (truthy) This is a know issue with GitHub actions yaml because `on` is a yaml keyword, or something like that. --- .github/workflows/rust.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5855ddacb..5f0971d85 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,4 +1,5 @@ -on: +--- # rust-bitcoin CI: If you edit this file please update README.md +on: # yamllint disable-line rule:truthy push: branches: - master From 6def5bc974b1907e39c44d7e2144af3603a6c1e1 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 13:08:20 +1000 Subject: [PATCH 9/9] CI: Use run_task from maintainer tools Use the shiny new `run_task.sh` script from maintainer tools. This patch should not change the test coverage in any way. --- .github/workflows/rust.yml | 56 +++++++-- contrib/crates.sh | 4 + contrib/get_matrix.sh | 12 -- contrib/run_task.sh | 247 ------------------------------------- contrib/test_vars.sh | 5 + 5 files changed, 58 insertions(+), 266 deletions(-) create mode 100755 contrib/crates.sh delete mode 100755 contrib/get_matrix.sh delete mode 100755 contrib/run_task.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5f0971d85..7f778e8fa 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,12 +30,18 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh stable + run: ./maintainer-tools/ci/run_task.sh stable Nightly: # 2 jobs, one per manifest. name: Test - nightly toolchain @@ -48,6 +54,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -55,7 +67,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh nightly + run: ./maintainer-tools/ci/run_task.sh nightly MSRV: # 2 jobs, one per manifest. name: Test - 1.56.1 toolchain @@ -67,6 +79,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable with: @@ -74,7 +92,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh msrv + run: ./maintainer-tools/ci/run_task.sh msrv Lint: name: Lint - nightly toolchain @@ -87,6 +105,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -96,7 +120,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh lint + run: ./maintainer-tools/ci/run_task.sh lint Docs: name: Docs - stable toolchain @@ -108,12 +132,18 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh docs + run: ./maintainer-tools/ci/run_task.sh docs Docsrs: name: Docs - nightly toolchain @@ -126,6 +156,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -133,7 +169,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh docsrs + run: ./maintainer-tools/ci/run_task.sh docsrs Bench: name: Bench - nightly toolchain @@ -146,6 +182,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -153,7 +195,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh bench + run: ./maintainer-tools/ci/run_task.sh bench Arch32bit: name: Test 32-bit version diff --git a/contrib/crates.sh b/contrib/crates.sh new file mode 100755 index 000000000..4686b3b06 --- /dev/null +++ b/contrib/crates.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Crates in this workspace to test (note "fuzz" is only built not tested). +CRATES=("base58" "bitcoin" "fuzz" "hashes" "internals" "io" "units") diff --git a/contrib/get_matrix.sh b/contrib/get_matrix.sh deleted file mode 100755 index d72a865bc..000000000 --- a/contrib/get_matrix.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -. contrib/test_vars.sh - -crates="`cargo metadata --no-deps --format-version 1 | jq -c '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/"))'`" -deps="`echo -n $DEPS | jq -R -c 'split(" ")'`" -# debug -echo "$crates" -echo "$deps" - -echo "crates=$crates" >> $GITHUB_OUTPUT -echo "deps=$deps" >> $GITHUB_OUTPUT diff --git a/contrib/run_task.sh b/contrib/run_task.sh deleted file mode 100755 index ff647eec6..000000000 --- a/contrib/run_task.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env bash - -set -euox pipefail - -REPO_DIR=$(git rev-parse --show-toplevel) -CRATES=("bitcoin" "hashes" "internals" "io" "units" "base58") - -# Make all cargo invocations verbose. -export CARGO_TERM_VERBOSE=true - -main() { - local task="$1" - - check_required_commands - - cargo --version - rustc --version - /usr/bin/env bash --version - locale - env - - case $task in - # 2 jobs each for these (one for each lock file). - stable) - # Test, run examples, do feature matrix. - # crate/contrib/test_vars.sh is sourced in this function. - build_and_test - ;; - - nightly) - build_and_test - ;; - - msrv) - build_and_test - ;; - - # 1 job each for these. - lint) - do_lint - do_dup_deps - ;; - - docs) - build_docs_with_stable_toolchain - ;; - - docsrs) - build_docs_with_nightly_toolchain - ;; - - bench) - do_bench - ;; - - *) - err "Error: unknown task $task" - ;; - esac -} - -# Build and test for each crate, done with each toolchain. -build_and_test() { - # Building the fuzz crate is more-or-less just a sanity check. - pushd "$REPO_DIR/fuzz" > /dev/null - cargo --locked build - popd > /dev/null - - for crate in "${CRATES[@]}"; do - pushd "$REPO_DIR/$crate" > /dev/null - - # Set crate specific variables. - . contrib/test_vars.sh || exit 1 - - do_test - do_feature_matrix - - popd > /dev/null - done -} - -do_test() { - # Use the current (recent/minimal) lock file. - local cargo="cargo --locked" - - # Defaults / sanity checks - $cargo build - $cargo test - - for example in $EXAMPLES; do # EXAMPLES is set in contrib/test_vars.sh - name="$(echo "$example" | cut -d ':' -f 1)" - features="$(echo "$example" | cut -d ':' -f 2)" - $cargo run --example "$name" --features="$features" - done - - if [ -e ./contrib/extra_tests.sh ]; - then - ./contrib/extra_tests.sh - fi -} - -# Each crate defines its own feature matrix test so feature combinations -# can be better controlled. -do_feature_matrix() { - local cargo="cargo --locked" - - $cargo build --no-default-features - $cargo test --no-default-features - - # All crates have a "std" feature and FEATURES_WITH_STD is set in - # contrib/test_vars.sh - loop_features "std" "$FEATURES_WITH_STD" - - # All but `bitcoin` crate have an "alloc" feature, this tests it - # along with any other features that should work with "std". - if [ -n "$FEATURES_WITHOUT_STD" ] - then - loop_features "" "$FEATURES_WITHOUT_STD" - fi -} - -# Build with each feature as well as all combinations of two features. -# -# Usage: loop_features "std" "this-feature that-feature other" -loop_features() { - local use="$1" - local features="$2" - local cargo="cargo --locked" - - # All the provided features including $use - $cargo build --no-default-features --features="$use $features" - $cargo test --no-default-features --features="$use $features" - - read -r -a array <<< "$features" - local len="${#array[@]}" - - if (( len > 1 )); then - for ((i = 0 ; i < len ; i++ )); - do - $cargo build --features="$use ${array[i]}" - $cargo test --features="$use ${array[i]}" - - if (( i < len - 1 )); then - for ((j = i + 1 ; j < len ; j++ )); - do - $cargo build --features="$use ${array[i]} ${array[j]}" - $cargo test --features="$use ${array[i]} ${array[j]}" - done - fi - done - fi -} - -# Lint the workspace. -do_lint() { - need_nightly - local cargo="cargo --locked" - - # Lint various feature combinations to try and catch mistakes in feature gating. - $cargo clippy --workspace --all-targets --keep-going -- -D warnings - $cargo clippy --workspace --all-targets --all-features --keep-going -- -D warnings - $cargo clippy --workspace --all-targets --no-default-features --keep-going -- -D warnings -} - -# We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies -# in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp). -do_dup_deps() { - # We can't use pipefail because these grep statements fail by design when there is no duplicate, - # the shell therefore won't pick up mistakes in your pipe - you are on your own. - set +o pipefail - - duplicate_dependencies=$( - # Only show the actual duplicated deps, not their reverse tree, then - # whitelist the 'syn' crate which is duplicated but it's not our fault. - # - # Temporarily allow 2 versions of `hashes`, `internals`, and `hex` while we upgrade. - cargo tree --target=all --all-features --duplicates \ - | grep '^[0-9A-Za-z]' \ - | grep -v 'syn' \ - | wc -l - ) - if [ "$duplicate_dependencies" -ne 0 ]; then - echo "Dependency tree is broken, contains duplicates" - cargo tree --target=all --all-features --duplicates - exit 1 - fi - - set -o pipefail -} - -# Build the docs with a nightly toolchain, in unison with the function -# below this checks that we feature guarded docs imports correctly. -build_docs_with_nightly_toolchain() { - need_nightly - local cargo="cargo --locked" - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features -} - -# Build the docs with a stable toolchain, in unison with the function -# above this checks that we feature guarded docs imports correctly. -build_docs_with_stable_toolchain() { - local cargo="cargo +stable --locked" - RUSTDOCFLAGS="-D warnings" $cargo doc --all-features -} - -# Bench only works with a non-stable toolchain (nightly, beta). -do_bench() { - for crate in bitcoin hashes; do - pushd "$REPO_DIR/$crate" > /dev/null - RUSTFLAGS='--cfg=bench' cargo bench - popd > /dev/null - done -} - -# Check all the commands we use are present in the current environment. -check_required_commands() { - need_cmd cargo - need_cmd rustc - need_cmd jq - need_cmd cut - need_cmd grep - need_cmd wc -} - -need_cmd() { - if ! command -v "$1" > /dev/null 2>&1 - then err "need '$1' (command not found)" - fi -} - -need_nightly() { - cargo_ver=$(cargo --version) - if echo "$cargo_ver" | grep -q -v nightly; then - err "Need a nightly compiler; have $(cargo --version)" - fi -} - -err() { - echo "$1" >&2 - exit 1 -} - -# -# Main script -# -main "$@" -exit 0 diff --git a/contrib/test_vars.sh b/contrib/test_vars.sh index 3f96f89e7..eb2b25119 100644 --- a/contrib/test_vars.sh +++ b/contrib/test_vars.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +# +# Used by labeler.yaml +# +# Not to be confused with the per crate `test_vars.sh` used by +# `rust-bitcoin-maintainer-tools-run_task.sh`. CRATES="`cargo metadata --no-deps --format-version 1 | jq -j -r '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/")) | join(" ")'`" DEPS="recent minimal"