From 6def5bc974b1907e39c44d7e2144af3603a6c1e1 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 13:08:20 +1000 Subject: [PATCH] CI: Use run_task from maintainer tools Use the shiny new `run_task.sh` script from maintainer tools. This patch should not change the test coverage in any way. --- .github/workflows/rust.yml | 56 +++++++-- contrib/crates.sh | 4 + contrib/get_matrix.sh | 12 -- contrib/run_task.sh | 247 ------------------------------------- contrib/test_vars.sh | 5 + 5 files changed, 58 insertions(+), 266 deletions(-) create mode 100755 contrib/crates.sh delete mode 100755 contrib/get_matrix.sh delete mode 100755 contrib/run_task.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5f0971d85..7f778e8fa 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,12 +30,18 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh stable + run: ./maintainer-tools/ci/run_task.sh stable Nightly: # 2 jobs, one per manifest. name: Test - nightly toolchain @@ -48,6 +54,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -55,7 +67,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh nightly + run: ./maintainer-tools/ci/run_task.sh nightly MSRV: # 2 jobs, one per manifest. name: Test - 1.56.1 toolchain @@ -67,6 +79,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable with: @@ -74,7 +92,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh msrv + run: ./maintainer-tools/ci/run_task.sh msrv Lint: name: Lint - nightly toolchain @@ -87,6 +105,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -96,7 +120,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh lint + run: ./maintainer-tools/ci/run_task.sh lint Docs: name: Docs - stable toolchain @@ -108,12 +132,18 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh docs + run: ./maintainer-tools/ci/run_task.sh docs Docsrs: name: Docs - nightly toolchain @@ -126,6 +156,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -133,7 +169,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh docsrs + run: ./maintainer-tools/ci/run_task.sh docsrs Bench: name: Bench - nightly toolchain @@ -146,6 +182,12 @@ jobs: steps: - name: "Checkout repo" uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: @@ -153,7 +195,7 @@ jobs: - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - name: "Run test script" - run: ./contrib/run_task.sh bench + run: ./maintainer-tools/ci/run_task.sh bench Arch32bit: name: Test 32-bit version diff --git a/contrib/crates.sh b/contrib/crates.sh new file mode 100755 index 000000000..4686b3b06 --- /dev/null +++ b/contrib/crates.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Crates in this workspace to test (note "fuzz" is only built not tested). +CRATES=("base58" "bitcoin" "fuzz" "hashes" "internals" "io" "units") diff --git a/contrib/get_matrix.sh b/contrib/get_matrix.sh deleted file mode 100755 index d72a865bc..000000000 --- a/contrib/get_matrix.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -. contrib/test_vars.sh - -crates="`cargo metadata --no-deps --format-version 1 | jq -c '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/"))'`" -deps="`echo -n $DEPS | jq -R -c 'split(" ")'`" -# debug -echo "$crates" -echo "$deps" - -echo "crates=$crates" >> $GITHUB_OUTPUT -echo "deps=$deps" >> $GITHUB_OUTPUT diff --git a/contrib/run_task.sh b/contrib/run_task.sh deleted file mode 100755 index ff647eec6..000000000 --- a/contrib/run_task.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env bash - -set -euox pipefail - -REPO_DIR=$(git rev-parse --show-toplevel) -CRATES=("bitcoin" "hashes" "internals" "io" "units" "base58") - -# Make all cargo invocations verbose. -export CARGO_TERM_VERBOSE=true - -main() { - local task="$1" - - check_required_commands - - cargo --version - rustc --version - /usr/bin/env bash --version - locale - env - - case $task in - # 2 jobs each for these (one for each lock file). - stable) - # Test, run examples, do feature matrix. - # crate/contrib/test_vars.sh is sourced in this function. - build_and_test - ;; - - nightly) - build_and_test - ;; - - msrv) - build_and_test - ;; - - # 1 job each for these. - lint) - do_lint - do_dup_deps - ;; - - docs) - build_docs_with_stable_toolchain - ;; - - docsrs) - build_docs_with_nightly_toolchain - ;; - - bench) - do_bench - ;; - - *) - err "Error: unknown task $task" - ;; - esac -} - -# Build and test for each crate, done with each toolchain. -build_and_test() { - # Building the fuzz crate is more-or-less just a sanity check. - pushd "$REPO_DIR/fuzz" > /dev/null - cargo --locked build - popd > /dev/null - - for crate in "${CRATES[@]}"; do - pushd "$REPO_DIR/$crate" > /dev/null - - # Set crate specific variables. - . contrib/test_vars.sh || exit 1 - - do_test - do_feature_matrix - - popd > /dev/null - done -} - -do_test() { - # Use the current (recent/minimal) lock file. - local cargo="cargo --locked" - - # Defaults / sanity checks - $cargo build - $cargo test - - for example in $EXAMPLES; do # EXAMPLES is set in contrib/test_vars.sh - name="$(echo "$example" | cut -d ':' -f 1)" - features="$(echo "$example" | cut -d ':' -f 2)" - $cargo run --example "$name" --features="$features" - done - - if [ -e ./contrib/extra_tests.sh ]; - then - ./contrib/extra_tests.sh - fi -} - -# Each crate defines its own feature matrix test so feature combinations -# can be better controlled. -do_feature_matrix() { - local cargo="cargo --locked" - - $cargo build --no-default-features - $cargo test --no-default-features - - # All crates have a "std" feature and FEATURES_WITH_STD is set in - # contrib/test_vars.sh - loop_features "std" "$FEATURES_WITH_STD" - - # All but `bitcoin` crate have an "alloc" feature, this tests it - # along with any other features that should work with "std". - if [ -n "$FEATURES_WITHOUT_STD" ] - then - loop_features "" "$FEATURES_WITHOUT_STD" - fi -} - -# Build with each feature as well as all combinations of two features. -# -# Usage: loop_features "std" "this-feature that-feature other" -loop_features() { - local use="$1" - local features="$2" - local cargo="cargo --locked" - - # All the provided features including $use - $cargo build --no-default-features --features="$use $features" - $cargo test --no-default-features --features="$use $features" - - read -r -a array <<< "$features" - local len="${#array[@]}" - - if (( len > 1 )); then - for ((i = 0 ; i < len ; i++ )); - do - $cargo build --features="$use ${array[i]}" - $cargo test --features="$use ${array[i]}" - - if (( i < len - 1 )); then - for ((j = i + 1 ; j < len ; j++ )); - do - $cargo build --features="$use ${array[i]} ${array[j]}" - $cargo test --features="$use ${array[i]} ${array[j]}" - done - fi - done - fi -} - -# Lint the workspace. -do_lint() { - need_nightly - local cargo="cargo --locked" - - # Lint various feature combinations to try and catch mistakes in feature gating. - $cargo clippy --workspace --all-targets --keep-going -- -D warnings - $cargo clippy --workspace --all-targets --all-features --keep-going -- -D warnings - $cargo clippy --workspace --all-targets --no-default-features --keep-going -- -D warnings -} - -# We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies -# in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp). -do_dup_deps() { - # We can't use pipefail because these grep statements fail by design when there is no duplicate, - # the shell therefore won't pick up mistakes in your pipe - you are on your own. - set +o pipefail - - duplicate_dependencies=$( - # Only show the actual duplicated deps, not their reverse tree, then - # whitelist the 'syn' crate which is duplicated but it's not our fault. - # - # Temporarily allow 2 versions of `hashes`, `internals`, and `hex` while we upgrade. - cargo tree --target=all --all-features --duplicates \ - | grep '^[0-9A-Za-z]' \ - | grep -v 'syn' \ - | wc -l - ) - if [ "$duplicate_dependencies" -ne 0 ]; then - echo "Dependency tree is broken, contains duplicates" - cargo tree --target=all --all-features --duplicates - exit 1 - fi - - set -o pipefail -} - -# Build the docs with a nightly toolchain, in unison with the function -# below this checks that we feature guarded docs imports correctly. -build_docs_with_nightly_toolchain() { - need_nightly - local cargo="cargo --locked" - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features -} - -# Build the docs with a stable toolchain, in unison with the function -# above this checks that we feature guarded docs imports correctly. -build_docs_with_stable_toolchain() { - local cargo="cargo +stable --locked" - RUSTDOCFLAGS="-D warnings" $cargo doc --all-features -} - -# Bench only works with a non-stable toolchain (nightly, beta). -do_bench() { - for crate in bitcoin hashes; do - pushd "$REPO_DIR/$crate" > /dev/null - RUSTFLAGS='--cfg=bench' cargo bench - popd > /dev/null - done -} - -# Check all the commands we use are present in the current environment. -check_required_commands() { - need_cmd cargo - need_cmd rustc - need_cmd jq - need_cmd cut - need_cmd grep - need_cmd wc -} - -need_cmd() { - if ! command -v "$1" > /dev/null 2>&1 - then err "need '$1' (command not found)" - fi -} - -need_nightly() { - cargo_ver=$(cargo --version) - if echo "$cargo_ver" | grep -q -v nightly; then - err "Need a nightly compiler; have $(cargo --version)" - fi -} - -err() { - echo "$1" >&2 - exit 1 -} - -# -# Main script -# -main "$@" -exit 0 diff --git a/contrib/test_vars.sh b/contrib/test_vars.sh index 3f96f89e7..eb2b25119 100644 --- a/contrib/test_vars.sh +++ b/contrib/test_vars.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +# +# Used by labeler.yaml +# +# Not to be confused with the per crate `test_vars.sh` used by +# `rust-bitcoin-maintainer-tools-run_task.sh`. CRATES="`cargo metadata --no-deps --format-version 1 | jq -j -r '.packages | map(.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr("'$PWD'/")) | join(" ")'`" DEPS="recent minimal"