From e386cbfadf7b95d47ec1aef54bbbd359d1254f4f Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Tue, 20 Feb 2024 16:39:32 +0000 Subject: [PATCH 1/5] ci: delete *test.sh files These are not run in CI since #2353 and are likely to go out of date. If we want a script that users can run locally then we should create a new script that wraps our current CI. --- .github/labeler.yml | 6 +- CONTRIBUTING.md | 13 ---- bitcoin/contrib/test.sh | 131 -------------------------------------- contrib/test.sh | 31 --------- fuzz/contrib/test.sh | 44 ------------- internals/contrib/test.sh | 64 ------------------- 6 files changed, 4 insertions(+), 285 deletions(-) delete mode 100755 bitcoin/contrib/test.sh delete mode 100755 contrib/test.sh delete mode 100755 fuzz/contrib/test.sh delete mode 100755 internals/contrib/test.sh diff --git a/.github/labeler.yml b/.github/labeler.yml index 986c15a0..884b5961 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -6,8 +6,10 @@ test: - any-glob-to-any-file: fuzz/** - any-glob-to-any-file: '*/tests/**' - any-glob-to-any-file: 'dep_test' - - any-glob-to-any-file: 'contrib/test.sh' - - any-glob-to-any-file: '*/contrib/test.sh' + - any-glob-to-any-file: 'contrib/run_task.sh' + - any-glob-to-any-file: 'contrib/test_vars.sh' + - any-glob-to-any-file: '*/contrib/extra_tests.sh' + - any-glob-to-any-file: '*/contrib/test_vars.sh' doc: - changed-files: - any-glob-to-any-file: '**/*.md' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 595d7d22..b1ae86c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -124,19 +124,6 @@ NB: reviewers may run more complex test/CI scripts, thus, satisfying all the requirements above is just a preliminary, but not necessary sufficient step for getting the PR accepted as a valid candidate PR for the `master` branch. -PR authors may also find it useful to run the following script locally in order -to check that each of the commits within the PR satisfies the requirements -above, before submitting the PR to review: -```shell script -RUSTUP_TOOLCHAIN=1.41.1 ./contrib/test.sh -``` -Please replace the value in `RUSTUP_TOOLCHAIN=1.41.1` with the current MSRV from -[README.md]. - -NB: Please keep in mind that the script above replaces `Cargo.lock` file, which -is necessary to support current MSRV, incompatible with `stable` and newer cargo -versions. - ### Peer review Anyone may participate in peer review which is expressed by comments in the pull diff --git a/bitcoin/contrib/test.sh b/bitcoin/contrib/test.sh deleted file mode 100755 index d1b51511..00000000 --- a/bitcoin/contrib/test.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -FEATURES="std rand-std rand serde secp-recovery bitcoinconsensus-std base64 bitcoinconsensus" - -cargo --version -rustc --version - -# Some tests require certain toolchain types. -NIGHTLY=false -STABLE=true -if cargo --version | grep nightly; then - STABLE=false - NIGHTLY=true -fi -if cargo --version | grep beta; then - STABLE=false -fi - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Defaults / sanity checks -cargo build -cargo test - -if [ "$DO_LINT" = true ] -then - cargo clippy --locked --all-features --all-targets -- -D warnings - cargo clippy --locked --example bip32 -- -D warnings - cargo clippy --locked --example handshake --features=rand-std -- -D warnings - cargo clippy --locked --example ecdsa-psbt --features=bitcoinconsensus -- -D warnings - cargo clippy --locked --example sign-tx-segwit-v0 --features=rand-std -- -D warnings - cargo clippy --locked --example sign-tx-taproot --features=rand-std -- -D warnings - cargo clippy --locked --example taproot-psbt --features=rand-std,bitcoinconsensus -- -D warnings - - # We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies - # in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp). - duplicate_dependencies=$( - # Only show the actual duplicated deps, not their reverse tree, then - # whitelist the 'syn' crate which is duplicated but it's not our fault. - # - # Whitelist `bitcoin_hashes` while we release it and until secp v0.28.0 comes out. - cargo tree --target=all --all-features --duplicates \ - | grep '^[0-9A-Za-z]' \ - | grep -v 'syn' \ - | grep -v 'bitcoin_hashes' \ - | wc -l - ) - if [ "$duplicate_dependencies" -ne 0 ]; then - echo "Dependency tree is broken, contains duplicates" - cargo tree --target=all --all-features --duplicates - exit 1 - fi -fi - -if [ "$DO_FEATURE_MATRIX" = true ]; then - cargo build --locked --no-default-features - cargo test --locked --no-default-features - - # All features - cargo build --locked --no-default-features --features="$FEATURES" - cargo test --locked --no-default-features --features="$FEATURES" - # Single features - for feature in ${FEATURES} - do - cargo build --locked --no-default-features --features="$feature" - cargo test --locked --no-default-features --features="$feature" - # All combos of two features - for featuretwo in ${FEATURES}; do - cargo build --locked --no-default-features --features="$feature $featuretwo" - cargo test --locked --no-default-features --features="$feature $featuretwo" - done - done -fi - -cargo run --locked --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd -cargo run --locked --no-default-features --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd - -cargo run --locked --example ecdsa-psbt --features=bitcoinconsensus -cargo run --locked --example sign-tx-segwit-v0 --features=rand-std -- -D warnings -cargo run --locked --example sign-tx-taproot --features=rand-std -- -D warnings -cargo run --locked --example taproot-psbt --features=rand-std,bitcoinconsensus - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCSRS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features -fi - -# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command -# above this checks that we feature guarded docs imports correctly. -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features -fi - -# Run formatter if told to. -if [ "$DO_FMT" = true ]; then - if [ "$NIGHTLY" = false ]; then - echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)" - exit 1 - fi - rustup component add rustfmt - cargo fmt --check -fi - -# Bench if told to, only works with non-stable toolchain (nightly, beta). -if [ "$DO_BENCH" = true ] -then - if [ "$STABLE" = true ]; then - if [ -n "$RUSTUP_TOOLCHAIN" ]; then - echo "RUSTUP_TOOLCHAIN is set to a stable toolchain but DO_BENCH requires a non-stable (beta, nightly) toolchain" - else - echo "DO_BENCH requires a non-stable (beta, nightly) toolchain" - fi - exit 1 - fi - RUSTFLAGS='--cfg=bench' cargo bench -fi - -# Use as dependency if told to -if [ "$AS_DEPENDENCY" = true ] -then - cargo new dep_test 2> /dev/null # Mute warning about workspace, fixed below. - cd dep_test - echo 'bitcoin = { path = "..", features = ["serde"] }\n\n' >> Cargo.toml - # Adding an empty workspace section excludes this crate from the rust-bitcoin workspace. - echo '[workspace]\n\n' >> Cargo.toml - - cargo test --verbose -fi diff --git a/contrib/test.sh b/contrib/test.sh deleted file mode 100755 index a825919c..00000000 --- a/contrib/test.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -CRATES="bitcoin hashes units internals fuzz" -DEPS="recent minimal" - -for dep in $DEPS -do - cp "Cargo-$dep.lock" Cargo.lock - for crate in ${CRATES} - do - ( - cd "$crate" - ./contrib/test.sh - ) - done - if [ "$dep" = recent ]; - then - # We always test committed dependencies but we want to warn if they could've been updated - cargo update - if diff Cargo-recent.lock Cargo.lock; - then - echo Dependencies are up to date - else - echo "::warning file=Cargo-recent.lock::Dependencies could be updated" - fi - fi -done - -exit 0 diff --git a/fuzz/contrib/test.sh b/fuzz/contrib/test.sh deleted file mode 100755 index becc2953..00000000 --- a/fuzz/contrib/test.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -FEATURES="" - -cargo --version -rustc --version - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Pin dependencies as required if we are using MSRV toolchain. -if cargo --version | grep "1\.41"; then - # 1.0.157 uses syn 2.0 which requires edition 2021 - cargo update -p serde --precise 1.0.156 - # 1.0.108 uses `matches!` macro so does not work with Rust 1.41.1, bad `syn` no biscuit. - cargo update -p syn --precise 1.0.107 - # Half 1.8 uses edition 2021 features - cargo update -p half --precise 1.7.1 -fi - -if [ "$DO_LINT" = true ] -then - cargo clippy --all-features --all-targets -- -D warnings -fi - -# Defaults / sanity checks -cargo build -cargo test - -# Address Sanitizer -if [ "$DO_ASAN" = true ]; then - cargo clean - CC='clang -fsanitize=address -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ - cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - cargo clean - CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ - cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu -fi - diff --git a/internals/contrib/test.sh b/internals/contrib/test.sh deleted file mode 100755 index e8e375d5..00000000 --- a/internals/contrib/test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -FEATURES="std alloc" - -cargo --version -rustc --version - -# Work out if we are using a nightly toolchain. -NIGHTLY=false -if cargo --version | grep nightly >/dev/null; then - NIGHTLY=true -fi - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Defaults / sanity checks -cargo --locked build -cargo --locked test - -if [ "$DO_LINT" = true ] -then - cargo clippy --locked --all-features --all-targets -- -D warnings -fi - -if [ "$DO_FEATURE_MATRIX" = true ]; then - # No features - cargo build --locked --no-default-features - cargo test --locked --no-default-features - - # All features - cargo build --locked --no-default-features --features="$FEATURES" - cargo test --locked --no-default-features --features="$FEATURES" - - # Single features - for feature in ${FEATURES} - do - cargo build --locked --no-default-features --features="$feature" - cargo test --locked --no-default-features --features="$feature" - done -fi - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCSRS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features -fi - -# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command -# above this checks that we feature guarded docs imports correctly. -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="-D warnings" cargo +stable doc --locked --all-features -fi - -# Run formatter if told to. -if [ "$DO_FMT" = true ]; then - if [ "$NIGHTLY" = false ]; then - echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)" - exit 1 - fi - rustup component add rustfmt - cargo fmt --check -fi From c97f2ccc6959128514f3c3023b455ef6f8c208a9 Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Tue, 20 Feb 2024 16:58:25 +0000 Subject: [PATCH 2/5] ci: require a nightly compiler rather than using +nightly If we pin the version of nightly to a specific date then `cargo +nightly` will stop working. And even locally, if you want to use a specific version of nightly, you can't if the script is overriding your version with "+nightly". Instead the user should set RUSTUP_TOOLCHAIN. --- contrib/run_task.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/contrib/run_task.sh b/contrib/run_task.sh index 105dd0cc..f7aa21ac 100755 --- a/contrib/run_task.sh +++ b/contrib/run_task.sh @@ -148,8 +148,10 @@ loop_features() { # Lint the workspace then the individual crate examples. do_lint() { + need_nightly + # Use the current (recent/minimal) lock file. - local cargo="cargo +nightly --locked" + local cargo="cargo --locked" $cargo clippy --workspace -- -D warnings @@ -181,7 +183,8 @@ do_dup_deps() { # Build the docs with a nightly toolchain, in unison with the function # below this checks that we feature guarded docs imports correctly. build_docs_with_nightly_toolchain() { - local cargo="cargo +nightly --locked" + need_nightly + local cargo="cargo --locked" RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features } @@ -242,6 +245,13 @@ need_cmd() { fi } +need_nightly() { + cargo_ver=$(cargo --version) + if echo "$cargo_ver" | grep -q -v nightly; then + err "Need a nightly compiler; have $(cargo --version)" + fi +} + err() { echo "$1" >&2 exit 1 From 85ead84a995bc37c5857e86e7bcf1a56883abf25 Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Tue, 20 Feb 2024 14:56:37 +0000 Subject: [PATCH 3/5] ci: pin nightly in current CI --- .github/nightly-version | 1 + .github/workflows/rust.yml | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 .github/nightly-version diff --git a/.github/nightly-version b/.github/nightly-version new file mode 100644 index 00000000..af49290f --- /dev/null +++ b/.github/nightly-version @@ -0,0 +1 @@ +nightly-2024-02-18 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4769e435..169e2e5f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,9 +13,13 @@ jobs: outputs: crates: ${{ steps.get_matrix.outputs.crates }} deps: ${{ steps.get_matrix.outputs.deps }} + nightly_version: ${{ steps.read_toolchain.outputs.nightly_version }} steps: - name: Checkout Crate uses: actions/checkout@v4 + - name: Read nightly version + id: read_toolchain + run: echo "nightly_version=$(cat .github/nightly-version)" >> $GITHUB_OUTPUT - name: Prepare tests id: get_matrix run: contrib/get_matrix.sh @@ -75,7 +79,9 @@ jobs: - name: Checkout Crate uses: actions/checkout@v4 - name: Checkout Toolchain - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} - name: Install clippy run: rustup component add clippy - name: Set dependencies @@ -139,6 +145,7 @@ jobs: run: cross test --target s390x-unknown-linux-gnu Embedded: + needs: Prepare runs-on: ubuntu-latest env: RUSTFLAGS: "-C link-arg=-Tlink.x" @@ -149,8 +156,9 @@ jobs: - name: Set up QEMU run: sudo apt update && sudo apt install -y qemu-system-arm gcc-arm-none-eabi - name: Checkout Toolchain - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@v1 with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} targets: thumbv7m-none-eabi - name: Install src run: rustup component add rust-src @@ -175,7 +183,9 @@ jobs: - name: Checkout Crate uses: actions/checkout@v4 - name: Checkout Toolchain - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} - name: Install src run: rustup component add rust-src - name: Running address sanitizer From f82567fda41a9f2180f7e9e13f39476b84655dea Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Tue, 20 Feb 2024 14:39:45 +0000 Subject: [PATCH 4/5] ci: rename a couple .yml files to indicate that they're scheduled A small attempt to organize the various github workflows --- .github/workflows/{kani.yml => cron-daily-kani.yml} | 0 .github/workflows/{rustfmt.yml => cron-weekly-rustfmt.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{kani.yml => cron-daily-kani.yml} (100%) rename .github/workflows/{rustfmt.yml => cron-weekly-rustfmt.yml} (100%) diff --git a/.github/workflows/kani.yml b/.github/workflows/cron-daily-kani.yml similarity index 100% rename from .github/workflows/kani.yml rename to .github/workflows/cron-daily-kani.yml diff --git a/.github/workflows/rustfmt.yml b/.github/workflows/cron-weekly-rustfmt.yml similarity index 100% rename from .github/workflows/rustfmt.yml rename to .github/workflows/cron-weekly-rustfmt.yml From ee113aa91ffed1a296b112133685b45bde546326 Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Tue, 20 Feb 2024 14:55:59 +0000 Subject: [PATCH 5/5] ci: add daily job to update nightly rustc --- .../workflows/cron-daily-update-nightly.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/cron-daily-update-nightly.yml diff --git a/.github/workflows/cron-daily-update-nightly.yml b/.github/workflows/cron-daily-update-nightly.yml new file mode 100644 index 00000000..4e8c9233 --- /dev/null +++ b/.github/workflows/cron-daily-update-nightly.yml @@ -0,0 +1,38 @@ +name: Update Nightly rustc +on: + schedule: + - cron: "0 0 * * *" # runs daily at 00:00 + workflow_dispatch: # allows manual triggering +jobs: + format: + name: Update nightly rustc + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + - name: Update rust.yml to use latest nightly + run: | + set -x + # Not every night has a nightly, so extract the date from whatever + # version of the compiler dtolnay/rust-toolchain gives us. + NIGHTLY_DATE=$(rustc +nightly --verbose --version | sed -ne 's/^commit-date: //p') + # Update the nightly version in the reference file. + echo "nightly-${NIGHTLY_DATE}" > .github/nightly-version + echo "nightly_date=${NIGHTLY_DATE}" >> $GITHUB_ENV + # Some days there is no new nightly. In this case don't make an empty PR. + if ! git diff --exit-code > /dev/null; then + echo "Updated nightly. Opening PR." + echo "changes_made=true" >> $GITHUB_ENV + else + echo "Attempted to update nightly but the latest-nightly date did not change. Not opening any PR." + echo "changes_made=false" >> $GITHUB_ENV + fi + - name: Create Pull Request + if: env.changes_made == 'true' + uses: peter-evans/create-pull-request@v6 + with: + author: Update Nightly Rustc Bot + title: Automated daily update to rustc (to nightly-${{ env.nightly_date }}) + body: | + Automated update to Github CI workflow `rust.yml` by [create-pull-request](https://github.com/peter-evans/create-pull-request) GitHub action + commit-message: Automated update to Github CI to rustc nightly-${{ env.nightly_date }}