Merge rust-bitcoin/rust-bitcoin#2489: ci: pin nightly compiler, add CI cronjob to update it.

ee113aa91f ci: add daily job to update nightly rustc (Andrew Poelstra)
f82567fda4 ci: rename a couple .yml files to indicate that they're scheduled (Andrew Poelstra)
85ead84a99 ci: pin nightly in current CI (Andrew Poelstra)
c97f2ccc69 ci: require a nightly compiler rather than using +nightly (Andrew Poelstra)
e386cbfadf ci: delete *test.sh files (Andrew Poelstra)

Pull request description:

  We should be pinning our nightly compiler in CI so that
  * running CI on old release branches will continue to work (not quite -- we'd want to do the same thing for Cargo.lock, but ok, one PR at a time)
  * when CI breaks due to nightly updates (usually new Clippy lints), we can address that in a dedicated "update nightly" PR rather than having every other PR suddenly break on us

  Co-authored with ChatGPT which found several typos and suggested most of the error-handling logic.

ACKs for top commit:
  tcharding:
    ACK ee113aa91f
  Kixunil:
    ACK ee113aa91f

Tree-SHA512: f198349291a7654f4e6f03998d02c1f7d2c7f999e0b5a89a915beb3e7c741148c2c65367b107c54c34d6669e6f0972699401ef85e76e76e5900c1fb5c844db4f

.github/labeler.yml

@@ -6,8 +6,10 @@ test:
     - any-glob-to-any-file: fuzz/**
     - any-glob-to-any-file: '*/tests/**'
     - any-glob-to-any-file: 'dep_test'
-    - any-glob-to-any-file: 'contrib/test.sh'
+    - any-glob-to-any-file: 'contrib/run_task.sh'
-    - any-glob-to-any-file: '*/contrib/test.sh'
+    - any-glob-to-any-file: 'contrib/test_vars.sh'
+    - any-glob-to-any-file: '*/contrib/extra_tests.sh'
+    - any-glob-to-any-file: '*/contrib/test_vars.sh'
 doc:
   - changed-files:
     - any-glob-to-any-file: '**/*.md'

.github/nightly-version

@@ -0,0 +1 @@
+nightly-2024-02-18

.github/workflows/cron-daily-update-nightly.yml

@@ -0,0 +1,38 @@
+name: Update Nightly rustc
+on:
+  schedule:
+    - cron: "0 0 * * *" # runs daily at 00:00
+  workflow_dispatch: # allows manual triggering
+jobs:
+  format:
+    name: Update nightly rustc
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@nightly
+      - name: Update rust.yml to use latest nightly
+        run: |
+          set -x
+          # Not every night has a nightly, so extract the date from whatever
+          # version of the compiler dtolnay/rust-toolchain gives us.
+          NIGHTLY_DATE=$(rustc +nightly --verbose --version | sed -ne 's/^commit-date: //p')
+          # Update the nightly version in the reference file.
+          echo "nightly-${NIGHTLY_DATE}" > .github/nightly-version
+          echo "nightly_date=${NIGHTLY_DATE}" >> $GITHUB_ENV
+          # Some days there is no new nightly. In this case don't make an empty PR.
+          if ! git diff --exit-code > /dev/null; then
+              echo "Updated nightly. Opening PR."
+              echo "changes_made=true" >> $GITHUB_ENV
+          else
+              echo "Attempted to update nightly but the latest-nightly date did not change. Not opening any PR."
+              echo "changes_made=false" >> $GITHUB_ENV
+          fi
+      - name: Create Pull Request
+        if: env.changes_made == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          author: Update Nightly Rustc Bot <bot@example.com>
+          title: Automated daily update to rustc (to nightly-${{ env.nightly_date }})
+          body: |
+           Automated update to Github CI workflow `rust.yml` by [create-pull-request](https://github.com/peter-evans/create-pull-request) GitHub action
+          commit-message: Automated update to Github CI to rustc nightly-${{ env.nightly_date }}

.github/workflows/rust.yml

@@ -13,9 +13,13 @@ jobs:
     outputs:
       crates: ${{ steps.get_matrix.outputs.crates }}
       deps: ${{ steps.get_matrix.outputs.deps }}
+      nightly_version: ${{ steps.read_toolchain.outputs.nightly_version }}
     steps:
       - name: Checkout Crate
         uses: actions/checkout@v4
+      - name: Read nightly version
+        id: read_toolchain
+        run: echo "nightly_version=$(cat .github/nightly-version)" >> $GITHUB_OUTPUT
       - name: Prepare tests
         id: get_matrix
         run: contrib/get_matrix.sh
@@ -75,7 +79,9 @@ jobs:
       - name: Checkout Crate
         uses: actions/checkout@v4
       - name: Checkout Toolchain
-        uses: dtolnay/rust-toolchain@nightly
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: ${{ needs.Prepare.outputs.nightly_version }}
       - name: Install clippy
         run: rustup component add clippy
       - name: Set dependencies
@@ -139,6 +145,7 @@ jobs:
         run: cross test --target s390x-unknown-linux-gnu
 
   Embedded:
+    needs: Prepare
     runs-on: ubuntu-latest
     env:
       RUSTFLAGS: "-C link-arg=-Tlink.x"
@@ -149,8 +156,9 @@ jobs:
       - name: Set up QEMU
         run: sudo apt update && sudo apt install -y qemu-system-arm gcc-arm-none-eabi
       - name: Checkout Toolchain
-        uses: dtolnay/rust-toolchain@nightly
+        uses: dtolnay/rust-toolchain@v1
         with:
+          toolchain: ${{ needs.Prepare.outputs.nightly_version }}
           targets: thumbv7m-none-eabi
       - name: Install src
         run: rustup component add rust-src
@@ -175,7 +183,9 @@ jobs:
       - name: Checkout Crate
         uses: actions/checkout@v4
       - name: Checkout Toolchain
-        uses: dtolnay/rust-toolchain@nightly
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: ${{ needs.Prepare.outputs.nightly_version }}
       - name: Install src
         run: rustup component add rust-src
       - name: Running address sanitizer

CONTRIBUTING.md

@@ -124,19 +124,6 @@ NB: reviewers may run more complex test/CI scripts, thus, satisfying all the
 requirements above is just a preliminary, but not necessary sufficient step for
 getting the PR accepted as a valid candidate PR for the `master` branch.
 
-PR authors may also find it useful to run the following script locally in order
-to check that each of the commits within the PR satisfies the requirements
-above, before submitting the PR to review:
-```shell script
-RUSTUP_TOOLCHAIN=1.41.1 ./contrib/test.sh
-```
-Please replace the value in `RUSTUP_TOOLCHAIN=1.41.1` with the current MSRV from
-[README.md].
-
-NB: Please keep in mind that the script above replaces `Cargo.lock` file, which
-is necessary to support current MSRV, incompatible with `stable` and newer cargo
-versions.
-
 ### Peer review
 
 Anyone may participate in peer review which is expressed by comments in the pull

bitcoin/contrib/test.sh

@@ -1,131 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-FEATURES="std rand-std rand serde secp-recovery bitcoinconsensus-std base64 bitcoinconsensus"
-
-cargo --version
-rustc --version
-
-# Some tests require certain toolchain types.
-NIGHTLY=false
-STABLE=true
-if cargo --version | grep nightly; then
-    STABLE=false
-    NIGHTLY=true
-fi
-if cargo --version | grep beta; then
-    STABLE=false
-fi
-
-# Make all cargo invocations verbose
-export CARGO_TERM_VERBOSE=true
-
-# Defaults / sanity checks
-cargo build
-cargo test
-
-if [ "$DO_LINT" = true ]
-then
-    cargo clippy --locked --all-features --all-targets -- -D warnings
-    cargo clippy --locked --example bip32 -- -D warnings
-    cargo clippy --locked --example handshake --features=rand-std -- -D warnings
-    cargo clippy --locked --example ecdsa-psbt --features=bitcoinconsensus -- -D warnings
-    cargo clippy --locked --example sign-tx-segwit-v0 --features=rand-std -- -D warnings
-    cargo clippy --locked --example sign-tx-taproot --features=rand-std -- -D warnings
-    cargo clippy --locked --example taproot-psbt --features=rand-std,bitcoinconsensus -- -D warnings
-
-    # We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies
-    # in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp).
-    duplicate_dependencies=$(
-        # Only show the actual duplicated deps, not their reverse tree, then
-        # whitelist the 'syn' crate which is duplicated but it's not our fault.
-        #
-        # Whitelist `bitcoin_hashes` while we release it and until secp v0.28.0 comes out.
-        cargo tree  --target=all --all-features --duplicates \
-            | grep '^[0-9A-Za-z]' \
-            | grep -v 'syn' \
-            | grep -v 'bitcoin_hashes' \
-            | wc -l
-                          )
-    if [ "$duplicate_dependencies" -ne 0 ]; then
-        echo "Dependency tree is broken, contains duplicates"
-        cargo tree  --target=all --all-features --duplicates
-        exit 1
-    fi
-fi
-
-if [ "$DO_FEATURE_MATRIX" = true ]; then
-    cargo build --locked --no-default-features
-    cargo test --locked --no-default-features
-
-    # All features
-    cargo build --locked --no-default-features --features="$FEATURES"
-    cargo test --locked --no-default-features --features="$FEATURES"
-    # Single features
-    for feature in ${FEATURES}
-    do
-        cargo build --locked --no-default-features --features="$feature"
-        cargo test --locked --no-default-features --features="$feature"
-		# All combos of two features
-		for featuretwo in ${FEATURES}; do
-			cargo build --locked --no-default-features --features="$feature $featuretwo"
-			cargo test --locked --no-default-features --features="$feature $featuretwo"
-		done
-    done
-fi
-
-cargo run --locked --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd
-cargo run --locked --no-default-features --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd
-
-cargo run --locked --example ecdsa-psbt --features=bitcoinconsensus
-cargo run --locked --example sign-tx-segwit-v0 --features=rand-std -- -D warnings
-cargo run --locked --example sign-tx-taproot --features=rand-std -- -D warnings
-cargo run --locked --example taproot-psbt --features=rand-std,bitcoinconsensus
-
-# Build the docs if told to (this only works with the nightly toolchain)
-if [ "$DO_DOCSRS" = true ]; then
-    RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features
-fi
-
-# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command
-# above this checks that we feature guarded docs imports correctly.
-if [ "$DO_DOCS" = true ]; then
-    RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features
-fi
-
-# Run formatter if told to.
-if [ "$DO_FMT" = true ]; then
-    if [ "$NIGHTLY" = false ]; then
-        echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)"
-        exit 1
-    fi
-    rustup component add rustfmt
-    cargo fmt --check
-fi
-
-# Bench if told to, only works with non-stable toolchain (nightly, beta).
-if [ "$DO_BENCH" = true ]
-then
-    if [ "$STABLE" = true ]; then
-        if [ -n "$RUSTUP_TOOLCHAIN" ]; then
-            echo "RUSTUP_TOOLCHAIN is set to a stable toolchain but DO_BENCH requires a non-stable (beta, nightly) toolchain"
-        else
-            echo "DO_BENCH requires a non-stable (beta, nightly) toolchain"
-        fi
-        exit 1
-    fi
-    RUSTFLAGS='--cfg=bench' cargo bench
-fi
-
-# Use as dependency if told to
-if [ "$AS_DEPENDENCY" = true ]
-then
-    cargo new dep_test 2> /dev/null # Mute warning about workspace, fixed below.
-    cd dep_test
-    echo 'bitcoin = { path = "..", features = ["serde"] }\n\n' >> Cargo.toml
-    # Adding an empty workspace section excludes this crate from the rust-bitcoin workspace.
-    echo '[workspace]\n\n' >> Cargo.toml
-
-    cargo test --verbose
-fi

contrib/run_task.sh

@@ -148,8 +148,10 @@ loop_features() {
 
 # Lint the workspace then the individual crate examples.
 do_lint() {
+    need_nightly
+
     # Use the current (recent/minimal) lock file.
-    local cargo="cargo +nightly --locked"
+    local cargo="cargo --locked"
 
     $cargo clippy --workspace -- -D warnings
 
@@ -181,7 +183,8 @@ do_dup_deps() {
 # Build the docs with a nightly toolchain, in unison with the function
 # below this checks that we feature guarded docs imports correctly.
 build_docs_with_nightly_toolchain() {
-    local cargo="cargo +nightly --locked"
+    need_nightly
+    local cargo="cargo --locked"
 
     RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features
 }
@@ -242,6 +245,13 @@ need_cmd() {
     fi
 }
 
+need_nightly() {
+    cargo_ver=$(cargo --version)
+    if echo "$cargo_ver" | grep -q -v nightly; then
+        err "Need a nightly compiler; have $(cargo --version)"
+    fi
+}
+
 err() {
     echo "$1" >&2
     exit 1

contrib/test.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-CRATES="bitcoin hashes units internals fuzz"
-DEPS="recent minimal"
-
-for dep in $DEPS
-do
-    cp "Cargo-$dep.lock" Cargo.lock
-    for crate in ${CRATES}
-    do
-        (
-            cd "$crate"
-            ./contrib/test.sh
-        )
-    done
-    if [ "$dep" = recent ];
-    then
-        # We always test committed dependencies but we want to warn if they could've been updated
-        cargo update
-        if diff Cargo-recent.lock Cargo.lock;
-        then
-            echo Dependencies are up to date
-        else
-            echo "::warning file=Cargo-recent.lock::Dependencies could be updated"
-        fi
-    fi
-done
-
-exit 0

fuzz/contrib/test.sh

@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-FEATURES=""
-
-cargo --version
-rustc --version
-
-# Make all cargo invocations verbose
-export CARGO_TERM_VERBOSE=true
-
-# Pin dependencies as required if we are using MSRV toolchain.
-if cargo --version | grep "1\.41"; then
-    # 1.0.157 uses syn 2.0 which requires edition 2021
-    cargo update -p serde --precise 1.0.156
-    # 1.0.108 uses `matches!` macro so does not work with Rust 1.41.1, bad `syn` no biscuit.
-    cargo update -p syn --precise 1.0.107
-    # Half 1.8 uses edition 2021 features
-    cargo update -p half --precise 1.7.1
-fi
-
-if [ "$DO_LINT" = true ]
-then
-    cargo clippy --all-features --all-targets -- -D warnings
-fi
-
-# Defaults / sanity checks
-cargo build
-cargo test
-
-# Address Sanitizer
-if [ "$DO_ASAN" = true ]; then
-    cargo clean
-    CC='clang -fsanitize=address -fno-omit-frame-pointer'                                        \
-    RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes'                    \
-    ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \
-    cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
-    cargo clean
-    CC='clang -fsanitize=memory -fno-omit-frame-pointer'                                         \
-    RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes'   \
-    cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu
-fi
-

internals/contrib/test.sh

@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-FEATURES="std alloc"
-
-cargo --version
-rustc --version
-
-# Work out if we are using a nightly toolchain.
-NIGHTLY=false
-if cargo --version | grep nightly >/dev/null; then
-    NIGHTLY=true
-fi
-
-# Make all cargo invocations verbose
-export CARGO_TERM_VERBOSE=true
-
-# Defaults / sanity checks
-cargo --locked build
-cargo --locked test
-
-if [ "$DO_LINT" = true ]
-then
-    cargo clippy --locked --all-features --all-targets -- -D warnings
-fi
-
-if [ "$DO_FEATURE_MATRIX" = true ]; then
-    # No features
-    cargo build --locked --no-default-features
-    cargo test --locked --no-default-features
-
-    # All features
-    cargo build --locked --no-default-features --features="$FEATURES"
-    cargo test --locked --no-default-features --features="$FEATURES"
-
-    # Single features
-    for feature in ${FEATURES}
-    do
-        cargo build --locked --no-default-features --features="$feature"
-        cargo test --locked --no-default-features --features="$feature"
-    done
-fi
-
-# Build the docs if told to (this only works with the nightly toolchain)
-if [ "$DO_DOCSRS" = true ]; then
-    RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features
-fi
-
-# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command
-# above this checks that we feature guarded docs imports correctly.
-if [ "$DO_DOCS" = true ]; then
-    RUSTDOCFLAGS="-D warnings" cargo +stable doc --locked --all-features
-fi
-
-# Run formatter if told to.
-if [ "$DO_FMT" = true ]; then
-    if [ "$NIGHTLY" = false ]; then
-        echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)"
-        exit 1
-    fi
-    rustup component add rustfmt
-    cargo fmt --check
-fi