Merge rust-bitcoin/rust-secp256k1#594: Rewrite revendoring script

2ae7ca9cf2 secp-sys: update README for new vendoring script (Andrew Poelstra)
4b02e9c405 run new vendor-libsecp.sh; fix upstream CHANGELOG. (Andrew Poelstra)
b58a60fd6c rewrite ./vendor-libsecp.sh (Andrew Poelstra)

Pull request description:

  For Nix purposes I need the revendoring script to work without network access and without user interaction. I also realized it would be convenient if the script could figure out what the right version prefix is supposed to be. Then I noticed some shellcheck issues.

  Anyway I just rewrote the whole thing. I'm now able to run this script within nix and vet that the current contents of the `depend/` directory are consistent with the secp256k1-HEAD-revision.txt, for all commits.

ACKs for top commit:
  tcharding:
    ACK 2ae7ca9cf2
  sanket1729:
    reACK 2ae7ca9cf2

Tree-SHA512: ea3028e3517b2dbe0f34bcf20685945ecf543fc42e01f10d435432ad290088586b2a2b0f0e94bc3ce59ec38727656eb04eef57c5df6a34da77070e0f288b1d84

secp256k1-sys/README.md

@@ -20,11 +20,12 @@ This prefix ensures that no symbol collision can happen:
 To update the vendored sources, use the `vendor-libsecp.sh` script:
 
 ```
-$ ./vendor-libsecp.sh depend <version-code> <rev>
+$ ./vendor-libsecp.sh <rev>
 ```
 
-- Where `<version-code>` is the secp256k1-sys version number underscored: `0_1_2`.
+Where `<rev>` is the git revision of libsecp256k1 to checkout. If you do not
-- Where `<rev>` is the git revision of libsecp256k1 to checkout.
+specify a revision, the script will simply clone the repo and use whatever
+revision the default branch is pointing to.
 
 
 ## Linking to external symbols

secp256k1-sys/depend/secp256k1-HEAD-revision.txt

@@ -1,2 +1,2 @@
-# This file was automatically created by ./vendor-libsecp.sh
+# This file was automatically created by vendor-libsecp.sh
 21ffe4b22a9683cf24ae0763359e401d1284cc7a

secp256k1-sys/depend/secp256k1/CHANGELOG.md

@@ -7,14 +7,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [0.2.0] - 2022-12-12
 
 ### Added
- - Added `rustsecp256k1_v0_8_0_selftest`, to be used in conjunction with `rustsecp256k1_v0_8_0_context_static`.
+ - Added `secp256k1_selftest`, to be used in conjunction with `secp256k1_context_static`.
 
 ### Changed
  - Enabled modules schnorrsig, extrakeys and ECDH by default in `./configure`.
 
 ### Deprecated
  - Deprecated context flags `SECP256K1_CONTEXT_VERIFY` and `SECP256K1_CONTEXT_SIGN`. Use `SECP256K1_CONTEXT_NONE` instead.
- - Renamed `rustsecp256k1_v0_8_0_context_no_precomp` to `rustsecp256k1_v0_8_0_context_static`.
+ - Renamed `secp256k1_context_no_precomp` to `secp256k1_context_static`.
 
 ### ABI Compatibility
 

secp256k1-sys/vendor-libsecp.sh

@@ -1,42 +1,96 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
-
+# Set default variables
-if [ -z "$1" ] | [ -z "$2" ]; then
+if [ -z "$SECP_VENDOR_GIT_ROOT" ]; then
-  echo "\$1 parameter must be the rust-secp256k1-sys depend directory"
+    SECP_VENDOR_GIT_ROOT="$(git rev-parse --show-toplevel)"
-  echo "\$2 parameter must be the rust-secp256k1-sys version code (M_m_p format)"
+else
-  echo "\$3 parameter (optional) can be the revision to check out"
+    SECP_VENDOR_GIT_ROOT="$(realpath "$SECP_VENDOR_GIT_ROOT")"
-  exit 1
 fi
+SECP_SYS="$SECP_VENDOR_GIT_ROOT"/secp256k1-sys
+DEFAULT_VERSION_CODE=$(grep version "$SECP_SYS/Cargo.toml" | sed 's/\./_/g' | sed 's/.*"\(.*\)".*/\1/')
+DEFAULT_DEPEND_DIR="$SECP_SYS/depend"
+DEFAULT_SECP_REPO=https://github.com/bitcoin-core/secp256k1.git
 
-PARENT_DIR=$1
+: "${SECP_VENDOR_VERSION_CODE:=$DEFAULT_VERSION_CODE}"
-VERSIONCODE=$2
+: "${SECP_VENDOR_DEPEND_DIR:=$DEFAULT_DEPEND_DIR}"
-REV=$3
+: "${SECP_VENDOR_SECP_REPO:=$DEFAULT_SECP_REPO}"
-DIR=secp256k1
+# CP_NOT_CLONE lets us just copy a directory rather than git cloning.
-ORIGDIR=$(pwd)
+# This is usually a bad idea, since it will bring in build artifacts or any other
+# junk from the source directory, but may be useful during development or CI.
+: "${SECP_VENDOR_CP_NOT_CLONE:=no}"
 
-while true; do
+echo "Using version code $SECP_VENDOR_VERSION_CODE. Set SECP_VENDOR_VERSION_CODE to override."
-    read -r -p "$PARENT_DIR/$DIR will be deleted [yn]: " yn
+echo "Using depend directory $SECP_VENDOR_DEPEND_DIR. Set SECP_VENDOR_DEPEND_DIR to override."
-    case $yn in
+echo "Using secp repository $SECP_VENDOR_SECP_REPO. Set SECP_VENDOR_SECP_REPO to override."
-        [Yy]* ) break;;
+
-        [Nn]* ) exit;;
+# Parse command-line options
-        * ) echo "Please answer yes or no.";;
+SECP_REV=""
+FORCE=no
+while (( "$#" )); do
+    case "$1" in
+    -f)
+        FORCE=yes
+        ;;
+    *)
+        if [ -z "$SECP_REV" ]; then
+            echo "Using secp256k1 revision $SECP_REV."
+            SECP_REV="$1"
+        else
+            echo "WARNING: ignoring unknown command-line argument $1"
+        fi
+        ;;
     esac
+    shift
 done
 
-cd "$PARENT_DIR" || exit 1
+if [ -z "$SECP_REV" ]; then
-rm -rf "$DIR"
+    echo "WARNING: No secp256k1 revision specified. Will use whatever we find at the git repo."
-git clone https://github.com/bitcoin-core/secp256k1.git "$DIR"
-cd "$DIR"
-if [ -n "$REV" ]; then
-    git checkout "$REV"
 fi
-HEAD=$(git rev-parse HEAD)
+echo
-cd ..
-echo "# This file was automatically created by $0" > ./secp256k1-HEAD-revision.txt
-echo "$HEAD" >> ./secp256k1-HEAD-revision.txt
 
-# We need to make some source changes to the files.
+# Check if we will do anything destructive.
+
+if [ "$FORCE" == "no" ]; then
+    if ! git diff --quiet -- "*.rs"; then
+        echo "ERROR: There appear to be modified source files. Check these in or pass -f (some source files will be modified to have symbols renamed)."
+        exit 2
+    fi
+    if ! git diff --quiet -- "$SECP_VENDOR_DEPEND_DIR"; then
+        echo "ERROR: The depend directory appears to be modified. Check it in or pass -f (this directory will be deleted)."
+        exit 2
+    fi
+fi
+
+DIR=./secp256k1
+
+pushd "$SECP_VENDOR_DEPEND_DIR" > /dev/null
+rm -rf "$DIR" || true
+
+# Clone the repo. As a special case, if the repo is a local path and we have
+# not specified a revision, just copy the directory rather than using 'git clone'.
+# This lets us use non-git repos or dirty source trees as secp sources.
+if [ "$SECP_VENDOR_CP_NOT_CLONE" == "yes" ]; then
+    cp -r "$SECP_VENDOR_SECP_REPO" "$DIR"
+    chmod -R +w "$DIR" # cp preserves write perms, which if missing will cause patch to fail
+else
+    git clone "$SECP_VENDOR_SECP_REPO" "$DIR"
+fi
+
+# Check out specified revision
+pushd "$DIR" > /dev/null
+if [ -n "$SECP_REV" ]; then
+    git checkout "$SECP_REV"
+fi
+SOURCE_REV=$(git rev-parse HEAD || echo "[unknown revision from $SECP_VENDOR_SECP_REPO]")
+rm -rf .git/ || true
+popd
+
+# Record revision
+echo "# This file was automatically created by $(basename "$0")" > ./secp256k1-HEAD-revision.txt
+echo "$SOURCE_REV" >> ./secp256k1-HEAD-revision.txt
+
+# Patch source files
 
 # To support compiling for WASM, we need to remove all methods that use malloc.
 # To compensate, the secp_context_create and _destroy methods are redefined in Rust.
@@ -46,28 +100,25 @@ patch "$DIR/src/scratch_impl.h" "./scratch_impl.h.patch"
 patch "$DIR/src/util.h" "./util.h.patch"
 
 # Prefix all methods with rustsecp and a version prefix
-find "$DIR" -not -path '*/\.*' -type f -print0 | xargs -0 sed -i "/^#include/! s/secp256k1_/rustsecp256k1_v${VERSIONCODE}_/g"
+find "$DIR" \
-
+    -not -path '*/\.*' \
+    -not -name "CHANGELOG.md" \
+    -type f \
+    -print0 | xargs -0 sed -i "/^#include/! s/secp256k1_/rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}_/g"
 # special rule for a method that is not prefixed in libsecp
-find "$DIR" -not -path '*/\.*' -type f -print0 | xargs -0 sed -i "/^#include/! s/ecdsa_signature_parse_der_lax/rustsecp256k1_v${VERSIONCODE}_ecdsa_signature_parse_der_lax/g"
+find "$DIR" \
-
+    -not -path '*/\.*' \
-# TODO: can be removed once 496c5b43b lands in secp-zkp
+    -type f \
-find "$DIR" -not -path '*/\.*' -type f -print0 | xargs -0 sed -i 's/^const int CURVE_B/static const int CURVE_B/g'
+    -print0 | xargs -0 sed -i "/^#include/! s/ecdsa_signature_parse_der_lax/rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}_ecdsa_signature_parse_der_lax/g"
-
-while true; do
-    read -r -p "Update Rust extern references and Cargo.toml as well? [yn]: " yn
-    case $yn in
-        [Yy]* ) break;;
-        [Nn]* ) exit;;
-        * ) echo "Please answer yes or no.";;
-    esac
-done
-
-cd "$ORIGDIR"
 
+cd "$SECP_SYS"
 # Update the `links = ` in the manifest file.
-sed -i -r "s/^links = \".*\"$/links = \"rustsecp256k1_v${VERSIONCODE}\"/" Cargo.toml
+sed -i -r "s/^links = \".*\"$/links = \"rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}\"/" Cargo.toml
-
 # Update the extern references in the Rust FFI source files.
-find "./src/" -name "*.rs" -type f -print0 | xargs -0 sed -i -r "s/rustsecp256k1_v[0-9]+_[0-9]+_[0-9]+_(.*)([\"\(])/rustsecp256k1_v${VERSIONCODE}_\1\2/g"
+find "./src/" \
+    -name "*.rs" \
+    -type f \
+    -print0 | xargs -0 sed -i -r "s/rustsecp256k1_v[0-9]+_[0-9]+_[0-9]+_(.*)([\"\(])/rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}_\1\2/g"
+
+popd > /dev/null