fuzz: auto-generate CI and Cargo.toml files

.github/workflows/fuzz.yml

@@ -1,3 +1,4 @@
+# Automatically generated by fuzz/generate-files.sh
 name: Fuzz
 
 on:
@@ -8,14 +9,32 @@ on:
   pull_request:
 
 jobs:
-
   fuzz:
     if: ${{ !github.event.act }}
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
-        fuzz_target: [deser_net_msg, deserialize_address, deserialize_amount, deserialize_block, deserialize_psbt, deserialize_script, deserialize_transaction, deserialize_prefilled_transaction, deserialize_witness, outpoint_string, script_bytes_to_asm_fmt]
+        fuzz_target: [
+bitcoin_outpoint_string,
+bitcoin_deserialize_amount,
+bitcoin_deserialize_transaction,
+bitcoin_deser_net_msg,
+bitcoin_deserialize_address,
+bitcoin_script_bytes_to_asm_fmt,
+bitcoin_deserialize_prefilled_transaction,
+bitcoin_deserialize_witness,
+bitcoin_deserialize_psbt,
+bitcoin_deserialize_block,
+bitcoin_deserialize_script,
+hashes_json,
+hashes_cbor,
+hashes_sha256,
+hashes_ripemd160,
+hashes_sha512_256,
+hashes_sha512,
+hashes_sha1,
+        ]
     steps:
       - name: Install test dependencies
         run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
@@ -35,7 +54,7 @@ jobs:
           profile: minimal
       - name: fuzz
         run: cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}"
-      - run: echo "${{ matrix.fuzz_target }}.rs" >executed_${{ matrix.fuzz_target }}
+      - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }}
       - uses: actions/upload-artifact@v2
         with:
           name: executed_${{ matrix.fuzz_target }}
@@ -51,5 +70,4 @@ jobs:
       - name: Display structure of downloaded files
         run: ls -R
       - run: find executed_* -type f -exec cat {} + | sort > executed
-      - run: ls fuzz/fuzz_targets | sort > expected
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed
-      - run: diff expected executed

fuzz/Cargo.toml

@@ -2,7 +2,7 @@
 name = "bitcoin-fuzz"
 edition = "2018"
 version = "0.0.1"
-authors = ["Automatically generated"]
+authors = ["Generated by fuzz/generate-files.sh"]
 publish = false
 
 [package.metadata]
@@ -23,74 +23,73 @@ serde_json = "1.0"
 serde_cbor = "0.9"
 
 [[bin]]
-name = "deserialize_block"
+name = "bitcoin_outpoint_string"
-path = "fuzz_targets/bitcoin/deserialize_block.rs"
-
-[[bin]]
-name = "deserialize_script"
-path = "fuzz_targets/bitcoin/deserialize_script.rs"
-
-[[bin]]
-name = "deserialize_transaction"
-path = "fuzz_targets/bitcoin/deserialize_transaction.rs"
-
-[[bin]]
-name = "deserialize_prefilled_transaction"
-path = "fuzz_targets/bitcoin/deserialize_prefilled_transaction.rs"
-
-[[bin]]
-name = "deserialize_address"
-path = "fuzz_targets/bitcoin/deserialize_address.rs"
-
-[[bin]]
-name = "deserialize_amount"
-path = "fuzz_targets/bitcoin/deserialize_amount.rs"
-
-[[bin]]
-name = "outpoint_string"
 path = "fuzz_targets/bitcoin/outpoint_string.rs"
 
 [[bin]]
-name = "deserialize_psbt"
+name = "bitcoin_deserialize_amount"
-path = "fuzz_targets/bitcoin/deserialize_psbt.rs"
+path = "fuzz_targets/bitcoin/deserialize_amount.rs"
 
 [[bin]]
-name = "deser_net_msg"
+name = "bitcoin_deserialize_transaction"
+path = "fuzz_targets/bitcoin/deserialize_transaction.rs"
+
+[[bin]]
+name = "bitcoin_deser_net_msg"
 path = "fuzz_targets/bitcoin/deser_net_msg.rs"
 
 [[bin]]
-name = "script_bytes_to_asm_fmt"
+name = "bitcoin_deserialize_address"
+path = "fuzz_targets/bitcoin/deserialize_address.rs"
+
+[[bin]]
+name = "bitcoin_script_bytes_to_asm_fmt"
 path = "fuzz_targets/bitcoin/script_bytes_to_asm_fmt.rs"
 
 [[bin]]
-name = "deserialize_witness"
+name = "bitcoin_deserialize_prefilled_transaction"
+path = "fuzz_targets/bitcoin/deserialize_prefilled_transaction.rs"
+
+[[bin]]
+name = "bitcoin_deserialize_witness"
 path = "fuzz_targets/bitcoin/deserialize_witness.rs"
 
+[[bin]]
+name = "bitcoin_deserialize_psbt"
+path = "fuzz_targets/bitcoin/deserialize_psbt.rs"
 
 [[bin]]
-name = "sha1"
+name = "bitcoin_deserialize_block"
-path = "fuzz_targets/hashes/sha1.rs"
+path = "fuzz_targets/bitcoin/deserialize_block.rs"
 
 [[bin]]
-name = "ripemd160"
+name = "bitcoin_deserialize_script"
-path = "fuzz_targets/hashes/ripemd160.rs"
+path = "fuzz_targets/bitcoin/deserialize_script.rs"
 
 [[bin]]
-name = "sha256"
+name = "hashes_json"
-path = "fuzz_targets/hashes/sha256.rs"
+path = "fuzz_targets/hashes/json.rs"
 
 [[bin]]
-name = "sha512"
+name = "hashes_cbor"
-path = "fuzz_targets/hashes/sha512.rs"
-
-[[bin]]
-name = "sha512_256"
-path = "fuzz_targets/hashes/sha512_256.rs"
-
-[[bin]]
-name = "cbor"
 path = "fuzz_targets/hashes/cbor.rs"
 
 [[bin]]
-name = "json"
+name = "hashes_sha256"
-path = "fuzz_targets/hashes/json.rs"
+path = "fuzz_targets/hashes/sha256.rs"
+
+[[bin]]
+name = "hashes_ripemd160"
+path = "fuzz_targets/hashes/ripemd160.rs"
+
+[[bin]]
+name = "hashes_sha512_256"
+path = "fuzz_targets/hashes/sha512_256.rs"
+
+[[bin]]
+name = "hashes_sha512"
+path = "fuzz_targets/hashes/sha512.rs"
+
+[[bin]]
+name = "hashes_sha1"
+path = "fuzz_targets/hashes/sha1.rs"

fuzz/fuzz-util.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+listTargetFiles() {
+  pushd "$REPO_DIR/fuzz" > /dev/null
+  find fuzz_targets/ -type f -name "*.rs"
+  popd > /dev/null
+}
+
+targetFileToName() {
+  echo "$1" \
+    | sed 's/^fuzz_targets\///' \
+    | sed 's/\.rs$//' \
+    | sed 's/\//_/g'
+}
+
+listTargetNames() {
+  for target in $(listTargetFiles); do
+    targetFileToName "$target"
+  done
+}
+
+

fuzz/generate-files.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -e
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# 1. Generate fuzz/Cargo.toml
+cat > "$REPO_DIR/fuzz/Cargo.toml" <<EOF
+[package]
+name = "bitcoin-fuzz"
+edition = "2018"
+version = "0.0.1"
+authors = ["Generated by fuzz/generate-files.sh"]
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[features]
+afl_fuzz = ["afl"]
+honggfuzz_fuzz = ["honggfuzz"]
+
+[dependencies]
+honggfuzz = { version = "0.5", optional = true, default-features = false }
+afl = { version = "0.4", optional = true }
+bitcoin = { version = "0.30.0", features = [ "serde" ] }
+
+rust-crypto = "0.2"
+serde = { version = "1.0.103", features = [ "derive" ] }
+serde_json = "1.0"
+serde_cbor = "0.9"
+EOF
+
+for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    cat >> "$REPO_DIR/fuzz/Cargo.toml" <<EOF
+
+[[bin]]
+name = "$targetName"
+path = "$targetFile"
+EOF
+done
+
+# 2. Generate .github/workflows/fuzz.yml
+cat > "$REPO_DIR/.github/workflows/fuzz.yml" <<EOF
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+
+on:
+  push:
+    branches:
+      - master
+      - 'test-ci/**'
+  pull_request:
+
+jobs:
+  fuzz:
+    if: \${{ !github.event.act }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        fuzz_target: [
+$(for name in $(listTargetNames); do echo "$name,"; done)
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-\${{ matrix.target }}-\${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58
+          override: true
+          profile: minimal
+      - name: fuzz
+        run: cd fuzz && ./fuzz.sh "\${{ matrix.fuzz_target }}"
+      - run: echo "\${{ matrix.fuzz_target }}" >executed_\${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: executed_\${{ matrix.fuzz_target }}
+          path: executed_\${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: \${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed
+EOF
+