Merge rust-bitcoin/rust-bitcoin#672: New Witness struct to improve ser/de perfomance

106acdc3ac Add fuzzing for Witness struct (Riccardo Casatta)
2fd0125bfa Introduce Witness struct mainly to improve ser/de performance while keeping most usability. (Riccardo Casatta)

Pull request description:

  At the moment the Witness struct is  `Vec<Vec<u8>>`, the vec inside a vec cause a lot of allocations, specifically:

  - empty witness -> 1 allocation, while an empty vec doesn't allocate, the outer vec is not empty
  - witness with n elements -> n+1 allocations

  The proposed Witness struct contains the serialized format of the witness. This reduces the allocations to:

  - empty witness -> 0 allocations
  - witness with n elements -> 1 allocation for most common cases (you don't know how many bytes is long the entire witness beforehand, thus you need to estimate a good value, not too big to avoid wasting space and not too low to avoid vector reallocation, I used 128 since it covers about 80% of cases on mainnet)

  The inconvenience is having slightly less comfortable access to the witness, but the iterator is efficient (no allocations) and you can always collect the iteration to have a Vec of slices. If you collect the iteration you end up doing allocation anyway, but the rationale is that it is an operation you need to do rarely while ser/de is done much more often.

  I had to add a bigger block to better see the improvement (ae860247e191e2136d7c87382f78c96e0908d700), these are the results of the benches on my machine:

  ```
  RCasatta/master_with_block
  test blockdata::block::benches::bench_block_deserialize                 ... bench:   5,496,821 ns/iter (+/- 298,859)
  test blockdata::block::benches::bench_block_serialize                   ... bench:     437,389 ns/iter (+/- 31,576)
  test blockdata::block::benches::bench_block_serialize_logic             ... bench:     108,759 ns/iter (+/- 5,807)
  test blockdata::transaction::benches::bench_transaction_deserialize     ... bench:         670 ns/iter (+/- 49)
  test blockdata::transaction::benches::bench_transaction_get_size        ... bench:           7 ns/iter (+/- 0)
  test blockdata::transaction::benches::bench_transaction_serialize       ... bench:          51 ns/iter (+/- 5)
  test blockdata::transaction::benches::bench_transaction_serialize_logic ... bench:          13 ns/iter (+/- 0)

  branch witness_with_block (this one)
  test blockdata::block::benches::bench_block_deserialize                 ... bench:   4,302,788 ns/iter (+/- 424,806)
  test blockdata::block::benches::bench_block_serialize                   ... bench:     366,493 ns/iter (+/- 42,216)
  test blockdata::block::benches::bench_block_serialize_logic             ... bench:      84,646 ns/iter (+/- 7,366)
  test blockdata::transaction::benches::bench_transaction_deserialize     ... bench:         648 ns/iter (+/- 77)
  test blockdata::transaction::benches::bench_transaction_get_size        ... bench:           7 ns/iter (+/- 0)
  test blockdata::transaction::benches::bench_transaction_serialize       ... bench:          50 ns/iter (+/- 5)
  test blockdata::transaction::benches::bench_transaction_serialize_logic ... bench:          14 ns/iter (+/- 0)
  ```

  With an increased performance to deserialize a block of about 21% and to serialize a block of about 16% (seems even higher than expected, need to do more tests to confirm, I'll appreciate tests results from reviewers)

ACKs for top commit:
  apoelstra:
    ACK 106acdc3ac
  sanket1729:
    ACK 106acdc3ac
  dr-orlovsky:
    utACK 106acdc3ac

Tree-SHA512: e4f23bdd55075c7ea788bc55846fd9e30f9cb76d5847cb259bddbf72523857715b0d4dbac505be3dfb9d4b1bcae289384ab39885b4887e188f8f1c06caf4049a

.github/workflows/fuzz.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        fuzz_target: [deser_net_msg, deserialize_address, deserialize_amount, deserialize_block, deserialize_psbt, deserialize_script, deserialize_transaction, outpoint_string, uint128_fuzz, script_bytes_to_asm_fmt]
+        fuzz_target: [deser_net_msg, deserialize_address, deserialize_amount, deserialize_block, deserialize_psbt, deserialize_script, deserialize_transaction, deserialize_witness, outpoint_string, uint128_fuzz, script_bytes_to_asm_fmt]
     steps:
       - name: Install test dependencies
         run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev

fuzz/Cargo.toml

@@ -59,3 +59,7 @@ path = "fuzz_targets/uint128_fuzz.rs"
 [[bin]]
 name = "script_bytes_to_asm_fmt"
 path = "fuzz_targets/script_bytes_to_asm_fmt.rs"
+
+[[bin]]
+name = "deserialize_witness"
+path = "fuzz_targets/deserialize_witness.rs"

fuzz/fuzz_targets/deserialize_transaction.rs

@@ -10,7 +10,7 @@ fn do_test(data: &[u8]) {
             let len = ser.len();
             let calculated_weight = tx.get_weight();
             for input in &mut tx.input {
-                input.witness = vec![];
+                input.witness = bitcoin::blockdata::witness::Witness::default();
             }
             let no_witness_len = bitcoin::consensus::encode::serialize(&tx).len();
             // For 0-input transactions, `no_witness_len` will be incorrect because

fuzz/fuzz_targets/deserialize_witness.rs

@@ -0,0 +1,59 @@
+extern crate bitcoin;
+
+use bitcoin::consensus::{serialize, deserialize};
+use bitcoin::blockdata::witness::Witness;
+
+fn do_test(data: &[u8]) {
+    let w: Result<Witness, _> = deserialize(data);
+    if let Ok(witness) = w {
+        let serialized = serialize(&witness);
+        assert_eq!(data, serialized);
+    }
+}
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+    fuzz!(|data| {
+        do_test(&data);
+    });
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+    loop {
+        fuzz!(|data| {
+            do_test(data);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    fn extend_vec_from_hex(hex: &str, out: &mut Vec<u8>) {
+        let mut b = 0;
+        for (idx, c) in hex.as_bytes().iter().enumerate() {
+            b <<= 4;
+            match *c {
+                b'A'..=b'F' => b |= c - b'A' + 10,
+                b'a'..=b'f' => b |= c - b'a' + 10,
+                b'0'..=b'9' => b |= c - b'0',
+                _ => panic!("Bad hex"),
+            }
+            if (idx & 1) == 1 {
+                out.push(b);
+                b = 0;
+            }
+        }
+    }
+
+    #[test]
+    fn duplicate_crash() {
+        let mut a = Vec::new();
+        extend_vec_from_hex("00", &mut a);
+        super::do_test(&a);
+    }
+}

src/blockdata/block.rs

@@ -199,9 +199,10 @@ impl Block {
                         o.script_pubkey[0..6] == [0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed] }) {
                     let commitment = WitnessCommitment::from_slice(&coinbase.output[pos].script_pubkey.as_bytes()[6..38]).unwrap();
                     // witness reserved value is in coinbase input witness
-                    if coinbase.input[0].witness.len() == 1 && coinbase.input[0].witness[0].len() == 32 {
+                    let witness_vec: Vec<_> = coinbase.input[0].witness.iter().collect();
+                    if witness_vec.len() == 1 && witness_vec[0].len() == 32 {
                         match self.witness_root() {
-                            Some(witness_root) => return commitment == Self::compute_witness_commitment(&witness_root, coinbase.input[0].witness[0].as_slice()),
+                            Some(witness_root) => return commitment == Self::compute_witness_commitment(&witness_root, witness_vec[0]),
                             None => return false,
                         }
                     }

src/blockdata/constants.rs

@@ -29,6 +29,7 @@ use blockdata::opcodes;
 use blockdata::script;
 use blockdata::transaction::{OutPoint, Transaction, TxOut, TxIn};
 use blockdata::block::{Block, BlockHeader};
+use blockdata::witness::Witness;
 use network::constants::Network;
 use util::uint::Uint256;
 
@@ -93,7 +94,7 @@ fn bitcoin_genesis_tx() -> Transaction {
         previous_output: OutPoint::null(),
         script_sig: in_script,
         sequence: MAX_SEQUENCE,
-        witness: vec![],
+        witness: Witness::default(),
     });
 
     // Outputs

src/blockdata/mod.rs

@@ -23,4 +23,5 @@ pub mod opcodes;
 pub mod script;
 pub mod transaction;
 pub mod block;
+pub mod witness;
 

src/blockdata/transaction.rs

@@ -36,6 +36,7 @@ use util::endian;
 use blockdata::constants::WITNESS_SCALE_FACTOR;
 #[cfg(feature="bitcoinconsensus")] use blockdata::script;
 use blockdata::script::Script;
+use blockdata::witness::Witness;
 use consensus::{encode, Decodable, Encodable};
 use consensus::encode::MAX_VEC_SIZE;
 use hash_types::{SigHash, Txid, Wtxid};
@@ -197,7 +198,7 @@ pub struct TxIn {
     /// Encodable/Decodable, as it is (de)serialized at the end of the full
     /// Transaction. It *is* (de)serialized with the rest of the TxIn in other
     /// (de)serialization routines.
-    pub witness: Vec<Vec<u8>>
+    pub witness: Witness
 }
 
 impl Default for TxIn {
@@ -206,7 +207,7 @@ impl Default for TxIn {
             previous_output: OutPoint::default(),
             script_sig: Script::new(),
             sequence: u32::max_value(),
-            witness: Vec::new(),
+            witness: Witness::default(),
         }
     }
 }
@@ -280,7 +281,7 @@ impl Transaction {
         let cloned_tx = Transaction {
             version: self.version,
             lock_time: self.lock_time,
-            input: self.input.iter().map(|txin| TxIn { script_sig: Script::new(), witness: vec![], .. *txin }).collect(),
+            input: self.input.iter().map(|txin| TxIn { script_sig: Script::new(), witness: Witness::default(), .. *txin }).collect(),
             output: self.output.clone(),
         };
         cloned_tx.txid().into()
@@ -357,7 +358,7 @@ impl Transaction {
                 previous_output: self.input[input_index].previous_output,
                 script_sig: script_pubkey.clone(),
                 sequence: self.input[input_index].sequence,
-                witness: vec![],
+                witness: Witness::default(),
             }];
         } else {
             tx.input = Vec::with_capacity(self.input.len());
@@ -366,7 +367,7 @@ impl Transaction {
                     previous_output: input.previous_output,
                     script_sig: if n == input_index { script_pubkey.clone() } else { Script::new() },
                     sequence: if n != input_index && (sighash == EcdsaSigHashType::Single || sighash == EcdsaSigHashType::None) { 0 } else { input.sequence },
-                    witness: vec![],
+                    witness: Witness::default(),
                 });
             }
         }
@@ -480,10 +481,7 @@ impl Transaction {
                 input.script_sig.len());
             if !input.witness.is_empty() {
                 inputs_with_witnesses += 1;
-                input_weight += VarInt(input.witness.len() as u64).len();
+                input_weight += input.witness.serialized_len();
-                for elem in &input.witness {
-                    input_weight += VarInt(elem.len() as u64).len() + elem.len();
-                }
             }
         }
         let mut output_size = 0;
@@ -585,7 +583,7 @@ impl Decodable for TxIn {
             previous_output: Decodable::consensus_decode(&mut d)?,
             script_sig: Decodable::consensus_decode(&mut d)?,
             sequence: Decodable::consensus_decode(d)?,
-            witness: vec![],
+            witness: Witness::default(),
         })
     }
 }
@@ -1458,6 +1456,7 @@ mod tests {
         use hashes::hex::FromHex;
         use std::collections::HashMap;
         use blockdata::script;
+        use blockdata::witness::Witness;
 
         // a random recent segwit transaction from blockchain using both old and segwit inputs
         let mut spending: Transaction = deserialize(Vec::from_hex("020000000001031cfbc8f54fbfa4a33a30068841371f80dbfe166211242213188428f437445c91000000006a47304402206fbcec8d2d2e740d824d3d36cc345b37d9f65d665a99f5bd5c9e8d42270a03a8022013959632492332200c2908459547bf8dbf97c65ab1a28dec377d6f1d41d3d63e012103d7279dfb90ce17fe139ba60a7c41ddf605b25e1c07a4ddcb9dfef4e7d6710f48feffffff476222484f5e35b3f0e43f65fc76e21d8be7818dd6a989c160b1e5039b7835fc00000000171600140914414d3c94af70ac7e25407b0689e0baa10c77feffffffa83d954a62568bbc99cc644c62eb7383d7c2a2563041a0aeb891a6a4055895570000000017160014795d04cc2d4f31480d9a3710993fbd80d04301dffeffffff06fef72f000000000017a91476fd7035cd26f1a32a5ab979e056713aac25796887a5000f00000000001976a914b8332d502a529571c6af4be66399cd33379071c588ac3fda0500000000001976a914fc1d692f8de10ae33295f090bea5fe49527d975c88ac522e1b00000000001976a914808406b54d1044c429ac54c0e189b0d8061667e088ac6eb68501000000001976a914dfab6085f3a8fb3e6710206a5a959313c5618f4d88acbba20000000000001976a914eb3026552d7e3f3073457d0bee5d4757de48160d88ac0002483045022100bee24b63212939d33d513e767bc79300051f7a0d433c3fcf1e0e3bf03b9eb1d70220588dc45a9ce3a939103b4459ce47500b64e23ab118dfc03c9caa7d6bfc32b9c601210354fd80328da0f9ae6eef2b3a81f74f9a6f66761fadf96f1d1d22b1fd6845876402483045022100e29c7e3a5efc10da6269e5fc20b6a1cb8beb92130cc52c67e46ef40aaa5cac5f0220644dd1b049727d991aece98a105563416e10a5ac4221abac7d16931842d5c322012103960b87412d6e169f30e12106bdf70122aabb9eb61f455518322a18b920a4dfa887d30700")
@@ -1496,7 +1495,9 @@ mod tests {
         }).is_err());
 
         // test that we get a failure if we corrupt a signature
-        spending.input[1].witness[0][10] = 42;
+        let mut witness: Vec<_> = spending.input[1].witness.to_vec();
+        witness[0][10] = 42;
+        spending.input[1].witness = Witness::from_vec(witness);
         match spending.verify(|point: &OutPoint| {
             if let Some(tx) = spent3.remove(&point.txid) {
                 return tx.output.get(point.vout as usize).cloned();

src/blockdata/witness.rs

@@ -0,0 +1,395 @@
+//! Witness
+//!
+//! This module contains the [`Witness`] struct and related methods to operate on it
+//!
+
+use consensus::encode::{Error, MAX_VEC_SIZE};
+use consensus::{Decodable, Encodable, WriteExt};
+use io::{self, Read, Write};
+use prelude::*;
+use VarInt;
+
+#[cfg(feature = "serde")]
+use serde;
+
+/// The Witness is the data used to unlock bitcoins since the [segwit upgrade](https://github.com/bitcoin/bips/blob/master/bip-0143.mediawiki)
+///
+/// Can be logically seen as an array of byte-arrays `Vec<Vec<u8>>` and indeed you can convert from
+/// it [`Witness::from_vec`] and convert into it [`Witness::to_vec`].
+///
+/// For serialization and deserialization performance it is stored internally as a single `Vec`,
+/// saving some allocations.
+///
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
+pub struct Witness {
+    /// contains the witness Vec<Vec<u8>> serialization without the initial varint indicating the
+    /// number of elements (which is stored in `witness_elements`)
+    content: Vec<u8>,
+
+    /// Number of elements in the witness.
+    /// It is stored separately (instead of as VarInt in the initial part of content) so that method
+    /// like [`Witness::push`] doesn't have case requiring to shift the entire array
+    witness_elements: usize,
+
+    /// If `witness_elements > 0` it's a valid index pointing to the last witness element in `content`
+    /// (Including the varint specifying the length of the element)
+    last: usize,
+
+    /// If `witness_elements > 1` it's a valid index pointing to the second-to-last witness element in `content`
+    /// (Including the varint specifying the length of the element)
+    second_to_last: usize,
+}
+
+/// Support structure to allow efficient and convenient iteration over the Witness elements
+pub struct Iter<'a>(::core::slice::Iter<'a, u8>);
+
+impl Decodable for Witness {
+    fn consensus_decode<D: Read>(mut d: D) -> Result<Self, Error> {
+        let witness_elements = VarInt::consensus_decode(&mut d)?.0 as usize;
+        if witness_elements == 0 {
+            Ok(Witness::default())
+        } else {
+            let mut cursor = 0usize;
+            let mut last = 0usize;
+            let mut second_to_last = 0usize;
+
+            // this number should be determined as high enough to cover most witness, and low enough
+            // to avoid wasting space without reallocating
+            let mut content = vec![0u8; 128];
+
+            for _ in 0..witness_elements {
+                second_to_last = last;
+                last = cursor;
+                let element_size_varint = VarInt::consensus_decode(&mut d)?;
+                let element_size_varint_len = element_size_varint.len();
+                let element_size = element_size_varint.0 as usize;
+                let required_len = cursor
+                    .checked_add(element_size)
+                    .ok_or_else(|| self::Error::OversizedVectorAllocation {
+                        requested: usize::max_value(),
+                        max: MAX_VEC_SIZE,
+                    })?
+                    .checked_add(element_size_varint_len)
+                    .ok_or_else(|| self::Error::OversizedVectorAllocation {
+                        requested: usize::max_value(),
+                        max: MAX_VEC_SIZE,
+                    })?;
+
+                if required_len > MAX_VEC_SIZE {
+                    return Err(self::Error::OversizedVectorAllocation {
+                        requested: required_len,
+                        max: MAX_VEC_SIZE,
+                    });
+                }
+
+                resize_if_needed(&mut content, required_len);
+                element_size_varint
+                    .consensus_encode(&mut content[cursor..cursor + element_size_varint_len])?;
+                cursor += element_size_varint_len;
+                d.read_exact(&mut content[cursor..cursor + element_size])?;
+                cursor += element_size;
+            }
+            content.truncate(cursor);
+            Ok(Witness {
+                content,
+                witness_elements,
+                last,
+                second_to_last,
+            })
+        }
+    }
+}
+
+fn resize_if_needed(vec: &mut Vec<u8>, required_len: usize) {
+    if required_len >= vec.len() {
+        let mut new_len = vec.len().max(1);
+        while new_len <= required_len {
+            new_len *= 2;
+        }
+        vec.resize(new_len, 0);
+    }
+}
+
+impl Encodable for Witness {
+    fn consensus_encode<W: Write>(&self, mut writer: W) -> Result<usize, io::Error> {
+        let len = VarInt(self.witness_elements as u64);
+        len.consensus_encode(&mut writer)?;
+        writer.emit_slice(&self.content[..])?;
+        Ok(self.content.len() + len.len())
+    }
+}
+
+impl Witness {
+    /// Creates [`Witness`] object from an array of byte-arrays
+    pub fn from_vec(vec: Vec<Vec<u8>>) -> Self {
+        let witness_elements = vec.len();
+
+        let content_size: usize = vec
+            .iter()
+            .map(|el| el.len() + VarInt(el.len() as u64).len())
+            .sum();
+        let mut content = vec![0u8; content_size];
+        let mut cursor = 0usize;
+        let mut last = 0;
+        let mut second_to_last = 0;
+        for el in vec {
+            second_to_last = last;
+            last = cursor;
+            let el_len_varint = VarInt(el.len() as u64);
+            el_len_varint
+                .consensus_encode(&mut content[cursor..cursor + el_len_varint.len()])
+                .expect("writers on vec don't errors, space granted by content_size");
+            cursor += el_len_varint.len();
+            content[cursor..cursor + el.len()].copy_from_slice(&el);
+            cursor += el.len();
+        }
+
+        Witness {
+            witness_elements,
+            content,
+            last,
+            second_to_last,
+        }
+    }
+
+    /// Convenience method to create an array of byte-arrays from this witness
+    pub fn to_vec(&self) -> Vec<Vec<u8>> {
+        self.iter().map(|s| s.to_vec()).collect()
+    }
+
+    /// Returns `true` if the witness contains no element
+    pub fn is_empty(&self) -> bool {
+        self.witness_elements == 0
+    }
+
+    /// Returns a struct implementing [`Iterator`]
+    pub fn iter(&self) -> Iter {
+        Iter(self.content.iter())
+    }
+
+    /// Returns the number of elements this witness holds
+    pub fn len(&self) -> usize {
+        self.witness_elements as usize
+    }
+
+    /// Returns the bytes required when this Witness is consensus encoded
+    pub fn serialized_len(&self) -> usize {
+        self.iter()
+            .map(|el| VarInt(el.len() as u64).len() + el.len())
+            .sum::<usize>()
+            + VarInt(self.witness_elements as u64).len()
+    }
+
+    /// Clear the witness
+    pub fn clear(&mut self) {
+        self.content.clear();
+        self.witness_elements = 0;
+        self.last = 0;
+        self.second_to_last = 0;
+    }
+
+    /// Push a new element on the witness, requires an allocation
+    pub fn push<T: AsRef<[u8]>>(&mut self, new_element: T) {
+        let new_element = new_element.as_ref();
+        self.witness_elements += 1;
+        self.second_to_last = self.last;
+        self.last = self.content.len();
+        let element_len_varint = VarInt(new_element.len() as u64);
+        let current_content_len = self.content.len();
+        self.content.resize(
+            current_content_len + element_len_varint.len() + new_element.len(),
+            0,
+        );
+        let end_varint = current_content_len + element_len_varint.len();
+        element_len_varint
+            .consensus_encode(&mut self.content[current_content_len..end_varint])
+            .expect("writers on vec don't error, space granted through previous resize");
+        self.content[end_varint..].copy_from_slice(new_element);
+    }
+
+    fn element_at(&self, index: usize) -> Option<&[u8]> {
+        let varint = VarInt::consensus_decode(&self.content[index..]).ok()?;
+        let start = index + varint.len();
+        Some(&self.content[start..start + varint.0 as usize])
+    }
+
+    /// Return the last element in the witness, if any
+    pub fn last(&self) -> Option<&[u8]> {
+        if self.witness_elements == 0 {
+            None
+        } else {
+            self.element_at(self.last)
+        }
+    }
+
+    /// Return the second-to-last element in the witness, if any
+    pub fn second_to_last(&self) -> Option<&[u8]> {
+        if self.witness_elements <= 1 {
+            None
+        } else {
+            self.element_at(self.second_to_last)
+        }
+    }
+}
+
+impl Default for Witness {
+    fn default() -> Self {
+        // from https://doc.rust-lang.org/std/vec/struct.Vec.html#method.new
+        // The vector will not allocate until elements are pushed onto it.
+        Witness {
+            content: Vec::new(),
+            witness_elements: 0,
+            last: 0,
+            second_to_last: 0,
+        }
+    }
+}
+
+impl<'a> Iterator for Iter<'a> {
+    type Item = &'a [u8];
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let varint = VarInt::consensus_decode(self.0.as_slice()).ok()?;
+        self.0.nth(varint.len() - 1)?; // VarInt::len returns at least 1
+        let len = varint.0 as usize;
+        let slice = &self.0.as_slice()[..len];
+        if len > 0 {
+            // we don't need to advance if the element is empty
+            self.0.nth(len - 1)?;
+        }
+        Some(slice)
+    }
+}
+
+// Serde keep backward compatibility with old Vec<Vec<u8>> format
+#[cfg(feature = "serde")]
+impl serde::Serialize for Witness {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let vec: Vec<_> = self.to_vec();
+        serde::Serialize::serialize(&vec, serializer)
+    }
+}
+#[cfg(feature = "serde")]
+impl<'de> serde::Deserialize<'de> for Witness {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let vec: Vec<Vec<u8>> = serde::Deserialize::deserialize(deserializer)?;
+        Ok(Witness::from_vec(vec))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use blockdata::witness::Witness;
+    use consensus::{deserialize, serialize};
+    use hashes::hex::{FromHex, ToHex};
+    use Transaction;
+
+    #[test]
+    fn test_push() {
+        let mut witness = Witness::default();
+        assert_eq!(witness.last(), None);
+        assert_eq!(witness.second_to_last(), None);
+        witness.push(&vec![0u8]);
+        let expected = Witness {
+            witness_elements: 1,
+            content: vec![1u8, 0],
+            last: 0,
+            second_to_last: 0,
+        };
+        assert_eq!(witness, expected);
+        assert_eq!(witness.last(), Some(&[0u8][..]));
+        assert_eq!(witness.second_to_last(), None);
+        witness.push(&vec![2u8, 3u8]);
+        let expected = Witness {
+            witness_elements: 2,
+            content: vec![1u8, 0, 2, 2, 3],
+            last: 2,
+            second_to_last: 0,
+        };
+        assert_eq!(witness, expected);
+        assert_eq!(witness.last(), Some(&[2u8, 3u8][..]));
+        assert_eq!(witness.second_to_last(), Some(&[0u8][..]));
+    }
+
+    #[test]
+    fn test_witness() {
+        let w0 =
+            Vec::from_hex("03d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f2105")
+                .unwrap();
+        let w1 = Vec::from_hex("000000").unwrap();
+        let witness_vec = vec![w0.clone(), w1.clone()];
+        let witness_serialized: Vec<u8> = serialize(&witness_vec);
+        let witness = Witness {
+            content: witness_serialized[1..].to_vec(),
+            witness_elements: 2,
+            last: 34,
+            second_to_last: 0,
+        };
+        for (i, el) in witness.iter().enumerate() {
+            assert_eq!(witness_vec[i], el);
+        }
+        assert_eq!(witness.last(), Some(&w1[..]));
+        assert_eq!(witness.second_to_last(), Some(&w0[..]));
+
+        let w_into = Witness::from_vec(witness_vec);
+        assert_eq!(w_into, witness);
+
+        assert_eq!(witness_serialized, serialize(&witness));
+    }
+
+    #[test]
+    fn test_tx() {
+        let s = "02000000000102b44f26b275b8ad7b81146ba3dbecd081f9c1ea0dc05b97516f56045cfcd3df030100000000ffffffff1cb4749ae827c0b75f3d0a31e63efc8c71b47b5e3634a4c698cd53661cab09170100000000ffffffff020b3a0500000000001976a9143ea74de92762212c96f4dd66c4d72a4deb20b75788ac630500000000000016001493a8dfd1f0b6a600ab01df52b138cda0b82bb7080248304502210084622878c94f4c356ce49c8e33a063ec90f6ee9c0208540888cfab056cd1fca9022014e8dbfdfa46d318c6887afd92dcfa54510e057565e091d64d2ee3a66488f82c0121026e181ffb98ebfe5a64c983073398ea4bcd1548e7b971b4c175346a25a1c12e950247304402203ef00489a0d549114977df2820fab02df75bebb374f5eee9e615107121658cfa02204751f2d1784f8e841bff6d3bcf2396af2f1a5537c0e4397224873fbd3bfbe9cf012102ae6aa498ce2dd204e9180e71b4fb1260fe3d1a95c8025b34e56a9adf5f278af200000000";
+        let tx_bytes = Vec::from_hex(s).unwrap();
+        let tx: Transaction = deserialize(&tx_bytes).unwrap();
+
+        let expected_wit = ["304502210084622878c94f4c356ce49c8e33a063ec90f6ee9c0208540888cfab056cd1fca9022014e8dbfdfa46d318c6887afd92dcfa54510e057565e091d64d2ee3a66488f82c01", "026e181ffb98ebfe5a64c983073398ea4bcd1548e7b971b4c175346a25a1c12e95"];
+        for (i, wit_el) in tx.input[0].witness.iter().enumerate() {
+            assert_eq!(expected_wit[i], wit_el.to_hex());
+        }
+        assert_eq!(
+            expected_wit[1],
+            tx.input[0].witness.last().unwrap().to_hex()
+        );
+        assert_eq!(
+            expected_wit[0],
+            tx.input[0].witness.second_to_last().unwrap().to_hex()
+        );
+        let tx_bytes_back = serialize(&tx);
+        assert_eq!(tx_bytes_back, tx_bytes);
+    }
+
+    #[test]
+    fn fuzz_cases() {
+        let s = "26ff0000000000c94ce592cf7a4cbb68eb00ce374300000057cd0000000000000026";
+        let bytes = Vec::from_hex(s).unwrap();
+        assert!(deserialize::<Witness>(&bytes).is_err()); // OversizedVectorAllocation
+
+        let s = "24000000ffffffffffffffffffffffff";
+        let bytes = Vec::from_hex(s).unwrap();
+        assert!(deserialize::<Witness>(&bytes).is_err()); // OversizedVectorAllocation
+    }
+
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde() {
+        use serde_json;
+
+        let old_witness_format = vec![vec![0u8], vec![2]];
+        let new_witness_format = Witness::from_vec(old_witness_format.clone());
+
+        let old = serde_json::to_string(&old_witness_format).unwrap();
+        let new = serde_json::to_string(&new_witness_format).unwrap();
+
+        assert_eq!(old, new);
+
+        let back = serde_json::from_str(&new).unwrap();
+        assert_eq!(new_witness_format, back);
+    }
+}

src/util/bip143.rs

@@ -22,11 +22,10 @@
 use hashes::Hash;
 use hash_types::SigHash;
 use blockdata::script::Script;
+use blockdata::witness::Witness;
 use blockdata::transaction::{Transaction, TxIn, EcdsaSigHashType};
 use consensus::{encode, Encodable};
 
-use prelude::*;
-
 use io;
 use core::ops::{Deref, DerefMut};
 use util::sighash;
@@ -177,10 +176,10 @@ impl<R: DerefMut<Target=Transaction>> SigHashCache<R> {
     ///     let prevout_script = Script::new();
     ///     let _sighash = sig_hasher.signature_hash(inp, &prevout_script, 42, EcdsaSigHashType::All);
     ///     // ... sign the sighash
-    ///     sig_hasher.access_witness(inp).push(Vec::new());
+    ///     sig_hasher.access_witness(inp).push(&[]);
     /// }
     /// ```
-    pub fn access_witness(&mut self, input_index: usize) -> &mut Vec<Vec<u8>> {
+    pub fn access_witness(&mut self, input_index: usize) -> &mut Witness {
         self.cache.witness_mut(input_index).unwrap()
     }
 }

src/util/psbt/mod.rs

@@ -27,6 +27,7 @@ use consensus::encode::MAX_VEC_SIZE;
 use prelude::*;
 
 use io;
+use blockdata::witness::Witness;
 
 mod error;
 pub use self::error::Error;
@@ -111,7 +112,7 @@ impl PartiallySignedTransaction {
 
         for (vin, psbtin) in tx.input.iter_mut().zip(self.inputs.into_iter()) {
             vin.script_sig = psbtin.final_script_sig.unwrap_or_else(Script::new);
-            vin.witness = psbtin.final_script_witness.unwrap_or_else(Vec::new);
+            vin.witness = Witness::from_vec(psbtin.final_script_witness.unwrap_or_else(Vec::new));
         }
 
         tx
@@ -261,6 +262,7 @@ mod tests {
     use super::PartiallySignedTransaction;
     use util::psbt::raw::ProprietaryKey;
     use std::collections::BTreeMap;
+    use blockdata::witness::Witness;
 
     #[test]
     fn trivial_psbt() {
@@ -344,7 +346,7 @@ mod tests {
                     },
                     script_sig: Script::new(),
                     sequence: 4294967294,
-                    witness: vec![],
+                    witness: Witness::default(),
                 }],
                 output: vec![
                     TxOut {
@@ -419,7 +421,7 @@ mod tests {
                 },
                 script_sig: hex_script!("160014be18d152a9b012039daf3da7de4f53349eecb985"),
                 sequence: 4294967295,
-                witness: vec![Vec::from_hex("03d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f2105").unwrap()],
+                witness: Witness::from_vec(vec![Vec::from_hex("03d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f2105").unwrap()]),
             }],
             output: vec![
                 TxOut {
@@ -458,7 +460,7 @@ mod tests {
             unsigned_tx: {
                 let mut unsigned = tx.clone();
                 unsigned.input[0].script_sig = Script::new();
-                unsigned.input[0].witness = Vec::new();
+                unsigned.input[0].witness = Witness::default();
                 unsigned
             },
             proprietary: proprietary.clone(),
@@ -513,6 +515,7 @@ mod tests {
         use util::psbt::raw;
         use util::psbt::{PartiallySignedTransaction, Error};
         use std::collections::BTreeMap;
+        use blockdata::witness::Witness;
 
         #[test]
         #[should_panic(expected = "InvalidMagic")]
@@ -605,7 +608,7 @@ mod tests {
                         },
                         script_sig: Script::new(),
                         sequence: 4294967294,
-                        witness: vec![],
+                        witness: Witness::default(),
                     }],
                     output: vec![
                         TxOut {
@@ -636,10 +639,10 @@ mod tests {
                             },
                             script_sig: hex_script!("160014be18d152a9b012039daf3da7de4f53349eecb985"),
                             sequence: 4294967295,
-                            witness: vec![
+                            witness: Witness::from_vec(vec![
                                 Vec::from_hex("304402202712be22e0270f394f568311dc7ca9a68970b8025fdd3b240229f07f8a5f3a240220018b38d7dcd314e734c9276bd6fb40f673325bc4baa144c800d2f2f02db2765c01").unwrap(),
                                 Vec::from_hex("03d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f2105").unwrap(),
-                            ],
+                            ]),
                         },
                         TxIn {
                             previous_output: OutPoint {
@@ -650,10 +653,10 @@ mod tests {
                             },
                             script_sig: hex_script!("160014fe3e9ef1a745e974d902c4355943abcb34bd5353"),
                             sequence: 4294967295,
-                            witness: vec![
+                            witness: Witness::from_vec(vec![
                                 Vec::from_hex("3045022100d12b852d85dcd961d2f5f4ab660654df6eedcc794c0c33ce5cc309ffb5fce58d022067338a8e0e1725c197fb1a88af59f51e44e4255b20167c8684031c05d1f2592a01").unwrap(),
                                 Vec::from_hex("0223b72beef0965d10be0778efecd61fcac6f79a4ea169393380734464f84f2ab3").unwrap(),
-                            ],
+                            ]),
                         }],
                         output: vec![
                             TxOut {
@@ -837,7 +840,7 @@ mod tests {
                     },
                     script_sig: Script::new(),
                     sequence: 4294967294,
-                    witness: vec![],
+                    witness: Witness::default(),
                 }],
                 output: vec![
                     TxOut {
@@ -868,10 +871,10 @@ mod tests {
                         },
                         script_sig: hex_script!("160014be18d152a9b012039daf3da7de4f53349eecb985"),
                         sequence: 4294967295,
-                        witness: vec![
+                        witness: Witness::from_vec(vec![
                             Vec::from_hex("304402202712be22e0270f394f568311dc7ca9a68970b8025fdd3b240229f07f8a5f3a240220018b38d7dcd314e734c9276bd6fb40f673325bc4baa144c800d2f2f02db2765c01").unwrap(),
                             Vec::from_hex("03d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f2105").unwrap(),
-                        ],
+                        ]),
                     },
                     TxIn {
                         previous_output: OutPoint {
@@ -882,10 +885,10 @@ mod tests {
                         },
                         script_sig: hex_script!("160014fe3e9ef1a745e974d902c4355943abcb34bd5353"),
                         sequence: 4294967295,
-                        witness: vec![
+                        witness: Witness::from_vec(vec![
                             Vec::from_hex("3045022100d12b852d85dcd961d2f5f4ab660654df6eedcc794c0c33ce5cc309ffb5fce58d022067338a8e0e1725c197fb1a88af59f51e44e4255b20167c8684031c05d1f2592a01").unwrap(),
                             Vec::from_hex("0223b72beef0965d10be0778efecd61fcac6f79a4ea169393380734464f84f2ab3").unwrap(),
-                        ],
+                        ]),
                     }],
                     output: vec![
                         TxOut {

src/util/sighash.rs

@@ -21,6 +21,7 @@
 //!
 
 pub use blockdata::transaction::EcdsaSigHashType;
+use blockdata::witness::Witness;
 use consensus::{encode, Encodable};
 use core::fmt;
 use core::ops::{Deref, DerefMut};
@@ -30,8 +31,6 @@ use util::taproot::{TapLeafHash, TapSighashHash};
 use SigHash;
 use {Script, Transaction, TxOut};
 
-use prelude::*;
-
 use super::taproot::LeafVersion;
 
 /// Efficiently calculates signature hash message for legacy, segwit and taproot inputs.
@@ -704,10 +703,10 @@ impl<R: DerefMut<Target = Transaction>> SigHashCache<R> {
     ///     let prevout_script = Script::new();
     ///     let _sighash = sig_hasher.segwit_signature_hash(inp, &prevout_script, 42, EcdsaSigHashType::All);
     ///     // ... sign the sighash
-    ///     sig_hasher.witness_mut(inp).unwrap().push(Vec::new());
+    ///     sig_hasher.witness_mut(inp).unwrap().push(&Vec::new());
     /// }
     /// ```
-    pub fn witness_mut(&mut self, input_index: usize) -> Option<&mut Vec<Vec<u8>>> {
+    pub fn witness_mut(&mut self, input_index: usize) -> Option<&mut Witness> {
         self.tx.input.get_mut(input_index).map(|i| &mut i.witness)
     }
 }