From 18d8b0e469720485ef7cfb9647e3ba66a044fa59 Mon Sep 17 00:00:00 2001 From: Steven Roose Date: Fri, 20 Oct 2023 00:51:40 +0100 Subject: [PATCH] Replace VarInt type with ReadExt and WriteExt functions At some stage we named the compact encoding `VarInt` (which makes sense because the compact size encoding is a variable length integer encoding). However it turns out the term "varint" is used in Core for a different encoding so this may lead to confusion. While we fix this naming thing observe also that the `VarInt` type is unnecessarily complicated, all we need to be able to do is encode and decode integers in compact form as specified by Core. We can do this simply by extending our `WriteExt` and `ReadExt` traits. Add `emit_compact_size` and `read_compact_size` to emit and read compact endcodings respectively. Includes addition of `internals::compact_size::encoded_size_const`. Patch originally written by Steven, Tobin cherry-picked and did a bunch of impovements after the varint vs compact_size thing (#1016). ref: https://en.bitcoin.it/wiki/Protocol_documentation#Variable_length_integer Co-developed-by: Tobin C. Harding --- bitcoin/src/bip152.rs | 18 +- bitcoin/src/bip158.rs | 21 ++- bitcoin/src/blockdata/block.rs | 6 +- bitcoin/src/blockdata/transaction.rs | 38 ++-- bitcoin/src/blockdata/witness.rs | 22 +-- bitcoin/src/consensus/encode.rs | 258 ++++++++++++--------------- bitcoin/src/lib.rs | 1 - bitcoin/src/merkle_tree/block.rs | 6 +- bitcoin/src/p2p/address.rs | 13 +- bitcoin/src/p2p/message.rs | 6 +- bitcoin/src/psbt/raw.rs | 8 +- bitcoin/src/psbt/serialize.rs | 7 +- bitcoin/src/sign_message.rs | 5 +- bitcoin/src/taproot/mod.rs | 4 +- internals/src/compact_size.rs | 15 +- 15 files changed, 199 insertions(+), 229 deletions(-) diff --git a/bitcoin/src/bip152.rs b/bitcoin/src/bip152.rs index cb6fb9ec3..b19ed3109 100644 --- a/bitcoin/src/bip152.rs +++ b/bitcoin/src/bip152.rs @@ -12,7 +12,7 @@ use hashes::{sha256, siphash24}; use internals::{impl_array_newtype, ToU64 as _}; use io::{BufRead, Write}; -use crate::consensus::encode::{self, Decodable, Encodable, VarInt}; +use crate::consensus::encode::{self, Decodable, Encodable, WriteExt, ReadExt}; use crate::internal_macros::{impl_array_newtype_stringify, impl_consensus_encoding}; use crate::prelude::Vec; use crate::transaction::TxIdentifier; @@ -76,14 +76,14 @@ impl convert::AsRef for PrefilledTransaction { impl Encodable for PrefilledTransaction { #[inline] fn consensus_encode(&self, w: &mut W) -> Result { - Ok(VarInt::from(self.idx).consensus_encode(w)? + self.tx.consensus_encode(w)?) + Ok(w.emit_compact_size(self.idx)? + self.tx.consensus_encode(w)?) } } impl Decodable for PrefilledTransaction { #[inline] fn consensus_decode(r: &mut R) -> Result { - let idx = VarInt::consensus_decode(r)?.0; + let idx = r.read_compact_size()?; let idx = u16::try_from(idx) .map_err(|_| encode::Error::ParseFailed("BIP152 prefilled tx index out of bounds"))?; let tx = Transaction::consensus_decode(r)?; @@ -289,10 +289,10 @@ impl Encodable for BlockTransactionsRequest { fn consensus_encode(&self, w: &mut W) -> Result { let mut len = self.block_hash.consensus_encode(w)?; // Manually encode indexes because they are differentially encoded VarInts. - len += VarInt(self.indexes.len().to_u64()).consensus_encode(w)?; + len += w.emit_compact_size(self.indexes.len())?; let mut last_idx = 0; for idx in &self.indexes { - len += VarInt(*idx - last_idx).consensus_encode(w)?; + len += w.emit_compact_size(*idx - last_idx)?; last_idx = *idx + 1; // can panic here } Ok(len) @@ -305,12 +305,12 @@ impl Decodable for BlockTransactionsRequest { block_hash: BlockHash::consensus_decode(r)?, indexes: { // Manually decode indexes because they are differentially encoded VarInts. - let nb_indexes = VarInt::consensus_decode(r)?.0 as usize; + let nb_indexes = r.read_compact_size()? as usize; // Since the number of indices ultimately represent transactions, // we can limit the number of indices to the maximum number of // transactions that would be allowed in a vector. - let byte_size = (nb_indexes) + let byte_size = nb_indexes .checked_mul(mem::size_of::()) .ok_or(encode::Error::ParseFailed("invalid length"))?; if byte_size > encode::MAX_VEC_SIZE { @@ -323,8 +323,8 @@ impl Decodable for BlockTransactionsRequest { let mut indexes = Vec::with_capacity(nb_indexes); let mut last_index: u64 = 0; for _ in 0..nb_indexes { - let differential: VarInt = Decodable::consensus_decode(r)?; - last_index = match last_index.checked_add(differential.0) { + let differential = r.read_compact_size()?; + last_index = match last_index.checked_add(differential) { Some(i) => i, None => return Err(encode::Error::ParseFailed("block index overflow")), }; diff --git a/bitcoin/src/bip158.rs b/bitcoin/src/bip158.rs index 414a0457e..6ea426174 100644 --- a/bitcoin/src/bip158.rs +++ b/bitcoin/src/bip158.rs @@ -45,8 +45,7 @@ use internals::{write_err, ToU64 as _}; use io::{BufRead, Write}; use crate::block::{Block, BlockHash}; -use crate::consensus::encode::VarInt; -use crate::consensus::{Decodable, Encodable}; +use crate::consensus::{ReadExt, WriteExt}; use crate::internal_macros::impl_hashencode; use crate::prelude::{BTreeSet, Borrow, Vec}; use crate::script::{Script, ScriptExt as _}; @@ -286,9 +285,9 @@ impl GcsFilterReader { I::Item: Borrow<[u8]>, R: BufRead + ?Sized, { - let n_elements: VarInt = Decodable::consensus_decode(reader).unwrap_or(VarInt(0)); + let n_elements = reader.read_compact_size().unwrap_or(0); // map hashes to [0, n_elements << grp] - let nm = n_elements.0 * self.m; + let nm = n_elements * self.m; let mut mapped = query.map(|e| map_to_range(self.filter.hash(e.borrow()), nm)).collect::>(); // sort @@ -296,14 +295,14 @@ impl GcsFilterReader { if mapped.is_empty() { return Ok(true); } - if n_elements.0 == 0 { + if n_elements == 0 { return Ok(false); } // find first match in two sorted arrays in one read pass let mut reader = BitStreamReader::new(reader); let mut data = self.filter.golomb_rice_decode(&mut reader)?; - let mut remaining = n_elements.0 - 1; + let mut remaining = n_elements - 1; for p in mapped { loop { match data.cmp(&p) { @@ -329,9 +328,9 @@ impl GcsFilterReader { I::Item: Borrow<[u8]>, R: BufRead + ?Sized, { - let n_elements: VarInt = Decodable::consensus_decode(reader).unwrap_or(VarInt(0)); + let n_elements = reader.read_compact_size().unwrap_or(0); // map hashes to [0, n_elements << grp] - let nm = n_elements.0 * self.m; + let nm = n_elements * self.m; let mut mapped = query.map(|e| map_to_range(self.filter.hash(e.borrow()), nm)).collect::>(); // sort @@ -340,14 +339,14 @@ impl GcsFilterReader { if mapped.is_empty() { return Ok(true); } - if n_elements.0 == 0 { + if n_elements == 0 { return Ok(false); } // figure if all mapped are there in one read pass let mut reader = BitStreamReader::new(reader); let mut data = self.filter.golomb_rice_decode(&mut reader)?; - let mut remaining = n_elements.0 - 1; + let mut remaining = n_elements - 1; for p in mapped { loop { match data.cmp(&p) { @@ -404,7 +403,7 @@ impl<'a, W: Write> GcsFilterWriter<'a, W> { mapped.sort_unstable(); // write number of elements as varint - let mut wrote = VarInt::from(mapped.len()).consensus_encode(self.writer)?; + let mut wrote = self.writer.emit_compact_size(mapped.len())?; // write out deltas of sorted values into a Golomb-Rice coded bit stream let mut writer = BitStreamWriter::new(self.writer); diff --git a/bitcoin/src/blockdata/block.rs b/bitcoin/src/blockdata/block.rs index 6c6e73622..9661b7612 100644 --- a/bitcoin/src/blockdata/block.rs +++ b/bitcoin/src/blockdata/block.rs @@ -10,6 +10,7 @@ use core::fmt; use hashes::{sha256d, HashEngine}; +use internals::compact_size; use io::{BufRead, Write}; use super::Weight; @@ -21,7 +22,6 @@ use crate::pow::{CompactTarget, Target, Work}; use crate::prelude::Vec; use crate::script::{self, ScriptExt as _}; use crate::transaction::{Transaction, Wtxid}; -use crate::VarInt; hashes::hash_newtype! { /// A bitcoin block hash. @@ -330,7 +330,7 @@ impl Block { fn base_size(&self) -> usize { let mut size = Header::SIZE; - size += VarInt::from(self.txdata.len()).size(); + size += compact_size::encoded_size(self.txdata.len()); size += self.txdata.iter().map(|tx| tx.base_size()).sum::(); size @@ -343,7 +343,7 @@ impl Block { pub fn total_size(&self) -> usize { let mut size = Header::SIZE; - size += VarInt::from(self.txdata.len()).size(); + size += compact_size::encoded_size(self.txdata.len()); size += self.txdata.iter().map(|tx| tx.total_size()).sum::(); size diff --git a/bitcoin/src/blockdata/transaction.rs b/bitcoin/src/blockdata/transaction.rs index 0b51b984b..a4a5b6071 100644 --- a/bitcoin/src/blockdata/transaction.rs +++ b/bitcoin/src/blockdata/transaction.rs @@ -15,7 +15,7 @@ use core::{cmp, fmt, str}; #[cfg(feature = "arbitrary")] use arbitrary::{Arbitrary, Unstructured}; use hashes::sha256d; -use internals::{write_err, ToU64 as _}; +use internals::{compact_size, write_err, ToU64}; use io::{BufRead, Write}; use primitives::Sequence; use units::parse; @@ -29,7 +29,7 @@ use crate::script::{Script, ScriptBuf, ScriptExt as _, ScriptExtPriv as _}; #[cfg(doc)] use crate::sighash::{EcdsaSighashType, TapSighashType}; use crate::witness::Witness; -use crate::{Amount, FeeRate, SignedAmount, VarInt}; +use crate::{Amount, FeeRate, SignedAmount}; #[rustfmt::skip] // Keep public re-exports separate. #[doc(inline)] @@ -301,7 +301,7 @@ impl TxIn { pub fn base_size(&self) -> usize { let mut size = OutPoint::SIZE; - size += VarInt::from(self.script_sig.len()).size(); + size += compact_size::encoded_size(self.script_sig.len()); size += self.script_sig.len(); size + Sequence::SIZE @@ -414,7 +414,7 @@ impl<'a> Arbitrary<'a> for TxOut { /// Returns the total number of bytes that this script pubkey would contribute to a transaction. fn size_from_script_pubkey(script_pubkey: &Script) -> usize { let len = script_pubkey.len(); - Amount::SIZE + VarInt::from(len).size() + len + Amount::SIZE + compact_size::encoded_size(len) + len } /// Bitcoin transaction. @@ -616,10 +616,10 @@ impl Transaction { pub fn base_size(&self) -> usize { let mut size: usize = 4; // Serialized length of a u32 for the version number. - size += VarInt::from(self.input.len()).size(); + size += compact_size::encoded_size(self.input.len()); size += self.input.iter().map(|input| input.base_size()).sum::(); - size += VarInt::from(self.output.len()).size(); + size += compact_size::encoded_size(self.output.len()); size += self.output.iter().map(|output| output.size()).sum::(); size + absolute::LockTime::SIZE @@ -638,14 +638,14 @@ impl Transaction { size += 2; // 1 byte for the marker and 1 for the flag. } - size += VarInt::from(self.input.len()).size(); + size += compact_size::encoded_size(self.input.len()); size += self .input .iter() .map(|input| if uses_segwit { input.total_size() } else { input.base_size() }) .sum::(); - size += VarInt::from(self.output.len()).size(); + size += compact_size::encoded_size(self.output.len()); size += self.output.iter().map(|output| output.size()).sum::(); size + absolute::LockTime::SIZE @@ -1176,7 +1176,7 @@ where let (output_count, output_scripts_size) = output_script_lens.into_iter().fold( (0, 0), |(output_count, total_scripts_size), script_len| { - let script_size = script_len + VarInt(script_len.to_u64()).size(); + let script_size = script_len + compact_size::encoded_size(script_len); (output_count + 1, total_scripts_size + script_size) }, ); @@ -1206,8 +1206,8 @@ const fn predict_weight_internal( // version: 4 + // count varints: - VarInt(input_count as u64).size() + - VarInt(output_count as u64).size() + + compact_size::encoded_size_const(input_count as u64) + + compact_size::encoded_size_const(output_count as u64) + output_size + // lock_time 4; @@ -1247,7 +1247,7 @@ pub const fn predict_weight_from_slices( i = 0; while i < output_script_lens.len() { let script_len = output_script_lens[i]; - output_scripts_size += script_len + VarInt(script_len as u64).size(); + output_scripts_size += script_len + compact_size::encoded_size_const(script_len as u64); i += 1; } @@ -1366,13 +1366,13 @@ impl InputWeightPrediction { T::Item: Borrow, { let (count, total_size) = - witness_element_lengths.into_iter().fold((0, 0), |(count, total_size), elem_len| { + witness_element_lengths.into_iter().fold((0usize, 0), |(count, total_size), elem_len| { let elem_len = *elem_len.borrow(); - let elem_size = elem_len + VarInt(elem_len.to_u64()).size(); + let elem_size = elem_len + compact_size::encoded_size(elem_len); (count + 1, total_size + elem_size) }); - let witness_size = if count > 0 { total_size + VarInt(count as u64).size() } else { 0 }; - let script_size = input_script_len + VarInt(input_script_len.to_u64()).size(); + let witness_size = if count > 0 { total_size + compact_size::encoded_size(count) } else { 0 }; + let script_size = input_script_len + compact_size::encoded_size(input_script_len); InputWeightPrediction { script_size, witness_size } } @@ -1388,16 +1388,16 @@ impl InputWeightPrediction { // for loops not supported in const fn while i < witness_element_lengths.len() { let elem_len = witness_element_lengths[i]; - let elem_size = elem_len + VarInt(elem_len as u64).size(); + let elem_size = elem_len + compact_size::encoded_size_const(elem_len as u64); total_size += elem_size; i += 1; } let witness_size = if !witness_element_lengths.is_empty() { - total_size + VarInt(witness_element_lengths.len() as u64).size() + total_size + compact_size::encoded_size_const(witness_element_lengths.len() as u64) } else { 0 }; - let script_size = input_script_len + VarInt(input_script_len as u64).size(); + let script_size = input_script_len + compact_size::encoded_size_const(input_script_len as u64); InputWeightPrediction { script_size, witness_size } } diff --git a/bitcoin/src/blockdata/witness.rs b/bitcoin/src/blockdata/witness.rs index cfcf167cb..8b469115e 100644 --- a/bitcoin/src/blockdata/witness.rs +++ b/bitcoin/src/blockdata/witness.rs @@ -12,14 +12,14 @@ use arbitrary::{Arbitrary, Unstructured}; use internals::compact_size; use io::{BufRead, Write}; -use crate::consensus::encode::{Error, MAX_VEC_SIZE}; +use crate::consensus::encode::{Error, ReadExt, MAX_VEC_SIZE}; use crate::consensus::{Decodable, Encodable, WriteExt}; use crate::crypto::ecdsa; use crate::prelude::Vec; #[cfg(doc)] use crate::script::ScriptExt as _; use crate::taproot::{self, TAPROOT_ANNEX_PREFIX}; -use crate::{Script, VarInt}; +use crate::Script; /// The Witness is the data used to unlock bitcoin since the [segwit upgrade]. /// @@ -137,7 +137,7 @@ pub struct Iter<'a> { impl Decodable for Witness { fn consensus_decode(r: &mut R) -> Result { - let witness_elements = VarInt::consensus_decode(r)?.0 as usize; + let witness_elements = r.read_compact_size()? as usize; // Minimum size of witness element is 1 byte, so if the count is // greater than MAX_VEC_SIZE we must return an error. if witness_elements > MAX_VEC_SIZE { @@ -159,16 +159,15 @@ impl Decodable for Witness { let mut content = vec![0u8; cursor + 128]; for i in 0..witness_elements { - let element_size_varint = VarInt::consensus_decode(r)?; - let element_size_varint_len = element_size_varint.size(); - let element_size = element_size_varint.0 as usize; + let element_size = r.read_compact_size()? as usize; + let element_size_len = compact_size::encoded_size(element_size); let required_len = cursor .checked_add(element_size) .ok_or(self::Error::OversizedVectorAllocation { requested: usize::MAX, max: MAX_VEC_SIZE, })? - .checked_add(element_size_varint_len) + .checked_add(element_size_len) .ok_or(self::Error::OversizedVectorAllocation { requested: usize::MAX, max: MAX_VEC_SIZE, @@ -186,10 +185,7 @@ impl Decodable for Witness { encode_cursor(&mut content, 0, i, cursor - witness_index_space); resize_if_needed(&mut content, required_len); - element_size_varint.consensus_encode( - &mut &mut content[cursor..cursor + element_size_varint_len], - )?; - cursor += element_size_varint_len; + cursor += (&mut content[cursor..cursor + element_size_len]).emit_compact_size(element_size)?; r.read_exact(&mut content[cursor..cursor + element_size])?; cursor += element_size; } @@ -234,12 +230,10 @@ fn resize_if_needed(vec: &mut Vec, required_len: usize) { impl Encodable for Witness { // `self.content` includes the varints so encoding here includes them, as expected. fn consensus_encode(&self, w: &mut W) -> Result { - let len = VarInt::from(self.witness_elements); - len.consensus_encode(w)?; let content_with_indices_len = self.content.len(); let indices_size = self.witness_elements * 4; let content_len = content_with_indices_len - indices_size; - Ok(w.emit_slice(&self.content[..content_len])? + len.size()) + Ok(w.emit_compact_size(self.witness_elements)? + w.emit_slice(&self.content[..content_len])?) } } diff --git a/bitcoin/src/consensus/encode.rs b/bitcoin/src/consensus/encode.rs index 538a02460..feeea2248 100644 --- a/bitcoin/src/consensus/encode.rs +++ b/bitcoin/src/consensus/encode.rs @@ -18,7 +18,7 @@ use core::{fmt, mem}; use hashes::{sha256, sha256d, GeneralHash, Hash}; use hex::error::{InvalidCharError, OddLengthStringError}; -use internals::{write_err, ToU64 as _}; +use internals::{compact_size, write_err, ToU64}; use io::{BufRead, Cursor, Read, Write}; use crate::bip152::{PrefilledTransaction, ShortId}; @@ -205,6 +205,11 @@ pub trait WriteExt: Write { /// Outputs an 8-bit signed integer. fn emit_i8(&mut self, v: i8) -> Result<(), io::Error>; + /// Outputs a variable sized integer ([`CompactSize`]). + /// + /// [`CompactSize`]: + fn emit_compact_size(&mut self, v: impl ToU64) -> Result; + /// Outputs a boolean. fn emit_bool(&mut self, v: bool) -> Result<(), io::Error>; @@ -232,6 +237,11 @@ pub trait ReadExt: Read { /// Reads an 8-bit signed integer. fn read_i8(&mut self) -> Result; + /// Reads a variable sized integer ([`CompactSize`]). + /// + /// [`CompactSize`]: + fn read_compact_size(&mut self) -> Result; + /// Reads a boolean. fn read_bool(&mut self) -> Result; @@ -278,6 +288,12 @@ impl WriteExt for W { self.write_all(v)?; Ok(v.len()) } + #[inline] + fn emit_compact_size(&mut self, v: impl ToU64) -> Result { + let encoded = compact_size::encode(v.to_u64()); + self.emit_slice(&encoded)?; + Ok(encoded.len()) + } } impl ReadExt for R { @@ -306,6 +322,36 @@ impl ReadExt for R { fn read_slice(&mut self, slice: &mut [u8]) -> Result<(), Error> { self.read_exact(slice).map_err(Error::Io) } + #[inline] + fn read_compact_size(&mut self) -> Result { + match self.read_u8()? { + 0xFF => { + let x = self.read_u64()?; + if x < 0x1_0000_0000 { // I.e., would have fit in a `u32`. + Err(Error::NonMinimalVarInt) + } else { + Ok(x) + } + } + 0xFE => { + let x = self.read_u32()?; + if x < 0x1_0000 { // I.e., would have fit in a `u16`. + Err(Error::NonMinimalVarInt) + } else { + Ok(x as u64) + } + } + 0xFD => { + let x = self.read_u16()?; + if x < 0xFD { // Could have been encoded as a `u8`. + Err(Error::NonMinimalVarInt) + } else { + Ok(x as u64) + } + } + n => Ok(n as u64), + } + } } /// Maximum size, in bytes, of a vector we are allowed to decode. @@ -376,10 +422,6 @@ pub trait Decodable: Sized { } } -/// A variable-length unsigned integer. -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug)] -pub struct VarInt(pub u64); - /// Data and a 4-byte checksum. #[derive(PartialEq, Eq, Clone, Debug)] pub struct CheckedData { @@ -437,96 +479,21 @@ impl_int_encodable!(i16, read_i16, emit_i16); impl_int_encodable!(i32, read_i32, emit_i32); impl_int_encodable!(i64, read_i64, emit_i64); -#[allow(clippy::len_without_is_empty)] // VarInt has no concept of 'is_empty'. -impl VarInt { - /// Returns the number of bytes this varint contributes to a transaction size. - /// - /// Returns 1 for 0..=0xFC, 3 for 0xFD..=(2^16-1), 5 for 0x10000..=(2^32-1), and 9 otherwise. - #[inline] - pub const fn size(&self) -> usize { - match self.0 { - 0..=0xFC => 1, - 0xFD..=0xFFFF => 3, - 0x10000..=0xFFFFFFFF => 5, - _ => 9, - } +/// Returns 1 for 0..=0xFC, 3 for 0xFD..=(2^16-1), 5 for 0x10000..=(2^32-1), and 9 otherwise. +#[inline] +pub const fn varint_size_u64(v: u64) -> usize { + match v { + 0..=0xFC => 1, + 0xFD..=0xFFFF => 3, + 0x10000..=0xFFFFFFFF => 5, + _ => 9, } } -/// Implements `From for VarInt`. -/// -/// `VarInt`s are consensus encoded as `u64`s so we store them as such. Casting from any integer size smaller than or equal to `u64` is always safe and the cast value is correctly handled by `consensus_encode`. -macro_rules! impl_var_int_from { - ($($ty:tt),*) => { - $( - /// Creates a `VarInt` from a `usize` by casting the to a `u64`. - impl From<$ty> for VarInt { - fn from(x: $ty) -> Self { VarInt(x.to_u64()) } - } - )* - } -} -impl_var_int_from!(u8, u16, u32, u64, usize); - -impl Encodable for VarInt { - #[inline] - fn consensus_encode(&self, w: &mut W) -> Result { - match self.0 { - 0..=0xFC => { - (self.0 as u8).consensus_encode(w)?; - Ok(1) - } - 0xFD..=0xFFFF => { - w.emit_u8(0xFD)?; - (self.0 as u16).consensus_encode(w)?; - Ok(3) - } - 0x10000..=0xFFFFFFFF => { - w.emit_u8(0xFE)?; - (self.0 as u32).consensus_encode(w)?; - Ok(5) - } - _ => { - w.emit_u8(0xFF)?; - self.0.consensus_encode(w)?; - Ok(9) - } - } - } -} - -impl Decodable for VarInt { - #[inline] - fn consensus_decode(r: &mut R) -> Result { - let n = ReadExt::read_u8(r)?; - match n { - 0xFF => { - let x = ReadExt::read_u64(r)?; - if x < 0x100000000 { - Err(self::Error::NonMinimalVarInt) - } else { - Ok(VarInt::from(x)) - } - } - 0xFE => { - let x = ReadExt::read_u32(r)?; - if x < 0x10000 { - Err(self::Error::NonMinimalVarInt) - } else { - Ok(VarInt::from(x)) - } - } - 0xFD => { - let x = ReadExt::read_u16(r)?; - if x < 0xFD { - Err(self::Error::NonMinimalVarInt) - } else { - Ok(VarInt::from(x)) - } - } - n => Ok(VarInt::from(n)), - } - } +/// Returns 1 for 0..=0xFC, 3 for 0xFD..=(2^16-1), 5 for 0x10000..=(2^32-1), and 9 otherwise. +#[inline] +pub fn varint_size(v: impl ToU64) -> usize { + varint_size_u64(v.to_u64()) } impl Encodable for bool { @@ -547,9 +514,7 @@ impl Decodable for bool { impl Encodable for String { #[inline] fn consensus_encode(&self, w: &mut W) -> Result { - let b = self.as_bytes(); - let vi_len = VarInt(b.len().to_u64()).consensus_encode(w)?; - Ok(vi_len + w.emit_slice(b)?) + consensus_encode_with_size(self.as_bytes(), w) } } @@ -564,9 +529,7 @@ impl Decodable for String { impl Encodable for Cow<'static, str> { #[inline] fn consensus_encode(&self, w: &mut W) -> Result { - let b = self.as_bytes(); - let vi_len = VarInt(b.len().to_u64()).consensus_encode(w)?; - Ok(vi_len + w.emit_slice(b)?) + consensus_encode_with_size(self.as_bytes(), w) } } @@ -587,7 +550,8 @@ macro_rules! impl_array { &self, w: &mut W, ) -> core::result::Result { - Ok(w.emit_slice(&self[..])?) + let n = w.emit_slice(&self[..])?; + Ok(n) } } @@ -644,7 +608,7 @@ macro_rules! impl_vec { w: &mut W, ) -> core::result::Result { let mut len = 0; - len += VarInt(self.len().to_u64()).consensus_encode(w)?; + len += w.emit_compact_size(self.len())?; for c in self.iter() { len += c.consensus_encode(w)?; } @@ -657,7 +621,7 @@ macro_rules! impl_vec { fn consensus_decode_from_finite_reader( r: &mut R, ) -> core::result::Result { - let len = VarInt::consensus_decode_from_finite_reader(r)?.0; + let len = r.read_compact_size()?; // Do not allocate upfront more items than if the sequence of type // occupied roughly quarter a block. This should never be the case // for normal data, but even if that's not true - `push` will just @@ -685,7 +649,6 @@ impl_vec!(TxIn); impl_vec!(Vec); impl_vec!(u64); impl_vec!(TapLeafHash); -impl_vec!(VarInt); impl_vec!(ShortId); impl_vec!(PrefilledTransaction); @@ -700,8 +663,7 @@ pub(crate) fn consensus_encode_with_size( data: &[u8], w: &mut W, ) -> Result { - let vi_len = VarInt(data.len().to_u64()).consensus_encode(w)?; - Ok(vi_len + w.emit_slice(data)?) + Ok(w.emit_compact_size(data.len())? + w.emit_slice(data)?) } struct ReadBytesFromFiniteReaderOpts { @@ -744,7 +706,7 @@ impl Encodable for Vec { impl Decodable for Vec { #[inline] fn consensus_decode_from_finite_reader(r: &mut R) -> Result { - let len = VarInt::consensus_decode(r)?.0 as usize; + let len = r.read_compact_size()? as usize; // most real-world vec of bytes data, wouldn't be larger than 128KiB let opts = ReadBytesFromFiniteReaderOpts { len, chunk_size: 128 * 1024 }; read_bytes_from_finite_reader(r, opts) @@ -900,6 +862,8 @@ mod tests { use core::mem::discriminant; use super::*; + #[cfg(feature = "std")] + use crate::p2p::{message_blockdata::Inventory, Address}; #[test] fn serialize_int_test() { @@ -950,46 +914,51 @@ mod tests { assert_eq!(serialize(&723401728380766730i64), [10u8, 10, 10, 10, 10, 10, 10, 10]); } - #[test] - fn serialize_varint_test() { - assert_eq!(serialize(&VarInt(10)), [10u8]); - assert_eq!(serialize(&VarInt(0xFC)), [0xFCu8]); - assert_eq!(serialize(&VarInt(0xFD)), [0xFDu8, 0xFD, 0]); - assert_eq!(serialize(&VarInt(0xFFF)), [0xFDu8, 0xFF, 0xF]); - assert_eq!(serialize(&VarInt(0xF0F0F0F)), [0xFEu8, 0xF, 0xF, 0xF, 0xF]); - assert_eq!( - serialize(&VarInt(0xF0F0F0F0F0E0)), - [0xFFu8, 0xE0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0, 0] - ); - assert_eq!( - test_varint_encode(0xFF, &0x100000000_u64.to_le_bytes()).unwrap(), - VarInt(0x100000000) - ); - assert_eq!(test_varint_encode(0xFE, &0x10000_u64.to_le_bytes()).unwrap(), VarInt(0x10000)); - assert_eq!(test_varint_encode(0xFD, &0xFD_u64.to_le_bytes()).unwrap(), VarInt(0xFD)); - - // Test that length calc is working correctly - test_varint_len(VarInt(0), 1); - test_varint_len(VarInt(0xFC), 1); - test_varint_len(VarInt(0xFD), 3); - test_varint_len(VarInt(0xFFFF), 3); - test_varint_len(VarInt(0x10000), 5); - test_varint_len(VarInt(0xFFFFFFFF), 5); - test_varint_len(VarInt(0xFFFFFFFF + 1), 9); - test_varint_len(VarInt(u64::MAX), 9); - } - - fn test_varint_len(varint: VarInt, expected: usize) { - let mut encoder = vec![]; - assert_eq!(varint.consensus_encode(&mut encoder).unwrap(), expected); - assert_eq!(varint.size(), expected); - } - - fn test_varint_encode(n: u8, x: &[u8]) -> Result { + fn test_varint_encode(n: u8, x: &[u8]) -> Result { let mut input = [0u8; 9]; input[0] = n; input[1..x.len() + 1].copy_from_slice(x); - deserialize_partial::(&input).map(|t| t.0) + (&input[..]).read_compact_size() + } + + #[test] + fn serialize_varint_test() { + fn encode(v: u64) -> Vec { + let mut buf = Vec::new(); + buf.emit_compact_size(v).unwrap(); + buf + } + + assert_eq!(encode(10), [10u8]); + assert_eq!(encode(0xFC), [0xFCu8]); + assert_eq!(encode(0xFD), [0xFDu8, 0xFD, 0]); + assert_eq!(encode(0xFFF), [0xFDu8, 0xFF, 0xF]); + assert_eq!(encode(0xF0F0F0F), [0xFEu8, 0xF, 0xF, 0xF, 0xF]); + assert_eq!( + encode(0xF0F0F0F0F0E0), + vec![0xFFu8, 0xE0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0, 0], + ); + assert_eq!( + test_varint_encode(0xFF, &0x100000000_u64.to_le_bytes()).unwrap(), + 0x100000000, + ); + assert_eq!(test_varint_encode(0xFE, &0x10000_u64.to_le_bytes()).unwrap(), 0x10000); + assert_eq!(test_varint_encode(0xFD, &0xFD_u64.to_le_bytes()).unwrap(), 0xFD); + + // Test that length calc is working correctly + fn test_varint_len(varint: u64, expected: usize) { + let mut encoder = vec![]; + assert_eq!(encoder.emit_compact_size(varint).unwrap(), expected); + assert_eq!(varint_size(varint), expected); + } + test_varint_len(0, 1); + test_varint_len(0xFC, 1); + test_varint_len(0xFD, 3); + test_varint_len(0xFFFF, 3); + test_varint_len(0x10000, 5); + test_varint_len(0xFFFFFFFF, 5); + test_varint_len(0xFFFFFFFF + 1, 9); + test_varint_len(u64::MAX, 9); } #[test] @@ -1197,8 +1166,9 @@ mod tests { T: fmt::Debug, { let rand_io_err = Error::Io(io::Error::new(io::ErrorKind::Other, "")); - let varint = VarInt((super::MAX_VEC_SIZE / mem::size_of::()).to_u64()); - let err = deserialize::>(&serialize(&varint)).unwrap_err(); + let mut buf = Vec::new(); + buf.emit_compact_size(super::MAX_VEC_SIZE / mem::size_of::()).unwrap(); + let err = deserialize::>(&buf).unwrap_err(); assert_eq!(discriminant(&err), discriminant(&rand_io_err)); } diff --git a/bitcoin/src/lib.rs b/bitcoin/src/lib.rs index 546c97e95..46f352a95 100644 --- a/bitcoin/src/lib.rs +++ b/bitcoin/src/lib.rs @@ -131,7 +131,6 @@ pub use crate::{ blockdata::transaction::{self, OutPoint, Transaction, TxIn, TxOut, Txid, Wtxid}, blockdata::weight::Weight, blockdata::witness::{self, Witness}, - consensus::encode::VarInt, crypto::ecdsa, crypto::key::{self, PrivateKey, PubkeyHash, PublicKey, CompressedPublicKey, WPubkeyHash, XOnlyPublicKey}, crypto::sighash::{self, LegacySighash, SegwitV0Sighash, TapSighash, TapSighashTag}, diff --git a/bitcoin/src/merkle_tree/block.rs b/bitcoin/src/merkle_tree/block.rs index 6034dc29c..c54b7f299 100644 --- a/bitcoin/src/merkle_tree/block.rs +++ b/bitcoin/src/merkle_tree/block.rs @@ -16,7 +16,7 @@ use io::{BufRead, Write}; use self::MerkleBlockError::*; use crate::block::{self, Block}; -use crate::consensus::encode::{self, Decodable, Encodable, MAX_VEC_SIZE}; +use crate::consensus::encode::{self, Decodable, Encodable, MAX_VEC_SIZE, WriteExt, ReadExt}; use crate::merkle_tree::{MerkleNode as _, TxMerkleNode}; use crate::prelude::Vec; use crate::transaction::{Transaction, Txid}; @@ -405,7 +405,7 @@ impl Encodable for PartialMerkleTree { ret += self.hashes.consensus_encode(w)?; let nb_bytes_for_bits = (self.bits.len() + 7) / 8; - ret += encode::VarInt::from(nb_bytes_for_bits).consensus_encode(w)?; + ret += w.emit_compact_size(nb_bytes_for_bits)?; for chunk in self.bits.chunks(8) { let mut byte = 0u8; for (i, bit) in chunk.iter().enumerate() { @@ -424,7 +424,7 @@ impl Decodable for PartialMerkleTree { let num_transactions: u32 = Decodable::consensus_decode(r)?; let hashes: Vec = Decodable::consensus_decode(r)?; - let nb_bytes_for_bits = encode::VarInt::consensus_decode(r)?.0 as usize; + let nb_bytes_for_bits = r.read_compact_size()? as usize; if nb_bytes_for_bits > MAX_VEC_SIZE { return Err(encode::Error::OversizedVectorAllocation { requested: nb_bytes_for_bits, diff --git a/bitcoin/src/p2p/address.rs b/bitcoin/src/p2p/address.rs index 8bcee95fc..a531618e0 100644 --- a/bitcoin/src/p2p/address.rs +++ b/bitcoin/src/p2p/address.rs @@ -10,7 +10,7 @@ use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, ToSoc use io::{BufRead, Read, Write}; -use crate::consensus::encode::{self, Decodable, Encodable, ReadExt, VarInt, WriteExt}; +use crate::consensus::encode::{self, Decodable, Encodable, ReadExt, WriteExt}; use crate::p2p::ServiceFlags; /// A message which can be sent on the Bitcoin network @@ -146,10 +146,7 @@ impl Encodable for AddrV2 { network: u8, bytes: &[u8], ) -> Result { - let len = network.consensus_encode(w)? - + VarInt::from(bytes.len()).consensus_encode(w)? - + w.emit_slice(bytes)?; - Ok(len) + Ok(network.consensus_encode(w)? + encode::consensus_encode_with_size(bytes, w)?) } Ok(match *self { AddrV2::Ipv4(ref addr) => encode_addr(w, 1, &addr.octets())?, @@ -166,7 +163,7 @@ impl Encodable for AddrV2 { impl Decodable for AddrV2 { fn consensus_decode(r: &mut R) -> Result { let network_id = u8::consensus_decode(r)?; - let len = VarInt::consensus_decode(r)?.0; + let len = r.read_compact_size()?; if len > 512 { return Err(encode::Error::ParseFailed("IP must be <= 512 bytes")); } @@ -271,7 +268,7 @@ impl Encodable for AddrV2Message { fn consensus_encode(&self, w: &mut W) -> Result { let mut len = 0; len += self.time.consensus_encode(w)?; - len += VarInt(self.services.to_u64()).consensus_encode(w)?; + len += w.emit_compact_size(self.services.to_u64())?; len += self.addr.consensus_encode(w)?; w.write_all(&self.port.to_be_bytes())?; @@ -285,7 +282,7 @@ impl Decodable for AddrV2Message { fn consensus_decode(r: &mut R) -> Result { Ok(AddrV2Message { time: Decodable::consensus_decode(r)?, - services: ServiceFlags::from(VarInt::consensus_decode(r)?.0), + services: ServiceFlags::from(r.read_compact_size()?), addr: Decodable::consensus_decode(r)?, port: u16::swap_bytes(Decodable::consensus_decode(r)?), }) diff --git a/bitcoin/src/p2p/message.rs b/bitcoin/src/p2p/message.rs index 33392cd8a..3dccfb85c 100644 --- a/bitcoin/src/p2p/message.rs +++ b/bitcoin/src/p2p/message.rs @@ -11,7 +11,7 @@ use hashes::sha256d; use internals::ToU64 as _; use io::{BufRead, Write}; -use crate::consensus::encode::{self, CheckedData, Decodable, Encodable, VarInt}; +use crate::consensus::encode::{self, CheckedData, Decodable, Encodable, WriteExt, ReadExt}; use crate::merkle_tree::MerkleBlock; use crate::p2p::address::{AddrV2Message, Address}; use crate::p2p::{ @@ -337,7 +337,7 @@ impl<'a> Encodable for HeaderSerializationWrapper<'a> { #[inline] fn consensus_encode(&self, w: &mut W) -> Result { let mut len = 0; - len += VarInt::from(self.0.len()).consensus_encode(w)?; + len += w.emit_compact_size(self.0.len())?; for header in self.0.iter() { len += header.consensus_encode(w)?; len += 0u8.consensus_encode(w)?; @@ -410,7 +410,7 @@ impl Decodable for HeaderDeserializationWrapper { fn consensus_decode_from_finite_reader( r: &mut R, ) -> Result { - let len = VarInt::consensus_decode(r)?.0; + let len = r.read_compact_size()?; // should be above usual number of items to avoid // allocation let mut ret = Vec::with_capacity(core::cmp::min(1024 * 16, len as usize)); diff --git a/bitcoin/src/psbt/raw.rs b/bitcoin/src/psbt/raw.rs index d6c7a6019..70f499ac1 100644 --- a/bitcoin/src/psbt/raw.rs +++ b/bitcoin/src/psbt/raw.rs @@ -12,7 +12,7 @@ use io::{BufRead, Write}; use super::serialize::{Deserialize, Serialize}; use crate::consensus::encode::{ - self, deserialize, serialize, Decodable, Encodable, ReadExt, VarInt, WriteExt, MAX_VEC_SIZE, + self, deserialize, serialize, Decodable, Encodable, ReadExt, WriteExt, MAX_VEC_SIZE, }; use crate::prelude::{DisplayHex, Vec}; use crate::psbt::Error; @@ -73,7 +73,7 @@ impl fmt::Display for Key { impl Key { pub(crate) fn decode(r: &mut R) -> Result { - let VarInt(byte_size): VarInt = Decodable::consensus_decode(r)?; + let byte_size = r.read_compact_size()?; if byte_size == 0 { return Err(Error::NoMorePairs); @@ -103,9 +103,7 @@ impl Key { impl Serialize for Key { fn serialize(&self) -> Vec { let mut buf = Vec::new(); - VarInt::from(self.key_data.len() + 1) - .consensus_encode(&mut buf) - .expect("in-memory writers don't error"); + buf.emit_compact_size(self.key_data.len() + 1).expect("in-memory writers don't error"); self.type_value.consensus_encode(&mut buf).expect("in-memory writers don't error"); diff --git a/bitcoin/src/psbt/serialize.rs b/bitcoin/src/psbt/serialize.rs index 9fa465d1f..896cbeb3f 100644 --- a/bitcoin/src/psbt/serialize.rs +++ b/bitcoin/src/psbt/serialize.rs @@ -6,6 +6,7 @@ //! according to the BIP-174 specification. use hashes::{hash160, ripemd160, sha256, sha256d}; +use internals::compact_size; use secp256k1::XOnlyPublicKey; use super::map::{Input, Map, Output, PsbtSighashType}; @@ -22,7 +23,7 @@ use crate::taproot::{ }; use crate::transaction::{Transaction, TxOut}; use crate::witness::Witness; -use crate::VarInt; + /// A trait for serializing a value as raw data for insertion into PSBT /// key-value maps. pub(crate) trait Serialize { @@ -355,8 +356,8 @@ impl Serialize for TapTree { let capacity = self .script_leaves() .map(|l| { - l.script().len() + VarInt::from(l.script().len()).size() // script version - + 1 // Merkle branch + l.script().len() + compact_size::encoded_size(l.script().len()) // script version + + 1 // merkle branch + 1 // leaf version }) .sum::(); diff --git a/bitcoin/src/sign_message.rs b/bitcoin/src/sign_message.rs index 8626e118f..5f2f183db 100644 --- a/bitcoin/src/sign_message.rs +++ b/bitcoin/src/sign_message.rs @@ -7,7 +7,7 @@ use hashes::{sha256d, HashEngine}; -use crate::consensus::{encode, Encodable}; +use crate::consensus::encode::WriteExt; #[rustfmt::skip] #[doc(inline)] @@ -209,8 +209,7 @@ pub fn signed_msg_hash(msg: impl AsRef<[u8]>) -> sha256d::Hash { let msg_bytes = msg.as_ref(); let mut engine = sha256d::Hash::engine(); engine.input(BITCOIN_SIGNED_MSG_PREFIX); - let msg_len = encode::VarInt::from(msg_bytes.len()); - msg_len.consensus_encode(&mut engine).expect("engines don't error"); + engine.emit_compact_size(msg_bytes.len()).expect("engines don't error"); engine.input(msg_bytes); sha256d::Hash::from_engine(engine) } diff --git a/bitcoin/src/taproot/mod.rs b/bitcoin/src/taproot/mod.rs index 3227159d9..31de88a20 100644 --- a/bitcoin/src/taproot/mod.rs +++ b/bitcoin/src/taproot/mod.rs @@ -1153,8 +1153,8 @@ impl ControlBlock { /// Serializes the control block. /// /// This would be required when using [`ControlBlock`] as a witness element while spending an - /// output via script path. This serialization does not include the [`crate::VarInt`] prefix that would - /// be applied when encoding this element as a witness. + /// output via script path. This serialization does not include the varint prefix that would be + /// applied when encoding this element as a witness. pub fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity(self.size()); self.encode(&mut buf).expect("writers don't error"); diff --git a/internals/src/compact_size.rs b/internals/src/compact_size.rs index e147d404a..0e8a0daa2 100644 --- a/internals/src/compact_size.rs +++ b/internals/src/compact_size.rs @@ -30,7 +30,20 @@ const MAX_ENCODING_SIZE: usize = 9; /// - 9 otherwise. #[inline] pub fn encoded_size(value: impl ToU64) -> usize { - match value.to_u64() { + encoded_size_const(value.to_u64()) +} + +/// Returns the number of bytes used to encode this `CompactSize` value (in const context). +/// +/// # Returns +/// +/// - 1 for 0..=0xFC +/// - 3 for 0xFD..=(2^16-1) +/// - 5 for 0x10000..=(2^32-1) +/// - 9 otherwise. +#[inline] +pub const fn encoded_size_const(value: u64) -> usize { + match value { 0..=0xFC => 1, 0xFD..=0xFFFF => 3, 0x10000..=0xFFFFFFFF => 5,