milksad
/
rust-bitcoin-unsafe-fast
14
0
Fork 0
Code Issues Pull Requests Activity

Merge pull request #264 from TheBlueMatt/2019-05-faster-enc-dec 

Speed up consensus::[d]encode significantly
This commit is contained in:
Andrew Poelstra 2019-06-05 11:46:56 +00:00 committed by GitHub
parent 4139f2a1ca 2b6058e335
commit 08c756d20e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 120 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fuzz/travis-fuzz.sh
View File

@ -7,7 +7,7 @@ for TARGET in fuzz_targets/*; do
if [ -d hfuzz_input/$FILE ]; then
HFUZZ_INPUT_ARGS="-f hfuzz_input/$FILE/input"
fi
HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" HFUZZ_RUN_ARGS="-N1000000 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run $FILE
HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" HFUZZ_RUN_ARGS="-N200000 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run $FILE
if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then
cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT

221
src/consensus/encode.rs
View File

@ -29,8 +29,6 @@
//! big-endian decimals, etc.)
//!
use std::collections::HashMap;
use std::hash::Hash;
use std::{mem, u32};
use std::error;
@ -47,6 +45,10 @@ use secp256k1;
use util::base58;
use util::psbt;
use blockdata::transaction::{TxOut, Transaction, TxIn};
use network::message_blockdata::Inventory;
use network::address::Address;
/// Encoding error
#[derive(Debug)]
pub enum Error {
@ -262,6 +264,9 @@ pub trait Encoder {
/// Output a boolean
fn emit_bool(&mut self, v: bool) -> Result<(), Error>;
/// Output a byte slice
fn emit_slice(&mut self, v: &[u8]) -> Result<(), Error>;
}
/// A simple Decoder trait
@ -286,6 +291,9 @@ pub trait Decoder {
/// Read a boolean
fn read_bool(&mut self) -> Result<bool, Error>;
/// Read a byte slice
fn read_slice(&mut self, slice: &mut [u8]) -> Result<(), Error>;
}
macro_rules! encoder_fn {
@ -326,6 +334,10 @@ impl<W: Write> Encoder for W {
fn emit_bool(&mut self, v: bool) -> Result<(), Error> {
self.write_i8(if v {1} else {0}).map_err(Error::Io)
}
#[inline]
fn emit_slice(&mut self, v: &[u8]) -> Result<(), Error> {
self.write_all(v).map_err(Error::Io)
}
}
impl<R: Read> Decoder for R {
@ -348,6 +360,10 @@ impl<R: Read> Decoder for R {
fn read_bool(&mut self) -> Result<bool, Error> {
Decoder::read_i8(self).map(|bit| bit != 0)
}
#[inline]
fn read_slice(&mut self, slice: &mut [u8]) -> Result<(), Error> {
self.read_exact(slice).map_err(Error::Io)
}
}
/// Maximum size, in bytes, of a vector we are allowed to decode
@ -475,7 +491,9 @@ impl<D: Decoder> Decodable<D> for bool {
impl<S: Encoder> Encodable<S> for String {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
self.as_bytes().consensus_encode(s)
let b = self.as_bytes();
VarInt(b.len() as u64).consensus_encode(s)?;
s.emit_slice(&b)
}
}
@ -491,21 +509,18 @@ impl<D: Decoder> Decodable<D> for String {
// Arrays
macro_rules! impl_array {
( $size:expr ) => (
impl<S: Encoder, T: Encodable<S>> Encodable<S> for [T; $size] {
impl<S: Encoder> Encodable<S> for [u8; $size] {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
for i in self.iter() { i.consensus_encode(s)?; }
Ok(())
s.emit_slice(&self[..])
}
}
impl<D: Decoder, T:Decodable<D> + Copy> Decodable<D> for [T; $size] {
impl<D: Decoder> Decodable<D> for [u8; $size] {
#[inline]
fn consensus_decode(d: &mut D) -> Result<[T; $size], self::Error> {
// Set everything to the first decode
let mut ret = [Decodable::consensus_decode(d)?; $size];
// Set the rest
for item in ret.iter_mut().take($size).skip(1) { *item = Decodable::consensus_decode(d)?; }
fn consensus_decode(d: &mut D) -> Result<[u8; $size], self::Error> {
let mut ret = [0; $size];
d.read_slice(&mut ret)?;
Ok(ret)
}
}
@ -520,54 +535,105 @@ impl_array!(16);
impl_array!(32);
impl_array!(33);
impl<S: Encoder, T: Encodable<S>> Encodable<S> for [T] {
impl<D: Decoder> Decodable<D> for [u16; 8] {
#[inline]
fn consensus_decode(d: &mut D) -> Result<[u16; 8], self::Error> {
let mut res = [0; 8];
for i in 0..8 {
res[i] = Decodable::consensus_decode(d)?;
}
Ok(res)
}
}
impl<S: Encoder> Encodable<S> for [u16; 8] {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
VarInt(self.len() as u64).consensus_encode(s)?;
for c in self.iter() { c.consensus_encode(s)?; }
Ok(())
}
}
// Cannot decode a slice
// Vectors
impl<S: Encoder, T: Encodable<S>> Encodable<S> for Vec<T> {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> { (&self[..]).consensus_encode(s) }
}
impl<D: Decoder, T: Decodable<D>> Decodable<D> for Vec<T> {
#[inline]
fn consensus_decode(d: &mut D) -> Result<Vec<T>, self::Error> {
let len = VarInt::consensus_decode(d)?.0;
let byte_size = (len as usize)
.checked_mul(mem::size_of::<T>())
.ok_or(self::Error::ParseFailed("Invalid length"))?;
if byte_size > MAX_VEC_SIZE {
return Err(self::Error::OversizedVectorAllocation { requested: byte_size, max: MAX_VEC_SIZE })
macro_rules! impl_vec {
($type: ty) => {
impl<S: Encoder> Encodable<S> for Vec<$type> {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
VarInt(self.len() as u64).consensus_encode(s)?;
for c in self.iter() { c.consensus_encode(s)?; }
Ok(())
}
}
let mut ret = Vec::with_capacity(len as usize);
for _ in 0..len { ret.push(Decodable::consensus_decode(d)?); }
Ok(ret)
impl<D: Decoder> Decodable<D> for Vec<$type> {
#[inline]
fn consensus_decode(d: &mut D) -> Result<Vec<$type>, self::Error> {
let len = VarInt::consensus_decode(d)?.0;
let byte_size = (len as usize)
.checked_mul(mem::size_of::<$type>())
.ok_or(self::Error::ParseFailed("Invalid length"))?;
if byte_size > MAX_VEC_SIZE {
return Err(self::Error::OversizedVectorAllocation { requested: byte_size, max: MAX_VEC_SIZE })
}
let mut ret = Vec::with_capacity(len as usize);
for _ in 0..len { ret.push(Decodable::consensus_decode(d)?); }
Ok(ret)
}
}
}
}
impl_vec!(sha256d::Hash);
impl_vec!(Transaction);
impl_vec!(TxOut);
impl_vec!(TxIn);
impl_vec!(Inventory);
impl_vec!(Vec<u8>);
impl_vec!((u32, Address));
impl_vec!(u64);
impl<S: Encoder> Encodable<S> for Vec<u8> {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
VarInt(self.len() as u64).consensus_encode(s)?;
s.emit_slice(&self)
}
}
impl<S: Encoder, T: Encodable<S>> Encodable<S> for Box<[T]> {
impl<D: Decoder> Decodable<D> for Vec<u8> {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> { (&self[..]).consensus_encode(s) }
}
impl<D: Decoder, T: Decodable<D>> Decodable<D> for Box<[T]> {
#[inline]
fn consensus_decode(d: &mut D) -> Result<Box<[T]>, self::Error> {
fn consensus_decode(d: &mut D) -> Result<Vec<u8>, self::Error> {
let len = VarInt::consensus_decode(d)?.0;
let len = len as usize;
if len > MAX_VEC_SIZE {
return Err(self::Error::OversizedVectorAllocation { requested: len, max: MAX_VEC_SIZE })
}
let mut ret = Vec::with_capacity(len);
for _ in 0..len { ret.push(Decodable::consensus_decode(d)?); }
ret.resize(len, 0);
d.read_slice(&mut ret)?;
Ok(ret)
}
}
impl<S: Encoder> Encodable<S> for Box<[u8]> {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
VarInt(self.len() as u64).consensus_encode(s)?;
s.emit_slice(&self)
}
}
impl<D: Decoder> Decodable<D> for Box<[u8]> {
#[inline]
fn consensus_decode(d: &mut D) -> Result<Box<[u8]>, self::Error> {
let len = VarInt::consensus_decode(d)?.0;
let len = len as usize;
if len > MAX_VEC_SIZE {
return Err(self::Error::OversizedVectorAllocation { requested: len, max: MAX_VEC_SIZE })
}
let mut ret = Vec::with_capacity(len);
ret.resize(len, 0);
d.read_slice(&mut ret)?;
Ok(ret.into_boxed_slice())
}
}
@ -585,11 +651,7 @@ impl<S: Encoder> Encodable<S> for CheckedData {
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
(self.0.len() as u32).consensus_encode(s)?;
sha2_checksum(&self.0).consensus_encode(s)?;
// We can't just pass to the slice encoder since it'll insert a length
for ch in &self.0 {
ch.consensus_encode(s)?;
}
Ok(())
s.emit_slice(&self.0)
}
}
@ -599,7 +661,8 @@ impl<D: Decoder> Decodable<D> for CheckedData {
let len: u32 = Decodable::consensus_decode(d)?;
let checksum: [u8; 4] = Decodable::consensus_decode(d)?;
let mut ret = Vec::with_capacity(len as usize);
for _ in 0..len { ret.push(Decodable::consensus_decode(d)?); }
ret.resize(len as usize, 0);
d.read_slice(&mut ret)?;
let expected_checksum = sha2_checksum(&ret);
if expected_checksum != checksum {
Err(self::Error::InvalidChecksum {
@ -640,54 +703,6 @@ tuple_encode!(T0, T1, T2, T3);
tuple_encode!(T0, T1, T2, T3, T4, T5);
tuple_encode!(T0, T1, T2, T3, T4, T5, T6, T7);
// References
impl<S: Encoder, T: Encodable<S>> Encodable<S> for Box<T> {
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> { (**self).consensus_encode(s) }
}
impl<D: Decoder, T: Decodable<D>> Decodable<D> for Box<T> {
#[inline]
fn consensus_decode(d: &mut D) -> Result<Box<T>, self::Error> {
Decodable::consensus_decode(d).map(Box::new)
}
}
// HashMap
impl<S, K, V> Encodable<S> for HashMap<K, V>
where S: Encoder,
K: Encodable<S> + Eq + Hash,
V: Encodable<S>
{
#[inline]
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
VarInt(self.len() as u64).consensus_encode(s)?;
for (key, value) in self.iter() {
key.consensus_encode(s)?;
value.consensus_encode(s)?;
}
Ok(())
}
}
impl<D, K, V> Decodable<D> for HashMap<K, V>
where D: Decoder,
K: Decodable<D> + Eq + Hash,
V: Decodable<D>
{
#[inline]
fn consensus_decode(d: &mut D) -> Result<HashMap<K, V>, self::Error> {
let len = VarInt::consensus_decode(d)?.0;
let mut ret = HashMap::with_capacity(len as usize);
for _ in 0..len {
ret.insert(Decodable::consensus_decode(d)?,
Decodable::consensus_decode(d)?);
}
Ok(ret)
}
}
impl<S: Encoder> Encodable<S> for sha256d::Hash {
fn consensus_encode(&self, s: &mut S) -> Result<(), self::Error> {
self.into_inner().consensus_encode(s)
@ -806,7 +821,6 @@ mod tests {
#[test]
fn serialize_vector_test() {
assert_eq!(serialize(&vec![1u8, 2, 3]), vec![3u8, 1, 2, 3]);
assert_eq!(serialize(&[1u8, 2, 3][..]), vec![3u8, 1, 2, 3]);
// TODO: test vectors of more interesting objects
}
@ -815,13 +829,6 @@ mod tests {
assert_eq!(serialize(&"Andrew".to_string()), vec![6u8, 0x41, 0x6e, 0x64, 0x72, 0x65, 0x77]);
}
#[test]
fn serialize_box_test() {
assert_eq!(serialize(&Box::new(1u8)), vec![1u8]);
assert_eq!(serialize(&Box::new(1u16)), vec![1u8, 0]);
assert_eq!(serialize(&Box::new(1u64)), vec![1u8, 0, 0, 0, 0, 0, 0, 0]);
}
#[test]
fn deserialize_int_test() {
// bool
@ -882,13 +889,5 @@ mod tests {
let cd: Result<CheckedData, _> = deserialize(&[5u8, 0, 0, 0, 162, 107, 175, 90, 1, 2, 3, 4, 5]);
assert_eq!(cd.ok(), Some(CheckedData(vec![1u8, 2, 3, 4, 5])));
}
#[test]
fn deserialize_box_test() {
let zero: Result<Box<u8>, _> = deserialize(&[0u8]);
let one: Result<Box<u8>, _> = deserialize(&[1u8]);
assert_eq!(zero.ok(), Some(Box::new(0)));
assert_eq!(one.ok(), Some(Box::new(1)));
}
}

5
src/network/message.rs
View File

@ -99,6 +99,8 @@ pub enum NetworkMessage {
Block(block::Block),
/// `headers`
Headers(Vec<block::BlockHeader>),
/// `sendheaders`
SendHeaders,
/// `getaddr`
GetAddr,
// TODO: checkorder,
@ -142,6 +144,7 @@ impl RawNetworkMessage {
NetworkMessage::Tx(_) => "tx",
NetworkMessage::Block(_) => "block",
NetworkMessage::Headers(_) => "headers",
NetworkMessage::SendHeaders => "sendheaders",
NetworkMessage::GetAddr => "getaddr",
NetworkMessage::Ping(_) => "ping",
NetworkMessage::Pong(_) => "pong",
@ -194,6 +197,7 @@ impl<S: Encoder> Encodable<S> for RawNetworkMessage {
NetworkMessage::CFCheckpt(ref dat) => serialize(dat),
NetworkMessage::Alert(ref dat) => serialize(dat),
NetworkMessage::Verack
| NetworkMessage::SendHeaders
| NetworkMessage::MemPool
| NetworkMessage::GetAddr => vec![],
}).consensus_encode(s)
@ -243,6 +247,7 @@ impl<D: Decoder> Decodable<D> for RawNetworkMessage {
"headers" =>
NetworkMessage::Headers(<HeaderDeserializationWrapper as Decodable<Cursor<Vec<u8>>>>
::consensus_decode(&mut mem_d)?.0),
"sendheaders" => NetworkMessage::SendHeaders,
"getaddr" => NetworkMessage::GetAddr,
"ping" => NetworkMessage::Ping(Decodable::consensus_decode(&mut mem_d)?),
"pong" => NetworkMessage::Pong(Decodable::consensus_decode(&mut mem_d)?),

5
src/util/uint.rs
View File

@ -348,7 +348,10 @@ macro_rules! construct_uint {
impl<D: ::consensus::encode::Decoder> ::consensus::encode::Decodable<D> for $name {
fn consensus_decode(d: &mut D) -> Result<$name, encode::Error> {
use consensus::encode::Decodable;
let ret: [u64; $n_words] = Decodable::consensus_decode(d)?;
let mut ret: [u64; $n_words] = [0; $n_words];
for i in 0..$n_words {
ret[i] = Decodable::consensus_decode(d)?;
}
Ok($name(ret))
}
}