From a2ce000b2b113a1f1183354d012bf74ebae905d0 Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Fri, 1 Aug 2014 09:01:39 -0700 Subject: [PATCH] Revamp Serializable interface to be similar to Encoder/Encodable This is a massive simplification, fixes a couple endianness bugs (though not all of them I don't think), should give a speedup, gets rid of the `serialize_iter` crap. --- src/blockdata/block.rs | 34 +- src/blockdata/blockchain.rs | 145 +++--- src/blockdata/constants.rs | 13 +- src/blockdata/script.rs | 25 +- src/blockdata/transaction.rs | 85 +-- src/blockdata/utxoset.rs | 53 +- src/internal_macros.rs | 36 +- src/network/address.rs | 103 ++-- src/network/constants.rs | 23 +- src/network/encodable.rs | 558 ++++++++++++++++++++ src/network/message.rs | 116 +++-- src/network/message_blockdata.rs | 114 ++-- src/network/message_network.rs | 82 ++- src/network/mod.rs | 1 + src/network/serialize.rs | 870 +++++-------------------------- src/network/socket.rs | 57 +- src/util/hash.rs | 67 +-- src/util/iter.rs | 71 --- src/util/mod.rs | 1 - src/util/patricia_tree.rs | 149 +++--- src/util/thinvec.rs | 44 +- src/util/uint.rs | 43 +- 22 files changed, 1278 insertions(+), 1412 deletions(-) create mode 100644 src/network/encodable.rs diff --git a/src/blockdata/block.rs b/src/blockdata/block.rs index 528691ee..c0329f15 100644 --- a/src/blockdata/block.rs +++ b/src/blockdata/block.rs @@ -20,13 +20,13 @@ //! these blocks and the blockchain. //! -use std::io::IoResult; use std::num::{Zero, from_u64}; use util::error::{BitcoinResult, SpvBadTarget, SpvBadProofOfWork}; use util::hash::Sha256dHash; use util::uint::Uint256; -use network::serialize::{Serializable, SerializeIter, VarInt}; +use network::encodable::{ConsensusEncodable, VarInt}; +use network::serialize::BitcoinHash; use blockdata::transaction::Transaction; /// A block header, which contains all the block's information except @@ -101,7 +101,7 @@ impl BlockHeader { if target != required_target { return Err(SpvBadTarget); } - let ref hash = self.bitcoin_hash().as_uint256(); + let ref hash = self.bitcoin_hash().into_uint256(); if hash <= target { Ok(()) } else { Err(SpvBadProofOfWork) } } @@ -117,10 +117,23 @@ impl BlockHeader { } } -impl_serializable!(BlockHeader, version, prev_blockhash, merkle_root, time, bits, nonce) +impl BitcoinHash for BlockHeader { + fn bitcoin_hash(&self) -> Sha256dHash { + use network::serialize::serialize; + Sha256dHash::from_data(serialize(self).unwrap().as_slice()) + } +} + +impl BitcoinHash for Block { + fn bitcoin_hash(&self) -> Sha256dHash { + self.header.bitcoin_hash() + } +} + +impl_consensus_encoding!(BlockHeader, version, prev_blockhash, merkle_root, time, bits, nonce) impl_json!(BlockHeader, version, prev_blockhash, merkle_root, time, bits, nonce) -impl_serializable!(Block, header, txdata) -impl_serializable!(LoneBlockHeader, header, tx_count) +impl_consensus_encoding!(Block, header, txdata) +impl_consensus_encoding!(LoneBlockHeader, header, tx_count) #[cfg(test)] mod tests { @@ -128,7 +141,7 @@ mod tests { use serialize::hex::FromHex; use blockdata::block::Block; - use network::serialize::Serializable; + use network::serialize::{deserialize, serialize}; #[test] fn block_test() { @@ -138,8 +151,8 @@ mod tests { let prevhash = "4ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000".from_hex().unwrap(); let merkle = "bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914c".from_hex().unwrap(); - let decode: IoResult = Serializable::deserialize(some_block.iter().map(|n| *n)); - let bad_decode: IoResult = Serializable::deserialize(cutoff_block.iter().map(|n| *n)); + let decode: IoResult = deserialize(some_block.clone()); + let bad_decode: IoResult = deserialize(cutoff_block); assert!(decode.is_ok()); assert!(bad_decode.is_err()); @@ -153,8 +166,7 @@ mod tests { assert_eq!(real_decode.header.nonce, 2067413810); // [test] TODO: check the transaction data - let reserialize = real_decode.serialize(); - assert_eq!(reserialize.as_slice(), some_block.as_slice()); + assert_eq!(serialize(&real_decode), Ok(some_block)); } } diff --git a/src/blockdata/blockchain.rs b/src/blockdata/blockchain.rs index a4bf9d64..91df7c93 100644 --- a/src/blockdata/blockchain.rs +++ b/src/blockdata/blockchain.rs @@ -22,7 +22,6 @@ //! to make sure we are holding the only references. //! -use std::io::{IoResult, IoError, OtherIoError}; use std::num::Zero; use std::kinds::marker; @@ -31,15 +30,15 @@ use blockdata::transaction::Transaction; use blockdata::constants::{DIFFCHANGE_INTERVAL, DIFFCHANGE_TIMESPAN, TARGET_BLOCK_SPACING, max_target, genesis_block}; use network::constants::{Network, BitcoinTestnet}; -use network::serialize::{Serializable, SerializeIter}; +use network::encodable::{ConsensusDecodable, ConsensusEncodable}; +use network::serialize::{BitcoinHash, SimpleDecoder, SimpleEncoder}; use util::BitArray; use util::error::{BitcoinResult, BlockNotFound, DuplicateHash, PrevHashNotFound}; use util::uint::Uint256; use util::hash::Sha256dHash; -use util::misc::prepend_err; use util::patricia_tree::PatriciaTree; -type BlockTree = PatriciaTree, Uint256>; +type BlockTree = PatriciaTree>; type NodePtr = *const BlockchainNode; /// A link in the blockchain @@ -80,32 +79,35 @@ impl BlockchainNode { } } -impl Serializable for BlockchainNode { - fn serialize(&self) -> Vec { - let mut ret = vec![]; - ret.extend(self.block.serialize().move_iter()); - ret.extend(self.total_work.serialize().move_iter()); - ret.extend(self.required_difficulty.serialize().move_iter()); - ret.extend(self.height.serialize().move_iter()); - ret.extend(self.has_txdata.serialize().move_iter()); +impl, E> ConsensusEncodable for BlockchainNode { + #[inline] + fn consensus_encode(&self, s: &mut S) -> Result<(), E> { + try!(self.block.consensus_encode(s)); + try!(self.total_work.consensus_encode(s)); + try!(self.required_difficulty.consensus_encode(s)); + try!(self.height.consensus_encode(s)); + try!(self.has_txdata.consensus_encode(s)); // Don't serialize the prev or next pointers - ret + Ok(()) } +} - fn deserialize>(mut iter: I) -> IoResult { +impl, E> ConsensusDecodable for BlockchainNode { + #[inline] + fn consensus_decode(d: &mut D) -> Result { Ok(BlockchainNode { - block: try!(prepend_err("block", Serializable::deserialize(iter.by_ref()))), - total_work: try!(prepend_err("total_work", Serializable::deserialize(iter.by_ref()))), - required_difficulty: try!(prepend_err("req_difficulty", Serializable::deserialize(iter.by_ref()))), - height: try!(prepend_err("height", Serializable::deserialize(iter.by_ref()))), - has_txdata: try!(prepend_err("has_txdata", Serializable::deserialize(iter.by_ref()))), + block: try!(ConsensusDecodable::consensus_decode(d)), + total_work: try!(ConsensusDecodable::consensus_decode(d)), + required_difficulty: try!(ConsensusDecodable::consensus_decode(d)), + height: try!(ConsensusDecodable::consensus_decode(d)), + has_txdata: try!(ConsensusDecodable::consensus_decode(d)), prev: RawPtr::null(), next: RawPtr::null() }) } +} - // Override Serialize::hash to return the blockheader hash, since the - // hash of the node itself is pretty much meaningless. +impl BitcoinHash for BlockchainNode { fn bitcoin_hash(&self) -> Sha256dHash { self.block.header.bitcoin_hash() } @@ -120,55 +122,39 @@ pub struct Blockchain { genesis_hash: Sha256dHash } -impl Serializable for Blockchain { - fn serialize(&self) -> Vec { - let mut ret = vec![]; - ret.extend(self.network.serialize().move_iter()); - ret.extend(self.tree.serialize().move_iter()); - ret.extend(self.best_hash.serialize().move_iter()); - ret.extend(self.genesis_hash.serialize().move_iter()); - ret +impl, E> ConsensusEncodable for Blockchain { + #[inline] + fn consensus_encode(&self, s: &mut S) -> Result<(), E> { + try!(self.network.consensus_encode(s)); + try!(self.tree.consensus_encode(s)); + try!(self.best_hash.consensus_encode(s)); + try!(self.genesis_hash.consensus_encode(s)); + Ok(()) } +} - fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> { - SerializeIter { - data_iter: None, - sub_iter_iter: box vec![ &self.network as &Serializable, - &self.tree as &Serializable, - &self.best_hash as &Serializable, - &self.genesis_hash as &Serializable ].move_iter(), - sub_iter: None, - sub_started: false - } - } +impl, E> ConsensusDecodable for Blockchain { + fn consensus_decode(d: &mut D) -> Result { + let network: Network = try!(ConsensusDecodable::consensus_decode(d)); + let mut tree: BlockTree = try!(ConsensusDecodable::consensus_decode(d)); + let best_hash: Sha256dHash = try!(ConsensusDecodable::consensus_decode(d)); + let genesis_hash: Sha256dHash = try!(ConsensusDecodable::consensus_decode(d)); - fn deserialize>(mut iter: I) -> IoResult { - let network: Network = try!(prepend_err("network", Serializable::deserialize(iter.by_ref()))); - let mut tree: BlockTree = try!(prepend_err("tree", Serializable::deserialize(iter.by_ref()))); - let best_hash: Sha256dHash = try!(prepend_err("best_hash", Serializable::deserialize(iter.by_ref()))); - let genesis_hash: Sha256dHash = try!(prepend_err("genesis_hash", Serializable::deserialize(iter.by_ref()))); // Lookup best tip - let best = match tree.lookup(&best_hash.as_uint256(), 256) { + let best = match tree.lookup(&best_hash.into_uint256(), 256) { Some(node) => &**node as NodePtr, - None => { return Err(IoError { - kind: OtherIoError, - desc: "best tip reference not found in tree", - detail: Some(format!("best tip {:x} not found", best_hash)) - }); + None => { + return Err(d.error(format!("best tip {:x} not in tree", best_hash).as_slice())); } }; // Lookup genesis - if tree.lookup(&genesis_hash.as_uint256(), 256).is_none() { - return Err(IoError { - kind: OtherIoError, - desc: "genesis block not found in tree", - detail: Some(format!("genesis {:x} not found", genesis_hash)) - }); + if tree.lookup(&genesis_hash.into_uint256(), 256).is_none() { + return Err(d.error(format!("genesis {:x} not in tree", genesis_hash).as_slice())); } // Reconnect all prev pointers let raw_tree = &tree as *const _; for node in tree.mut_iter() { - let hash = node.block.header.prev_blockhash.as_uint256(); + let hash = node.block.header.prev_blockhash.into_uint256(); let prevptr = match unsafe { (*raw_tree).lookup(&hash, 256) } { Some(node) => &**node as NodePtr, @@ -187,11 +173,8 @@ impl Serializable for Blockchain { // Check that "genesis" is the genesis if (*scan).bitcoin_hash() != genesis_hash { - return Err(IoError { - kind: OtherIoError, - desc: "best tip did not link back to genesis", - detail: Some(format!("no path from tip {:x} to genesis {:x}", best_hash, genesis_hash)) - }); + return Err(d.error(format!("no path from tip {:x} to genesis {:x}", + best_hash, genesis_hash).as_slice())); } } @@ -372,7 +355,7 @@ impl Blockchain { network: network, tree: { let mut pat = PatriciaTree::new(); - pat.insert(&genhash.as_uint256(), 256, new_node); + pat.insert(&genhash.into_uint256(), 256, new_node); pat }, best_hash: genhash, @@ -417,17 +400,17 @@ impl Blockchain { /// Looks up a block in the chain and returns the BlockchainNode containing it pub fn get_block<'a>(&'a self, hash: Sha256dHash) -> Option<&'a BlockchainNode> { - self.tree.lookup(&hash.as_uint256(), 256).map(|node| &**node) + self.tree.lookup(&hash.into_uint256(), 256).map(|node| &**node) } /// Locates a block in the chain and overwrites its txdata pub fn add_txdata(&mut self, block: Block) -> BitcoinResult<()> { - self.replace_txdata(&block.header.bitcoin_hash().as_uint256(), block.txdata, true) + self.replace_txdata(&block.header.bitcoin_hash().into_uint256(), block.txdata, true) } /// Locates a block in the chain and removes its txdata pub fn remove_txdata(&mut self, hash: Sha256dHash) -> BitcoinResult<()> { - self.replace_txdata(&hash.as_uint256(), vec![], false) + self.replace_txdata(&hash.into_uint256(), vec![], false) } /// Adds a block header to the chain @@ -447,13 +430,13 @@ impl Blockchain { if hash == chain.best_hash { Some(chain.best_tip) } else { - chain.tree.lookup(&hash.as_uint256(), 256).map(|boxptr| &**boxptr as NodePtr) + chain.tree.lookup(&hash.into_uint256(), 256).map(|boxptr| &**boxptr as NodePtr) } } // Check for multiple inserts (bitcoind from c9a09183 to 3c85d2ec doesn't // handle locator hashes properly and may return blocks multiple times, // and this may also happen in case of a reorg. - if self.tree.lookup(&block.header.bitcoin_hash().as_uint256(), 256).is_some() { + if self.tree.lookup(&block.header.bitcoin_hash().into_uint256(), 256).is_some() { return Err(DuplicateHash); } // Construct node, if possible @@ -532,7 +515,7 @@ impl Blockchain { // Insert the new block let raw_ptr = &*new_block as NodePtr; - self.tree.insert(&new_block.block.header.bitcoin_hash().as_uint256(), 256, new_block); + self.tree.insert(&new_block.block.header.bitcoin_hash().into_uint256(), 256, new_block); // Replace the best tip if necessary if unsafe { (*raw_ptr).total_work > (*self.best_tip).total_work } { self.set_best_tip(raw_ptr); @@ -582,7 +565,7 @@ impl Blockchain { /// An iterator over all blocks in the chain starting from `start_hash` pub fn iter<'a>(&'a self, start_hash: Sha256dHash) -> BlockIter<'a> { - let start = match self.tree.lookup(&start_hash.as_uint256(), 256) { + let start = match self.tree.lookup(&start_hash.into_uint256(), 256) { Some(boxptr) => &**boxptr as NodePtr, None => RawPtr::null() }; @@ -594,7 +577,7 @@ impl Blockchain { /// An iterator over all blocks in reverse order to the genesis, starting with `start_hash` pub fn rev_iter<'a>(&'a self, start_hash: Sha256dHash) -> RevBlockIter<'a> { - let start = match self.tree.lookup(&start_hash.as_uint256(), 256) { + let start = match self.tree.lookup(&start_hash.into_uint256(), 256) { Some(boxptr) => &**boxptr as NodePtr, None => RawPtr::null() }; @@ -606,7 +589,7 @@ impl Blockchain { /// An iterator over all blocks -not- in the best chain, in reverse order, starting from `start_hash` pub fn rev_stale_iter<'a>(&'a self, start_hash: Sha256dHash) -> RevStaleBlockIter<'a> { - let start = match self.tree.lookup(&start_hash.as_uint256(), 256) { + let start = match self.tree.lookup(&start_hash.into_uint256(), 256) { Some(boxptr) => { // If we are already on the main chain, we have a dead iterator if boxptr.is_on_main_chain(self) { @@ -626,28 +609,26 @@ impl Blockchain { #[cfg(test)] mod tests { - use std::prelude::*; use std::io::IoResult; use blockdata::blockchain::Blockchain; use blockdata::constants::genesis_block; use network::constants::Bitcoin; - use network::serialize::Serializable; + use network::serialize::{BitcoinHash, deserialize, serialize}; #[test] fn blockchain_serialize_test() { let empty_chain = Blockchain::new(Bitcoin); - assert_eq!(empty_chain.best_tip().header.bitcoin_hash().serialize(), - genesis_block(Bitcoin).header.bitcoin_hash().serialize()); + assert_eq!(empty_chain.best_tip().header.bitcoin_hash(), + genesis_block(Bitcoin).header.bitcoin_hash()); - let serial = empty_chain.serialize(); - assert_eq!(serial, empty_chain.serialize_iter().collect()); + let serial = serialize(&empty_chain); + let deserial: IoResult = deserialize(serial.unwrap()); - let deserial: IoResult = Serializable::deserialize(serial.iter().map(|n| *n)); assert!(deserial.is_ok()); let read_chain = deserial.unwrap(); - assert_eq!(read_chain.best_tip().header.bitcoin_hash().serialize(), - genesis_block(Bitcoin).header.bitcoin_hash().serialize()); + assert_eq!(read_chain.best_tip().header.bitcoin_hash(), + genesis_block(Bitcoin).header.bitcoin_hash()); } } diff --git a/src/blockdata/constants.rs b/src/blockdata/constants.rs index 59c032ff..9380d1b8 100644 --- a/src/blockdata/constants.rs +++ b/src/blockdata/constants.rs @@ -113,11 +113,12 @@ pub fn genesis_block(network: Network) -> Block { #[cfg(test)] mod test { - use network::serialize::Serializable; + use serialize::hex::FromHex; + use network::constants::{Bitcoin, BitcoinTestnet}; + use network::serialize::{BitcoinHash, serialize}; use blockdata::constants::{genesis_block, bitcoin_genesis_tx}; use blockdata::constants::{MAX_SEQUENCE, COIN_VALUE}; - use util::misc::hex_bytes; use util::hash::zero_hash; #[test] @@ -128,13 +129,13 @@ mod test { assert_eq!(gen.input.len(), 1); assert_eq!(gen.input[0].prev_hash.as_slice(), zero_hash().as_slice()); assert_eq!(gen.input[0].prev_index, 0xFFFFFFFF); - assert_eq!(gen.input[0].script_sig.serialize().as_slice(), - hex_bytes("4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73").unwrap().as_slice()); + assert_eq!(serialize(&gen.input[0].script_sig), + Ok("4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73".from_hex().unwrap())); assert_eq!(gen.input[0].sequence, MAX_SEQUENCE); assert_eq!(gen.output.len(), 1); - assert_eq!(gen.output[0].script_pubkey.serialize().as_slice(), - hex_bytes("434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac").unwrap().as_slice()); + assert_eq!(serialize(&gen.output[0].script_pubkey), + Ok("434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac".from_hex().unwrap())); assert_eq!(gen.output[0].value, 50 * COIN_VALUE); assert_eq!(gen.lock_time, 0); diff --git a/src/blockdata/script.rs b/src/blockdata/script.rs index 3b6074f6..b838d5dd 100644 --- a/src/blockdata/script.rs +++ b/src/blockdata/script.rs @@ -25,11 +25,11 @@ //! use std::char::from_digit; -use std::io::IoResult; use serialize::json; -use network::serialize::Serializable; use blockdata::opcodes; +use network::encodable::{ConsensusDecodable, ConsensusEncodable}; +use network::serialize::{SimpleDecoder, SimpleEncoder}; use util::thinvec::ThinVec; #[deriving(PartialEq, Show, Clone)] @@ -136,15 +136,18 @@ impl json::ToJson for Script { } // Network serialization -impl Serializable for Script { - fn serialize(&self) -> Vec { +impl, E> ConsensusEncodable for Script { + #[inline] + fn consensus_encode(&self, s: &mut S) -> Result<(), E> { let &Script(ref data) = self; - data.serialize() + data.consensus_encode(s) } +} - fn deserialize>(iter: I) -> IoResult