Revamp Serializable interface to be similar to Encoder/Encodable

This is a massive simplification, fixes a couple endianness bugs (though
not all of them I don't think), should give a speedup, gets rid of the
`serialize_iter` crap.

src/blockdata/block.rs

@@ -20,13 +20,13 @@
 //! these blocks and the blockchain.
 //!
 
-use std::io::IoResult;
 use std::num::{Zero, from_u64};
 
 use util::error::{BitcoinResult, SpvBadTarget, SpvBadProofOfWork};
 use util::hash::Sha256dHash;
 use util::uint::Uint256;
-use network::serialize::{Serializable, SerializeIter, VarInt};
+use network::encodable::{ConsensusEncodable, VarInt};
+use network::serialize::BitcoinHash;
 use blockdata::transaction::Transaction;
 
 /// A block header, which contains all the block's information except
@@ -101,7 +101,7 @@ impl BlockHeader {
     if target != required_target {
       return Err(SpvBadTarget);
     }
-    let ref hash = self.bitcoin_hash().as_uint256();
+    let ref hash = self.bitcoin_hash().into_uint256();
     if hash <= target { Ok(()) } else { Err(SpvBadProofOfWork) }
   }
 
@@ -117,10 +117,23 @@ impl BlockHeader {
   }
 }
 
-impl_serializable!(BlockHeader, version, prev_blockhash, merkle_root, time, bits, nonce)
+impl BitcoinHash for BlockHeader {
+  fn bitcoin_hash(&self) -> Sha256dHash {
+    use network::serialize::serialize;
+    Sha256dHash::from_data(serialize(self).unwrap().as_slice())
+  }
+}
+
+impl BitcoinHash for Block {
+  fn bitcoin_hash(&self) -> Sha256dHash {
+    self.header.bitcoin_hash()
+  }
+}
+
+impl_consensus_encoding!(BlockHeader, version, prev_blockhash, merkle_root, time, bits, nonce)
 impl_json!(BlockHeader, version, prev_blockhash, merkle_root, time, bits, nonce)
-impl_serializable!(Block, header, txdata)
+impl_consensus_encoding!(Block, header, txdata)
-impl_serializable!(LoneBlockHeader, header, tx_count)
+impl_consensus_encoding!(LoneBlockHeader, header, tx_count)
 
 #[cfg(test)]
 mod tests {
@@ -128,7 +141,7 @@ mod tests {
   use serialize::hex::FromHex;
 
   use blockdata::block::Block;
-  use network::serialize::Serializable;
+  use network::serialize::{deserialize, serialize};
 
   #[test]
   fn block_test() {
@@ -138,8 +151,8 @@ mod tests {
     let prevhash = "4ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000".from_hex().unwrap();
     let merkle = "bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914c".from_hex().unwrap();
 
-    let decode: IoResult<Block> = Serializable::deserialize(some_block.iter().map(|n| *n));
+    let decode: IoResult<Block> = deserialize(some_block.clone());
-    let bad_decode: IoResult<Block> = Serializable::deserialize(cutoff_block.iter().map(|n| *n));
+    let bad_decode: IoResult<Block> = deserialize(cutoff_block);
 
     assert!(decode.is_ok());
     assert!(bad_decode.is_err());
@@ -153,8 +166,7 @@ mod tests {
     assert_eq!(real_decode.header.nonce, 2067413810);
     // [test] TODO: check the transaction data
   
-    let reserialize = real_decode.serialize();
+    assert_eq!(serialize(&real_decode), Ok(some_block));
-    assert_eq!(reserialize.as_slice(), some_block.as_slice());
   }
 }
 

src/blockdata/blockchain.rs

@@ -22,7 +22,6 @@
 //! to make sure we are holding the only references.
 //!
 
-use std::io::{IoResult, IoError, OtherIoError};
 use std::num::Zero;
 use std::kinds::marker;
 
@@ -31,15 +30,15 @@ use blockdata::transaction::Transaction;
 use blockdata::constants::{DIFFCHANGE_INTERVAL, DIFFCHANGE_TIMESPAN,
                            TARGET_BLOCK_SPACING, max_target, genesis_block};
 use network::constants::{Network, BitcoinTestnet};
-use network::serialize::{Serializable, SerializeIter};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
+use network::serialize::{BitcoinHash, SimpleDecoder, SimpleEncoder};
 use util::BitArray;
 use util::error::{BitcoinResult, BlockNotFound, DuplicateHash, PrevHashNotFound};
 use util::uint::Uint256;
 use util::hash::Sha256dHash;
-use util::misc::prepend_err;
 use util::patricia_tree::PatriciaTree;
 
-type BlockTree = PatriciaTree<Box<BlockchainNode>, Uint256>;
+type BlockTree = PatriciaTree<Uint256, Box<BlockchainNode>>;
 type NodePtr = *const BlockchainNode;
 
 /// A link in the blockchain
@@ -80,32 +79,35 @@ impl BlockchainNode {
   }
 }
 
-impl Serializable for BlockchainNode {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for BlockchainNode {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
-    let mut ret = vec![];
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    ret.extend(self.block.serialize().move_iter());
+    try!(self.block.consensus_encode(s));
-    ret.extend(self.total_work.serialize().move_iter());
+    try!(self.total_work.consensus_encode(s));
-    ret.extend(self.required_difficulty.serialize().move_iter());
+    try!(self.required_difficulty.consensus_encode(s));
-    ret.extend(self.height.serialize().move_iter());
+    try!(self.height.consensus_encode(s));
-    ret.extend(self.has_txdata.serialize().move_iter());
+    try!(self.has_txdata.consensus_encode(s));
     // Don't serialize the prev or next pointers
-    ret
+    Ok(())
+  }
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<BlockchainNode> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for BlockchainNode {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<BlockchainNode, E> {
     Ok(BlockchainNode {
-      block: try!(prepend_err("block", Serializable::deserialize(iter.by_ref()))),
+      block: try!(ConsensusDecodable::consensus_decode(d)),
-      total_work: try!(prepend_err("total_work", Serializable::deserialize(iter.by_ref()))),
+      total_work: try!(ConsensusDecodable::consensus_decode(d)),
-      required_difficulty: try!(prepend_err("req_difficulty", Serializable::deserialize(iter.by_ref()))),
+      required_difficulty: try!(ConsensusDecodable::consensus_decode(d)),
-      height: try!(prepend_err("height", Serializable::deserialize(iter.by_ref()))),
+      height: try!(ConsensusDecodable::consensus_decode(d)),
-      has_txdata: try!(prepend_err("has_txdata", Serializable::deserialize(iter.by_ref()))),
+      has_txdata: try!(ConsensusDecodable::consensus_decode(d)),
       prev: RawPtr::null(),
       next: RawPtr::null()
     })
   }
+}
 
-  // Override Serialize::hash to return the blockheader hash, since the
+impl BitcoinHash for BlockchainNode {
-  // hash of the node itself is pretty much meaningless.
   fn bitcoin_hash(&self) -> Sha256dHash {
     self.block.header.bitcoin_hash()
   }
@@ -120,55 +122,39 @@ pub struct Blockchain {
   genesis_hash: Sha256dHash
 }
 
-impl Serializable for Blockchain {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for Blockchain {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
-    let mut ret = vec![];
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    ret.extend(self.network.serialize().move_iter());
+    try!(self.network.consensus_encode(s));
-    ret.extend(self.tree.serialize().move_iter());
+    try!(self.tree.consensus_encode(s));
-    ret.extend(self.best_hash.serialize().move_iter());
+    try!(self.best_hash.consensus_encode(s));
-    ret.extend(self.genesis_hash.serialize().move_iter());
+    try!(self.genesis_hash.consensus_encode(s));
-    ret
+    Ok(())
-  }
-
-  fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> {
-    SerializeIter {
-      data_iter: None,
-      sub_iter_iter: box vec![ &self.network as &Serializable,
-                               &self.tree as &Serializable,
-                               &self.best_hash as &Serializable,
-                               &self.genesis_hash as &Serializable ].move_iter(),
-      sub_iter: None,
-      sub_started: false
   }
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<Blockchain> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for Blockchain {
-    let network: Network = try!(prepend_err("network", Serializable::deserialize(iter.by_ref())));
+  fn consensus_decode(d: &mut D) -> Result<Blockchain, E> {
-    let mut tree: BlockTree = try!(prepend_err("tree", Serializable::deserialize(iter.by_ref())));
+    let network: Network = try!(ConsensusDecodable::consensus_decode(d));
-    let best_hash: Sha256dHash = try!(prepend_err("best_hash", Serializable::deserialize(iter.by_ref())));
+    let mut tree: BlockTree = try!(ConsensusDecodable::consensus_decode(d));
-    let genesis_hash: Sha256dHash = try!(prepend_err("genesis_hash", Serializable::deserialize(iter.by_ref())));
+    let best_hash: Sha256dHash = try!(ConsensusDecodable::consensus_decode(d));
+    let genesis_hash: Sha256dHash = try!(ConsensusDecodable::consensus_decode(d));
+
     // Lookup best tip
-    let best = match tree.lookup(&best_hash.as_uint256(), 256) {
+    let best = match tree.lookup(&best_hash.into_uint256(), 256) {
       Some(node) => &**node as NodePtr,
-      None => { return Err(IoError {
+      None => {
-          kind: OtherIoError,
+        return Err(d.error(format!("best tip {:x} not in tree", best_hash).as_slice()));
-          desc: "best tip reference not found in tree",
-          detail: Some(format!("best tip {:x} not found", best_hash))
-        });
       }
     };
     // Lookup genesis
-    if tree.lookup(&genesis_hash.as_uint256(), 256).is_none() {
+    if tree.lookup(&genesis_hash.into_uint256(), 256).is_none() {
-      return Err(IoError {
+      return Err(d.error(format!("genesis {:x} not in tree", genesis_hash).as_slice()));
-        kind: OtherIoError,
-        desc: "genesis block not found in tree",
-        detail: Some(format!("genesis {:x} not found", genesis_hash))
-      });
     }
     // Reconnect all prev pointers
     let raw_tree = &tree as *const _;
     for node in tree.mut_iter() {
-      let hash = node.block.header.prev_blockhash.as_uint256();
+      let hash = node.block.header.prev_blockhash.into_uint256();
       let prevptr =
         match unsafe { (*raw_tree).lookup(&hash, 256) } {
           Some(node) => &**node as NodePtr,
@@ -187,11 +173,8 @@ impl Serializable for Blockchain {
 
       // Check that "genesis" is the genesis
       if (*scan).bitcoin_hash() != genesis_hash {
-        return Err(IoError {
+        return Err(d.error(format!("no path from tip {:x} to genesis {:x}",
-            kind: OtherIoError,
+                                   best_hash, genesis_hash).as_slice()));
-            desc: "best tip did not link back to genesis",
-            detail: Some(format!("no path from tip {:x} to genesis {:x}", best_hash, genesis_hash))
-        });
       }
     }
 
@@ -372,7 +355,7 @@ impl Blockchain {
       network: network,
       tree: {
         let mut pat = PatriciaTree::new();
-        pat.insert(&genhash.as_uint256(), 256, new_node);
+        pat.insert(&genhash.into_uint256(), 256, new_node);
         pat
       },
       best_hash: genhash,
@@ -417,17 +400,17 @@ impl Blockchain {
 
   /// Looks up a block in the chain and returns the BlockchainNode containing it
   pub fn get_block<'a>(&'a self, hash: Sha256dHash) -> Option<&'a BlockchainNode> {
-    self.tree.lookup(&hash.as_uint256(), 256).map(|node| &**node)
+    self.tree.lookup(&hash.into_uint256(), 256).map(|node| &**node)
   }
 
   /// Locates a block in the chain and overwrites its txdata
   pub fn add_txdata(&mut self, block: Block) -> BitcoinResult<()> {
-    self.replace_txdata(&block.header.bitcoin_hash().as_uint256(), block.txdata, true)
+    self.replace_txdata(&block.header.bitcoin_hash().into_uint256(), block.txdata, true)
   }
 
   /// Locates a block in the chain and removes its txdata
   pub fn remove_txdata(&mut self, hash: Sha256dHash) -> BitcoinResult<()> {
-    self.replace_txdata(&hash.as_uint256(), vec![], false)
+    self.replace_txdata(&hash.into_uint256(), vec![], false)
   }
 
   /// Adds a block header to the chain
@@ -447,13 +430,13 @@ impl Blockchain {
       if hash == chain.best_hash { 
         Some(chain.best_tip)
       } else {
-        chain.tree.lookup(&hash.as_uint256(), 256).map(|boxptr| &**boxptr as NodePtr)
+        chain.tree.lookup(&hash.into_uint256(), 256).map(|boxptr| &**boxptr as NodePtr)
       }
     }
     // Check for multiple inserts (bitcoind from c9a09183 to 3c85d2ec doesn't
     // handle locator hashes properly and may return blocks multiple times,
     // and this may also happen in case of a reorg.
-    if self.tree.lookup(&block.header.bitcoin_hash().as_uint256(), 256).is_some() {
+    if self.tree.lookup(&block.header.bitcoin_hash().into_uint256(), 256).is_some() {
       return Err(DuplicateHash);
     }
     // Construct node, if possible
@@ -532,7 +515,7 @@ impl Blockchain {
 
     // Insert the new block
     let raw_ptr = &*new_block as NodePtr;
-    self.tree.insert(&new_block.block.header.bitcoin_hash().as_uint256(), 256, new_block);
+    self.tree.insert(&new_block.block.header.bitcoin_hash().into_uint256(), 256, new_block);
     // Replace the best tip if necessary
     if unsafe { (*raw_ptr).total_work > (*self.best_tip).total_work } {
       self.set_best_tip(raw_ptr);
@@ -582,7 +565,7 @@ impl Blockchain {
 
   /// An iterator over all blocks in the chain starting from `start_hash`
   pub fn iter<'a>(&'a self, start_hash: Sha256dHash) -> BlockIter<'a> {
-    let start = match self.tree.lookup(&start_hash.as_uint256(), 256) {
+    let start = match self.tree.lookup(&start_hash.into_uint256(), 256) {
         Some(boxptr) => &**boxptr as NodePtr,
         None => RawPtr::null()
       };
@@ -594,7 +577,7 @@ impl Blockchain {
 
   /// An iterator over all blocks in reverse order to the genesis, starting with `start_hash`
   pub fn rev_iter<'a>(&'a self, start_hash: Sha256dHash) -> RevBlockIter<'a> {
-    let start = match self.tree.lookup(&start_hash.as_uint256(), 256) {
+    let start = match self.tree.lookup(&start_hash.into_uint256(), 256) {
         Some(boxptr) => &**boxptr as NodePtr,
         None => RawPtr::null()
       };
@@ -606,7 +589,7 @@ impl Blockchain {
 
   /// An iterator over all blocks -not- in the best chain, in reverse order, starting from `start_hash`
   pub fn rev_stale_iter<'a>(&'a self, start_hash: Sha256dHash) -> RevStaleBlockIter<'a> {
-    let start = match self.tree.lookup(&start_hash.as_uint256(), 256) {
+    let start = match self.tree.lookup(&start_hash.into_uint256(), 256) {
         Some(boxptr) => {
           // If we are already on the main chain, we have a dead iterator
           if boxptr.is_on_main_chain(self) {
@@ -626,28 +609,26 @@ impl Blockchain {
 
 #[cfg(test)]
 mod tests {
-  use std::prelude::*;
   use std::io::IoResult;
 
   use blockdata::blockchain::Blockchain;
   use blockdata::constants::genesis_block;
   use network::constants::Bitcoin;
-  use network::serialize::Serializable;
+  use network::serialize::{BitcoinHash, deserialize, serialize};
 
   #[test]
   fn blockchain_serialize_test() {
     let empty_chain = Blockchain::new(Bitcoin);
-    assert_eq!(empty_chain.best_tip().header.bitcoin_hash().serialize(),
+    assert_eq!(empty_chain.best_tip().header.bitcoin_hash(),
-               genesis_block(Bitcoin).header.bitcoin_hash().serialize());
+               genesis_block(Bitcoin).header.bitcoin_hash());
 
-    let serial = empty_chain.serialize();
+    let serial = serialize(&empty_chain);
-    assert_eq!(serial, empty_chain.serialize_iter().collect());
+    let deserial: IoResult<Blockchain> = deserialize(serial.unwrap());
 
-    let deserial: IoResult<Blockchain> = Serializable::deserialize(serial.iter().map(|n| *n));
     assert!(deserial.is_ok());
     let read_chain = deserial.unwrap();
-    assert_eq!(read_chain.best_tip().header.bitcoin_hash().serialize(),
+    assert_eq!(read_chain.best_tip().header.bitcoin_hash(),
-               genesis_block(Bitcoin).header.bitcoin_hash().serialize());
+               genesis_block(Bitcoin).header.bitcoin_hash());
   }
 }
 

src/blockdata/constants.rs

@@ -113,11 +113,12 @@ pub fn genesis_block(network: Network) -> Block {
 
 #[cfg(test)]
 mod test {
-  use network::serialize::Serializable;
+  use serialize::hex::FromHex;
+
   use network::constants::{Bitcoin, BitcoinTestnet};
+  use network::serialize::{BitcoinHash, serialize};
   use blockdata::constants::{genesis_block, bitcoin_genesis_tx};
   use blockdata::constants::{MAX_SEQUENCE, COIN_VALUE};
-  use util::misc::hex_bytes;
   use util::hash::zero_hash;
 
   #[test]
@@ -128,13 +129,13 @@ mod test {
     assert_eq!(gen.input.len(), 1);
     assert_eq!(gen.input[0].prev_hash.as_slice(), zero_hash().as_slice());
     assert_eq!(gen.input[0].prev_index, 0xFFFFFFFF);
-    assert_eq!(gen.input[0].script_sig.serialize().as_slice(),
+    assert_eq!(serialize(&gen.input[0].script_sig),
-               hex_bytes("4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73").unwrap().as_slice());
+               Ok("4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73".from_hex().unwrap()));
 
     assert_eq!(gen.input[0].sequence, MAX_SEQUENCE);
     assert_eq!(gen.output.len(), 1);
-    assert_eq!(gen.output[0].script_pubkey.serialize().as_slice(),
+    assert_eq!(serialize(&gen.output[0].script_pubkey),
-               hex_bytes("434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac").unwrap().as_slice());
+               Ok("434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac".from_hex().unwrap()));
     assert_eq!(gen.output[0].value, 50 * COIN_VALUE);
     assert_eq!(gen.lock_time, 0);
 

src/blockdata/script.rs

@@ -25,11 +25,11 @@
 //!
 
 use std::char::from_digit;
-use std::io::IoResult;
 use serialize::json;
 
-use network::serialize::Serializable;
 use blockdata::opcodes;
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
+use network::serialize::{SimpleDecoder, SimpleEncoder};
 use util::thinvec::ThinVec;
 
 #[deriving(PartialEq, Show, Clone)]
@@ -136,15 +136,18 @@ impl json::ToJson for Script {
 }
 
 // Network serialization
-impl Serializable for Script {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for Script {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
     let &Script(ref data) = self;
-    data.serialize()
+    data.consensus_encode(s)
+  }
 }
 
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<Script> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for Script {
-    let raw = Serializable::deserialize(iter);
+  #[inline]
-    raw.map(|ok| Script(ok))
+  fn consensus_decode(d: &mut D) -> Result<Script, E> {
+    Ok(Script(try!(ConsensusDecodable::consensus_decode(d))))
   }
 }
 
@@ -152,7 +155,7 @@ impl Serializable for Script {
 mod test {
   use std::io::IoResult;
 
-  use network::serialize::Serializable;
+  use network::serialize::{deserialize, serialize};
   use blockdata::script::Script;
   use blockdata::opcodes;
   use util::misc::hex_bytes;
@@ -189,9 +192,9 @@ mod test {
   #[test]
   fn script_serialize() {
     let hex_script = hex_bytes("6c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52").unwrap();
-    let script: IoResult<Script> = Serializable::deserialize(hex_script.iter().map(|n| *n));
+    let script: IoResult<Script> = deserialize(hex_script.clone());
     assert!(script.is_ok());
-    assert_eq!(script.unwrap().serialize().as_slice(), hex_script.as_slice());
+    assert_eq!(serialize(&script.unwrap()), Ok(hex_script));
   }
 }
 

src/blockdata/transaction.rs

@@ -23,12 +23,10 @@
 //! This module provides the structures and functions needed to support transactions.
 //!
 
-use std::io::IoResult;
 use util::hash::Sha256dHash;
-use network::serialize::{Serializable, SerializeIter};
 use blockdata::script::Script;
-#[cfg(test)]
+use network::encodable::ConsensusEncodable;
-use util::misc::hex_bytes;
+use network::serialize::BitcoinHash;
 
 /// A transaction input, which defines old coins to be consumed
 #[deriving(Clone, PartialEq, Show)]
@@ -70,23 +68,41 @@ pub struct Transaction {
   pub output: Vec<TxOut>
 }
 
-impl_serializable!(TxIn, prev_hash, prev_index, script_sig, sequence)
+impl BitcoinHash for Transaction {
+  fn bitcoin_hash(&self) -> Sha256dHash {
+    use network::serialize::serialize;
+    Sha256dHash::from_data(serialize(self).unwrap().as_slice())
+  }
+}
+
+impl_consensus_encoding!(TxIn, prev_hash, prev_index, script_sig, sequence)
 impl_json!(TxIn, prev_hash, prev_index, script_sig, sequence)
-impl_serializable!(TxOut, value, script_pubkey)
+impl_consensus_encoding!(TxOut, value, script_pubkey)
 impl_json!(TxOut, value, script_pubkey)
-impl_serializable!(Transaction, version, input, output, lock_time)
+impl_consensus_encoding!(Transaction, version, input, output, lock_time)
 impl_json!(Transaction, version, input, output, lock_time)
 
+
+#[cfg(test)]
+mod tests {
+  use super::{Transaction, TxIn};
+
+  use std::io::IoResult;
+
+  use network::serialize::BitcoinHash;
+  use network::serialize::deserialize;
+  use util::misc::hex_bytes;
+
   #[test]
   fn test_txin() {
-  let txin: IoResult<TxIn> = Serializable::deserialize(hex_bytes("a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff").unwrap().iter().map(|n| *n));
+    let txin: IoResult<TxIn> = deserialize(hex_bytes("a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff").unwrap());
     assert!(txin.is_ok());
   }
 
   #[test]
   fn test_transaction() {
     let hex_tx = hex_bytes("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
-  let tx: IoResult<Transaction> = Serializable::deserialize(hex_tx.iter().map(|n| *n));
+    let tx: IoResult<Transaction> = deserialize(hex_tx);
     assert!(tx.is_ok());
     let realtx = tx.unwrap();
     // All these tests aren't really needed because if they fail, the hash check at the end
@@ -104,6 +120,5 @@ fn test_transaction() {
     assert_eq!(realtx.bitcoin_hash().le_hex_string(),
                "a6eab3c14ab5272a58a5ba91505ba1a4b6d7a3a9fcbd187b6cd99a7b6d548cb7".to_string());
   }
-
+}
-
 

src/blockdata/utxoset.rs

@@ -19,14 +19,13 @@
 //!
 
 use std::collections::HashMap;
-use std::io::IoResult;
 use std::mem;
 
 use blockdata::transaction::{Transaction, TxOut};
 use blockdata::constants::genesis_block;
 use blockdata::block::Block;
 use network::constants::Network;
-use network::serialize::{Serializable, SerializeIter};
+use network::serialize::BitcoinHash;
 use util::hash::{DumbHasher, Sha256dHash};
 use util::uint::Uint128;
 use util::thinvec::ThinVec;
@@ -45,7 +44,7 @@ pub struct UtxoSet {
   n_utxos: u64
 }
 
-impl_serializable!(UtxoSet, last_hash, n_utxos, spent_txos, spent_idx, table)
+impl_consensus_encoding!(UtxoSet, last_hash, n_utxos, spent_txos, spent_idx, table)
 
 impl UtxoSet {
   /// Constructs a new UTXO set
@@ -67,14 +66,17 @@ impl UtxoSet {
   fn add_utxos(&mut self, tx: &Transaction) -> Option<UtxoNode> {
     let txid = tx.bitcoin_hash();
     // Locate node if it's already there
+    let new_node = unsafe {
       let mut new_node = ThinVec::with_capacity(tx.output.len() as u32);
       for (vout, txo) in tx.output.iter().enumerate() {
         // Unsafe since we are not uninitializing the old data in the vector
-      unsafe { new_node.init(vout as uint, Some(txo.clone())); }
+        new_node.init(vout as uint, Some(txo.clone()));
       }
+      new_node
+    };
     // Get the old value, if any (this is suprisingly possible, c.f. BIP30
     // and the other comments in this file referring to it)
-    let ret = self.table.swap(txid.as_uint128(), new_node);
+    let ret = self.table.swap(txid.into_uint128(), new_node);
     if ret.is_none() {
       self.n_utxos += tx.output.len() as u64;
     }
@@ -86,7 +88,7 @@ impl UtxoSet {
     // This whole function has awkward scoping thx to lexical borrow scoping :(
     let (ret, should_delete) = {
       // Locate the UTXO, failing if not found
-      let node = match self.table.find_mut(&txid.as_uint128()) {
+      let node = match self.table.find_mut(&txid.into_uint128()) {
         Some(node) => node,
         None => return None
       };
@@ -104,7 +106,7 @@ impl UtxoSet {
 
     // Delete the whole node if it is no longer being used
     if should_delete {
-      self.table.remove(&txid.as_uint128());
+      self.table.remove(&txid.into_uint128());
     }
 
     self.n_utxos -= if ret.is_some() { 1 } else { 0 };
@@ -114,7 +116,7 @@ impl UtxoSet {
   /// Get a reference to a UTXO in the set
   pub fn get_utxo<'a>(&'a mut self, txid: Sha256dHash, vout: u32) -> Option<&'a TxOut> {
     // Locate the UTXO, failing if not found
-    let node = match self.table.find_mut(&txid.as_uint128()) {
+    let node = match self.table.find_mut(&txid.into_uint128()) {
       Some(node) => node,
       None => return None
     };
@@ -227,7 +229,7 @@ impl UtxoSet {
         for ((txid, n), txo) in extract_vec.move_iter() {
           // Remove the tx's utxo list and patch the txo into place
           let new_node =
-              match self.table.pop(&txid.as_uint128()) {
+              match self.table.pop(&txid.into_uint128()) {
                 Some(mut thinvec) => {
                   let old_len = thinvec.len() as u32;
                   if old_len < n + 1 {
@@ -242,16 +244,18 @@ impl UtxoSet {
                   thinvec
                 }
                 None => {
+                  unsafe {
                     let mut thinvec = ThinVec::with_capacity(n + 1);
-                  for i in range(0, n + 1) {
+                    for i in range(0, n) {
-                    unsafe { thinvec.init(i as uint, None); }
+                      thinvec.init(i as uint, None);
                     }
-                  unsafe { *thinvec.get_mut(n as uint) = Some(txo); }
+                    thinvec.init(n as uint, Some(txo));
                     thinvec
                   }
+                }
               };
           // Ram it back into the tree
-          self.table.insert(txid.as_uint128(), new_node);
+          self.table.insert(txid.into_uint128(), new_node);
         }
       }
       skipped_genesis = true;
@@ -284,13 +288,13 @@ mod tests {
   use blockdata::block::Block;
   use blockdata::utxoset::UtxoSet;
   use network::constants::Bitcoin;
-  use network::serialize::Serializable;
+  use network::serialize::{BitcoinHash, deserialize, serialize};
 
   #[test]
   fn utxoset_serialize_test() {
     let mut empty_set = UtxoSet::new(Bitcoin, 100);
 
-    let new_block: Block = Serializable::deserialize("010000004ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914cd74d6e49ffff001d323b3a7b0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d026e04ffffffff0100f2052a0100000043410446ef0102d1ec5240f0d061a4246c1bdef63fc3dbab7733052fbbf0ecd8f41fc26bf049ebb4f9527f374280259e7cfa99c48b0e3f39c51347a19a5819651503a5ac00000000010000000321f75f3139a013f50f315b23b0c9a2b6eac31e2bec98e5891c924664889942260000000049483045022100cb2c6b346a978ab8c61b18b5e9397755cbd17d6eb2fe0083ef32e067fa6c785a02206ce44e613f31d9a6b0517e46f3db1576e9812cc98d159bfdaf759a5014081b5c01ffffffff79cda0945903627c3da1f85fc95d0b8ee3e76ae0cfdc9a65d09744b1f8fc85430000000049483045022047957cdd957cfd0becd642f6b84d82f49b6cb4c51a91f49246908af7c3cfdf4a022100e96b46621f1bffcf5ea5982f88cef651e9354f5791602369bf5a82a6cd61a62501fffffffffe09f5fe3ffbf5ee97a54eb5e5069e9da6b4856ee86fc52938c2f979b0f38e82000000004847304402204165be9a4cbab8049e1af9723b96199bfd3e85f44c6b4c0177e3962686b26073022028f638da23fc003760861ad481ead4099312c60030d4cb57820ce4d33812a5ce01ffffffff01009d966b01000000434104ea1feff861b51fe3f5f8a3b12d0f4712db80e919548a80839fc47c6a21e66d957e9c5d8cd108c7a2d2324bad71f9904ac0ae7336507d785b17a2c115e427a32fac00000000".from_hex().unwrap().iter().map(|n| *n)).unwrap();
+    let new_block: Block = deserialize("010000004ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914cd74d6e49ffff001d323b3a7b0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d026e04ffffffff0100f2052a0100000043410446ef0102d1ec5240f0d061a4246c1bdef63fc3dbab7733052fbbf0ecd8f41fc26bf049ebb4f9527f374280259e7cfa99c48b0e3f39c51347a19a5819651503a5ac00000000010000000321f75f3139a013f50f315b23b0c9a2b6eac31e2bec98e5891c924664889942260000000049483045022100cb2c6b346a978ab8c61b18b5e9397755cbd17d6eb2fe0083ef32e067fa6c785a02206ce44e613f31d9a6b0517e46f3db1576e9812cc98d159bfdaf759a5014081b5c01ffffffff79cda0945903627c3da1f85fc95d0b8ee3e76ae0cfdc9a65d09744b1f8fc85430000000049483045022047957cdd957cfd0becd642f6b84d82f49b6cb4c51a91f49246908af7c3cfdf4a022100e96b46621f1bffcf5ea5982f88cef651e9354f5791602369bf5a82a6cd61a62501fffffffffe09f5fe3ffbf5ee97a54eb5e5069e9da6b4856ee86fc52938c2f979b0f38e82000000004847304402204165be9a4cbab8049e1af9723b96199bfd3e85f44c6b4c0177e3962686b26073022028f638da23fc003760861ad481ead4099312c60030d4cb57820ce4d33812a5ce01ffffffff01009d966b01000000434104ea1feff861b51fe3f5f8a3b12d0f4712db80e919548a80839fc47c6a21e66d957e9c5d8cd108c7a2d2324bad71f9904ac0ae7336507d785b17a2c115e427a32fac00000000".from_hex().unwrap()).unwrap();
 
     // Make sure we can't add the block directly, since we are missing the inputs
     assert!(!empty_set.update(&new_block));
@@ -324,10 +328,9 @@ mod tests {
     }
 
     // Serialize/deserialize the resulting UTXO set
-    let serial = empty_set.serialize();
+    let serial = serialize(&empty_set).unwrap();
-    assert_eq!(serial, empty_set.serialize_iter().collect());
 
-    let deserial: IoResult<UtxoSet> = Serializable::deserialize(serial.iter().map(|n| *n));
+    let deserial: IoResult<UtxoSet> = deserialize(serial.clone());
     assert!(deserial.is_ok());
 
     // Check that all outputs are there
@@ -347,7 +350,7 @@ mod tests {
       }
     }
 
-    let deserial_again: IoResult<UtxoSet> = Serializable::deserialize(serial.iter().map(|n| *n));
+    let deserial_again: IoResult<UtxoSet> = deserialize(serial);
     let mut read_again = deserial_again.unwrap();
     assert!(read_again.rewind(&new_block));
     assert_eq!(read_again.n_utxos(), 0);

src/internal_macros.rs

@@ -14,30 +14,23 @@
 
 #![macro_escape]
 
-macro_rules! impl_serializable(
+macro_rules! impl_consensus_encoding(
   ($thing:ident, $($field:ident),+) => (
-    impl Serializable for $thing {
+    impl<S: ::network::serialize::SimpleEncoder<E>, E> ::network::encodable::ConsensusEncodable<S, E> for $thing {
-      fn serialize(&self) -> Vec<u8> {
+      #[inline]
-        let mut ret = vec![];
+      fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-        $( ret.extend(self.$field.serialize().move_iter()); )+
+        $( try!(self.$field.consensus_encode(s)); )+
-        ret
+        Ok(())
-      }
-
-      fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> {
-        SerializeIter {
-          data_iter: None,
-          sub_iter_iter: box vec![ $( &self.$field as &Serializable, )+ ].move_iter(),
-          sub_iter: None,
-          sub_started: false
       }
     }
 
-      fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<$thing> {
+    impl<D: ::network::serialize::SimpleDecoder<E>, E> ::network::encodable::ConsensusDecodable<D, E> for $thing {
-        use util::misc::prepend_err;
+      #[inline]
-        let ret = Ok($thing {
+      fn consensus_decode(d: &mut D) -> Result<$thing, E> {
-          $( $field: try!(prepend_err(stringify!($field), Serializable::deserialize(iter.by_ref()))), )+
+        use network::encodable::ConsensusDecodable;
-        });
+        Ok($thing {
-        ret
+          $( $field: try!(ConsensusDecodable::consensus_decode(d)), )+
+        })
       }
     }
   );
@@ -50,7 +43,6 @@ macro_rules! impl_json(
         use std::collections::TreeMap;
         use serialize::json::{ToJson, Object};
         let mut ret = TreeMap::new();
-        ret.insert("hash".to_string(), self.bitcoin_hash().to_json());
         $( ret.insert(stringify!($field).to_string(), self.$field.to_json()); )+
         Object(ret)
       }

src/network/address.rs

@@ -19,9 +19,9 @@
 //!
 
 use std::fmt;
-use std::io::{IoResult, standard_error, InvalidInput};
 
-use network::serialize::Serializable;
+use network::serialize::{SimpleEncoder, SimpleDecoder};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
 
 /// A message which can be sent on the Bitcoin network
 pub struct Address {
@@ -33,31 +33,31 @@ pub struct Address {
   pub port: u16
 }
 
-impl Serializable for Address {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for Address {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
-    let mut rv = vec!();
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    rv.extend(self.services.serialize().move_iter());
+    try!(self.services.consensus_encode(s));
-    rv.extend(self.address.iter().map(|n| *n));
+    try!(self.address.consensus_encode(s));
     // Explicitly code the port since it needs to be big-endian
-    rv.extend([(self.port / 256) as u8, (self.port % 256) as u8].iter().map(|n| *n));
+    try!(((self.port / 0x100) as u8).consensus_encode(s));
-    rv
+    try!(((self.port % 0x100) as u8).consensus_encode(s));
+    Ok(())
+  }
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<Address> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for Address {
-    let ret = Address {
+  #[inline]
-      services: try!(Serializable::deserialize(iter.by_ref())),
+  fn consensus_decode(d: &mut D) -> Result<Address, E> {
-      address: try!(Serializable::deserialize(iter.by_ref())),
+    Ok(Address {
+      services: try!(ConsensusDecodable::consensus_decode(d)),
+      address: try!(ConsensusDecodable::consensus_decode(d)),
       // Explicitly code the port since it needs to be big-endian
       port: {
-        let b1 = iter.next();
+        let b1: u8 = try!(ConsensusDecodable::consensus_decode(d));
-        let b2 = iter.next();
+        let b2: u8 = try!(ConsensusDecodable::consensus_decode(d));
-        if b1.is_none() || b2.is_none() {
+        (b1 as u16 * 0x100) + (b2 as u16)
-          return Err(standard_error(InvalidInput));
       }
-        (b1.unwrap() as u16) * 0x100 + (b2.unwrap() as u16)
+    })
-      }
-    };
-    Ok(ret)
   }
 }
 
@@ -69,32 +69,39 @@ impl fmt::Show for Address {
   }
 }
 
+#[cfg(test)]
+mod test {
+  use super::Address;
+
+  use std::io::IoResult;
+
+  use network::serialize::{deserialize, serialize};
+
   #[test]
   fn serialize_address_test() {
-  assert!(Address {
+    assert_eq!(serialize(&Address {
       services: 1,
       address: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1],
       port: 8333
-  }.serialize() == Vec::from_slice([1u8, 0, 0, 0, 0, 0, 0, 0,
+    }),
-                                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1,
+    Ok(vec![1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                    0x20, 0x8d]));
+            0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1, 0x20, 0x8d]));
   }
 
   #[test]
   fn deserialize_address_test() {
-  let mut addr: IoResult<Address> = Serializable::deserialize([1u8, 0, 0, 0, 0, 0, 0, 0,
+    let mut addr: IoResult<Address> = deserialize(vec![1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1,
+                                                       0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0,
-                                    0x20, 0x8d].iter().map(|n| *n));
+                                                       0, 1, 0x20, 0x8d]);
     assert!(addr.is_ok())
     let full = addr.unwrap();
     assert!(full.services == 1);
     assert!(full.address == [0u8,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1]);
     assert!(full.port == 8333);
 
-  addr = Serializable::deserialize([1u8, 0, 0, 0, 0, 0, 0, 0,
+    addr = deserialize(vec![1u8, 0, 0, 0, 0, 0, 0, 0,
-                                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1].iter().map(|n| *n));
+                            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x0a, 0, 0, 1]);
     assert!(addr.is_err());
   }
-
+}
-
 

src/network/constants.rs

@@ -18,10 +18,8 @@
 //! protocol, such as protocol versioning and magic header bytes.
 //!
 
-use std::io::{IoResult, InvalidInput, standard_error};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
-
+use network::serialize::{SimpleEncoder, SimpleDecoder};
-use network::serialize::Serializable;
-use util::misc::prepend_err;
 
 /// The cryptocurrency to operate on
 #[deriving(Encodable, Decodable, PartialEq, Eq, Clone, Show, Hash)]
@@ -46,18 +44,21 @@ pub fn magic(network: Network) -> u32 {
   }
 }
 
-// This affects the representation of the `Network` in text files
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for Network {
-impl Serializable for Network {
+  #[inline]
-  fn serialize(&self) -> Vec<u8> {
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    magic(*self).serialize()
+    magic(*self).consensus_encode(s)
+  }
 }
 
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<Network> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for Network {
-    let magic: u32 = try!(prepend_err("magic", Serializable::deserialize(iter)));
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<Network, E> {
+    let magic: u32 = try!(ConsensusDecodable::consensus_decode(d));
     match magic {
       0xD9B4BEF9 => Ok(Bitcoin),
       0x0709110B => Ok(BitcoinTestnet),
-      _ => Err(standard_error(InvalidInput))
+      x => Err(d.error(format!("Unknown network (magic {:x})", x).as_slice()))
     }
   }
 }

src/network/encodable.rs

@@ -0,0 +1,558 @@
+// Rust Bitcoin Library
+// Written in 2014 by
+//   Andrew Poelstra <apoelstra@wpsoftware.net>
+//
+// To the extent possible under law, the author(s) have dedicated all
+// copyright and related and neighboring rights to this software to
+// the public domain worldwide. This software is distributed without
+// any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication
+// along with this software.
+// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+//
+
+//! # Consensus-encodable types
+//!
+//! This is basically a replacement of the `Encodable` trait which does
+//! normalization for endianness, etc., to ensure that the encoding
+//! matches for endianness, etc., to ensure that the encoding matches
+//! the network consensus encoding.
+//!
+//! Essentially, anything that must go on the -disk- or -network- must
+//! be encoded using the `ConsensusEncodable` trait, since this data
+//! must be the same for all systems. Any data going to the -user-, e.g.
+//! over JSONRPC, should use the ordinary `Encodable` trait. (This
+//! should also be the same across systems, of course, but has some
+//! critical differences from the network format, e.g. scripts come
+//! with an opcode decode, hashes are big-endian, numbers are typically
+//! big-endian decimals, etc.)
+//!
+
+use std::collections::HashMap;
+use std::default::Default;
+use std::hash::{Hash, Hasher};
+use std::u32;
+
+use util::thinvec::ThinVec;
+use util::hash::Sha256dHash;
+use network::serialize::{SimpleDecoder, SimpleEncoder};
+
+/// Data which can be encoded in a consensus-consistent way
+pub trait ConsensusEncodable<S:SimpleEncoder<E>, E> {
+  /// Encode an object with a well-defined format
+  fn consensus_encode(&self, e: &mut S) -> Result<(), E>;
+}
+
+/// Data which can be encoded in a consensus-consistent way
+pub trait ConsensusDecodable<D:SimpleDecoder<E>, E> {
+  /// Decode an object with a well-defined format
+  fn consensus_decode(d: &mut D) -> Result<Self, E>;
+}
+
+/// A variable-length unsigned integer
+#[deriving(PartialEq, Show)]
+pub struct VarInt(pub u64);
+
+/// Data which must be preceded by a 4-byte checksum
+#[deriving(PartialEq, Clone, Show)]
+pub struct CheckedData(pub Vec<u8>);
+
+// Primitive types
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for u8 {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_u8(*self) }
+}
+
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for u16 {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_u16(self.to_le()) }
+}
+
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for u32 {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_u32(self.to_le()) }
+}
+
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for u64 {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_u64(self.to_le()) }
+}
+
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for i32 {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_i32(self.to_le()) }
+}
+
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for i64 {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_i64(self.to_le()) }
+}
+
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for VarInt {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    let &VarInt(n) = self;
+    match n {
+      0..0xFC             => { (n as u8).consensus_encode(s) }
+      0xFD..0xFFFF        => { try!(s.emit_u8(0xFD)); (n as u16).consensus_encode(s) }
+      0x10000..0xFFFFFFFF => { try!(s.emit_u8(0xFE)); (n as u32).consensus_encode(s) }
+      _                   => { try!(s.emit_u8(0xFF)); (n as u64).consensus_encode(s) }
+    }
+  }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for u8 {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<u8, E> { d.read_u8() }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for u16 {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<u16, E> { d.read_u16().map(|n| Int::from_le(n)) }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for u32 {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<u32, E> { d.read_u32().map(|n| Int::from_le(n)) }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for u64 {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<u64, E> { d.read_u64().map(|n| Int::from_le(n)) }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for i32 {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<i32, E> { d.read_i32().map(|n| Int::from_le(n)) }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for i64 {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<i64, E> { d.read_i64().map(|n| Int::from_le(n)) }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for VarInt {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<VarInt, E> {
+    let n = try!(d.read_u8());
+    match n {
+      0xFF => d.read_u64().map(|n| VarInt(Int::from_le(n))),
+      0xFE => d.read_u32().map(|n| VarInt(Int::from_le(n) as u64)),
+      0xFD => d.read_u16().map(|n| VarInt(Int::from_le(n) as u64)),
+      n => Ok(VarInt(n as u64))
+    }
+  }
+}
+
+// Booleans
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for bool {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { s.emit_u8(if *self {1} else {0}) }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for bool {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<bool, E> { d.read_u8().map(|n| n != 0) }
+}
+
+// Strings
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for String {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    self.as_bytes().consensus_encode(s)
+  }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for String {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<String, E> {
+    String::from_utf8(try!(ConsensusDecodable::consensus_decode(d))).map_err(|_| d.error("String was not valid UTF8"))
+  }
+}
+
+
+// Arrays
+macro_rules! impl_array(
+  ( $size:expr ) => (
+    impl<S:SimpleEncoder<E>, E, T:ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for [T, ..$size] {
+      #[inline]
+      fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+        for i in self.iter() { try!(i.consensus_encode(s)); }
+        Ok(())
+      }
+    }
+
+    impl<D:SimpleDecoder<E>, E, T:ConsensusDecodable<D, E>+Copy> ConsensusDecodable<D, E> for [T, ..$size] {
+      #[inline]
+      fn consensus_decode(d: &mut D) -> Result<[T, ..$size], E> {
+        // Set everything to the first decode
+        let mut ret = [try!(ConsensusDecodable::consensus_decode(d)), ..$size];
+        // Set the rest
+        for i in range(1, $size) { ret[i] = try!(ConsensusDecodable::consensus_decode(d)); }
+        Ok(ret)
+      }
+    }
+  );
+)
+
+impl_array!(2)
+impl_array!(4)
+impl_array!(8)
+impl_array!(12)
+impl_array!(16)
+impl_array!(32)
+
+impl<'a, S:SimpleEncoder<E>, E, T:ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for &'a [T] {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    try!(VarInt(self.len() as u64).consensus_encode(s));
+    for c in self.iter() { try!(c.consensus_encode(s)); }
+    Ok(())
+  }
+}
+
+// Cannot decode a slice
+
+// Vectors
+impl<S:SimpleEncoder<E>, E, T:ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for Vec<T> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { self.as_slice().consensus_encode(s) }
+}
+
+impl<D:SimpleDecoder<E>, E, T:ConsensusDecodable<D, E>> ConsensusDecodable<D, E> for Vec<T> {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<Vec<T>, E> {
+    let VarInt(len): VarInt = try!(ConsensusDecodable::consensus_decode(d));
+    let mut ret = Vec::with_capacity(len as uint);
+    for _ in range(0, len) { ret.push(try!(ConsensusDecodable::consensus_decode(d))); }
+    Ok(ret)
+  }
+}
+
+impl<S:SimpleEncoder<E>, E, T:ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for ThinVec<T> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { self.as_slice().consensus_encode(s) }
+}
+
+impl<D:SimpleDecoder<E>, E, T:ConsensusDecodable<D, E>> ConsensusDecodable<D, E> for ThinVec<T> {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<ThinVec<T>, E> {
+    let VarInt(len): VarInt = try!(ConsensusDecodable::consensus_decode(d));
+    if len > u32::MAX as u64 {
+      return Err(d.error("ThinVec length out of range!"));
+    }
+    unsafe {
+      let mut ret = ThinVec::with_capacity(len as u32);
+      // Huge danger: if this fails, the remaining uninitialized part of the ThinVec
+      // will be freed. This is ok, but only because the memory is u8, which has no
+      // destructor...and assuming there are no trap representations...very fragile.
+      for i in range(0, len as uint) { ret.init(i, try!(ConsensusDecodable::consensus_decode(d))); }
+      Ok(ret)
+    }
+  }
+}
+
+// Options (encoded as vectors of length 0 or 1)
+impl<S:SimpleEncoder<E>, E, T:ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for Option<T> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    match *self {
+      Some(ref data) => {
+        try!(1u8.consensus_encode(s));
+        try!(data.consensus_encode(s));
+      }
+      None => { try!(0u8.consensus_encode(s)); }
+    }
+    Ok(())
+  }
+}
+
+impl<D:SimpleDecoder<E>, E, T:ConsensusDecodable<D, E>> ConsensusDecodable<D, E> for Option<T> {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<Option<T>, E> {
+    let bit: u8 = try!(ConsensusDecodable::consensus_decode(d));
+    Ok(if bit != 0 {
+      Some(try!(ConsensusDecodable::consensus_decode(d)))
+    } else {
+      None
+    })
+  }
+}
+
+
+/// Do a double-SHA256 on some data and return the first 4 bytes
+fn sha2_checksum(data: &[u8]) -> [u8, ..4] {
+  let checksum = Sha256dHash::from_data(data);
+  [checksum[0u], checksum[1u], checksum[2u], checksum[3u]]
+}
+
+// Checked data
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for CheckedData {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    let &CheckedData(ref data) = self;
+    try!((data.len() as u32).consensus_encode(s));
+    try!(sha2_checksum(data.as_slice()).consensus_encode(s))
+    // We can't just pass to the slice encoder since it'll insert a length
+    for ch in data.iter() {
+      try!(ch.consensus_encode(s));
+    }
+    Ok(())
+  }
+}
+
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for CheckedData {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<CheckedData, E> {
+    let len: u32 = try!(ConsensusDecodable::consensus_decode(d));
+    let checksum: [u8, ..4] = try!(ConsensusDecodable::consensus_decode(d));
+    let mut ret = Vec::with_capacity(len as uint);
+    for _ in range(0, len) { ret.push(try!(ConsensusDecodable::consensus_decode(d))); }
+    let expected_checksum = sha2_checksum(ret.as_slice());
+    if expected_checksum != checksum {
+      Err(d.error("bad checksum"))
+    } else {
+      Ok(CheckedData(ret))
+    }
+  }
+}
+
+// Tuples
+impl<S:SimpleEncoder<E>, E, T: ConsensusEncodable<S, E>, U: ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for (T, U) {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    let &(ref s1, ref s2) = self;
+    try!(s1.consensus_encode(s));
+    try!(s2.consensus_encode(s));
+    Ok(())
+  }
+}
+
+impl<D:SimpleDecoder<E>, E, T: ConsensusDecodable<D, E>, U: ConsensusDecodable<D, E>> ConsensusDecodable<D, E> for (T, U) {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<(T, U), E> {
+    Ok((try!(ConsensusDecodable::consensus_decode(d)),
+        try!(ConsensusDecodable::consensus_decode(d))))
+  }
+}
+
+// References
+impl<S:SimpleEncoder<E>, E, T: ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for Box<T> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> { (**self).consensus_encode(s) }
+}
+
+impl<D:SimpleDecoder<E>, E, T: ConsensusDecodable<D, E>> ConsensusDecodable<D, E> for Box<T> {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<Box<T>, E> {
+    ConsensusDecodable::consensus_decode(d).map(|res| box res)
+  }
+}
+
+
+// HashMap
+impl<S:SimpleEncoder<E>, E,
+     K:ConsensusEncodable<S,E>+Eq+Hash<u64>,
+     V:ConsensusEncodable<S,E>,
+     H:Hasher<u64>+Default> ConsensusEncodable<S, E> for HashMap<K, V, H> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
+    try!(VarInt(self.len() as u64).consensus_encode(s));
+    for (key, value) in self.iter() {
+      try!(key.consensus_encode(s));
+      try!(value.consensus_encode(s));
+    }
+    Ok(())
+  }
+}
+
+impl<D:SimpleDecoder<E>, E,
+     K:ConsensusDecodable<D,E>+Eq+Hash<u64>,
+     V:ConsensusDecodable<D,E>,
+     H:Hasher<u64>+Default> ConsensusDecodable<D, E> for HashMap<K, V, H> {
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<HashMap<K, V, H>, E> {
+    let VarInt(len): VarInt = try!(ConsensusDecodable::consensus_decode(d));
+
+    let mut ret = HashMap::with_capacity_and_hasher(len as uint, Default::default());
+    for _ in range(0, len) {
+      ret.insert(try!(ConsensusDecodable::consensus_decode(d)),
+                 try!(ConsensusDecodable::consensus_decode(d)));
+    }
+    Ok(ret)
+  }
+}
+
+
+
+// Tests
+#[cfg(test)]
+mod tests {
+  use super::{CheckedData, VarInt};
+
+  use std::io::IoResult;
+
+  use network::serialize::{deserialize, serialize};
+
+  #[test]
+  fn serialize_int_test() {
+    // bool
+    assert_eq!(serialize(&false), Ok(vec![0u8]));
+    assert_eq!(serialize(&true), Ok(vec![1u8]));
+    // u8
+    assert_eq!(serialize(&1u8), Ok(vec![1u8]));
+    assert_eq!(serialize(&0u8), Ok(vec![0u8]));
+    assert_eq!(serialize(&255u8), Ok(vec![255u8]));
+    // u16
+    assert_eq!(serialize(&1u16), Ok(vec![1u8, 0]));
+    assert_eq!(serialize(&256u16), Ok(vec![0u8, 1]));
+    assert_eq!(serialize(&5000u16), Ok(vec![136u8, 19]));
+    // u32
+    assert_eq!(serialize(&1u32), Ok(vec![1u8, 0, 0, 0]));
+    assert_eq!(serialize(&256u32), Ok(vec![0u8, 1, 0, 0]));
+    assert_eq!(serialize(&5000u32), Ok(vec![136u8, 19, 0, 0]));
+    assert_eq!(serialize(&500000u32), Ok(vec![32u8, 161, 7, 0]));
+    assert_eq!(serialize(&168430090u32), Ok(vec![10u8, 10, 10, 10]));
+    // TODO: test negative numbers
+    assert_eq!(serialize(&1i32), Ok(vec![1u8, 0, 0, 0]));
+    assert_eq!(serialize(&256i32), Ok(vec![0u8, 1, 0, 0]));
+    assert_eq!(serialize(&5000i32), Ok(vec![136u8, 19, 0, 0]));
+    assert_eq!(serialize(&500000i32), Ok(vec![32u8, 161, 7, 0]));
+    assert_eq!(serialize(&168430090i32), Ok(vec![10u8, 10, 10, 10]));
+    // u64
+    assert_eq!(serialize(&1u64), Ok(vec![1u8, 0, 0, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&256u64), Ok(vec![0u8, 1, 0, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&5000u64), Ok(vec![136u8, 19, 0, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&500000u64), Ok(vec![32u8, 161, 7, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&723401728380766730u64), Ok(vec![10u8, 10, 10, 10, 10, 10, 10, 10]));
+    // TODO: test negative numbers
+    assert_eq!(serialize(&1i64), Ok(vec![1u8, 0, 0, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&256i64), Ok(vec![0u8, 1, 0, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&5000i64), Ok(vec![136u8, 19, 0, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&500000i64), Ok(vec![32u8, 161, 7, 0, 0, 0, 0, 0]));
+    assert_eq!(serialize(&723401728380766730i64), Ok(vec![10u8, 10, 10, 10, 10, 10, 10, 10]));
+  }
+
+  #[test]
+  fn serialize_varint_test() {
+    assert_eq!(serialize(&VarInt(10)), Ok(vec![10u8]));
+    assert_eq!(serialize(&VarInt(0xFC)), Ok(vec![0xFCu8]));
+    assert_eq!(serialize(&VarInt(0xFD)), Ok(vec![0xFDu8, 0xFD, 0]));
+    assert_eq!(serialize(&VarInt(0xFFF)), Ok(vec![0xFDu8, 0xFF, 0xF]));
+    assert_eq!(serialize(&VarInt(0xF0F0F0F)), Ok(vec![0xFEu8, 0xF, 0xF, 0xF, 0xF]));
+    assert_eq!(serialize(&VarInt(0xF0F0F0F0F0E0)), Ok(vec![0xFFu8, 0xE0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0, 0]));
+  }
+
+  #[test]
+  fn serialize_checkeddata_test() {
+    let cd = CheckedData(vec![1u8, 2, 3, 4, 5]);
+    assert_eq!(serialize(&cd), Ok(vec![5, 0, 0, 0, 162, 107, 175, 90, 1, 2, 3, 4, 5]));
+  }
+
+  #[test]
+  fn serialize_vector_test() {
+    assert_eq!(serialize(&vec![1u8, 2, 3]), Ok(vec![3u8, 1, 2, 3]));
+    assert_eq!(serialize(&&[1u8, 2, 3]), Ok(vec![3u8, 1, 2, 3]));
+    // TODO: test vectors of more interesting objects
+  }
+
+  #[test]
+  fn serialize_strbuf_test() {
+    assert_eq!(serialize(&"Andrew".to_string()), Ok(vec![6u8, 0x41, 0x6e, 0x64, 0x72, 0x65, 0x77]));
+  }
+
+  #[test]
+  fn serialize_box_test() {
+    assert_eq!(serialize(&box 1u8), Ok(vec![1u8]));
+    assert_eq!(serialize(&box 1u16), Ok(vec![1u8, 0]));
+    assert_eq!(serialize(&box 1u64), Ok(vec![1u8, 0, 0, 0, 0, 0, 0, 0]));
+  }
+
+  #[test]
+  fn serialize_option_test() {
+    let none: Option<u8> = None;
+    let none_ser = serialize(&none);
+    let some_ser = serialize(&Some(0xFFu8));
+    assert_eq!(none_ser, Ok(vec![0]));
+    assert_eq!(some_ser, Ok(vec![1, 0xFF]));
+  }
+
+  #[test]
+  fn deserialize_int_test() {
+    // bool
+    assert_eq!(deserialize(vec![58u8, 0]), Ok(true));
+    assert_eq!(deserialize(vec![58u8]), Ok(true));
+    assert_eq!(deserialize(vec![1u8]), Ok(true));
+    assert_eq!(deserialize(vec![0u8]), Ok(false));
+    assert_eq!(deserialize(vec![0u8, 1]), Ok(false));
+
+    // u8
+    assert_eq!(deserialize(vec![58u8]), Ok(58u8));
+
+    // u16
+    assert_eq!(deserialize(vec![0x01u8, 0x02]), Ok(0x0201u16));
+    assert_eq!(deserialize(vec![0xABu8, 0xCD]), Ok(0xCDABu16));
+    assert_eq!(deserialize(vec![0xA0u8, 0x0D]), Ok(0xDA0u16));
+    let failure16: IoResult<u16> = deserialize(vec![1u8]);
+    assert!(failure16.is_err());
+
+    // u32
+    assert_eq!(deserialize(vec![0xABu8, 0xCD, 0, 0]), Ok(0xCDABu32));
+    assert_eq!(deserialize(vec![0xA0u8, 0x0D, 0xAB, 0xCD]), Ok(0xCDAB0DA0u32));
+    let failure32: IoResult<u32> = deserialize(vec![1u8, 2, 3]);
+    assert!(failure32.is_err());
+    // TODO: test negative numbers
+    assert_eq!(deserialize(vec![0xABu8, 0xCD, 0, 0]), Ok(0xCDABi32));
+    assert_eq!(deserialize(vec![0xA0u8, 0x0D, 0xAB, 0x2D]), Ok(0x2DAB0DA0i32));
+    let failurei32: IoResult<i32> = deserialize(vec![1u8, 2, 3]);
+    assert!(failurei32.is_err());
+
+    // u64
+    assert_eq!(deserialize(vec![0xABu8, 0xCD, 0, 0, 0, 0, 0, 0]), Ok(0xCDABu64));
+    assert_eq!(deserialize(vec![0xA0u8, 0x0D, 0xAB, 0xCD, 0x99, 0, 0, 0x99]), Ok(0x99000099CDAB0DA0u64));
+    let failure64: IoResult<u64> = deserialize(vec![1u8, 2, 3, 4, 5, 6, 7]);
+    assert!(failure64.is_err());
+    // TODO: test negative numbers
+    assert_eq!(deserialize(vec![0xABu8, 0xCD, 0, 0, 0, 0, 0, 0]), Ok(0xCDABi64));
+    assert_eq!(deserialize(vec![0xA0u8, 0x0D, 0xAB, 0xCD, 0x99, 0, 0, 0x99]), Ok(0x99000099CDAB0DA0i64));
+    let failurei64: IoResult<i64> = deserialize(vec![1u8, 2, 3, 4, 5, 6, 7]);
+    assert!(failurei64.is_err());
+  }
+
+  #[test]
+  fn deserialize_vec_test() {
+    assert_eq!(deserialize(vec![3u8, 2, 3, 4]), Ok(vec![2u8, 3, 4]));
+    assert_eq!(deserialize(vec![4u8, 2, 3, 4, 5, 6]), Ok(vec![2u8, 3, 4, 5]));
+  }
+
+  #[test]
+  fn deserialize_strbuf_test() {
+    assert_eq!(deserialize(vec![6u8, 0x41, 0x6e, 0x64, 0x72, 0x65, 0x77]), Ok(String::from_str("Andrew")));
+  }
+
+  #[test]
+  fn deserialize_checkeddata_test() {
+    let cd: IoResult<CheckedData> = deserialize(vec![5u8, 0, 0, 0, 162, 107, 175, 90, 1, 2, 3, 4, 5]);
+    assert_eq!(cd, Ok(CheckedData(vec![1u8, 2, 3, 4, 5])));
+  }
+
+  #[test]
+  fn deserialize_option_test() {
+    let none: IoResult<Option<u8>> = deserialize(vec![0u8]);
+    let good: IoResult<Option<u8>> = deserialize(vec![1u8, 0xFF]);
+    let bad: IoResult<Option<u8>> = deserialize(vec![2u8]);
+    assert!(bad.is_err());
+    assert_eq!(none, Ok(None));
+    assert_eq!(good, Ok(Some(0xFF)));
+  }
+
+  #[test]
+  fn deserialize_box_test() {
+    let zero: IoResult<Box<u8>> = deserialize(vec![0u8]);
+    let one: IoResult<Box<u8>> = deserialize(vec![1u8]);
+    assert_eq!(zero, Ok(box 0));
+    assert_eq!(one, Ok(box 1));
+  }
+}
+

src/network/message.rs

@@ -20,37 +20,38 @@
 //!
 
 use collections::Vec;
-use std::io::{IoError, IoResult, InvalidInput, OtherIoError, standard_error};
+use std::io::{IoError, IoResult, OtherIoError};
+use std::io::MemReader;
 
 use blockdata::block;
 use network::address::Address;
 use network::message_network;
 use network::message_blockdata;
-use network::serialize::{Serializable, CheckedData};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
-use util::iter::FixedTakeable;
+use network::encodable::CheckedData;
+use network::serialize::{serialize, RawDecoder, SimpleEncoder, SimpleDecoder};
 use util::misc::prepend_err;
 
 /// Serializer for command string
 #[deriving(PartialEq, Clone, Show)]
 pub struct CommandString(pub String);
 
-impl Serializable for CommandString {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for CommandString {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
     let &CommandString(ref inner_str) = self;
     let mut rawbytes = [0u8, ..12]; 
     rawbytes.copy_from(inner_str.as_bytes().as_slice());
-    Vec::from_slice(rawbytes.as_slice())
+    rawbytes.consensus_encode(s)
+  }
 }
 
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<CommandString> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for CommandString {
-    let mut fixiter = iter.fixed_take(12);
+  #[inline]
-    let rv: String = FromIterator::from_iter(fixiter.by_ref().filter_map(|u| if u > 0 { Some(u as char) } else { None }));
+  fn consensus_decode(d: &mut D) -> Result<CommandString, E> {
-    // Once we've read the string, run out the iterator
+    let rawbytes: [u8, ..12] = try!(ConsensusDecodable::consensus_decode(d)); 
-    for _ in fixiter {}
+    let rv: String = FromIterator::from_iter(rawbytes.iter().filter_map(|&u| if u > 0 { Some(u as char) } else { None }));
-    match fixiter.is_err() {
+    Ok(CommandString(rv))
-      false => Ok(CommandString(rv)),
-      true => Err(standard_error(InvalidInput))
-    }
   }
 }
 
@@ -120,46 +121,48 @@ impl RawNetworkMessage {
   }
 }
 
-impl Serializable for RawNetworkMessage {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for RawNetworkMessage {
-  fn serialize(&self) -> Vec<u8> {
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    let mut ret = vec![];
+    try!(self.magic.consensus_encode(s));
-    ret.extend(self.magic.serialize().move_iter());
+    try!(CommandString(self.command()).consensus_encode(s));
-    ret.extend(CommandString(self.command()).serialize().move_iter());
+    match self.payload {
-    let payload_data = match self.payload {
+      Version(ref dat) => serialize(dat),
-      Version(ref dat) => dat.serialize(),
+      Verack           => Ok(vec![]),
-      Verack           => vec![],
+      Addr(ref dat)    => serialize(dat),
-      Addr(ref dat)    => dat.serialize(),
+      Inv(ref dat)     => serialize(dat),
-      Inv(ref dat)     => dat.serialize(),
+      GetData(ref dat) => serialize(dat),
-      GetData(ref dat) => dat.serialize(),
+      NotFound(ref dat) => serialize(dat),
-      NotFound(ref dat) => dat.serialize(),
+      GetBlocks(ref dat) => serialize(dat),
-      GetBlocks(ref dat) => dat.serialize(),
+      GetHeaders(ref dat) => serialize(dat),
-      GetHeaders(ref dat) => dat.serialize(),
+      Block(ref dat)   => serialize(dat),
-      Block(ref dat)   => dat.serialize(),
+      Headers(ref dat) => serialize(dat),
-      Headers(ref dat) => dat.serialize(),
+      Ping(ref dat)    => serialize(dat),
-      Ping(ref dat)    => dat.serialize(),
+      Pong(ref dat)    => serialize(dat),
-      Pong(ref dat)    => dat.serialize()
+    }.unwrap();
-    };
+    Ok(())
-    ret.extend(CheckedData(payload_data).serialize().move_iter());
+  }
-    ret
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<RawNetworkMessage> {
+impl<D:SimpleDecoder<IoError>> ConsensusDecodable<D, IoError> for RawNetworkMessage {
-    let magic = try!(prepend_err("magic", Serializable::deserialize(iter.by_ref())));
+  fn consensus_decode(d: &mut D) -> IoResult<RawNetworkMessage> {
-    let CommandString(cmd): CommandString = try!(prepend_err("command", Serializable::deserialize(iter.by_ref())));
+    let magic = try!(ConsensusDecodable::consensus_decode(d));
-    let CheckedData(raw_payload): CheckedData = try!(prepend_err("payload", Serializable::deserialize(iter.by_ref())));
+    let CommandString(cmd): CommandString= try!(ConsensusDecodable::consensus_decode(d));
+    let CheckedData(raw_payload): CheckedData = try!(ConsensusDecodable::consensus_decode(d));
+
+    let mut mem_d = RawDecoder::new(MemReader::new(raw_payload));
     let payload = match cmd.as_slice() {
-      "version" => Version(try!(prepend_err("version", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "version" => Version(try!(prepend_err("version", ConsensusDecodable::consensus_decode(&mut mem_d)))),
       "verack"  => Verack,
-      "addr"    => Addr(try!(prepend_err("addr", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "addr"    => Addr(try!(prepend_err("addr", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "inv"     => Inv(try!(prepend_err("inv", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "inv"     => Inv(try!(prepend_err("inv", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "getdata" => GetData(try!(prepend_err("getdata", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "getdata" => GetData(try!(prepend_err("getdata", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "notfound" => NotFound(try!(prepend_err("notfound", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "notfound" => NotFound(try!(prepend_err("notfound", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "getblocks" => GetBlocks(try!(prepend_err("getblocks", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "getblocks" => GetBlocks(try!(prepend_err("getblocks", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "getheaders" => GetHeaders(try!(prepend_err("getheaders", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "getheaders" => GetHeaders(try!(prepend_err("getheaders", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "block"   => Block(try!(prepend_err("block", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "block"   => Block(try!(prepend_err("block", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "headers" => Headers(try!(prepend_err("headers", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "headers" => Headers(try!(prepend_err("headers", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "ping"    => Ping(try!(prepend_err("ping", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "ping"    => Ping(try!(prepend_err("ping", ConsensusDecodable::consensus_decode(&mut mem_d)))),
-      "pong"    => Ping(try!(prepend_err("pong", Serializable::deserialize(raw_payload.iter().map(|n| *n))))),
+      "pong"    => Ping(try!(prepend_err("pong", ConsensusDecodable::consensus_decode(&mut mem_d)))),
       cmd => {
         return Err(IoError {
                      kind: OtherIoError,
@@ -177,24 +180,25 @@ impl Serializable for RawNetworkMessage {
 
 #[cfg(test)]
 mod test {
+  use super::CommandString;
+
   use std::io::IoResult;
 
-  use network::message::CommandString;
+  use network::serialize::{deserialize, serialize};
-  use network::serialize::Serializable;
 
   #[test]
   fn serialize_commandstring_test() {
     let cs = CommandString(String::from_str("Andrew"));
-    assert_eq!(cs.serialize(), vec![0x41u8, 0x6e, 0x64, 0x72, 0x65, 0x77, 0, 0, 0, 0, 0, 0]);
+    assert_eq!(serialize(&cs), Ok(vec![0x41u8, 0x6e, 0x64, 0x72, 0x65, 0x77, 0, 0, 0, 0, 0, 0]));
   }
 
   #[test]
   fn deserialize_commandstring_test() {
-    let cs: IoResult<CommandString> = Serializable::deserialize([0x41u8, 0x6e, 0x64, 0x72, 0x65, 0x77, 0, 0, 0, 0, 0, 0].iter().map(|n| *n));
+    let cs: IoResult<CommandString> = deserialize(vec![0x41u8, 0x6e, 0x64, 0x72, 0x65, 0x77, 0, 0, 0, 0, 0, 0]);
     assert!(cs.is_ok());
     assert_eq!(cs.unwrap(), CommandString(String::from_str("Andrew")));
 
-    let short_cs: IoResult<CommandString> = Serializable::deserialize([0x41u8, 0x6e, 0x64, 0x72, 0x65, 0x77, 0, 0, 0, 0, 0].iter().map(|n| *n));
+    let short_cs: IoResult<CommandString> = deserialize(vec![0x41u8, 0x6e, 0x64, 0x72, 0x65, 0x77, 0, 0, 0, 0, 0]);
     assert!(short_cs.is_err());
   }
 

src/network/message_blockdata.rs

@@ -18,14 +18,9 @@
 //! Bitcoin data (blocks and transactions) around.
 //!
 
-use std::io::{IoResult, IoError, InvalidInput};
-#[cfg(test)]
-use serialize::hex::FromHex;
-#[cfg(test)]
-use util::hash::zero_hash;
-
 use network::constants;
-use network::serialize::{Serializable, SerializeIter};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
+use network::serialize::{SimpleDecoder, SimpleEncoder};
 use util::hash::Sha256dHash;
 
 #[deriving(Clone, PartialEq, Show)]
@@ -87,7 +82,7 @@ impl GetBlocksMessage {
   }
 }
 
-impl_serializable!(GetBlocksMessage, version, locator_hashes, stop_hash)
+impl_consensus_encoding!(GetBlocksMessage, version, locator_hashes, stop_hash)
 
 impl GetHeadersMessage {
   /// Construct a new `getheaders` message
@@ -100,45 +95,53 @@ impl GetHeadersMessage {
   }
 }
 
-impl_serializable!(GetHeadersMessage, version, locator_hashes, stop_hash)
+impl_consensus_encoding!(GetHeadersMessage, version, locator_hashes, stop_hash)
 
-impl Serializable for Inventory {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for Inventory {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
-    let int_type: u32 = match self.inv_type {
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-      InvError => 0, 
+    try!(match self.inv_type {
+      InvError => 0u32, 
       InvTransaction => 1,
       InvBlock => 2
-    };
+    }.consensus_encode(s));
-    let mut rv = vec!();
+    self.hash.consensus_encode(s)
-    rv.extend(int_type.serialize().move_iter());
+  }
-    rv.extend(self.hash.serialize().move_iter());
-    rv
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<Inventory> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for Inventory {
-    let int_type: u32 = try!(Serializable::deserialize(iter.by_ref()));
+  #[inline]
+  fn consensus_decode(d: &mut D) -> Result<Inventory, E> {
+    let int_type: u32 = try!(ConsensusDecodable::consensus_decode(d));
     Ok(Inventory {
       inv_type: match int_type {
         0 => InvError,
         1 => InvTransaction,
         2 => InvBlock,
-        _ => { return Err(IoError {
+        // TODO do not fail here
-          kind: InvalidInput,
+        _ => { fail!("bad inventory type field") }
-          desc: "bad inventory type field",
-          detail: None
-        })}
       },
-      hash: try!(Serializable::deserialize(iter.by_ref()))
+      hash: try!(ConsensusDecodable::consensus_decode(d))
     })
   }
 }
 
+#[cfg(test)]
+mod tests {
+  use super::{GetHeadersMessage, GetBlocksMessage};
+
+  use std::io::IoResult;
+  use serialize::hex::FromHex;
+
+  use network::serialize::{deserialize, serialize};
+  use util::hash::zero_hash;
+
   #[test]
   fn getblocks_message_test() {
     let from_sat = "72110100014a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b0000000000000000000000000000000000000000000000000000000000000000".from_hex().unwrap();
     let genhash = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b".from_hex().unwrap();
 
-  let decode: IoResult<GetBlocksMessage> = Serializable::deserialize(from_sat.iter().map(|n| *n));
+    let decode: IoResult<GetBlocksMessage> = deserialize(from_sat.clone());
     assert!(decode.is_ok());
     let real_decode = decode.unwrap();
     assert_eq!(real_decode.version, 70002);
@@ -146,8 +149,7 @@ fn getblocks_message_test() {
     assert_eq!(real_decode.locator_hashes[0].as_slice(), genhash.as_slice());
     assert_eq!(real_decode.stop_hash.as_slice(), zero_hash().as_slice());
 
-  let reserialize = real_decode.serialize();
+    assert_eq!(serialize(&real_decode), Ok(from_sat));
-  assert_eq!(reserialize.as_slice(), from_sat.as_slice());
   }
 
   #[test]
@@ -155,7 +157,7 @@ fn getheaders_message_test() {
     let from_sat = "72110100014a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b0000000000000000000000000000000000000000000000000000000000000000".from_hex().unwrap();
     let genhash = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b".from_hex().unwrap();
 
-  let decode: IoResult<GetHeadersMessage> = Serializable::deserialize(from_sat.iter().map(|n| *n));
+    let decode: IoResult<GetHeadersMessage> = deserialize(from_sat.clone());
     assert!(decode.is_ok());
     let real_decode = decode.unwrap();
     assert_eq!(real_decode.version, 70002);
@@ -163,7 +165,7 @@ fn getheaders_message_test() {
     assert_eq!(real_decode.locator_hashes[0].as_slice(), genhash.as_slice());
     assert_eq!(real_decode.stop_hash.as_slice(), zero_hash().as_slice());
 
-  let reserialize = real_decode.serialize();
+    assert_eq!(serialize(&real_decode), Ok(from_sat));
-  assert_eq!(reserialize.as_slice(), from_sat.as_slice());
+  }
 }
 

src/network/message_network.rs

@@ -19,12 +19,9 @@
 //!
 
 use std::io::IoResult;
-#[cfg(test)]
-use serialize::hex::FromHex;
 
 use network::constants;
 use network::address::Address;
-use network::serialize::Serializable;
 use network::socket::Socket;
 
 /// Some simple messages
@@ -83,44 +80,25 @@ impl VersionMessage {
   }
 }
 
-impl Serializable for VersionMessage {
+impl_consensus_encoding!(VersionMessage, version, services, timestamp,
-  fn serialize(&self) -> Vec<u8> {
+                                         receiver, sender, nonce,
-    let mut rv = vec!();
+                                         user_agent, start_height, relay)
-    rv.extend(self.version.serialize().move_iter());
-    rv.extend(self.services.serialize().move_iter());
-    rv.extend(self.timestamp.serialize().move_iter());
-    rv.extend(self.receiver.serialize().move_iter());
-    rv.extend(self.sender.serialize().move_iter());
-    rv.extend(self.nonce.serialize().move_iter());
-    rv.extend(self.user_agent.serialize().move_iter());
-    rv.extend(self.start_height.serialize().move_iter());
-    if self.version >= 70001 {
-      rv.extend(self.relay.serialize().move_iter());
-    }
-    rv
-  }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<VersionMessage> {
+#[cfg(test)]
-    Ok(VersionMessage {
+mod tests {
-      version: try!(Serializable::deserialize(iter.by_ref())),
+  use super::VersionMessage;
-      services: try!(Serializable::deserialize(iter.by_ref())),
+
-      timestamp: try!(Serializable::deserialize(iter.by_ref())),
+  use std::io::IoResult;
-      receiver: try!(Serializable::deserialize(iter.by_ref())),
+  use serialize::hex::FromHex;
-      sender: try!(Serializable::deserialize(iter.by_ref())),
+
-      nonce: try!(Serializable::deserialize(iter.by_ref())),
+  use network::serialize::{deserialize, serialize};
-      user_agent: try!(Serializable::deserialize(iter.by_ref())),
-      start_height: try!(Serializable::deserialize(iter.by_ref())),
-      relay: try!(Serializable::deserialize(iter.by_ref()))
-    })
-  }
-}
 
   #[test]
   fn version_message_test() {
     // This message is from my satoshi node, morning of May 27 2014
     let from_sat = "721101000100000000000000e6e0845300000000010000000000000000000000000000000000ffff0000000000000100000000000000fd87d87eeb4364f22cf54dca59412db7208d47d920cffce83ee8102f5361746f7368693a302e392e39392f2c9f040001".from_hex().unwrap();
 
-  let decode: IoResult<VersionMessage> = Serializable::deserialize(from_sat.iter().map(|n| *n));
+    let decode: IoResult<VersionMessage> = deserialize(from_sat.clone());
     assert!(decode.is_ok());
     let real_decode = decode.unwrap();
     assert_eq!(real_decode.version, 70002);
@@ -132,8 +110,8 @@ fn version_message_test() {
     assert_eq!(real_decode.start_height, 302892);
     assert_eq!(real_decode.relay, true);
 
-  let reserialize = real_decode.serialize();
+    assert_eq!(serialize(&real_decode), Ok(from_sat));
-  assert_eq!(reserialize.as_slice(), from_sat.as_slice());
+  }
 }
 
 

src/network/mod.rs

@@ -19,6 +19,7 @@
 //!
 
 pub mod constants;
+pub mod encodable;
 pub mod socket;
 pub mod serialize;
 

src/network/serialize.rs

@@ -20,742 +20,160 @@
 //!
 
 use collections::Vec;
-use collections::bitv::{Bitv, from_bytes};
+use std::io::{IoError, IoResult, OtherIoError, MemReader, MemWriter};
-use std::default::Default;
-use std::hash::{Hash, Hasher};
-use std::collections::HashMap;
-use std::io::{IoError, IoResult, EndOfFile, InvalidInput, OtherIoError, standard_error};
-use std::io::{BufferedReader, BufferedWriter, File, Truncate, Write};
-use std::io::fs::rename;
-use std::mem::transmute;
-use std::u32;
-use serialize::{Encoder, Encodable};
 
-use util::iter::{FixedTake, FixedTakeable, NullIterator};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
 use util::hash::Sha256dHash;
-use util::thinvec::ThinVec;
 
-/// An iterator which returns serialized data one byte at a time
+/// Objects which are referred to by hash
-pub struct SerializeIter<'a> {
+pub trait BitcoinHash {
-  /// Iterator over actual data
+  /// Produces a Sha256dHash which can be used to refer to the object
-  pub data_iter: Option<Box<Iterator<u8>>>,
+  fn bitcoin_hash(&self) -> Sha256dHash;
-  /// Objects which are serialized through their own `SerializeIter`s
-  pub sub_iter_iter: Box<Iterator<&'a Serializable>>,
-  /// Current subiterator
-  pub sub_iter: Option<Box<SerializeIter<'a>>>,
-  /// Whether we have started using sub_iter_iter
-  pub sub_started: bool
 }
 
-impl<'a> Iterator<u8> for SerializeIter<'a> {
+impl BitcoinHash for Vec<u8> {
-  fn next(&mut self) -> Option<u8> { 
-    let mut ret = None;
-    // Try to use the data iterator
-    if self.data_iter.is_some() {
-      // Unwrap the current data iterator to use it
-      let mut it = self.data_iter.take_unwrap();
-      ret = it.next();
-      // Delete the current data iterator if it's exhausted, by putting
-      // it back only when it's -not- exhausted
-      if ret.is_some() { self.data_iter = Some(it); }
-    }
-    // Failing that, start using the subobject iterator
-    if ret.is_none() && !self.sub_started {
-      // Unwrap the current data iterator to use it
-      self.sub_started = true;
-      self.sub_iter = self.sub_iter_iter.next().map(|obj| box obj.serialize_iter());
-    }
-    // If it doesn't work, find one that does
-    while ret.is_none() && self.sub_iter.is_some() {
-      let mut iter = self.sub_iter.take_unwrap();
-      ret = iter.next();
-      self.sub_iter = if ret.is_none() {
-          self.sub_iter_iter.next().map(|obj| box obj.serialize_iter())
-        } else {
-          Some(iter)
-        }
-    }
-    // Eventually we got Some(u8) --- or None and we're exhausted
-    ret
-  }
-}
-
-#[deriving(PartialEq, Clone, Show)]
-/// Data which must be preceded by a 4-byte checksum
-pub struct CheckedData(pub Vec<u8>);
-
-/// An object which can be (de)serialized. If the object can be sent on the
-/// Bitcoin network, the serialization must be the standard p2p network
-/// serialization.
-pub trait Serializable {
-  /// Turn an object into a bytestring that can be put on the wire
-  fn serialize(&self) -> Vec<u8>;
-  /// Serialize an object, returning an iterator rather than complete vector
-  fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> {
-    SerializeIter {
-      data_iter: Some(box self.serialize().move_iter() as Box<Iterator<u8>>),
-      sub_iter_iter: box NullIterator::<&Serializable>::new(),
-      sub_iter: None,
-      sub_started: false
-    }
-  }
-  /// Read an object off the wire
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<Self>;
-  /// Obtain a hash of the object
   fn bitcoin_hash(&self) -> Sha256dHash {
-    Sha256dHash::from_data(self.serialize().as_slice())
+    Sha256dHash::from_data(self.as_slice())
-  }
-  /// Dump the object to a file
-  fn serialize_file(&self, p: &Path) -> IoResult<()> {
-    let tmp_path = p.with_extension("0");
-    {
-      let file = File::open_mode(&tmp_path, Truncate, Write);
-      let mut writer = BufferedWriter::new(file);
-      for ch in self.serialize_iter() {
-        try!(writer.write_u8(ch));
-      }
-      try!(writer.flush());
-    }
-    rename(&tmp_path, p)
-  }
-  /// Read the object from a file
-  fn deserialize_file(p: &Path) -> IoResult<Self> {
-    let file = try!(File::open(p));
-    let mut reader = BufferedReader::new(file);
-    let mut error: IoResult<u8> = Ok(0);
-    // This is kinda a hacky way to catch file read errors
-    let ret = Serializable::deserialize(reader.bytes().filter_map(|res| {
-        if res.is_err() {
-          error = res;
-          None
-        } else {
-          res.ok()
-        }
-      }));
-    // Return file error if there was one, else parse error
-    match error {
-      Ok(_) => ret,
-      Err(e) => Err(e)
-    }
-  }
-}
-
-/// A variable-length unsigned integer
-#[deriving(PartialEq, Show)]
-pub enum VarInt {
-  /// 8-bit int
-  VarU8(u8),
-  /// 16-bit int
-  VarU16(u16),
-  /// 32-bit int
-  VarU32(u32),
-  /// 64-bit int
-  VarU64(u64)
-}
-
-impl<S:Encoder<E>, E> Encodable<S, E> for VarInt {
-  fn encode(&self, s: &mut S) -> Result<(), E> {
-    s.emit_u64(varint_to_u64(*self))
   }
 }
 
-// Utility functions
+/// Encode an object into a vector
-/// Convert a Rust uint to a Bitcoin network Varint
+pub fn serialize<T: ConsensusEncodable<RawEncoder<MemWriter>, IoError>>(obj: &T) -> IoResult<Vec<u8>> {
-pub fn u64_to_varint(n: u64) -> VarInt {
+  let mut encoder = RawEncoder::new(MemWriter::new());
-  match n {
+  try!(obj.consensus_encode(&mut encoder));
-    n if n < 0xFD => VarU8(n as u8),
+  Ok(encoder.unwrap().unwrap())
-    n if n <= 0xFFFF => VarU16(n as u16),
+}
-    n if n <= 0xFFFFFFFF => VarU32(n as u32),
+
-    n => VarU64(n)
+/// Deserialize an object from a vector
+pub fn deserialize<T: ConsensusDecodable<RawDecoder<MemReader>, IoError>>(data: Vec<u8>) -> IoResult<T> {
+  let mut decoder = RawDecoder::new(MemReader::new(data));
+  ConsensusDecodable::consensus_decode(&mut decoder)
+}
+
+/// An encoder for raw binary data
+pub struct RawEncoder<W> {
+  writer: W
+}
+
+/// An decoder for raw binary data
+pub struct RawDecoder<R> {
+  reader: R
+}
+
+impl<W:Writer> RawEncoder<W> {
+  /// Constructor
+  pub fn new(writer: W) -> RawEncoder<W> {
+    RawEncoder { writer: writer }
+  }
+
+  /// Returns the underlying Writer
+  pub fn unwrap(self) -> W {
+    self.writer
   }
 }
 
-/// Convert a Bitcoin network Varint to a Rust uint
+impl<R:Reader> RawDecoder<R> {
-pub fn varint_to_u64(n: VarInt) -> u64 {
+  /// Constructor
-  match n {
+  pub fn new(reader: R) -> RawDecoder<R> {
-    VarU8(m) => m as u64,
+    RawDecoder { reader: reader }
-    VarU16(m) => m as u64,
+  }
-    VarU32(m) => m as u64,
+
-    VarU64(m) => m,
+  /// Returns the underlying Reader
+  pub fn unwrap(self) -> R {
+    self.reader
   }
 }
 
-fn read_uint_le<I: Iterator<u8>>(mut iter: FixedTake<I>) -> Option<u64> {
+/// A simple Encoder trait
-  let (rv, _) = iter.fold((0u64, 1u64), |(old, mult), next| (old + next as u64 * mult, mult * 0x100));
+pub trait SimpleEncoder<E> {
-  match iter.is_err() {
+  fn emit_u64(&mut self, v: u64) -> Result<(), E>;
-    false => Some(rv),
+  fn emit_u32(&mut self, v: u32) -> Result<(), E>;
-    true => None
+  fn emit_u16(&mut self, v: u16) -> Result<(), E>;
-  }
+  fn emit_u8(&mut self, v: u8) -> Result<(), E>;
+
+  fn emit_i64(&mut self, v: i64) -> Result<(), E>;
+  fn emit_i32(&mut self, v: i32) -> Result<(), E>;
+  fn emit_i16(&mut self, v: i16) -> Result<(), E>;
+  fn emit_i8(&mut self, v: i8) -> Result<(), E>;
+
+  fn emit_bool(&mut self, v: bool) -> Result<(), E>;
 }
 
-/// Do a double-SHA256 on some data and return the first 4 bytes
+/// A simple Decoder trait
-fn sha2_checksum(data: &[u8]) -> u32 {
+pub trait SimpleDecoder<E> {
-  let checksum = Sha256dHash::from_data(data);
+  fn read_u64(&mut self) -> Result<u64, E>;
-  read_uint_le(checksum.as_slice().iter().map(|n| *n).fixed_take(4)).unwrap() as u32
+  fn read_u32(&mut self) -> Result<u32, E>;
+  fn read_u16(&mut self) -> Result<u16, E>;
+  fn read_u8(&mut self) -> Result<u8, E>;
+
+  fn read_i64(&mut self) -> Result<i64, E>;
+  fn read_i32(&mut self) -> Result<i32, E>;
+  fn read_i16(&mut self) -> Result<i16, E>;
+  fn read_i8(&mut self) -> Result<i8, E>;
+
+  fn read_bool(&mut self) -> Result<bool, E>;
+
+  fn error(&mut self, err: &str) -> E;
 }
 
-/// Primitives
+// TODO: trait reform: impl SimpleEncoder for every Encoder, ditto for Decoder
-impl Serializable for bool {
+
-  fn serialize(&self) -> Vec<u8> {
+impl<W:Writer> SimpleEncoder<IoError> for RawEncoder<W> {
-    if *self { Vec::from_slice(&[1u8]) } else { Vec::from_slice(&[0u8]) }
+  #[inline]
+  fn emit_u64(&mut self, v: u64) -> IoResult<()> { self.writer.write_le_u64(v) }
+  #[inline]
+  fn emit_u32(&mut self, v: u32) -> IoResult<()> { self.writer.write_le_u32(v) }
+  #[inline]
+  fn emit_u16(&mut self, v: u16) -> IoResult<()> { self.writer.write_le_u16(v) }
+  #[inline]
+  fn emit_u8(&mut self, v: u8) -> IoResult<()> { self.writer.write_u8(v) }
+
+  #[inline]
+  fn emit_i64(&mut self, v: i64) -> IoResult<()> { self.writer.write_le_i64(v) }
+  #[inline]
+  fn emit_i32(&mut self, v: i32) -> IoResult<()> { self.writer.write_le_i32(v) }
+  #[inline]
+  fn emit_i16(&mut self, v: i16) -> IoResult<()> { self.writer.write_le_i16(v) }
+  #[inline]
+  fn emit_i8(&mut self, v: i8) -> IoResult<()> { self.writer.write_i8(v) }
+
+  #[inline]
+  fn emit_bool(&mut self, v: bool) -> IoResult<()> { self.writer.write_i8(if v {1} else {0}) }
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<bool> {
+impl<R:Reader> SimpleDecoder<IoError> for RawDecoder<R> {
-    match iter.next() {
+  #[inline]
-      Some(u) => Ok(u != 0),
+  fn read_u64(&mut self) -> IoResult<u64> { self.reader.read_le_u64() }
-      None    => Err(standard_error(EndOfFile))
+  #[inline]
-    }
+  fn read_u32(&mut self) -> IoResult<u32> { self.reader.read_le_u32() }
-  }
+  #[inline]
-}
+  fn read_u16(&mut self) -> IoResult<u16> { self.reader.read_le_u16() }
+  #[inline]
+  fn read_u8(&mut self) -> IoResult<u8> { self.reader.read_u8() }
 
-impl Serializable for u8 {
+  #[inline]
-  fn serialize(&self) -> Vec<u8> {
+  fn read_i64(&mut self) -> IoResult<i64> { self.reader.read_le_i64() }
-    Vec::from_slice(&[*self])
+  #[inline]
-  }
+  fn read_i32(&mut self) -> IoResult<i32> { self.reader.read_le_i32() }
+  #[inline]
+  fn read_i16(&mut self) -> IoResult<i16> { self.reader.read_le_i16() }
+  #[inline]
+  fn read_i8(&mut self) -> IoResult<i8> { self.reader.read_i8() }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<u8> {
+  #[inline]
-    match iter.next() {
+  fn read_bool(&mut self) -> IoResult<bool> { self.reader.read_u8().map(|res| res != 0) }
-      Some(u) => Ok(u as u8),
-      None    => Err(standard_error(EndOfFile))
-    }
-  }
-}
 
-impl Serializable for u16 {
+  #[inline]
-  fn serialize(&self) -> Vec<u8> {
+  fn error(&mut self, err: &str) -> IoError {
-    unsafe { Vec::from_slice(transmute::<_, [u8, ..2]>(self.to_le())) }
+    IoError {
-  }
-
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<u16> {
-    match read_uint_le(iter.fixed_take(2)) {
-      Some(u) => Ok(u as u16),
-      None    => Err(standard_error(EndOfFile))
-    }
-  }
-}
-
-impl Serializable for u32 {
-  fn serialize(&self) -> Vec<u8> {
-    unsafe { Vec::from_slice(transmute::<_, [u8, ..4]>(self.to_le())) }
-  }
-
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<u32> {
-    match read_uint_le(iter.fixed_take(4)) {
-      Some(u) => Ok(u as u32),
-      None    => Err(standard_error(EndOfFile))
-    }
-  }
-}
-
-impl Serializable for i32 {
-  fn serialize(&self) -> Vec<u8> {
-    unsafe { Vec::from_slice(transmute::<_, [u8, ..4]>(self.to_le())) }
-  }
-
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<i32> {
-    match read_uint_le(iter.fixed_take(4)) {
-      Some(u) => Ok(u as i32),
-      None    => Err(standard_error(EndOfFile))
-    }
-  }
-}
-
-impl Serializable for u64 {
-  fn serialize(&self) -> Vec<u8> {
-    unsafe { Vec::from_slice(transmute::<_, [u8, ..8]>(self.to_le())) }
-  }
-
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<u64> {
-    match read_uint_le(iter.fixed_take(8)) {
-      Some(u) => Ok(u as u64),
-      None    => Err(standard_error(EndOfFile))
-    }
-  }
-}
-
-impl Serializable for i64 {
-  fn serialize(&self) -> Vec<u8> {
-    unsafe { Vec::from_slice(transmute::<_, [u8, ..8]>(self.to_le())) }
-  }
-
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<i64> {
-    match read_uint_le(iter.fixed_take(8)) {
-      Some(u) => Ok(u as i64),
-      None    => Err(standard_error(EndOfFile))
-    }
-  }
-}
-
-impl Serializable for VarInt {
-  fn serialize(&self) -> Vec<u8> {
-    match *self {
-      VarU8(n)  => Vec::from_slice(&[n]),
-      VarU16(n) => { let mut rv = n.serialize(); rv.insert(0, 0xFD); rv },
-      VarU32(n) => { let mut rv = n.serialize(); rv.insert(0, 0xFE); rv },
-      VarU64(n) => { let mut rv = n.serialize(); rv.insert(0, 0xFF); rv },
-    }
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<VarInt> {
-    match iter.next() {
-      Some(n) if n < 0xFD => Ok(VarU8(n)),
-      Some(n) if n == 0xFD => Ok(VarU16(try!(Serializable::deserialize(iter)))),
-      Some(n) if n == 0xFE => Ok(VarU32(try!(Serializable::deserialize(iter)))),
-      Some(n) if n == 0xFF => Ok(VarU64(try!(Serializable::deserialize(iter)))),
-      _ => unreachable!()
-    }
-  }
-}
-
-macro_rules! serialize_fixvec(
-  ($($size:expr),+) => (
-    $(
-      impl Serializable for [u8, ..$size] {
-        fn serialize(&self) -> Vec<u8> {
-          Vec::from_slice(self.as_slice())
-        }
-
-        fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<[u8, ..$size]> {
-          let mut v = [0u8, ..$size];
-          let mut fixiter = iter.fixed_take($size);
-          let mut n = 0;
-          for ch in fixiter {
-            v[n] = ch;
-            n += 1;
-          }
-          match fixiter.is_err() {
-            false => Ok(v),
-            true => Err(standard_error(EndOfFile))
-          }
-        }
-      }
-    )+
-
-    #[test]
-    fn test_fixvec() {
-      $(
-        let vec = [5u8, ..$size];
-        let short_vec = [5u8, ..($size - 1)];
-        assert_eq!(vec.as_slice(), vec.serialize().as_slice());
-
-        let decode: IoResult<[u8, ..$size]> = Serializable::deserialize(vec.iter().map(|n| *n));
-        let short_decode: IoResult<[u8, ..$size]> = Serializable::deserialize(short_vec.iter().map(|n| *n));
-
-        assert!(decode.is_ok());
-        assert!(short_decode.is_err());
-        assert_eq!(decode.unwrap().as_slice(), vec.as_slice());
-      )+
-    }
-  );
-)
-// we need to do this in one call so that we can do a test for
-// every value; we can't define a new test fn for each invocation
-// because there are no gensyms.
-serialize_fixvec!(4, 8, 12, 16, 32)
-
-impl Serializable for CheckedData {
-  fn serialize(&self) -> Vec<u8> {
-    let &CheckedData(ref data) = self;
-    let mut ret = (data.len() as u32).serialize();
-    ret.extend(sha2_checksum(data.as_slice()).serialize().move_iter());
-    ret.extend(data.iter().map(|n| *n));
-    ret
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<CheckedData> {
-    let length: u32 = try!(Serializable::deserialize(iter.by_ref()));
-    let checksum: u32 = try!(Serializable::deserialize(iter.by_ref()));
-
-    let mut fixiter = iter.fixed_take(length as uint);
-    let v: Vec<u8> =  FromIterator::from_iter(fixiter.by_ref());
-    if fixiter.is_err() {
-      return Err(IoError {
-        kind: EndOfFile,
-        desc: "overrun",
-        detail: Some(format!("checksummed data length given as {:}, but read fewer bytes", length))
-      });
-    }
-
-    let expected_checksum = sha2_checksum(v.as_slice());
-    if checksum == expected_checksum {
-      Ok(CheckedData(v))
-    } else {
-      Err(IoError {
       kind: OtherIoError,
-        desc: "bad checksum",
+      desc: "parse error",
-        detail: Some(format!("checksum {:4x} did not match expected {:4x}", checksum, expected_checksum)),
+      detail: Some(err.to_string())
-      })
     }
   }
 }
 
-impl Serializable for String {
+// Aren't really any tests here.. the main functions are serialize and
-  fn serialize(&self) -> Vec<u8> {
+// deserialize, which get the crap tested out of them it every other
-    let mut rv = u64_to_varint(self.len() as u64).serialize();
+// module.
-    rv.push_all(self.as_bytes());
-    rv
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<String> {
-    let length: VarInt = try!(Serializable::deserialize(iter.by_ref()));
-    let mut fixiter = iter.fixed_take(varint_to_u64(length) as uint);
-    let rv: String = FromIterator::from_iter(fixiter.by_ref().map(|u| u as char));
-    match fixiter.is_err() {
-      false => Ok(rv),
-      true => Err(standard_error(EndOfFile))
-    }
-  }
-}
-
-impl<T: Serializable> Serializable for Vec<T> {
-  fn serialize(&self) -> Vec<u8> {
-    let n_elems = u64_to_varint(self.len() as u64);
-    let mut rv = n_elems.serialize();
-    for elem in self.iter() {
-      rv.extend(elem.serialize().move_iter());
-    }
-    rv
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<Vec<T>> {
-    let mut n_elems = varint_to_u64(try!(Serializable::deserialize(iter.by_ref())));
-    let mut v: Vec<T> = vec![];
-    while n_elems > 0 {
-      v.push(try!(Serializable::deserialize(iter.by_ref())));
-      n_elems -= 1;
-    }
-    Ok(v)
-  }
-}
-
-impl <K: Serializable+Eq+Hash<u64>, T: Serializable, H: Hasher<u64>+Default> Serializable for HashMap<K, T, H> {
-  fn serialize(&self) -> Vec<u8> {
-    let n_elems = u64_to_varint(self.len() as u64);
-    let mut rv = n_elems.serialize();
-    for (key, value) in self.iter() {
-      rv.extend(key.serialize().move_iter());
-      rv.extend(value.serialize().move_iter());
-    }
-    rv
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<HashMap<K, T, H>> {
-    let mut n_elems = varint_to_u64(try!(Serializable::deserialize(iter.by_ref())));
-    let mut ret = HashMap::with_capacity_and_hasher(n_elems as uint, Default::default());
-    while n_elems > 0 {
-      let key: K = try!(Serializable::deserialize(iter.by_ref()));
-      let value: T = try!(Serializable::deserialize(iter.by_ref()));
-      ret.insert(key, value);
-      n_elems -= 1;
-    }
-    Ok(ret)
-  }
-}
-
-impl<T: Serializable> Serializable for ThinVec<T> {
-  fn serialize(&self) -> Vec<u8> {
-    let n_elems = u64_to_varint(self.len() as u64);
-    let mut rv = n_elems.serialize();
-    for elem in self.iter() {
-      rv.extend(elem.serialize().move_iter());
-    }
-    rv
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<ThinVec<T>> {
-    let n_elems = varint_to_u64(try!(Serializable::deserialize(iter.by_ref())));
-    if n_elems >= u32::MAX as u64 {
-      return Err(IoError {
-        kind: InvalidInput,
-        desc: "vector length too large",
-        detail: Some(format!("tried to read ThinVec with len {} > 4bn", n_elems))
-      });
-    }
-
-    let mut v: ThinVec<T> = ThinVec::with_capacity(n_elems as u32);
-    for i in range(0, n_elems) {
-      unsafe {
-        v.init(i as uint, try!(Serializable::deserialize(iter.by_ref())));
-      };
-    }
-    Ok(v)
-  }
-}
-
-impl<T:Serializable, U:Serializable> Serializable for (T, U) {
-  fn serialize(&self) -> Vec<u8> {
-    let &(ref self1, ref self2) = self;
-    let mut ret = vec![];
-    ret.extend(self1.serialize().move_iter());
-    ret.extend(self2.serialize().move_iter());
-    ret
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<(T, U)> {
-    // FIXME: assign then return is a workaround for https://github.com/rust-lang/rust/issues/15763
-    let ret = Ok((try!(Serializable::deserialize(iter.by_ref())),
-                  try!(Serializable::deserialize(iter.by_ref()))));
-    ret
-  }
-}
-
-impl<T:Serializable+'static> Serializable for Option<T> {
-  fn serialize(&self) -> Vec<u8> {
-    match self {
-      &Some(ref dat) => {
-        let mut ret = vec![1];
-        ret.extend(dat.serialize().move_iter());
-        ret
-      },
-      &None => vec![0]
-    }
-  }
-
-  fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> {
-    match self {
-      &Some(ref dat) => SerializeIter {
-        data_iter: Some(box Some(1u8).move_iter() as Box<Iterator<u8>>),
-        sub_iter_iter: box vec![ dat as &Serializable ].move_iter(),
-        sub_iter: None,
-        sub_started: false
-      },
-      &None => SerializeIter {
-        data_iter: Some(box Some(0u8).move_iter() as Box<Iterator<u8>>),
-        sub_iter_iter: box NullIterator::<&Serializable>::new(),
-        sub_iter: None,
-        sub_started: false
-      }
-    }
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<Option<T>> {
-    match iter.next() {
-      Some(0) => Ok(None),
-      Some(1) => Ok(Some(try!(Serializable::deserialize(iter)))),
-      _ => Err(standard_error(InvalidInput))
-    }
-  }
-}
-
-impl <T:Serializable> Serializable for Box<T> {
-  fn serialize(&self) -> Vec<u8> {
-    (**self).serialize()
-  }
-
-  fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> {
-    (**self).serialize_iter()
-  }
-
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<Box<T>> {
-    let ret: T = try!(Serializable::deserialize(iter));
-    Ok(box ret)
-  }
-}
-
-impl Serializable for Bitv {
-  fn serialize(&self) -> Vec<u8> {
-    let n_elems = u64_to_varint(self.len() as u64);
-    let mut rv = n_elems.serialize();
-    for elem in self.to_bytes().iter() {
-      rv.extend(elem.serialize().move_iter());
-    }
-    rv
-  }
-
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<Bitv> {
-    let n_elems = varint_to_u64(try!(Serializable::deserialize(iter.by_ref())));
-    let mut v: Vec<u8> = vec![];
-    for _ in range(0, (n_elems + 7) / 8) {
-      v.push(try!(Serializable::deserialize(iter.by_ref())));
-    }
-    let mut ret = from_bytes(v.as_slice());
-    ret.truncate(n_elems as uint);  // from_bytes will round up to 8
-    Ok(ret)
-  }
-}
-
-#[test]
-fn serialize_iter_test() {
-  assert_eq!(true.serialize(), true.serialize_iter().collect());
-  assert_eq!(1u8.serialize(), 1u8.serialize_iter().collect());
-  assert_eq!(300u32.serialize(), 300u32.serialize_iter().collect());
-  assert_eq!(20u64.serialize(), 20u64.serialize_iter().collect());
-}
-
-#[test]
-fn serialize_int_test() {
-  // bool
-  assert_eq!(false.serialize(), Vec::from_slice([0u8]));
-  assert_eq!(true.serialize(), Vec::from_slice([1u8]));
-  // u8
-  assert_eq!(1u8.serialize(), Vec::from_slice([1u8]));
-  assert_eq!(0u8.serialize(), Vec::from_slice([0u8]));
-  assert_eq!(255u8.serialize(), Vec::from_slice([255u8]));
-  // u16
-  assert_eq!(1u16.serialize(), Vec::from_slice([1u8, 0]));
-  assert_eq!(256u16.serialize(), Vec::from_slice([0u8, 1]));
-  assert_eq!(5000u16.serialize(), Vec::from_slice([136u8, 19]));
-  // u32
-  assert_eq!(1u32.serialize(), Vec::from_slice([1u8, 0, 0, 0]));
-  assert_eq!(256u32.serialize(), Vec::from_slice([0u8, 1, 0, 0]));
-  assert_eq!(5000u32.serialize(), Vec::from_slice([136u8, 19, 0, 0]));
-  assert_eq!(500000u32.serialize(), Vec::from_slice([32u8, 161, 7, 0]));
-  assert_eq!(168430090u32.serialize(), Vec::from_slice([10u8, 10, 10, 10]));
-  // TODO: test negative numbers
-  assert_eq!(1i32.serialize(), Vec::from_slice([1u8, 0, 0, 0]));
-  assert_eq!(256i32.serialize(), Vec::from_slice([0u8, 1, 0, 0]));
-  assert_eq!(5000i32.serialize(), Vec::from_slice([136u8, 19, 0, 0]));
-  assert_eq!(500000i32.serialize(), Vec::from_slice([32u8, 161, 7, 0]));
-  assert_eq!(168430090i32.serialize(), Vec::from_slice([10u8, 10, 10, 10]));
-  // u64
-  assert_eq!(1u64.serialize(), Vec::from_slice([1u8, 0, 0, 0, 0, 0, 0, 0]));
-  assert_eq!(256u64.serialize(), Vec::from_slice([0u8, 1, 0, 0, 0, 0, 0, 0]));
-  assert_eq!(5000u64.serialize(), Vec::from_slice([136u8, 19, 0, 0, 0, 0, 0, 0]));
-  assert_eq!(500000u64.serialize(), Vec::from_slice([32u8, 161, 7, 0, 0, 0, 0, 0]));
-  assert_eq!(723401728380766730u64.serialize(), Vec::from_slice([10u8, 10, 10, 10, 10, 10, 10, 10]));
-  // TODO: test negative numbers
-  assert_eq!(1i64.serialize(), Vec::from_slice([1u8, 0, 0, 0, 0, 0, 0, 0]));
-  assert_eq!(256i64.serialize(), Vec::from_slice([0u8, 1, 0, 0, 0, 0, 0, 0]));
-  assert_eq!(5000i64.serialize(), Vec::from_slice([136u8, 19, 0, 0, 0, 0, 0, 0]));
-  assert_eq!(500000i64.serialize(), Vec::from_slice([32u8, 161, 7, 0, 0, 0, 0, 0]));
-  assert_eq!(723401728380766730i64.serialize(), Vec::from_slice([10u8, 10, 10, 10, 10, 10, 10, 10]));
-}
-
-#[test]
-fn serialize_varint_test() {
-  assert_eq!(VarU8(10).serialize(), Vec::from_slice([10u8]));
-  assert_eq!(VarU8(0xFC).serialize(), Vec::from_slice([0xFCu8]));
-  assert_eq!(VarU16(0xFD).serialize(), Vec::from_slice([0xFDu8, 0xFD, 0]));
-  assert_eq!(VarU16(0xFFF).serialize(), Vec::from_slice([0xFDu8, 0xFF, 0xF]));
-  assert_eq!(VarU32(0xF0F0F0F).serialize(), Vec::from_slice([0xFEu8, 0xF, 0xF, 0xF, 0xF]));
-  assert_eq!(VarU64(0xF0F0F0F0F0E0).serialize(), Vec::from_slice([0xFFu8, 0xE0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0, 0]));
-}
-
-#[test]
-fn serialize_vector_test() {
-  assert_eq!(Vec::from_slice([1u8, 2, 3]).serialize(), Vec::from_slice([3u8, 1, 2, 3]));
-  // TODO: test vectors of more interesting objects
-}
-
-#[test]
-fn serialize_strbuf_test() {
-  assert_eq!(String::from_str("Andrew").serialize(), Vec::from_slice([6u8, 0x41, 0x6e, 0x64, 0x72, 0x65, 0x77]));
-}
-
-#[test]
-fn serialize_checkeddata_test() {
-  let cd = CheckedData(vec![1u8, 2, 3, 4, 5]);
-  assert_eq!(cd.serialize(), vec![5, 0, 0, 0, 162, 107, 175, 90, 1, 2, 3, 4, 5]);
-}
-
-#[test]
-fn serialize_box_test() {
-  assert_eq!((box 1u8).serialize(), vec![1u8]);
-  assert_eq!((box 1u16).serialize(), vec![1u8, 0]);
-  assert_eq!((box 1u64).serialize(), vec![1u8, 0, 0, 0, 0, 0, 0, 0]);
-}
-
-#[test]
-fn serialize_option_test() {
-  let none: Option<u8> = None;
-  let none_ser = none.serialize();
-  let some_ser = Some(0xFFu8).serialize();
-  assert_eq!(none_ser, vec![0]);
-  assert_eq!(some_ser, vec![1, 0xFF]);
-
-  assert_eq!(none.serialize(), none.serialize_iter().collect());
-  assert_eq!(Some(true).serialize(), Some(true).serialize_iter().collect());
-}
-
-#[test]
-fn serialize_bitv_test() {
-  let bv = Bitv::with_capacity(10, true);
-  assert_eq!(bv.serialize(), vec![10, 0xFF, 0xC0]);
-  assert_eq!(bv.serialize(), bv.serialize_iter().collect());
-}
-
-#[test]
-fn deserialize_int_test() {
-  // bool
-  assert_eq!(Serializable::deserialize([58u8, 0].iter().map(|n| *n)), Ok(true));
-  assert_eq!(Serializable::deserialize([58u8].iter().map(|n| *n)), Ok(true));
-  assert_eq!(Serializable::deserialize([1u8].iter().map(|n| *n)), Ok(true));
-  assert_eq!(Serializable::deserialize([0u8].iter().map(|n| *n)), Ok(false));
-  assert_eq!(Serializable::deserialize([0u8, 1].iter().map(|n| *n)), Ok(false));
-
-  // u8
-  assert_eq!(Serializable::deserialize([58u8].iter().map(|n| *n)), Ok(58u8));
-
-  // u16
-  assert_eq!(Serializable::deserialize([0x01u8, 0x02].iter().map(|n| *n)), Ok(0x0201u16));
-  assert_eq!(Serializable::deserialize([0xABu8, 0xCD].iter().map(|n| *n)), Ok(0xCDABu16));
-  assert_eq!(Serializable::deserialize([0xA0u8, 0x0D].iter().map(|n| *n)), Ok(0xDA0u16));
-  let failure16: IoResult<u16> = Serializable::deserialize([1u8].iter().map(|n| *n));
-  assert!(failure16.is_err());
-
-  // u32
-  assert_eq!(Serializable::deserialize([0xABu8, 0xCD, 0, 0].iter().map(|n| *n)), Ok(0xCDABu32));
-  assert_eq!(Serializable::deserialize([0xA0u8, 0x0D, 0xAB, 0xCD].iter().map(|n| *n)), Ok(0xCDAB0DA0u32));
-  let failure32: IoResult<u32> = Serializable::deserialize([1u8, 2, 3].iter().map(|n| *n));
-  assert!(failure32.is_err());
-  // TODO: test negative numbers
-  assert_eq!(Serializable::deserialize([0xABu8, 0xCD, 0, 0].iter().map(|n| *n)), Ok(0xCDABi32));
-  assert_eq!(Serializable::deserialize([0xA0u8, 0x0D, 0xAB, 0x2D].iter().map(|n| *n)), Ok(0x2DAB0DA0i32));
-  let failurei32: IoResult<i32> = Serializable::deserialize([1u8, 2, 3].iter().map(|n| *n));
-  assert!(failurei32.is_err());
-
-  // u64
-  assert_eq!(Serializable::deserialize([0xABu8, 0xCD, 0, 0, 0, 0, 0, 0].iter().map(|n| *n)), Ok(0xCDABu64));
-  assert_eq!(Serializable::deserialize([0xA0u8, 0x0D, 0xAB, 0xCD, 0x99, 0, 0, 0x99].iter().map(|n| *n)), Ok(0x99000099CDAB0DA0u64));
-  let failure64: IoResult<u64> = Serializable::deserialize([1u8, 2, 3, 4, 5, 6, 7].iter().map(|n| *n));
-  assert!(failure64.is_err());
-  // TODO: test negative numbers
-  assert_eq!(Serializable::deserialize([0xABu8, 0xCD, 0, 0, 0, 0, 0, 0].iter().map(|n| *n)), Ok(0xCDABi64));
-  assert_eq!(Serializable::deserialize([0xA0u8, 0x0D, 0xAB, 0xCD, 0x99, 0, 0, 0x99].iter().map(|n| *n)), Ok(0x99000099CDAB0DA0i64));
-  let failurei64: IoResult<i64> = Serializable::deserialize([1u8, 2, 3, 4, 5, 6, 7].iter().map(|n| *n));
-  assert!(failurei64.is_err());
-}
-
-#[test]
-fn deserialize_vec_test() {
-  assert_eq!(Serializable::deserialize([3u8, 2, 3, 4].iter().map(|n| *n)), Ok(vec![2u8, 3, 4]));
-  assert_eq!(Serializable::deserialize([4u8, 2, 3, 4, 5, 6].iter().map(|n| *n)), Ok(vec![2u8, 3, 4, 5]));
-}
-
-#[test]
-fn deserialize_strbuf_test() {
-  assert_eq!(Serializable::deserialize([6u8, 0x41, 0x6e, 0x64, 0x72, 0x65, 0x77].iter().map(|n| *n)), Ok(String::from_str("Andrew")));
-}
-
-#[test]
-fn deserialize_checkeddata_test() {
-  let cd: IoResult<CheckedData> = Serializable::deserialize([5u8, 0, 0, 0, 162, 107, 175, 90, 1, 2, 3, 4, 5].iter().map(|n| *n));
-  assert!(cd.is_ok());
-  assert_eq!(cd.unwrap(), CheckedData(Vec::from_slice([1u8, 2, 3, 4, 5])));
-}
-
-#[test]
-fn deserialize_option_test() {
-  let none: IoResult<Option<u8>> = Serializable::deserialize([0u8].iter().map(|n| *n));
-  let good: IoResult<Option<u8>> = Serializable::deserialize([1u8, 0xFF].iter().map(|n| *n));
-  let bad: IoResult<Option<u8>> = Serializable::deserialize([2u8].iter().map(|n| *n));
-  assert!(bad.is_err());
-  assert_eq!(none, Ok(None));
-  assert_eq!(good, Ok(Some(0xFF)));
-}
-
-#[test]
-fn deserialize_box_test() {
-  let zero: IoResult<Box<u8>> = Serializable::deserialize([0u8].iter().map(|n| *n));
-  let one: IoResult<Box<u8>> = Serializable::deserialize([1u8].iter().map(|n| *n));
-  assert_eq!(zero, Ok(box 0));
-  assert_eq!(one, Ok(box 1));
-}
-
-#[test]
-fn deserialize_bitv_test() {
-  let bv: IoResult<Bitv> = Serializable::deserialize([10u8, 0xFF, 0xC0].iter().map(|n| *n));
-  assert!(bv.is_ok());
-  assert_eq!(bv.unwrap(), Bitv::with_capacity(10, true));
-}
 

src/network/socket.rs

@@ -27,9 +27,10 @@ use std::sync::{Arc, Mutex};
 
 use network::constants;
 use network::address::Address;
+use network::encodable::{ConsensusEncodable, ConsensusDecodable};
 use network::message::{RawNetworkMessage, NetworkMessage, Version};
-use network::serialize::Serializable;
 use network::message_network::VersionMessage;
+use network::serialize::{RawEncoder, RawDecoder};
 use util::misc::prepend_err;
 
 /// Format an IP address in the 16-byte bitcoin protocol serialization
@@ -185,7 +186,7 @@ impl Socket {
       None => Err(standard_error(NotConnected)),
       Some(ref mut writer) => {
         let message = RawNetworkMessage { magic: self.magic, payload: payload };
-        try!(writer.write(message.serialize().as_slice()));
+        try!(message.consensus_encode(&mut RawEncoder::new(writer.by_ref())));
         writer.flush()
       }
     }
@@ -198,33 +199,17 @@ impl Socket {
     match *reader_lock.deref_mut() {
       None => Err(standard_error(NotConnected)),
       Some(ref mut buf) => {
-        let mut read_err = None;
         // We need a new scope since the closure in here borrows read_err,
         // and we try to read it afterward. Letting `iter` go out fixes it.
-        let ret: IoResult<RawNetworkMessage> = {
+        let mut decoder = RawDecoder::new(buf.by_ref());
-          // Set up iterator so we will catch network errors properly
+        let decode: IoResult<RawNetworkMessage> = ConsensusDecodable::consensus_decode(&mut decoder);
-          let iter = buf.bytes().filter_map(|res|
+        match decode {
-            match res {
+          // Check for parse errors...
-              Ok(ch) => Some(ch),
-              Err(e) => { read_err = Some(e); None }
-            });
-          // Receive message
-          Serializable::deserialize(iter)
-        };
-        // Return
-        match read_err {
-          // Network errors get priority since they are probably more meaningful
-          Some(e) => {
-            prepend_err("network", Err(e))
-          },
-          _ => {
-            match ret {
-              // Next come parse errors
           Err(e) => {
-                prepend_err("decode", Err(e))
+            prepend_err("network_decode", Err(e))
           },
           Ok(ret) => {
-                // Finally magic (this should come before parse error, but we can't
+            // Then for magic (this should come before parse error, but we can't
             // get to it if the deserialization failed). TODO restructure this
             if ret.magic != self.magic {
               Err(IoError {
@@ -241,7 +226,5 @@ impl Socket {
     }
   }
 }
-  }
-}
 
 

src/util/hash.rs

@@ -20,7 +20,7 @@ use core::char::from_digit;
 use core::cmp::min;
 use std::default::Default;
 use std::fmt;
-use std::io::{IoResult, IoError, InvalidInput};
+use std::io::MemWriter;
 use std::mem::transmute;
 use std::hash::{Hash, Hasher};
 use serialize::json::{mod, ToJson};
@@ -28,8 +28,8 @@ use serialize::json::{mod, ToJson};
 use crypto::digest::Digest;
 use crypto::sha2;
 
-use network::serialize::Serializable;
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
-use util::iter::FixedTakeable;
+use network::serialize::{RawEncoder, BitcoinHash, SimpleDecoder, SimpleEncoder};
 use util::uint::Uint128;
 use util::uint::Uint256;
 
@@ -108,15 +108,16 @@ impl Sha256dHash {
     from_bytes(self.as_slice())
   }
 
-  /// Converts a hash to a Uint256, interpreting it as a little endian encoding.
+  /// Converts a hash to a Uint256, interpreting it as a little endian number.
-  pub fn as_uint256(&self) -> Uint256 {
+  pub fn into_uint256(self) -> Uint256 {
-    let &Sha256dHash(data) = self;
+    let Sha256dHash(data) = self;
     unsafe { Uint256(transmute(data)) }
   }
 
-  /// Converts a hash to a Uint128, interpreting it as a little endian encoding.
+  /// Converts a hash to a Uint128, interpreting it as a little endian number.
-  pub fn as_uint128(&self) -> Uint128 {
+  pub fn into_uint128(self) -> Uint128 {
-    let &Sha256dHash(data) = self;
+    let Sha256dHash(data) = self;
+    // TODO: this function won't work correctly on big-endian machines
     unsafe { Uint128(transmute([data[16], data[17], data[18], data[19], data[20],
                                 data[21], data[22], data[23], data[24], data[25],
                                 data[26], data[27], data[28], data[29], data[30],
@@ -156,9 +157,17 @@ impl PartialEq for Sha256dHash {
 
 impl Eq for Sha256dHash {}
 
+impl Index<uint, u8> for Sha256dHash {
+  #[inline]
+  fn index<'a>(&'a self, idx: &uint) -> &'a u8 {
+    let &Sha256dHash(ref data) = self;
+    &data[*idx]
+  }
+}
+
 // Note that this outputs hashes as big endian hex numbers, so this should be
 // used only for user-facing stuff. Internal and network serialization is
-// little-endian and should be done using the consensus `network::serialize`
+// little-endian and should be done using the consensus `encodable::ConsensusEncodable`
 // interface.
 impl ToJson for Sha256dHash {
   fn to_json(&self) -> json::Json {
@@ -166,26 +175,17 @@ impl ToJson for Sha256dHash {
   }
 }
 
-impl Serializable for Sha256dHash {
+impl<S:SimpleEncoder<E>, E> ConsensusEncodable<S, E> for Sha256dHash {
-  fn serialize(&self) -> Vec<u8> {
+  #[inline]
-    let &Sha256dHash(ref data) = self;
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    data.iter().map(|n| *n).collect()
+    self.into_uint256().consensus_encode(s)
+  }
 }
 
-  fn deserialize<I: Iterator<u8>>(iter: I) -> IoResult<Sha256dHash> {
+impl<D:SimpleDecoder<E>, E> ConsensusDecodable<D, E> for Sha256dHash {
-    let Sha256dHash(mut ret) = zero_hash();
+  #[inline]
-    let mut fixediter = iter.enumerate().fixed_take(32);
+  fn consensus_decode(d: &mut D) -> Result<Sha256dHash, E> {
-    for (n, data) in fixediter {
+    Ok(Sha256dHash(try!(ConsensusDecodable::consensus_decode(d))))
-      ret[n] = data;
-    }
-    match fixediter.is_err() {
-      false => Ok(Sha256dHash(ret)),
-      true => Err(IoError {
-        kind: InvalidInput,
-        desc: "unexpected end of input",
-        detail: Some(format!("Need 32 bytes, was {:} short.", fixediter.remaining()))
-      })
-    }
   }
 }
 
@@ -215,7 +215,7 @@ pub trait MerkleRoot {
   fn merkle_root(&self) -> Sha256dHash;
 }
 
-impl<'a, T: Serializable> MerkleRoot for &'a [T] {
+impl<'a, T: BitcoinHash> MerkleRoot for &'a [T] {
   fn merkle_root(&self) -> Sha256dHash {
     fn merkle_root(data: Vec<Sha256dHash>) -> Sha256dHash {
       // Base case
@@ -230,9 +230,10 @@ impl<'a, T: Serializable> MerkleRoot for &'a [T] {
       for idx in range(0, (data.len() + 1) / 2) {
         let idx1 = 2 * idx;
         let idx2 = min(idx1 + 1, data.len() - 1);
-        let to_hash = data[idx1].bitcoin_hash().serialize()
+        let mut encoder = RawEncoder::new(MemWriter::new());
-                        .append(data[idx2].bitcoin_hash().serialize().as_slice());
+        data[idx1].consensus_encode(&mut encoder).unwrap();
-        next.push(to_hash.bitcoin_hash());
+        data[idx2].consensus_encode(&mut encoder).unwrap();
+        next.push(encoder.unwrap().unwrap().bitcoin_hash());
       }
       merkle_root(next)
     }
@@ -240,7 +241,7 @@ impl<'a, T: Serializable> MerkleRoot for &'a [T] {
   }
 }
 
-impl <T: Serializable> MerkleRoot for Vec<T> {
+impl <T: BitcoinHash> MerkleRoot for Vec<T> {
   fn merkle_root(&self) -> Sha256dHash {
     self.as_slice().merkle_root()
   }

src/util/iter.rs

@@ -17,77 +17,6 @@
 //! Iterator adaptors needed by Bitcoin but not provided by the Rust
 //! standard library.
 
-/// An iterator that just returns None
-pub struct NullIterator<T>;
-impl<T> Iterator<T> for NullIterator<T> {
-  #[inline]
-  fn next(&mut self) -> Option<T> { None }
-  #[inline]
-  fn size_hint(&self) -> (uint, Option<uint>) { (0, Some(0)) }
-}
-
-impl<T> NullIterator<T> {
-  /// Creates a new NullIterator
-  pub fn new() -> NullIterator<T> { NullIterator }
-}
-
-/// An Iterator which will give n elements of the contained iterator
-/// before returning None. If the contained iterator returns None too
-/// early,
-pub struct FixedTake<I> {
-  iter: I,
-  n_elems: uint,
-  is_err: bool
-}
-
-impl<T, I: Iterator<T>> Iterator<T> for FixedTake<I> {
-  fn next(&mut self) -> Option<T> {
-    if self.n_elems == 0 {
-      None
-    } else {
-      self.n_elems -= 1;
-      match self.iter.next() {
-        Some(e) => Some(e),
-        None => {
-          self.is_err = true;
-          None
-        }
-      }
-    }
-  }
-}
-
-impl<I> FixedTake<I> {
-  /// Constructs a FixedTake iterator from an underlying iterator
-  pub fn new(iter: I, n_elems: uint) -> FixedTake<I> {
-    FixedTake { iter: iter, n_elems: n_elems, is_err: false }
-  }
-
-  /// Indicates whether the underlying iterator has ended early
-  pub fn is_err(&self) -> bool {
-    self.is_err
-  }
-
-  /// Number of remaining elements
-  pub fn remaining(&self) -> uint {
-    self.n_elems
-  }
-}
-
-/// An iterator that returns at most `n_elems` elements, entering an error
-/// state if the underlying iterator yields fewer than `n_elems` elements.
-pub trait FixedTakeable<I> {
-  /// Returns an iterator similar to Take but which detects if the original 
-  /// iterator runs out early
-  fn fixed_take(self, n_elems: uint) -> FixedTake<I>;
-}
-
-impl<T, I: Iterator<T>> FixedTakeable<I> for I {
-  fn fixed_take(self, n_elems: uint) -> FixedTake<I> {
-    FixedTake::new(self, n_elems)
-  }
-}
-
 /// An iterator that returns pairs of elements
 pub struct Pair<A, I> {
   iter: I,

src/util/mod.rs

@@ -39,4 +39,3 @@ pub trait BitArray {
   fn trailing_zeros(&self) -> uint;
 }
 
-

src/util/patricia_tree.rs

@@ -21,28 +21,26 @@
 //!
 
 use core::fmt::Show;
-use core::iter::ByRef;
 use core::cmp;
 use std::kinds::marker;
 use std::num::{Zero, One};
-use std::io::{IoResult, InvalidInput, standard_error};
 
-use network::serialize::{Serializable, SerializeIter};
+use network::encodable::{ConsensusDecodable, ConsensusEncodable};
+use network::serialize::{SimpleDecoder, SimpleEncoder};
 use util::BitArray;
-use util::misc::prepend_err;
 
 /// Patricia troo
-pub struct PatriciaTree<T, K> {
+pub struct PatriciaTree<K, V> {
-  data: Option<T>,
+  data: Option<V>,
-  child_l: Option<Box<PatriciaTree<T, K>>>,
+  child_l: Option<Box<PatriciaTree<K, V>>>,
-  child_r: Option<Box<PatriciaTree<T, K>>>,
+  child_r: Option<Box<PatriciaTree<K, V>>>,
   skip_prefix: K,
   skip_len: u8
 }
 
-impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree<T, K> {
+impl<K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>, V> PatriciaTree<K, V> {
   /// Constructs a new Patricia tree
-  pub fn new() -> PatriciaTree<T, K> {
+  pub fn new() -> PatriciaTree<K, V> {
     PatriciaTree {
       data: None,
       child_l: None,
@@ -53,7 +51,7 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
   }
 
   /// Lookup a value by exactly matching `key` and return a referenc
-  pub fn lookup_mut<'a>(&'a mut self, key: &K, key_len: uint) -> Option<&'a mut T> {
+  pub fn lookup_mut<'a>(&'a mut self, key: &K, key_len: uint) -> Option<&'a mut V> {
     // Caution: `lookup_mut` never modifies its self parameter (in fact its
     // internal recursion uses a non-mutable self, so we are OK to just
     // transmute our self pointer into a mutable self before passing it in.
@@ -62,7 +60,7 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
   }
 
   /// Lookup a value by exactly matching `key` and return a mutable reference
-  pub fn lookup<'a>(&'a self, key: &K, key_len: uint) -> Option<&'a T> {
+  pub fn lookup<'a>(&'a self, key: &K, key_len: uint) -> Option<&'a V> {
     let mut node = self;
     let mut key_idx = 0;
 
@@ -98,18 +96,18 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
   /// Inserts a value with key `key`, returning true on success. If a value is already
   /// stored against `key`, do nothing and return false.
   #[inline]
-  pub fn insert(&mut self, key: &K, key_len: uint, value: T) -> bool {
+  pub fn insert(&mut self, key: &K, key_len: uint, value: V) -> bool {
     self.real_insert(key, key_len, value, false)
   }
 
   /// Inserts a value with key `key`, returning true on success. If a value is already
   /// stored against `key`, overwrite it and return false.
   #[inline]
-  pub fn insert_or_update(&mut self, key: &K, key_len: uint, value: T) -> bool {
+  pub fn insert_or_update(&mut self, key: &K, key_len: uint, value: V) -> bool {
     self.real_insert(key, key_len, value, true)
   }
 
-  fn real_insert(&mut self, key: &K, key_len: uint, value: T, overwrite: bool) -> bool {
+  fn real_insert(&mut self, key: &K, key_len: uint, value: V, overwrite: bool) -> bool {
     let mut node = self;
     let mut idx = 0;
     loop {
@@ -213,10 +211,10 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
 
   /// Deletes a value with key `key`, returning it on success. If no value with
   /// the given key is found, return None
-  pub fn delete(&mut self, key: &K, key_len: uint) -> Option<T> {
+  pub fn delete(&mut self, key: &K, key_len: uint) -> Option<V> {
     /// Return value is (deletable, actual return value), where `deletable` is true
     /// is true when the entire node can be deleted (i.e. it has no children)
-    fn recurse<T, K:BitArray+Eq+Zero+One+Add<K,K>+Shr<uint,K>+Shl<uint,K>>(tree: &mut PatriciaTree<T, K>, key: &K, key_len: uint) -> (bool, Option<T>) {
+    fn recurse<K:BitArray+Eq+Zero+One+Add<K,K>+Shr<uint,K>+Shl<uint,K>, V>(tree: &mut PatriciaTree<K, V>, key: &K, key_len: uint) -> (bool, Option<V>) {
       // If the search key is shorter than the node prefix, there is no
       // way we can match, so fail.
       if key_len < tree.skip_len as uint {
@@ -322,7 +320,7 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
 
   /// Count all the nodes
   pub fn node_count(&self) -> uint {
-    fn recurse<T, K>(node: &Option<Box<PatriciaTree<T, K>>>) -> uint {
+    fn recurse<K, V>(node: &Option<Box<PatriciaTree<K, V>>>) -> uint {
       match node {
         &Some(ref node) => { 1 + recurse(&node.child_l) + recurse(&node.child_r) }
         &None => 0
@@ -332,7 +330,7 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
   }
 
   /// Returns an iterator over all elements in the tree
-  pub fn iter<'a>(&'a self) -> Items<'a, T, K> {
+  pub fn iter<'a>(&'a self) -> Items<'a, K, V> {
     Items {
       node: Some(self),
       parents: vec![],
@@ -341,7 +339,7 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
   }
 
   /// Returns a mutable iterator over all elements in the tree
-  pub fn mut_iter<'a>(&'a mut self) -> MutItems<'a, T, K> {
+  pub fn mut_iter<'a>(&'a mut self) -> MutItems<'a, K, V> {
     MutItems {
       node: self as *mut _,
       parents: vec![],
@@ -351,10 +349,10 @@ impl<T, K:BitArray+Eq+Zero+One+BitXor<K,K>+Shl<uint,K>+Shr<uint,K>> PatriciaTree
   }
 }
 
-impl<T:Show, K:BitArray> PatriciaTree<T, K> {
+impl<K:BitArray, V:Show> PatriciaTree<K, V> {
   /// Print the entire tree
   pub fn print<'a>(&'a self) {
-    fn recurse<'a, T:Show, K:BitArray>(tree: &'a PatriciaTree<T, K>, depth: uint) {
+    fn recurse<'a, K:BitArray, V:Show>(tree: &'a PatriciaTree<K, V>, depth: uint) {
       for i in range(0, tree.skip_len as uint) {
         print!("{:}", if tree.skip_prefix.bit(i) { 1u } else { 0 });
       }
@@ -386,74 +384,48 @@ impl<T:Show, K:BitArray> PatriciaTree<T, K> {
   }
 }
 
-impl<T:Serializable+'static, K:BitArray+Serializable+'static> Serializable for PatriciaTree<T, K> {
+impl<S:SimpleEncoder<E>, E, K:ConsensusEncodable<S, E>, V:ConsensusEncodable<S, E>> ConsensusEncodable<S, E> for PatriciaTree<K, V> {
-  fn serialize(&self) -> Vec<u8> {
+  fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-    // Depth-first serialization
+    // Depth-first serialization: serialize self, then children
-    let mut ret = vec![];
+    try!(self.skip_prefix.consensus_encode(s));
-    // Serialize self, then children
+    try!(self.skip_len.consensus_encode(s));
-    ret.extend(self.skip_prefix.serialize().move_iter());
+    try!(self.data.consensus_encode(s));
-    ret.extend(self.skip_len.serialize().move_iter());
+    try!(self.child_l.consensus_encode(s));
-    ret.extend(self.data.serialize().move_iter());
+    try!(self.child_r.consensus_encode(s));
-    ret.extend(self.child_l.serialize().move_iter());
+    Ok(())
-    ret.extend(self.child_r.serialize().move_iter());
-    ret
-  }
-
-  fn serialize_iter<'a>(&'a self) -> SerializeIter<'a> {
-    SerializeIter {
-      data_iter: None,
-      sub_iter_iter: box vec![ &self.skip_prefix as &Serializable,
-                               &self.skip_len as &Serializable,
-                               &self.data as &Serializable,
-                               &self.child_l as &Serializable,
-                               &self.child_r as &Serializable ].move_iter(),
-      sub_iter: None,
-      sub_started: false
   }
 }
 
-  fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<PatriciaTree<T, K>> {
+impl<D:SimpleDecoder<E>, E, K:ConsensusDecodable<D, E>, V:ConsensusDecodable<D, E>> ConsensusDecodable<D, E> for PatriciaTree<K, V> {
-    // This goofy deserialization routine is to prevent an infinite
+  fn consensus_decode(d: &mut D) -> Result<PatriciaTree<K, V>, E> {
-    // regress of ByRef<ByRef<...<ByRef<I>>...>>, see #15188
-    fn recurse<T:Serializable+'static, K:Serializable, I: Iterator<u8>>(iter: &mut ByRef<I>) -> IoResult<PatriciaTree<T, K>> {
     Ok(PatriciaTree {
-        skip_prefix: try!(prepend_err("skip_prefix", Serializable::deserialize(iter.by_ref()))),
+      skip_prefix: try!(ConsensusDecodable::consensus_decode(d)),
-        skip_len: try!(prepend_err("skip_len", Serializable::deserialize(iter.by_ref()))),
+      skip_len: try!(ConsensusDecodable::consensus_decode(d)),
-        data: try!(prepend_err("data", Serializable::deserialize(iter.by_ref()))),
+      data: try!(ConsensusDecodable::consensus_decode(d)),
-        child_l: match iter.next() {
+      child_l: try!(ConsensusDecodable::consensus_decode(d)),
-                   Some(1) => Some(box try!(prepend_err("child_l", recurse(iter)))),
+      child_r: try!(ConsensusDecodable::consensus_decode(d))
-                   Some(0) => None,
-                   _ => { return Err(standard_error(InvalidInput)) }
-                 },
-        child_r: match iter.next() {
-                   Some(1) => Some(box try!(prepend_err("child_r", recurse(iter)))),
-                   Some(0) => None,
-                   _ => { return Err(standard_error(InvalidInput)) }
-                 }
     })
   }
-    recurse(&mut iter.by_ref())
-  }
 }
 
 /// Iterator
-pub struct Items<'tree, T, K> {
+pub struct Items<'tree, K, V> {
   started: bool,
-  node: Option<&'tree PatriciaTree<T, K>>,
+  node: Option<&'tree PatriciaTree<K, V>>,
-  parents: Vec<&'tree PatriciaTree<T, K>>
+  parents: Vec<&'tree PatriciaTree<K, V>>
 }
 
 /// Mutable iterator
-pub struct MutItems<'tree, T, K> {
+pub struct MutItems<'tree, K, V> {
   started: bool,
-  node: *mut PatriciaTree<T, K>,
+  node: *mut PatriciaTree<K, V>,
-  parents: Vec<*mut PatriciaTree<T, K>>,
+  parents: Vec<*mut PatriciaTree<K, V>>,
   marker: marker::ContravariantLifetime<'tree>
 }
 
-impl<'a, T, K> Iterator<&'a T> for Items<'a, T, K> {
+impl<'a, K, V> Iterator<&'a V> for Items<'a, K, V> {
-  fn next(&mut self) -> Option<&'a T> {
+  fn next(&mut self) -> Option<&'a V> {
-    fn borrow_opt<'a, T, K>(opt_ptr: &'a Option<Box<PatriciaTree<T, K>>>) -> Option<&'a PatriciaTree<T, K>> {
+    fn borrow_opt<'a, K, V>(opt_ptr: &'a Option<Box<PatriciaTree<K, V>>>) -> Option<&'a PatriciaTree<K, V>> {
       opt_ptr.as_ref().map(|b| &**b)
     }
 
@@ -495,9 +467,9 @@ impl<'a, T, K> Iterator<&'a T> for Items<'a, T, K> {
   }
 }
 
-impl<'a, T, K> Iterator<&'a mut T> for MutItems<'a, T, K> {
+impl<'a, K, V> Iterator<&'a mut V> for MutItems<'a, K, V> {
-  fn next(&mut self) -> Option<&'a mut T> {
+  fn next(&mut self) -> Option<&'a mut V> {
-    fn borrow_opt<'a, T, K>(opt_ptr: &'a Option<Box<PatriciaTree<T, K>>>) -> *mut PatriciaTree<T, K> {
+    fn borrow_opt<'a, K, V>(opt_ptr: &'a Option<Box<PatriciaTree<K, V>>>) -> *mut PatriciaTree<K, V> {
       match *opt_ptr {
         Some(ref data) => &**data as *const _ as *mut _,
         None => RawPtr::null()
@@ -553,11 +525,11 @@ mod tests {
   use std::io::IoResult;
   use std::num::Zero;
 
+  use network::serialize::{deserialize, serialize};
   use util::hash::Sha256dHash;
   use util::uint::Uint128;
   use util::uint::Uint256;
   use util::patricia_tree::PatriciaTree;
-  use network::serialize::Serializable;
 
   #[test]
   fn patricia_single_insert_lookup_delete_test() {
@@ -579,7 +551,7 @@ mod tests {
     let mut tree = PatriciaTree::new();
     let mut hashes = vec![];
     for i in range(0u32, 5000) {
-      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).as_uint128();
+      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).into_uint128();
       tree.insert(&hash, 250, i);
       hashes.push(hash);
     }
@@ -620,7 +592,7 @@ mod tests {
     let mut hashes = vec![];
     // Start by inserting a bunch of chunder
     for i in range(1u32, 500) {
-      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).as_uint128();
+      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).into_uint128();
       tree.insert(&hash, 128, i * 1000);
       hashes.push(hash);
     }
@@ -652,7 +624,7 @@ mod tests {
     let mut data = Vec::from_elem(n_elems, None);
     // Start by inserting a bunch of stuff
     for i in range(0, n_elems) {
-      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).as_uint128();
+      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).into_uint128();
       tree.insert(&hash, 128, i);
       *data.get_mut(i) = Some(());
     }
@@ -674,7 +646,7 @@ mod tests {
     let mut data = Vec::from_elem(n_elems, None);
     // Start by inserting a bunch of stuff
     for i in range(0, n_elems) {
-      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).as_uint128();
+      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).into_uint128();
       tree.insert(&hash, 128, i);
       *data.get_mut(i) = Some(());
     }
@@ -700,18 +672,15 @@ mod tests {
     let mut tree = PatriciaTree::new();
     let mut hashes = vec![];
     for i in range(0u32, 5000) {
-      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).as_uint128();
+      let hash = Sha256dHash::from_data(&[(i / 0x100) as u8, (i % 0x100) as u8]).into_uint128();
       tree.insert(&hash, 250, i);
       hashes.push(hash);
     }
 
     // Serialize it
-    let serialized = tree.serialize();
+    let serialized = serialize(&tree).unwrap();
-    // Check iterator
-    let serialized_1 = tree.serialize_iter().collect();
-    assert_eq!(serialized, serialized_1);
     // Deserialize it
-    let deserialized: IoResult<PatriciaTree<u32, Uint128>> = Serializable::deserialize(serialized.iter().map(|n| *n));
+    let deserialized: IoResult<PatriciaTree<Uint128, u32>> = deserialize(serialized);
     assert!(deserialized.is_ok());
     let new_tree = deserialized.unwrap();
 

src/util/thinvec.rs

@@ -38,7 +38,7 @@ impl<T> ThinVec<T> {
 
   /// Constructor with predetermined capacity
   #[inline]
-  pub fn with_capacity(capacity: u32) -> ThinVec<T> {
+  pub unsafe fn with_capacity(capacity: u32) -> ThinVec<T> {
     if mem::size_of::<T>() == 0 {
       ThinVec { ptr: RawPtr::null(), cap: capacity }
     } else if capacity == 0 {
@@ -46,7 +46,7 @@ impl<T> ThinVec<T> {
     } else {
       let size = (capacity as uint).checked_mul(&mem::size_of::<T>())
                    .expect("ThinVec::reserve: capacity overflow");
-      let ptr = unsafe { allocate(size, mem::min_align_of::<T>()) };
+      let ptr = allocate(size, mem::min_align_of::<T>());
       ThinVec { ptr: ptr as *mut T, cap: capacity }
     }
   }
@@ -159,17 +159,17 @@ impl<T> Vector<T> for ThinVec<T> {
 
 impl<T:Clone> Clone for ThinVec<T> {
   fn clone(&self) -> ThinVec<T> {
+    unsafe {
       let mut ret = ThinVec::with_capacity(self.len() as u32);
       // Copied from vec.rs, which claims this will be optimized to a memcpy
       // if T is Copy
       for i in range(0, self.len()) {
-      unsafe {
         ptr::write(ret.as_mut_slice().unsafe_mut_ref(i),
                    self.as_slice().unsafe_ref(i).clone());
       }
-    }
       ret
     }
+  }
 
    // TODO: clone_from
 }
@@ -179,10 +179,11 @@ impl<T> FromIterator<T> for ThinVec<T> {
   fn from_iter<I: Iterator<T>>(iter: I) -> ThinVec<T> {
     let (lower, _) = iter.size_hint();
     assert!(lower < u32::MAX as uint);
+    unsafe {
       let mut vector = ThinVec::with_capacity(lower as u32);
       for (n, elem) in iter.enumerate() {
         if n < lower {
-        unsafe { vector.init(n, elem) };
+          vector.init(n, elem);
         } else {
           vector.push(elem);
         }
@@ -190,6 +191,7 @@ impl<T> FromIterator<T> for ThinVec<T> {
       vector
     }
   }
+}
 
 impl<T> Extendable<T> for ThinVec<T> {
   #[inline]
@@ -252,14 +254,14 @@ mod tests {
   #[test]
   fn simple_destructor_thinvec_test() {
     let cap = 2;
+    unsafe {
       let mut thinvec = ThinVec::with_capacity(cap);
 
       for i in range(0, cap) {
-      unsafe { thinvec.init(i as uint, Some(box i)); }
+        thinvec.init(i as uint, Some(box i));
       }
 
       for i in range(0, cap) {
-      unsafe {
         assert_eq!(thinvec.get_mut(i as uint).take(), Some(box i));
         assert_eq!(thinvec.get_mut(i as uint).take(), None); 
       }

src/util/uint.rs

@@ -19,11 +19,9 @@
 //!
 
 use std::fmt;
-use std::io::IoResult;
 use std::num::{Zero, One};
-use std::mem::transmute;
 
-use network::serialize::Serializable;
+use network::serialize::RawEncoder;
 use util::BitArray;
 
 macro_rules! construct_uint(
@@ -308,8 +306,8 @@ macro_rules! construct_uint(
         let &$name(ref me) = self;
         let &$name(ref you) = other;
         for i in range(0, $n_words) {
-          if me[3 - i] < you[3 - i] { return Less; }
+          if me[$n_words - 1 - i] < you[$n_words - 1 - i] { return Less; }
-          if me[3 - i] > you[3 - i] { return Greater; }
+          if me[$n_words - 1 - i] > you[$n_words - 1 - i] { return Greater; }
         }
         return Equal;
       }
@@ -323,19 +321,28 @@ macro_rules! construct_uint(
 
     impl fmt::Show for $name {
       fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.serialize().as_slice())
+        use std::fmt::WriteError;
+        use network::encodable::ConsensusEncodable;
+        let mut encoder = RawEncoder::new(f.by_ref());
+        self.consensus_encode(&mut encoder).map_err(|_| WriteError)
       }
     }
 
-    impl Serializable for $name {
+    impl<S: ::network::serialize::SimpleEncoder<E>, E> ::network::encodable::ConsensusEncodable<S, E> for $name {
-      fn serialize(&self) -> Vec<u8> {
+      #[inline]
-        let vec = unsafe { transmute::<$name, [u8, ..($n_words*8)]>(*self) };
+      fn consensus_encode(&self, s: &mut S) -> Result<(), E> {
-        vec.serialize()
+        use network::encodable::ConsensusEncodable;
+        let &$name(ref data) = self;
+        for word in data.iter() { try!(word.consensus_encode(s)); }
+        Ok(())
+      }
     }
 
-      fn deserialize<I: Iterator<u8>>(mut iter: I) -> IoResult<$name> {
+    impl<D: ::network::serialize::SimpleDecoder<E>, E> ::network::encodable::ConsensusDecodable<D, E> for $name {
-        let ret: [u8, ..($n_words*8)] = try!(Serializable::deserialize(iter.by_ref()));
+      fn consensus_decode(d: &mut D) -> Result<$name, E> {
-        Ok(unsafe { transmute(ret) })
+        use network::encodable::ConsensusDecodable;
+        let ret: [u64, ..$n_words] = try!(ConsensusDecodable::consensus_decode(d));
+        Ok($name(ret))
       }
     }
   );
@@ -367,7 +374,7 @@ mod tests {
   use std::io::IoResult;
   use std::num::from_u64;
 
-  use network::serialize::Serializable;
+  use network::serialize::{deserialize, serialize};
   use util::uint::Uint256;
   use util::BitArray;
 
@@ -471,10 +478,10 @@ mod tests {
   pub fn uint256_serialize_test() {
     let start1 = Uint256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]);
     let start2 = Uint256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0xABCD, 0xFFFF]);
-    let serial1 = start1.serialize();
+    let serial1 = serialize(&start1).unwrap();
-    let serial2 = start2.serialize();
+    let serial2 = serialize(&start2).unwrap();
-    let end1: IoResult<Uint256> = Serializable::deserialize(serial1.iter().map(|n| *n));
+    let end1: IoResult<Uint256> = deserialize(serial1);
-    let end2: IoResult<Uint256> = Serializable::deserialize(serial2.iter().map(|n| *n));
+    let end2: IoResult<Uint256> = deserialize(serial2);
 
     assert_eq!(end1, Ok(start1));
     assert_eq!(end2, Ok(start2));