milksad
/
rust-bitcoin-unsafe-fast
14
0
Fork 0
Code Issues Pull Requests Activity

Make UtxoSet store (height, txo) pairs rather than just the raw txo 

Will take some experimentation to see if this is what I want the API
to be, if the memory usage is acceptable, etc.

This will force a total reindex for wizards-wallet users.

[breaking-change]
This commit is contained in:
Andrew Poelstra 2014-09-10 19:58:58 -05:00
parent 184721db61
commit e0c9844ff2
4 changed files with 54 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/blockdata/transaction.rs
View File

@ -161,7 +161,7 @@ impl TxIn {
index: uint) -> Result<(), TransactionError> { index: uint) -> Result<(), TransactionError> {
let txo = utxoset.get_utxo(self.prev_hash, self.prev_index); let txo = utxoset.get_utxo(self.prev_hash, self.prev_index);
match txo { match txo {
Some(txo) => { Some((_, txo)) => {
let mut p2sh_stack = Vec::new(); let mut p2sh_stack = Vec::new();
let mut p2sh_script = Script::new(); let mut p2sh_script = Script::new();
@ -242,7 +242,7 @@ impl Transaction {
// Run through the input // Run through the input
let txo = utxoset.get_utxo(input.prev_hash, input.prev_index); let txo = utxoset.get_utxo(input.prev_hash, input.prev_index);
match txo { match txo {
Some(txo) => { Some((_, txo)) => {
let mut p2sh_stack = Vec::new(); let mut p2sh_stack = Vec::new();
let mut p2sh_script = Script::new(); let mut p2sh_script = Script::new();

84
src/blockdata/utxoset.rs
View File

@ -59,8 +59,13 @@ pub enum UtxoSetError {
InvalidTx(Sha256dHash, TransactionError), InvalidTx(Sha256dHash, TransactionError),
} }
/// Vector of outputs; None indicates a nonexistent or already spent output struct UtxoNode {
type UtxoNode = ThinVec<Option<TxOut>>; /// Blockheight at which this UTXO appeared in the blockchain
height: u32,
/// Vector of outputs; None indicates a nonexistent or already spent output
outputs: ThinVec<Option<TxOut>>
}
impl_consensus_encoding!(UtxoNode, height, outputs)
/// An iterator over UTXOs /// An iterator over UTXOs
pub struct UtxoIterator<'a> { pub struct UtxoIterator<'a> {
@ -70,16 +75,17 @@ pub struct UtxoIterator<'a> {
tx_index: uint tx_index: uint
} }
impl<'a> Iterator<(Sha256dHash, uint, &'a TxOut)> for UtxoIterator<'a> { impl<'a> Iterator<(Sha256dHash, uint, &'a TxOut, uint)> for UtxoIterator<'a> {
fn next(&mut self) -> Option<(Sha256dHash, uint, &'a TxOut)> { fn next(&mut self) -> Option<(Sha256dHash, uint, &'a TxOut, uint)> {
while self.current.is_some() { while self.current.is_some() {
let current = self.current.unwrap(); let current = &self.current.unwrap().outputs;
while self.tx_index < current.len() { while self.tx_index < current.len() {
self.tx_index += 1; self.tx_index += 1;
if unsafe { current.get(self.tx_index - 1) }.is_some() { if unsafe { current.get(self.tx_index - 1) }.is_some() {
return Some((self.current_key, return Some((self.current_key,
self.tx_index, self.tx_index,
unsafe { current.get(self.tx_index - 1) }.as_ref().unwrap())); unsafe { current.get(self.tx_index - 1) }.as_ref().unwrap(),
self.current.unwrap().height as uint));
} }
} }
match self.tx_iter.next() { match self.tx_iter.next() {
@ -100,7 +106,7 @@ pub struct UtxoSet {
table: HashMap<Sha256dHash, UtxoNode, DumbHasher>, table: HashMap<Sha256dHash, UtxoNode, DumbHasher>,
last_hash: Sha256dHash, last_hash: Sha256dHash,
// A circular buffer of deleted utxos, grouped by block // A circular buffer of deleted utxos, grouped by block
spent_txos: Vec<Vec<((Sha256dHash, u32), TxOut)>>, spent_txos: Vec<Vec<((Sha256dHash, u32), (u32, TxOut))>>,
// The last index into the above buffer that was assigned to // The last index into the above buffer that was assigned to
spent_idx: u64, spent_idx: u64,
n_utxos: u64, n_utxos: u64,
@ -127,7 +133,7 @@ impl UtxoSet {
} }
/// Add all the UTXOs of a transaction to the set /// Add all the UTXOs of a transaction to the set
fn add_utxos(&mut self, tx: &Transaction) -> Option<UtxoNode> { fn add_utxos(&mut self, tx: &Transaction, height: u32) -> Option<UtxoNode> {
let txid = tx.bitcoin_hash(); let txid = tx.bitcoin_hash();
// Locate node if it's already there // Locate node if it's already there
let new_node = unsafe { let new_node = unsafe {
@ -142,7 +148,7 @@ impl UtxoSet {
new_node.init(vout as uint, Some(txo.clone())); new_node.init(vout as uint, Some(txo.clone()));
} }
} }
new_node UtxoNode { outputs: new_node, height: height }
}; };
// Get the old value, if any (this is suprisingly possible, c.f. BIP30 // Get the old value, if any (this is suprisingly possible, c.f. BIP30
// and the other comments in this file referring to it) // and the other comments in this file referring to it)
@ -154,9 +160,9 @@ impl UtxoSet {
} }
/// Remove a UTXO from the set and return it /// Remove a UTXO from the set and return it
fn take_utxo(&mut self, txid: Sha256dHash, vout: u32) -> Option<TxOut> { fn take_utxo(&mut self, txid: Sha256dHash, vout: u32) -> Option<(u32, TxOut)> {
// This whole function has awkward scoping thx to lexical borrow scoping :( // This whole function has awkward scoping thx to lexical borrow scoping :(
let (ret, should_delete) = { let (height, ret, should_delete) = {
// Locate the UTXO, failing if not found // Locate the UTXO, failing if not found
let node = match self.table.find_mut(&txid) { let node = match self.table.find_mut(&txid) {
Some(node) => node, Some(node) => node,
@ -165,13 +171,13 @@ impl UtxoSet {
let ret = { let ret = {
// Check that this specific output is there // Check that this specific output is there
if vout as uint >= node.len() { return None; } if vout as uint >= node.outputs.len() { return None; }
let replace = unsafe { node.get_mut(vout as uint) }; let replace = unsafe { node.outputs.get_mut(vout as uint) };
replace.take() replace.take()
}; };
let should_delete = node.iter().filter(|slot| slot.is_some()).count() == 0; let should_delete = node.outputs.iter().filter(|slot| slot.is_some()).count() == 0;
(ret, should_delete) (node.height, ret, should_delete)
}; };
// Delete the whole node if it is no longer being used // Delete the whole node if it is no longer being used
@ -180,24 +186,24 @@ impl UtxoSet {
} }
self.n_utxos -= if ret.is_some() { 1 } else { 0 }; self.n_utxos -= if ret.is_some() { 1 } else { 0 };
ret ret.map(|o| (height, o))
} }
/// Get a reference to a UTXO in the set /// Get a reference to a UTXO in the set
pub fn get_utxo<'a>(&'a self, txid: Sha256dHash, vout: u32) -> Option<&'a TxOut> { pub fn get_utxo<'a>(&'a self, txid: Sha256dHash, vout: u32) -> Option<(uint, &'a TxOut)> {
// Locate the UTXO, failing if not found // Locate the UTXO, failing if not found
let node = match self.table.find(&txid) { let node = match self.table.find(&txid) {
Some(node) => node, Some(node) => node,
None => return None None => return None
}; };
// Check that this specific output is there // Check that this specific output is there
if vout as uint >= node.len() { return None; } if vout as uint >= node.outputs.len() { return None; }
let replace = unsafe { node.get(vout as uint) }; let replace = unsafe { node.outputs.get(vout as uint) };
replace.as_ref() Some((node.height as uint, replace.as_ref().unwrap()))
} }
/// Apply the transactions contained in a block /// Apply the transactions contained in a block
pub fn update(&mut self, block: &Block, validation: ValidationLevel) pub fn update(&mut self, block: &Block, blockheight: uint, validation: ValidationLevel)
-> Result<(), UtxoSetError> { -> Result<(), UtxoSetError> {
// Make sure we are extending the UTXO set in order // Make sure we are extending the UTXO set in order
if validation >= ChainValidation && if validation >= ChainValidation &&
@ -223,7 +229,7 @@ impl UtxoSet {
// is invalid, -except- for two historic blocks which appeared in the // is invalid, -except- for two historic blocks which appeared in the
// blockchain before the dupes were noticed. // blockchain before the dupes were noticed.
// See bitcoind commit `ab91bf39` and BIP30. // See bitcoind commit `ab91bf39` and BIP30.
match self.add_utxos(tx) { match self.add_utxos(tx, blockheight as u32) {
Some(mut replace) => { Some(mut replace) => {
let blockhash = block.header.bitcoin_hash().be_hex_string(); let blockhash = block.header.bitcoin_hash().be_hex_string();
if blockhash == "00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec".to_string() || if blockhash == "00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec".to_string() ||
@ -233,10 +239,10 @@ impl UtxoSet {
} else { } else {
// Otherwise put the replaced txouts into the `deleted` cache // Otherwise put the replaced txouts into the `deleted` cache
// so that rewind will put them back. // so that rewind will put them back.
self.spent_txos.get_mut(spent_idx).reserve_additional(replace.len()); self.spent_txos.get_mut(spent_idx).reserve_additional(replace.outputs.len());
for (n, input) in replace.mut_iter().enumerate() { for (n, input) in replace.outputs.mut_iter().enumerate() {
match input.take() { match input.take() {
Some(txo) => { self.spent_txos.get_mut(spent_idx).push(((txid, n as u32), txo)); } Some(txo) => { self.spent_txos.get_mut(spent_idx).push(((txid, n as u32), (replace.height, txo))); }
None => {} None => {}
} }
} }
@ -343,22 +349,22 @@ impl UtxoSet {
if skipped_genesis { if skipped_genesis {
let mut extract_vec = vec![]; let mut extract_vec = vec![];
mem::swap(&mut extract_vec, self.spent_txos.get_mut(self.spent_idx as uint)); mem::swap(&mut extract_vec, self.spent_txos.get_mut(self.spent_idx as uint));
for ((txid, n), txo) in extract_vec.move_iter() { for ((txid, n), (height, txo)) in extract_vec.move_iter() {
// Remove the tx's utxo list and patch the txo into place // Remove the tx's utxo list and patch the txo into place
let new_node = let new_node =
match self.table.pop(&txid) { match self.table.pop(&txid) {
Some(mut thinvec) => { Some(mut node) => {
let old_len = thinvec.len() as u32; let old_len = node.outputs.len() as u32;
if old_len < n + 1 { if old_len < n + 1 {
unsafe { unsafe {
thinvec.reserve(n + 1); node.outputs.reserve(n + 1);
for i in range(old_len, n + 1) { for i in range(old_len, n + 1) {
thinvec.init(i as uint, None); node.outputs.init(i as uint, None);
} }
} }
} }
unsafe { *thinvec.get_mut(n as uint) = Some(txo); } unsafe { *node.outputs.get_mut(n as uint) = Some(txo); }
thinvec node
} }
None => { None => {
unsafe { unsafe {
@ -367,7 +373,7 @@ impl UtxoSet {
thinvec.init(i as uint, None); thinvec.init(i as uint, None);
} }
thinvec.init(n as uint, Some(txo)); thinvec.init(n as uint, Some(txo));
thinvec UtxoNode { outputs: thinvec, height: height }
} }
} }
}; };
@ -441,11 +447,11 @@ mod tests {
let new_block: Block = deserialize("010000004ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914cd74d6e49ffff001d323b3a7b0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d026e04ffffffff0100f2052a0100000043410446ef0102d1ec5240f0d061a4246c1bdef63fc3dbab7733052fbbf0ecd8f41fc26bf049ebb4f9527f374280259e7cfa99c48b0e3f39c51347a19a5819651503a5ac00000000010000000321f75f3139a013f50f315b23b0c9a2b6eac31e2bec98e5891c924664889942260000000049483045022100cb2c6b346a978ab8c61b18b5e9397755cbd17d6eb2fe0083ef32e067fa6c785a02206ce44e613f31d9a6b0517e46f3db1576e9812cc98d159bfdaf759a5014081b5c01ffffffff79cda0945903627c3da1f85fc95d0b8ee3e76ae0cfdc9a65d09744b1f8fc85430000000049483045022047957cdd957cfd0becd642f6b84d82f49b6cb4c51a91f49246908af7c3cfdf4a022100e96b46621f1bffcf5ea5982f88cef651e9354f5791602369bf5a82a6cd61a62501fffffffffe09f5fe3ffbf5ee97a54eb5e5069e9da6b4856ee86fc52938c2f979b0f38e82000000004847304402204165be9a4cbab8049e1af9723b96199bfd3e85f44c6b4c0177e3962686b26073022028f638da23fc003760861ad481ead4099312c60030d4cb57820ce4d33812a5ce01ffffffff01009d966b01000000434104ea1feff861b51fe3f5f8a3b12d0f4712db80e919548a80839fc47c6a21e66d957e9c5d8cd108c7a2d2324bad71f9904ac0ae7336507d785b17a2c115e427a32fac00000000".from_hex().unwrap()).unwrap(); let new_block: Block = deserialize("010000004ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914cd74d6e49ffff001d323b3a7b0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d026e04ffffffff0100f2052a0100000043410446ef0102d1ec5240f0d061a4246c1bdef63fc3dbab7733052fbbf0ecd8f41fc26bf049ebb4f9527f374280259e7cfa99c48b0e3f39c51347a19a5819651503a5ac00000000010000000321f75f3139a013f50f315b23b0c9a2b6eac31e2bec98e5891c924664889942260000000049483045022100cb2c6b346a978ab8c61b18b5e9397755cbd17d6eb2fe0083ef32e067fa6c785a02206ce44e613f31d9a6b0517e46f3db1576e9812cc98d159bfdaf759a5014081b5c01ffffffff79cda0945903627c3da1f85fc95d0b8ee3e76ae0cfdc9a65d09744b1f8fc85430000000049483045022047957cdd957cfd0becd642f6b84d82f49b6cb4c51a91f49246908af7c3cfdf4a022100e96b46621f1bffcf5ea5982f88cef651e9354f5791602369bf5a82a6cd61a62501fffffffffe09f5fe3ffbf5ee97a54eb5e5069e9da6b4856ee86fc52938c2f979b0f38e82000000004847304402204165be9a4cbab8049e1af9723b96199bfd3e85f44c6b4c0177e3962686b26073022028f638da23fc003760861ad481ead4099312c60030d4cb57820ce4d33812a5ce01ffffffff01009d966b01000000434104ea1feff861b51fe3f5f8a3b12d0f4712db80e919548a80839fc47c6a21e66d957e9c5d8cd108c7a2d2324bad71f9904ac0ae7336507d785b17a2c115e427a32fac00000000".from_hex().unwrap()).unwrap();
// Make sure we can't add the block directly, since we are missing the inputs // Make sure we can't add the block directly, since we are missing the inputs
assert!(empty_set.update(&new_block, TxoValidation).is_err()); assert!(empty_set.update(&new_block, 1, TxoValidation).is_err());
assert_eq!(empty_set.n_utxos(), 0); assert_eq!(empty_set.n_utxos(), 0);
// Add the block manually so that we'll have some UTXOs for the rest of the test // Add the block manually so that we'll have some UTXOs for the rest of the test
for tx in new_block.txdata.iter() { for tx in new_block.txdata.iter() {
empty_set.add_utxos(tx); empty_set.add_utxos(tx, 1);
} }
empty_set.last_hash = new_block.header.bitcoin_hash(); empty_set.last_hash = new_block.header.bitcoin_hash();
@ -455,19 +461,19 @@ mod tests {
let hash = tx.bitcoin_hash(); let hash = tx.bitcoin_hash();
for (n, out) in tx.output.iter().enumerate() { for (n, out) in tx.output.iter().enumerate() {
let n = n as u32; let n = n as u32;
assert_eq!(empty_set.get_utxo(hash, n), Some(&out.clone())); assert_eq!(empty_set.get_utxo(hash, n), Some((1, &out.clone())));
} }
} }
// Check again that we can't add the block, and that this doesn't mess up the // Check again that we can't add the block, and that this doesn't mess up the
// existing UTXOs // existing UTXOs
assert!(empty_set.update(&new_block, TxoValidation).is_err()); assert!(empty_set.update(&new_block, 2, TxoValidation).is_err());
assert_eq!(empty_set.n_utxos(), 2); assert_eq!(empty_set.n_utxos(), 2);
for tx in new_block.txdata.iter() { for tx in new_block.txdata.iter() {
let hash = tx.bitcoin_hash(); let hash = tx.bitcoin_hash();
for (n, out) in tx.output.iter().enumerate() { for (n, out) in tx.output.iter().enumerate() {
let n = n as u32; let n = n as u32;
assert_eq!(empty_set.get_utxo(hash, n), Some(&out.clone())); assert_eq!(empty_set.get_utxo(hash, n), Some((1, &out.clone())));
} }
} }
@ -488,7 +494,7 @@ mod tests {
assert_eq!(read_set.take_utxo(hash, 100 + n), None); assert_eq!(read_set.take_utxo(hash, 100 + n), None);
// Check take of real UTXO // Check take of real UTXO
let ret = read_set.take_utxo(hash, n); let ret = read_set.take_utxo(hash, n);
assert_eq!(ret, Some(out.clone())); assert_eq!(ret, Some((1, out.clone())));
// Try double-take // Try double-take
assert_eq!(read_set.take_utxo(hash, n), None); assert_eq!(read_set.take_utxo(hash, n), None);
} }

10
src/wallet/address_index.rs
View File

@ -32,7 +32,7 @@ use util::hash::Sha256dHash;
/// An address index /// An address index
#[deriving(Clone, PartialEq, Eq, Show)] #[deriving(Clone, PartialEq, Eq, Show)]
pub struct AddressIndex { pub struct AddressIndex {
index: HashMap<Script, Vec<(Sha256dHash, uint, TxOut)>>, index: HashMap<Script, Vec<(Sha256dHash, uint, TxOut, uint)>>,
network: Network, network: Network,
k1: u64, k1: u64,
k2: u64 k2: u64
@ -49,11 +49,11 @@ impl AddressIndex {
k1: k1, k1: k1,
k2: k2 k2: k2
}; };
for (key, idx, txo) in utxo_set.iter() { for (key, idx, txo, height) in utxo_set.iter() {
if ret.admissible_txo(txo) { if ret.admissible_txo(txo) {
ret.index.insert_or_update_with(txo.script_pubkey.clone(), ret.index.insert_or_update_with(txo.script_pubkey.clone(),
vec![(key, idx, txo.clone())], vec![(key, idx, txo.clone(), height)],
|_, v| v.push((key, idx, txo.clone()))); |_, v| v.push((key, idx, txo.clone(), height)));
} }
} }
ret ret
@ -77,7 +77,7 @@ impl AddressIndex {
/// Lookup a txout by its scriptpubkey. Returns a slice because there /// Lookup a txout by its scriptpubkey. Returns a slice because there
/// may be more than one for any given scriptpubkey. /// may be more than one for any given scriptpubkey.
#[inline] #[inline]
pub fn find_by_script<'a>(&'a self, pubkey: &Script) -> &'a [(Sha256dHash, uint, TxOut)] { pub fn find_by_script<'a>(&'a self, pubkey: &Script) -> &'a [(Sha256dHash, uint, TxOut, uint)] {
self.index.find(pubkey).map(|v| v.as_slice()).unwrap_or(&[]) self.index.find(pubkey).map(|v| v.as_slice()).unwrap_or(&[])
} }
} }

4
src/wallet/wallet.rs
View File

@ -273,7 +273,7 @@ impl Wallet {
let sk = try!(master.ckd_priv(cnum).map_err(Bip32Error)); let sk = try!(master.ckd_priv(cnum).map_err(Bip32Error));
let pk = ExtendedPubKey::from_private(&sk); let pk = ExtendedPubKey::from_private(&sk);
let addr = Address::from_key(pk.network, &pk.public_key); let addr = Address::from_key(pk.network, &pk.public_key);
for &(_, _, ref out) in index.find_by_script(&addr.script_pubkey()).iter() { for &(_, _, ref out, _) in index.find_by_script(&addr.script_pubkey()).iter() {
ret += out.value; ret += out.value;
} }
} }
@ -285,7 +285,7 @@ impl Wallet {
let sk = try!(master.ckd_priv(cnum).map_err(Bip32Error)); let sk = try!(master.ckd_priv(cnum).map_err(Bip32Error));
let pk = ExtendedPubKey::from_private(&sk); let pk = ExtendedPubKey::from_private(&sk);
let addr = Address::from_key(pk.network, &pk.public_key); let addr = Address::from_key(pk.network, &pk.public_key);
for &(_, _, ref out) in index.find_by_script(&addr.script_pubkey()).iter() { for &(_, _, ref out, _) in index.find_by_script(&addr.script_pubkey()).iter() {
ret += out.value; ret += out.value;
} }
} }