milksad
/
rust-bitcoin-unsafe-fast
14
0
Fork 0
Code Issues Pull Requests Activity

Merge rust-bitcoin/rust-bitcoin#1211: Move bip158 module to crate root 

01161e66ee Run formmater on bip158 (Tobin C. Harding)
95393aadbc Move bip158 module to crate root (Tobin C. Harding)

Pull request description:

  We are attempting to flatten the `util` module.

  Move the `bip158` module to the crate root out of `util`.

  Currently `src/util/` is ignored by the formatter so this move requires `bip158` module to be formatted. Formatting is done as a separate patch so reviewers can run `cargo +nightly fmt` and compare the diffs if so desired.

ACKs for top commit:
  apoelstra:
    ACK 01161e66ee
  sanket1729:
    ACK 01161e66ee

Tree-SHA512: 192279d8d1466aa939ecad1f7beae12c4c7b5067871f6f1297fc80094a4ab736de4c1ed82a0b1d9d8e1cdba15c0342b90722002a9ba02ab1eff901edbd0fb356
This commit is contained in:
Andrew Poelstra 2022-09-06 13:18:33 +00:00
parent 1d8d04dd59 01161e66ee
commit 2bb3552e4f
No known key found for this signature in database
GPG Key ID: C588D63CE41B97C1
3 changed files with 109 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
src/util/bip158.rs → src/bip158.rs
View File

@ -39,22 +39,20 @@
//! ``` //! ```
//! //!
use crate::prelude::*;
use crate::io;
use core::fmt::{self, Display, Formatter};
use core::cmp::{self, Ordering}; use core::cmp::{self, Ordering};
use core::fmt::{self, Display, Formatter};
use crate::hashes::{Hash, siphash24};
use crate::hash_types::{BlockHash, FilterHash, FilterHeader};
use crate::blockdata::block::Block; use crate::blockdata::block::Block;
use crate::blockdata::script::Script; use crate::blockdata::script::Script;
use crate::blockdata::transaction::OutPoint; use crate::blockdata::transaction::OutPoint;
use crate::consensus::{Decodable, Encodable};
use crate::consensus::encode::VarInt; use crate::consensus::encode::VarInt;
use crate::util::endian; use crate::consensus::{Decodable, Encodable};
use crate::hash_types::{BlockHash, FilterHash, FilterHeader};
use crate::hashes::{siphash24, Hash};
use crate::internal_macros::write_err; use crate::internal_macros::write_err;
use crate::io;
use crate::prelude::*;
use crate::util::endian;
/// Golomb encoding parameter as in BIP-158, see also https://gist.github.com/sipa/576d5f09c3b86c3b1b75598d799fc845 /// Golomb encoding parameter as in BIP-158, see also https://gist.github.com/sipa/576d5f09c3b86c3b1b75598d799fc845
const P: u8 = 19; const P: u8 = 19;
@ -93,16 +91,14 @@ impl std::error::Error for Error {
} }
impl From<io::Error> for Error { impl From<io::Error> for Error {
fn from(io: io::Error) -> Self { fn from(io: io::Error) -> Self { Error::Io(io) }
Error::Io(io)
}
} }
/// A block filter, as described by BIP 158. /// A block filter, as described by BIP 158.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub struct BlockFilter { pub struct BlockFilter {
/// Golomb encoded filter /// Golomb encoded filter
pub content: Vec<u8> pub content: Vec<u8>,
} }
impl FilterHash { impl FilterHash {
@ -117,14 +113,12 @@ impl FilterHash {
impl BlockFilter { impl BlockFilter {
/// Creates a new filter from pre-computed data. /// Creates a new filter from pre-computed data.
pub fn new (content: &[u8]) -> BlockFilter { pub fn new(content: &[u8]) -> BlockFilter { BlockFilter { content: content.to_vec() } }
BlockFilter { content: content.to_vec() }
}
/// Computes a SCRIPT_FILTER that contains spent and output scripts. /// Computes a SCRIPT_FILTER that contains spent and output scripts.
pub fn new_script_filter<M>(block: &Block, script_for_coin: M) -> Result<BlockFilter, Error> pub fn new_script_filter<M>(block: &Block, script_for_coin: M) -> Result<BlockFilter, Error>
where where
M: Fn(&OutPoint) -> Result<Script, Error> M: Fn(&OutPoint) -> Result<Script, Error>,
{ {
let mut out = Vec::new(); let mut out = Vec::new();
let mut writer = BlockFilterWriter::new(&mut out, block); let mut writer = BlockFilterWriter::new(&mut out, block);
@ -192,33 +186,35 @@ impl<'a, W: io::Write> BlockFilterWriter<'a, W> {
/// Adds consumed output scripts of a block to filter. /// Adds consumed output scripts of a block to filter.
pub fn add_input_scripts<M>(&mut self, script_for_coin: M) -> Result<(), Error> pub fn add_input_scripts<M>(&mut self, script_for_coin: M) -> Result<(), Error>
where M: Fn(&OutPoint) -> Result<Script, Error> { where
for script in self.block.txdata.iter() M: Fn(&OutPoint) -> Result<Script, Error>,
{
for script in self
.block
.txdata
.iter()
.skip(1) // skip coinbase .skip(1) // skip coinbase
.flat_map(|t| t.input.iter().map(|i| &i.previous_output)) .flat_map(|t| t.input.iter().map(|i| &i.previous_output))
.map(script_for_coin) { .map(script_for_coin)
{
match script { match script {
Ok(script) => self.add_element(script.as_bytes()), Ok(script) => self.add_element(script.as_bytes()),
Err(e) => return Err(e) Err(e) => return Err(e),
} }
} }
Ok(()) Ok(())
} }
/// Adds an arbitrary element to filter. /// Adds an arbitrary element to filter.
pub fn add_element(&mut self, data: &[u8]) { pub fn add_element(&mut self, data: &[u8]) { self.writer.add_element(data); }
self.writer.add_element(data);
}
/// Writes the block filter. /// Writes the block filter.
pub fn finish(&mut self) -> Result<usize, io::Error> { pub fn finish(&mut self) -> Result<usize, io::Error> { self.writer.finish() }
self.writer.finish()
}
} }
/// Reads and interprets a block filter. /// Reads and interprets a block filter.
pub struct BlockFilterReader { pub struct BlockFilterReader {
reader: GcsFilterReader reader: GcsFilterReader,
} }
impl BlockFilterReader { impl BlockFilterReader {
@ -252,7 +248,7 @@ impl BlockFilterReader {
/// Golomb-Rice encoded filter reader. /// Golomb-Rice encoded filter reader.
pub struct GcsFilterReader { pub struct GcsFilterReader {
filter: GcsFilter, filter: GcsFilter,
m: u64 m: u64,
} }
impl GcsFilterReader { impl GcsFilterReader {
@ -290,14 +286,13 @@ impl GcsFilterReader {
loop { loop {
match data.cmp(&p) { match data.cmp(&p) {
Ordering::Equal => return Ok(true), Ordering::Equal => return Ok(true),
Ordering::Less => { Ordering::Less =>
if remaining > 0 { if remaining > 0 {
data += self.filter.golomb_rice_decode(&mut reader)?; data += self.filter.golomb_rice_decode(&mut reader)?;
remaining -= 1; remaining -= 1;
} else { } else {
return Ok(false); return Ok(false);
} },
}
Ordering::Greater => break, Ordering::Greater => break,
} }
} }
@ -335,14 +330,13 @@ impl GcsFilterReader {
loop { loop {
match data.cmp(&p) { match data.cmp(&p) {
Ordering::Equal => break, Ordering::Equal => break,
Ordering::Less => { Ordering::Less =>
if remaining > 0 { if remaining > 0 {
data += self.filter.golomb_rice_decode(&mut reader)?; data += self.filter.golomb_rice_decode(&mut reader)?;
remaining -= 1; remaining -= 1;
} else { } else {
return Ok(false); return Ok(false);
} },
},
Ordering::Greater => return Ok(false), Ordering::Greater => return Ok(false),
} }
} }
@ -352,27 +346,20 @@ impl GcsFilterReader {
} }
/// Fast reduction of hash to [0, nm) range. /// Fast reduction of hash to [0, nm) range.
fn map_to_range(hash: u64, nm: u64) -> u64 { fn map_to_range(hash: u64, nm: u64) -> u64 { ((hash as u128 * nm as u128) >> 64) as u64 }
((hash as u128 * nm as u128) >> 64) as u64
}
/// Golomb-Rice encoded filter writer. /// Golomb-Rice encoded filter writer.
pub struct GcsFilterWriter<'a, W> { pub struct GcsFilterWriter<'a, W> {
filter: GcsFilter, filter: GcsFilter,
writer: &'a mut W, writer: &'a mut W,
elements: HashSet<Vec<u8>>, elements: HashSet<Vec<u8>>,
m: u64 m: u64,
} }
impl<'a, W: io::Write> GcsFilterWriter<'a, W> { impl<'a, W: io::Write> GcsFilterWriter<'a, W> {
/// Creates a new [`GcsFilterWriter`] wrapping a generic writer, with specific seed to siphash. /// Creates a new [`GcsFilterWriter`] wrapping a generic writer, with specific seed to siphash.
pub fn new(writer: &'a mut W, k0: u64, k1: u64, m: u64, p: u8) -> GcsFilterWriter<'a, W> { pub fn new(writer: &'a mut W, k0: u64, k1: u64, m: u64, p: u8) -> GcsFilterWriter<'a, W> {
GcsFilterWriter { GcsFilterWriter { filter: GcsFilter::new(k0, k1, p), writer, elements: HashSet::new(), m }
filter: GcsFilter::new(k0, k1, p),
writer,
elements: HashSet::new(),
m
}
} }
/// Adds data to the filter. /// Adds data to the filter.
@ -387,8 +374,11 @@ impl<'a, W: io::Write> GcsFilterWriter<'a, W> {
let nm = self.elements.len() as u64 * self.m; let nm = self.elements.len() as u64 * self.m;
// map hashes to [0, n_elements * M) // map hashes to [0, n_elements * M)
let mut mapped: Vec<_> = self.elements.iter() let mut mapped: Vec<_> = self
.map(|e| map_to_range(self.filter.hash(e.as_slice()), nm)).collect(); .elements
.iter()
.map(|e| map_to_range(self.filter.hash(e.as_slice()), nm))
.collect();
mapped.sort_unstable(); mapped.sort_unstable();
// write number of elements as varint // write number of elements as varint
@ -410,17 +400,19 @@ impl<'a, W: io::Write> GcsFilterWriter<'a, W> {
struct GcsFilter { struct GcsFilter {
k0: u64, // sip hash key k0: u64, // sip hash key
k1: u64, // sip hash key k1: u64, // sip hash key
p: u8 p: u8,
} }
impl GcsFilter { impl GcsFilter {
/// Creates a new [`GcsFilter`]. /// Creates a new [`GcsFilter`].
fn new(k0: u64, k1: u64, p: u8) -> GcsFilter { fn new(k0: u64, k1: u64, p: u8) -> GcsFilter { GcsFilter { k0, k1, p } }
GcsFilter { k0, k1, p }
}
/// Golomb-Rice encodes a number `n` to a bit stream (parameter 2^k). /// Golomb-Rice encodes a number `n` to a bit stream (parameter 2^k).
fn golomb_rice_encode<'a, W>(&self, writer: &mut BitStreamWriter<'a, W>, n: u64) -> Result<usize, io::Error> fn golomb_rice_encode<'a, W>(
&self,
writer: &mut BitStreamWriter<'a, W>,
n: u64,
) -> Result<usize, io::Error>
where where
W: io::Write, W: io::Write,
{ {
@ -439,7 +431,7 @@ impl GcsFilter {
/// Golomb-Rice decodes a number from a bit stream (parameter 2^k). /// Golomb-Rice decodes a number from a bit stream (parameter 2^k).
fn golomb_rice_decode<R>(&self, reader: &mut BitStreamReader<R>) -> Result<u64, io::Error> fn golomb_rice_decode<R>(&self, reader: &mut BitStreamReader<R>) -> Result<u64, io::Error>
where where
R: io::Read R: io::Read,
{ {
let mut q = 0u64; let mut q = 0u64;
while reader.read(1)? == 1 { while reader.read(1)? == 1 {
@ -465,11 +457,7 @@ pub struct BitStreamReader<'a, R> {
impl<'a, R: io::Read> BitStreamReader<'a, R> { impl<'a, R: io::Read> BitStreamReader<'a, R> {
/// Creates a new [`BitStreamReader`] that reads bitwise from a given `reader`. /// Creates a new [`BitStreamReader`] that reads bitwise from a given `reader`.
pub fn new(reader: &'a mut R) -> BitStreamReader<'a, R> { pub fn new(reader: &'a mut R) -> BitStreamReader<'a, R> {
BitStreamReader { BitStreamReader { buffer: [0u8], reader, offset: 8 }
buffer: [0u8],
reader,
offset: 8,
}
} }
/// Reads nbit bits, returning the bits in a `u64` starting with the rightmost bit. /// Reads nbit bits, returning the bits in a `u64` starting with the rightmost bit.
@ -485,7 +473,10 @@ impl<'a, R: io::Read> BitStreamReader<'a, R> {
/// ``` /// ```
pub fn read(&mut self, mut nbits: u8) -> Result<u64, io::Error> { pub fn read(&mut self, mut nbits: u8) -> Result<u64, io::Error> {
if nbits > 64 { if nbits > 64 {
return Err(io::Error::new(io::ErrorKind::Other, "can not read more than 64 bits at once")); return Err(io::Error::new(
io::ErrorKind::Other,
"can not read more than 64 bits at once",
));
} }
let mut data = 0u64; let mut data = 0u64;
while nbits > 0 { while nbits > 0 {
@ -513,17 +504,16 @@ pub struct BitStreamWriter<'a, W> {
impl<'a, W: io::Write> BitStreamWriter<'a, W> { impl<'a, W: io::Write> BitStreamWriter<'a, W> {
/// Creates a new [`BitStreamWriter`] that writes bitwise to a given `writer`. /// Creates a new [`BitStreamWriter`] that writes bitwise to a given `writer`.
pub fn new(writer: &'a mut W) -> BitStreamWriter<'a, W> { pub fn new(writer: &'a mut W) -> BitStreamWriter<'a, W> {
BitStreamWriter { BitStreamWriter { buffer: [0u8], writer, offset: 0 }
buffer: [0u8],
writer,
offset: 0,
}
} }
/// Writes nbits bits from data. /// Writes nbits bits from data.
pub fn write(&mut self, data: u64, mut nbits: u8) -> Result<usize, io::Error> { pub fn write(&mut self, data: u64, mut nbits: u8) -> Result<usize, io::Error> {
if nbits > 64 { if nbits > 64 {
return Err(io::Error::new(io::ErrorKind::Other, "can not write more than 64 bits at once")); return Err(io::Error::new(
io::ErrorKind::Other,
"can not write more than 64 bits at once",
));
} }
let mut wrote = 0; let mut wrote = 0;
while nbits > 0 { while nbits > 0 {
@ -553,60 +543,75 @@ impl<'a, W: io::Write> BitStreamWriter<'a, W> {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*;
use std::collections::HashMap; use std::collections::HashMap;
use serde_json::Value; use serde_json::Value;
use super::*;
use crate::consensus::encode::deserialize; use crate::consensus::encode::deserialize;
use crate::hash_types::BlockHash; use crate::hash_types::BlockHash;
use crate::hashes::hex::FromHex; use crate::hashes::hex::FromHex;
#[test] #[test]
fn test_blockfilters() { fn test_blockfilters() {
// test vectors from: https://github.com/jimpo/bitcoin/blob/c7efb652f3543b001b4dd22186a354605b14f47e/src/test/data/blockfilters.json // test vectors from: https://github.com/jimpo/bitcoin/blob/c7efb652f3543b001b4dd22186a354605b14f47e/src/test/data/blockfilters.json
let data = include_str!("../../test_data/blockfilters.json"); let data = include_str!("../test_data/blockfilters.json");
let testdata = serde_json::from_str::<Value>(data).unwrap().as_array().unwrap().clone(); let testdata = serde_json::from_str::<Value>(data).unwrap().as_array().unwrap().clone();
for t in testdata.iter().skip(1) { for t in testdata.iter().skip(1) {
let block_hash = BlockHash::from_hex(t.get(1).unwrap().as_str().unwrap()).unwrap(); let block_hash = BlockHash::from_hex(t.get(1).unwrap().as_str().unwrap()).unwrap();
let block: Block = deserialize(&Vec::from_hex(t.get(2).unwrap().as_str().unwrap()).unwrap()).unwrap(); let block: Block =
deserialize(&Vec::from_hex(t.get(2).unwrap().as_str().unwrap()).unwrap()).unwrap();
assert_eq!(block.block_hash(), block_hash); assert_eq!(block.block_hash(), block_hash);
let scripts = t.get(3).unwrap().as_array().unwrap(); let scripts = t.get(3).unwrap().as_array().unwrap();
let previous_filter_header = FilterHeader::from_hex(t.get(4).unwrap().as_str().unwrap()).unwrap(); let previous_filter_header =
FilterHeader::from_hex(t.get(4).unwrap().as_str().unwrap()).unwrap();
let filter_content = Vec::from_hex(t.get(5).unwrap().as_str().unwrap()).unwrap(); let filter_content = Vec::from_hex(t.get(5).unwrap().as_str().unwrap()).unwrap();
let filter_header = FilterHeader::from_hex(t.get(6).unwrap().as_str().unwrap()).unwrap(); let filter_header =
FilterHeader::from_hex(t.get(6).unwrap().as_str().unwrap()).unwrap();
let mut txmap = HashMap::new(); let mut txmap = HashMap::new();
let mut si = scripts.iter(); let mut si = scripts.iter();
for tx in block.txdata.iter().skip(1) { for tx in block.txdata.iter().skip(1) {
for input in tx.input.iter() { for input in tx.input.iter() {
txmap.insert(input.previous_output, Script::from(Vec::from_hex(si.next().unwrap().as_str().unwrap()).unwrap())); txmap.insert(
input.previous_output,
Script::from(Vec::from_hex(si.next().unwrap().as_str().unwrap()).unwrap()),
);
} }
} }
let filter = BlockFilter::new_script_filter(&block, let filter = BlockFilter::new_script_filter(&block, |o| {
|o| if let Some(s) = txmap.get(o) { if let Some(s) = txmap.get(o) {
Ok(s.clone()) Ok(s.clone())
} else { } else {
Err(Error::UtxoMissing(*o)) Err(Error::UtxoMissing(*o))
}).unwrap(); }
})
.unwrap();
let test_filter = BlockFilter::new(filter_content.as_slice()); let test_filter = BlockFilter::new(filter_content.as_slice());
assert_eq!(test_filter.content, filter.content); assert_eq!(test_filter.content, filter.content);
let block_hash = &block.block_hash(); let block_hash = &block.block_hash();
assert!(filter.match_all(block_hash, &mut txmap.iter() assert!(filter
.filter_map(|(_, s)| if !s.is_empty() { Some(s.as_bytes()) } else { None })).unwrap()); .match_all(
block_hash,
&mut txmap.iter().filter_map(|(_, s)| if !s.is_empty() {
Some(s.as_bytes())
} else {
None
})
)
.unwrap());
for script in txmap.values() { for script in txmap.values() {
let query = vec![script]; let query = vec![script];
if !script.is_empty () { if !script.is_empty() {
assert!(filter.match_any(block_hash, &mut query.iter() assert!(filter
.map(|s| s.as_bytes())).unwrap()); .match_any(block_hash, &mut query.iter().map(|s| s.as_bytes()))
.unwrap());
} }
} }
@ -649,12 +654,16 @@ mod test {
{ {
let query = vec![Vec::from_hex("abcdef").unwrap(), Vec::from_hex("eeeeee").unwrap()]; let query = vec![Vec::from_hex("abcdef").unwrap(), Vec::from_hex("eeeeee").unwrap()];
let reader = GcsFilterReader::new(0, 0, M, P); let reader = GcsFilterReader::new(0, 0, M, P);
assert!(reader.match_any(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice())).unwrap()); assert!(reader
.match_any(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
} }
{ {
let query = vec![Vec::from_hex("abcdef").unwrap(), Vec::from_hex("123456").unwrap()]; let query = vec![Vec::from_hex("abcdef").unwrap(), Vec::from_hex("123456").unwrap()];
let reader = GcsFilterReader::new(0, 0, M, P); let reader = GcsFilterReader::new(0, 0, M, P);
assert!(!reader.match_any(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice())).unwrap()); assert!(!reader
.match_any(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
} }
{ {
let reader = GcsFilterReader::new(0, 0, M, P); let reader = GcsFilterReader::new(0, 0, M, P);
@ -662,7 +671,9 @@ mod test {
for p in &patterns { for p in &patterns {
query.push(p.clone()); query.push(p.clone());
} }
assert!(reader.match_all(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice())).unwrap()); assert!(reader
.match_all(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
} }
{ {
let reader = GcsFilterReader::new(0, 0, M, P); let reader = GcsFilterReader::new(0, 0, M, P);
@ -671,7 +682,9 @@ mod test {
query.push(p.clone()); query.push(p.clone());
} }
query.push(Vec::from_hex("abcdef").unwrap()); query.push(Vec::from_hex("abcdef").unwrap());
assert!(!reader.match_all(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice())).unwrap()); assert!(!reader
.match_all(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
} }
} }
@ -690,7 +703,10 @@ mod test {
writer.flush().unwrap(); writer.flush().unwrap();
} }
let bytes = out; let bytes = out;
assert_eq!("01011010110000110000000001110000", format!("{:08b}{:08b}{:08b}{:08b}", bytes[0], bytes[1], bytes[2], bytes[3])); assert_eq!(
"01011010110000110000000001110000",
format!("{:08b}{:08b}{:08b}{:08b}", bytes[0], bytes[1], bytes[2], bytes[3])
);
{ {
let mut input = bytes.as_slice(); let mut input = bytes.as_slice();
let mut reader = BitStreamReader::new(&mut input); let mut reader = BitStreamReader::new(&mut input);

1
src/lib.rs
View File

@ -80,6 +80,7 @@ mod serde_utils;
#[macro_use] #[macro_use]
pub mod network; pub mod network;
pub mod address; pub mod address;
pub mod bip158;
pub mod blockdata; pub mod blockdata;
pub mod consensus; pub mod consensus;
pub mod error; pub mod error;

4
src/util/mod.rs
View File

@ -20,7 +20,6 @@ pub mod misc;
pub mod psbt; pub mod psbt;
pub mod taproot; pub mod taproot;
pub mod uint; pub mod uint;
pub mod bip158;
pub mod sighash; pub mod sighash;
pub(crate) mod endian; pub(crate) mod endian;
@ -117,3 +116,6 @@ pub(crate) fn read_to_end<D: io::Read>(mut d: D) -> Result<Vec<u8>, io::Error> {
pub mod address { pub mod address {
pub use crate::address::*; pub use crate::address::*;
} }
#[deprecated(since = "0.30.0", note = "Please use crate::bip158")]
pub use crate::bip158;