Reduce blockfilter memory (#302)

* use same Error type in all methods of BlockFilter

* reduce Blockfilter memory footprint

* amend the example use

* remove unused constant
This commit is contained in:
Tamás Blummer 2019-08-09 16:58:02 +02:00 committed by GitHub
parent 4a1830c423
commit 4ddf6f80b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 30 deletions

View File

@ -33,12 +33,12 @@
//! let filter = BlockFilter::new_script_filter (&block, get_script_for_coin)?; //! let filter = BlockFilter::new_script_filter (&block, get_script_for_coin)?;
//! //!
//! // or create a filter from known raw data //! // or create a filter from known raw data
//! let filter = BlockFilter::new(&block_hash, filter_type, content); //! let filter = BlockFilter::new(content);
//! //!
//! // read and evaluate a filter //! // read and evaluate a filter
//! //!
//! let query: Iterator<Item=Script> = // .. some scripts you care about //! let query: Iterator<Item=Script> = // .. some scripts you care about
//! if filter.match_any (&mut query.map(|s| s.as_bytes())) { //! if filter.match_any (&block_hash, &mut query.map(|s| s.as_bytes())) {
//! // get this block //! // get this block
//! } //! }
//! //!
@ -60,9 +60,6 @@ use consensus::{Decodable, Encodable};
use consensus::encode::VarInt; use consensus::encode::VarInt;
use util::hash::BitcoinHash; use util::hash::BitcoinHash;
/// BIP158 base filter type 0: input and output scripts
pub const SCRIPT_FILTER: u8 = 0;
/// Golomb encoding parameter as in BIP-158, see also https://gist.github.com/sipa/576d5f09c3b86c3b1b75598d799fc845 /// Golomb encoding parameter as in BIP-158, see also https://gist.github.com/sipa/576d5f09c3b86c3b1b75598d799fc845
const P: u8 = 19; const P: u8 = 19;
const M: u64 = 784931; const M: u64 = 784931;
@ -103,14 +100,8 @@ impl From<io::Error> for Error {
/// a computed or read block filter /// a computed or read block filter
pub struct BlockFilter { pub struct BlockFilter {
/// id of the block
pub block_hash: sha256d::Hash,
/// filte type (see SCRIPT_FILTER)
pub filter_type: u8,
/// Golomb encoded filter /// Golomb encoded filter
pub content: Vec<u8>, pub content: Vec<u8>
// a reader of the filter
filter_reader: BlockFilterReader
} }
impl BlockFilter { impl BlockFilter {
@ -124,9 +115,8 @@ impl BlockFilter {
} }
/// create a new filter from pre-computed data /// create a new filter from pre-computed data
pub fn new (block_hash: &sha256d::Hash, filter_type: u8, content: &[u8]) -> BlockFilter { pub fn new (content: &[u8]) -> BlockFilter {
let filter_reader = BlockFilterReader::new(block_hash); BlockFilter { content: content.to_vec() }
BlockFilter { block_hash: block_hash.clone(), filter_type, content: content.to_vec(), filter_reader }
} }
/// Compute a SCRIPT_FILTER that contains spent and output scripts /// Compute a SCRIPT_FILTER that contains spent and output scripts
@ -139,19 +129,19 @@ impl BlockFilter {
writer.add_input_scripts(script_for_coin)?; writer.add_input_scripts(script_for_coin)?;
writer.finish()?; writer.finish()?;
} }
let block_hash = block.bitcoin_hash(); Ok(BlockFilter { content: out.into_inner() })
let filter_reader = BlockFilterReader::new(&block_hash);
Ok(BlockFilter { block_hash, filter_type: SCRIPT_FILTER, content: out.into_inner(), filter_reader })
} }
/// match any query pattern /// match any query pattern
pub fn match_any(&self, query: &mut Iterator<Item=&[u8]>) -> Result<bool, io::Error> { pub fn match_any(&self, block_hash: &sha256d::Hash, query: &mut Iterator<Item=&[u8]>) -> Result<bool, Error> {
self.filter_reader.match_any(&mut Cursor::new(self.content.as_slice()), query) let filter_reader = BlockFilterReader::new(block_hash);
filter_reader.match_any(&mut Cursor::new(self.content.as_slice()), query)
} }
/// match all query pattern /// match all query pattern
pub fn match_all(&self, query: &mut Iterator<Item=&[u8]>) -> Result<bool, io::Error> { pub fn match_all(&self, block_hash: &sha256d::Hash, query: &mut Iterator<Item=&[u8]>) -> Result<bool, Error> {
self.filter_reader.match_all(&mut Cursor::new(self.content.as_slice()), query) let filter_reader = BlockFilterReader::new(block_hash);
filter_reader.match_all(&mut Cursor::new(self.content.as_slice()), query)
} }
} }
@ -224,12 +214,12 @@ impl BlockFilterReader {
} }
/// match any query pattern /// match any query pattern
pub fn match_any(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, io::Error> { pub fn match_any(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, Error> {
self.reader.match_any(reader, query) self.reader.match_any(reader, query)
} }
/// match all query pattern /// match all query pattern
pub fn match_all(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, io::Error> { pub fn match_all(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, Error> {
self.reader.match_all(reader, query) self.reader.match_all(reader, query)
} }
} }
@ -248,7 +238,7 @@ impl GCSFilterReader {
} }
/// match any query pattern /// match any query pattern
pub fn match_any(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, io::Error> { pub fn match_any(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, Error> {
let mut decoder = reader; let mut decoder = reader;
let n_elements: VarInt = Decodable::consensus_decode(&mut decoder).unwrap_or(VarInt(0)); let n_elements: VarInt = Decodable::consensus_decode(&mut decoder).unwrap_or(VarInt(0));
let ref mut reader = decoder; let ref mut reader = decoder;
@ -288,7 +278,7 @@ impl GCSFilterReader {
} }
/// match all query pattern /// match all query pattern
pub fn match_all(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, io::Error> { pub fn match_all(&self, reader: &mut io::Read, query: &mut Iterator<Item=&[u8]>) -> Result<bool, Error> {
let mut decoder = reader; let mut decoder = reader;
let n_elements: VarInt = Decodable::consensus_decode(&mut decoder).unwrap_or(VarInt(0)); let n_elements: VarInt = Decodable::consensus_decode(&mut decoder).unwrap_or(VarInt(0));
let ref mut reader = decoder; let ref mut reader = decoder;
@ -588,18 +578,18 @@ mod test {
Err(Error::UtxoMissing(o.clone())) Err(Error::UtxoMissing(o.clone()))
}).unwrap(); }).unwrap();
let test_filter = BlockFilter::new(&block.header.bitcoin_hash(), let test_filter = BlockFilter::new(filter_content.as_slice());
SCRIPT_FILTER, filter_content.as_slice());
assert_eq!(test_filter.content, filter.content); assert_eq!(test_filter.content, filter.content);
assert!(filter.match_all(&mut txmap.iter() let block_hash = &block.header.bitcoin_hash();
assert!(filter.match_all(&block_hash, &mut txmap.iter()
.filter_map(|(_, s)| if !s.is_empty() { Some(s.as_bytes()) } else { None })).unwrap()); .filter_map(|(_, s)| if !s.is_empty() { Some(s.as_bytes()) } else { None })).unwrap());
for (_, script) in &txmap { for (_, script) in &txmap {
let query = vec![script]; let query = vec![script];
if !script.is_empty () { if !script.is_empty () {
assert!(filter.match_any(&mut query.iter() assert!(filter.match_any(&block_hash, &mut query.iter()
.map(|s| s.as_bytes())).unwrap()); .map(|s| s.as_bytes())).unwrap());
} }
} }