From 9ca6c75b184898e5118f519cb38a2d80e66e2e80 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Fri, 22 Oct 2021 16:05:20 +0200 Subject: [PATCH 1/4] Bench StreamReader --- src/blockdata/block.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/blockdata/block.rs b/src/blockdata/block.rs index 31d7fe13..a0f69a89 100644 --- a/src/blockdata/block.rs +++ b/src/blockdata/block.rs @@ -475,12 +475,33 @@ mod tests { mod benches { use super::Block; use EmptyWrite; - use consensus::{deserialize, Encodable}; + use consensus::{deserialize, Encodable, serialize}; use hashes::hex::FromHex; use test::{black_box, Bencher}; + use network::stream_reader::StreamReader; const SOME_BLOCK: &'static str = "000000202aa2f2ca794ccbd40c16e2f3333f6b8b683f9e7179b2c4d7490600000000000010bc26e70a2f672ad420a6153dd0c28b40a6002c55531bfc99bf8994a8e8f67e5503bd5750d4061a4ed90a700f010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff3603da1b0e00045503bd5704c7dd8a0d0ced13bb5785010800000000000a636b706f6f6c122f4e696e6a61506f6f6c2f5345475749542fffffffff02b4e5a212000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9edf91c46b49eb8a29089980f02ee6b57e7d63d33b18b4fddac2bcd7db2a3983704012000000000000000000000000000000000000000000000000000000000000000000000000001000000017e4f81175332a733e26d4ba4e29f53f67b7a5d7c2adebb276e447ca71d130b55000000006b483045022100cac809cd1a3d9ad5d5e31a84e2e1d8ec5542841e4d14c6b52e8b38cbe1ff1728022064470b7fb0c2efeccb2e84bfa36ec5f9e434c84b1101c00f7ee32f726371b7410121020e62280798b6b8c37f068df0915b0865b63fabc401c2457cbc3ef96887dd3647ffffffff02ca2f780c000000001976a914c6b5545b3592cb477d709896fa705592c9b6113a88ac663b2a06000000001976a914e7c1345fc8f87c68170b3aa798a956c2fe6a9eff88ac0000000001000000011e99f5a785e677e017d36b50aa4fd10010ffd039f38f42f447ca8895250e121f01000000d90047304402200d3d296ad641a281dd5c0d68b9ab0d1ad5f7052bec148c1fb81fb1ba69181ec502201a372bb16fb8e054ee9bef41e300d292153830f841a4db0ab7f7407f6581b9bc01473044022002584f313ae990236b6bebb82fbbb006a2b02a448dd5c93434428991eae960d60220491d67d2660c4dde19025cf86e5164a559e2c79c3b98b40e146fab974acd24690147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a9140ffdcf96700455074292a821c74922e8652993998788997bc60000000017a9148ce5408cfeaddb7ccb2545ded41ef478109454848700000000010000000113100b09e6a78d63ec4850654ab0f68806de29710b09172eddfef730652b155501000000da00473044022015389408e3446a3f36a05060e0e4a3c8b92ff3901ba2511aa944ec91a537a1cb022045a33b6ec47605b1718ed2e753263e54918edbf6126508ff039621fb928d28a001483045022100bb952fde81f216f7063575c0bb2bedc050ce08c96d9b437ea922f5eb98c882da02201b7cbf3a2f94ea4c5eb7f0df3af2ebcafa8705af7f410ab5d3d4bac13d6bc6120147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a914d3db9a20312c3ab896a316eb108dbd01e47e17d687e0ba7ac60000000017a9148ce5408cfeaddb7ccb2545ded41ef47810945484870000000001000000016e3cca1599cde54878e2f27f434df69df0afd1f313cb6e38c08d3ffb57f97a6c01000000da0048304502210095623b70ec3194fa4037a1c1106c2580caedc390e25e5b330bbeb3111e8184bc02205ae973c4a4454be2a3a03beb66297143c1044a3c4743742c5cdd1d516a1ad3040147304402202f3d6d89996f5b42773dd6ebaf367f1af1f3a95c7c7b487ec040131c40f4a4a30220524ffbb0b563f37b3eb1341228f792e8f84111b7c4a9f49cdd998e052ee42efa0147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a9141ade6b95896dde8ec4dee9e59af8849d3797348e8728af7ac60000000017a9148ce5408cfeaddb7ccb2545ded41ef47810945484870000000001000000011d9dc3a5df9b5b2eeb2bd11a2db243be9e8cc23e2f180bf317d32a499904c15501000000db00483045022100ebbd1c9a8ce626edbb1a7881df81e872ef8c6424feda36faa8a5745157400c6a02206eb463bc8acd5ea06a289e86115e1daae0c2cf10d9cbbd199e1311170d5543ef01483045022100809411a917dc8cf4f3a777f0388fdea6de06243ef7691e500c60abd1c7f19ae602205255d2b1191d8adedb77b814ccb66471eb8486cb4ff8727824254ee5589f176b0147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a914759a49c772347be81c49517f9e1e6def6a88d4dd87800b85c60000000017a9148ce5408cfeaddb7ccb2545ded41ef47810945484870000000001000000018c51902affd8e5247dfcc2e5d0528a3815f53c8b6d2c200ff290b2b2b486d7704f0000006a47304402201be0d485f6a3ce871be80064c593c5327b3fd7e450f05ab7fae38385bc40cfbe02206e2a6c9970b5d1d10207892376733757486634fce4f352e772149c486857612101210350c33bc9a790c9495195761577b34912a949b73d5bc5ae5343f5ba08b33220ccffffffff0110270000000000001976a9142ab1c62710a7bdfdb4bb6394bbedc58b32b4d5a388ac0000000001000000018c51902affd8e5247dfcc2e5d0528a3815f53c8b6d2c200ff290b2b2b486d7704e0000006b483045022100ccc8c0ac90bdb0402842aec91830c765cdead7a728552a6a34de7d13a6dab28e02206c96f8640cf3444054e9632b197be30598a09c3d5defcd95750bdb922a60d64801210350c33bc9a790c9495195761577b34912a949b73d5bc5ae5343f5ba08b33220ccffffffff0110270000000000001976a9142ab1c62710a7bdfdb4bb6394bbedc58b32b4d5a388ac0000000001000000011b436669c06cbf3442e21a2fe3edc20cd3cf13c358c53234bc4d88bfd8c4bd2a000000006a47304402204a63410ee13db52c7609ab08e25b7fe3c608cc21cc1755ad13460685eb55193202204cd1ea80c06a81571119be0b8cccd96ef7cdd90f62c1fe2d538622feb08e22ba0121024baa8b67cc9ed8a97d90895e3716b25469b67cb26d3324d7aff213f507764765ffffffff010000000000000000306a2e516d64523365345261445653324d436a736e536171734a5753324465655446624238354541794a4d5843784c7934000000000100000001be4a95ed36316cada5118b1982e4cb4a07f93e7a4153e227466f1cb0776de995000000006b483045022100a22d5251deea0470806bab817013d675a63cd52218d6e477ab0c9d601d018b7f022042121b46afcdcd0c66f189398212b66085e88c6973ae560f1810c13e55e2bee40121024baa8b67cc9ed8a97d90895e3716b25469b67cb26d3324d7aff213f507764765ffffffff010000000000000000306a2e516d57484d57504e5248515872504c7338554c586b4d483746745356413675366b5a6b4a4e3851796e4e583751340000000001000000016c061a65b49edec21acdbc22f97dc853aa872302aeef13fabf0bf6807de1b8bd010000006b483045022100dd80381f2d158b4dad7f98d2d97317c533fb36e737542473feb05fa74d0b73bb02207097d4331196069167e525b61d132532292fd75cc039a5839c04c2545d427e2b0121035e9a597df8b417bef66811882a2844604fc591c427f642628f0fef46be19a4c9feffffff0280a4bf07000000001976a914573b9106e16ee0b5c143dc40f0724f77dd0e282088ac9533b22c000000001976a9149c4da607efb1d759d33da71778bc6cafa56acb5988acd31b0e0001000000017dae20994b69b28534e5b22f3d7c50f9d7541348cbf6f43fcc654263ebaf8f68000000006b483045022100a85300eb94b24b044877d0b0d61e08e16dbc82ec7d69c723a8a45519f95c35b002203d78376e6bee31b455c097557af7fe4d6b620bc74269e9a75e2aad2b545abddb012103b0d08aba2a5ac6cf2788fda941c386040e35e49d3a57d2aefb16c0438fb98acbfeffffff022222305f000000001976a914cfda30dd836b596db6a9c230c45ae2179107f04888ac80a4bf07000000001976a91442dfcf5823aacb185844e663873c35fb98bfd21b88acd31b0e000100000002ad3e85e4af30678a330f8941ed7a9ca17cd0236368d238cac4e9ff09c466fed1020000006b483045022100d1196c48a0392e09592f1b96b4aec32ab0cecb6fd17b1d0c85ab3250a2fe45d9022059217c82f684fcdecdbe660a2077ea956dfbbb964d2648bc1e8ae0f0fe565449012103b64e32e5f62e03701428fb1e3151e9a57f149c67708f6164a235c8199fe17cc2ffffffff34f0a71c1c2cd610522e9c18c67931cded5e9647d4419c49b99715e2a0795f3d020000006a4730440220316e81d8242abf3c5f885d200feca12c3adb63cf2cd4dc74602f7b8b0cba50340220210d525758df77ccdca6908311c1895275e07bbb29b45963a19252acde55873f012103b64e32e5f62e03701428fb1e3151e9a57f149c67708f6164a235c8199fe17cc2ffffffff0510270000000000001976a914449d2394dde057bc199f23fb8aa2e400f344611788ac10270000000000001976a914449d2394dde057bc199f23fb8aa2e400f344611788aca0860100000000001976a91413d35ad337dd80a055757e5ea0a45b59fee3060c88ac70110100000000001976a91413d35ad337dd80a055757e5ea0a45b59fee3060c88ac0000000000000000026a000000000001000000018e33fecc2ddbd86c5ea919f7bd5a5acf8a09f3e0cdaaaf4f08c5ef095161ef1100000000fdfe0000483045022100d2489b225d39b7d8b6767a6928c8029a2a1297c08fdf00d683ba0c1987e7d7000220176cb66c8a243806bb7421f658325a69a51c82c0c3314e37f2400f33626390210148304502210096cfa57662a545830d0e29610becd41ea031e256339913718ce18dbb1a27bdb00220482911c851d15adcd37097dff99a9ff1f97d953bcebc528835118f447412553e014c695221028d9889862b29430278c084b5c4090b7b807b31e047bcd212ebc2c4e43fc0e3c52103160949a7c8c81f2c25d7763f57eb1cb407d867c5b7c290331bd2dc4b1182c6d32103fbef3b60914bda9173765902013a251ec89450c75d0b5a96a143db1dabf98d9553aeffffffff0220e8891c0100000017a914d996715e081c50f8f6b1b4e7fb6ca214f9924fdf87809698000000000017a9145611d812263f32960228cb5f85329bce4770a218870000000001000000017720507dcbe6c69f652b0c0ce19406f482372d1a8abc05d45fb7acf97fb80eec00000000fdfe00004830450221009821d8e117de44b1202c829c0f5063997acf007cf9b561c6fb8d1212cddb6c40022010ff5067b0d9d4eca2da0ceb876e9a16f1a2142da866d3042a7bae8968813e8001483045022100dea759d14a8a1c5da5f3dcc5509871aaa2c1e3be03752c1b858d80fa4227163702205183d70cc28dcb6df9b037714c8b6442ef84e0ddce07711a30c731e9f0925090014c695221028d70ea66fe7a7def282df7b2b498007e5072933e42c18f63ce85975dcbcf1a8821037e8f842b1e47e21d88002c5aab2559212a4c2c9dbe5ef5347f2a29afd0510ec1210251259cb9fd4f6206488408286e4475c9c9fe887e57a3e32ae4da222778a2aedf53aeffffffff023380cb020000000017a9143b5a7e85b22656a34d43187ac8dd09acd7109d2487809698000000000017a914b9b4b555f594a34deec3ad61d5c5f3738b17ee158700000000"; + #[bench] + #[allow(deprecated)] + pub fn bench_stream_reader(bh: &mut Bencher) { + let raw_block = Vec::from_hex(SOME_BLOCK).unwrap(); + let mut block: Block = deserialize(&raw_block).unwrap(); + for _ in 0..8 { + // make a big block + block.txdata.extend(block.txdata.clone().into_iter()); + } + let big_block = serialize(&block); + assert_eq!(big_block.len(), 1_085_011); + let big_block = black_box(big_block); + + bh.iter(|| { + let mut reader = StreamReader::new(&big_block[..], None); + let block: Block = reader.read_next().unwrap(); + black_box(&block); + }); + } + #[bench] pub fn bench_block_serialize(bh: &mut Bencher) { let raw_block = Vec::from_hex(SOME_BLOCK).unwrap(); From 5dfb93df7108d109154727a0ba3dc5b0d7680784 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Thu, 21 Oct 2021 11:03:47 +0200 Subject: [PATCH 2/4] Deprecate StreamReader StreamReader before this commit is trying to repeatedly parse big object like blocks at every read, causing useless overhead. consensus_encode deal with partial data by simply blocking. After this changes it doesn't look what remain of the StreamReader is really giving value, so it's deprecated --- examples/handshake.rs | 9 ++-- src/network/stream_reader.rs | 86 ++++++++++++------------------------ 2 files changed, 33 insertions(+), 62 deletions(-) diff --git a/examples/handshake.rs b/examples/handshake.rs index 036d4223..9c4cf046 100644 --- a/examples/handshake.rs +++ b/examples/handshake.rs @@ -3,11 +3,10 @@ extern crate bitcoin; use std::net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{env, process}; -use std::io::Write; +use std::io::{Write, BufReader}; -use bitcoin::consensus::encode; +use bitcoin::consensus::{encode, Decodable}; use bitcoin::network::{address, constants, message, message_network}; -use bitcoin::network::stream_reader::StreamReader; use bitcoin::secp256k1; use bitcoin::secp256k1::rand::Rng; @@ -41,10 +40,10 @@ fn main() { // Setup StreamReader let read_stream = stream.try_clone().unwrap(); - let mut stream_reader = StreamReader::new(read_stream, None); + let mut stream_reader = BufReader::new(read_stream); loop { // Loop an retrieve new messages - let reply: message::RawNetworkMessage = stream_reader.read_next().unwrap(); + let reply = message::RawNetworkMessage::consensus_decode(&mut stream_reader).unwrap(); match reply.payload { message::NetworkMessage::Version(_) => { println!("Received version message: {:?}", reply.payload); diff --git a/src/network/stream_reader.rs b/src/network/stream_reader.rs index 4b5d61d3..3becfabc 100644 --- a/src/network/stream_reader.rs +++ b/src/network/stream_reader.rs @@ -14,16 +14,16 @@ //! Stream reader. //! -//! This module defines the `StreamReader` struct and its implementation which -//! is used for parsing an incoming stream into separate `RawNetworkMessage`s, -//! handling assembling messages from multiple packets, or dealing with partial -//! or multiple messages in the stream (e.g. when reading from a TCP socket). +//! Deprecated +//! +//! This module defines `StreamReader` struct and its implementation which is used +//! for parsing incoming stream into separate `Decodable`s, handling assembling +//! messages from multiple packets or dealing with partial or multiple messages in the stream +//! (like can happen with reading from TCP socket) //! -use prelude::*; - use core::fmt; -use io::{self, Read}; +use io::Read; use consensus::{encode, Decodable}; @@ -31,61 +31,36 @@ use consensus::{encode, Decodable}; pub struct StreamReader { /// Stream to read from pub stream: R, - /// I/O buffer - data: Vec, - /// Buffer containing unparsed message part - unparsed: Vec } impl fmt::Debug for StreamReader { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "StreamReader with buffer_size={} and unparsed content {:?}", - self.data.capacity(), self.unparsed) + write!(f, "StreamReader") } } impl StreamReader { - /// Constructs new stream reader for a given input stream `stream` with - /// optional parameter `buffer_size` determining reading buffer size - pub fn new(stream: R, buffer_size: Option) -> StreamReader { + /// Constructs new stream reader for a given input stream `stream` + #[deprecated(since="0.28.0", note="wrap you stream into a buffered reader if necessary and use consensus_encode directly")] + pub fn new(stream: R, _buffer_size: Option) -> StreamReader { StreamReader { stream, - data: vec![0u8; buffer_size.unwrap_or(64 * 1024)], - unparsed: vec![] } } - /// Reads stream and parses next message from its current input, - /// also taking into account previously unparsed partial message (if there was such). + /// Reads stream and parses next message from its current input + #[deprecated(since="0.28.0", note="wrap you stream into a buffered reader if necessary and use consensus_encode directly")] pub fn read_next(&mut self) -> Result { - loop { - match encode::deserialize_partial::(&self.unparsed) { - // In this case we just have an incomplete data, so we need to read more - Err(encode::Error::Io(ref err)) if err.kind () == io::ErrorKind::UnexpectedEof => { - let count = self.stream.read(&mut self.data)?; - if count > 0 { - self.unparsed.extend(self.data[0..count].iter()); - } - else { - return Err(encode::Error::Io(io::Error::from(io::ErrorKind::UnexpectedEof))); - } - }, - Err(err) => return Err(err), - // We have successfully read from the buffer - Ok((message, index)) => { - self.unparsed.drain(..index); - return Ok(message) - }, - } - } + Decodable::consensus_decode(&mut self.stream) } } +#[allow(deprecated)] #[cfg(test)] mod test { use std::thread; use std::time::Duration; - use io::{self, BufReader, Write}; + use io::{BufReader, Write}; use std::net::{TcpListener, TcpStream, Shutdown}; use std::thread::JoinHandle; use network::constants::ServiceFlags; @@ -193,22 +168,6 @@ mod test { } } - #[test] - fn parse_multipartmsg_test() { - let stream = io::empty(); - let mut reader = StreamReader::new(stream, None); - reader.unparsed = MSG_ALERT[..24].to_vec(); - let message: Result = reader.read_next(); - assert!(message.is_err()); - assert_eq!(reader.unparsed.len(), 24); - - reader.unparsed = MSG_ALERT.to_vec(); - let message = reader.read_next().unwrap(); - assert_eq!(reader.unparsed.len(), 0); - - check_alert_msg(&message); - } - #[test] fn read_singlemsg_test() { let stream = MSG_VERSION[..].to_vec(); @@ -346,4 +305,17 @@ mod test { // should be also ok for a non-witness block as commitment is optional in that case assert!(block.check_witness_commitment()); } + + #[test] + fn parse_multipartmsg_test() { + let mut multi = MSG_ALERT.to_vec(); + multi.extend(&MSG_ALERT[..]); + let mut reader = StreamReader::new(&multi[..], None); + let message: Result = reader.read_next(); + assert!(message.is_ok()); + check_alert_msg(&message.unwrap()); + let message: Result = reader.read_next(); + assert!(message.is_ok()); + check_alert_msg(&message.unwrap()); + } } From 9189539715d0f1d6c4944ece34d276c15f5eff9b Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 4 Jan 2022 10:31:45 +0100 Subject: [PATCH 3/4] Use BufReader internally in StreamReader to avoid performance regression on existing callers --- src/network/stream_reader.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/network/stream_reader.rs b/src/network/stream_reader.rs index 3becfabc..6d07ef66 100644 --- a/src/network/stream_reader.rs +++ b/src/network/stream_reader.rs @@ -23,14 +23,14 @@ //! use core::fmt; -use io::Read; +use io::{Read, BufReader}; use consensus::{encode, Decodable}; /// Struct used to configure stream reader function pub struct StreamReader { /// Stream to read from - pub stream: R, + pub stream: BufReader, } impl fmt::Debug for StreamReader { @@ -44,7 +44,7 @@ impl StreamReader { #[deprecated(since="0.28.0", note="wrap you stream into a buffered reader if necessary and use consensus_encode directly")] pub fn new(stream: R, _buffer_size: Option) -> StreamReader { StreamReader { - stream, + stream: BufReader::new(stream), } } From e860333bf3fdfb53626b69357900801f4b2790d2 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Wed, 5 Jan 2022 09:39:57 +0100 Subject: [PATCH 4/4] Fix typos --- src/network/stream_reader.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/network/stream_reader.rs b/src/network/stream_reader.rs index 6d07ef66..036fe244 100644 --- a/src/network/stream_reader.rs +++ b/src/network/stream_reader.rs @@ -41,7 +41,7 @@ impl fmt::Debug for StreamReader { impl StreamReader { /// Constructs new stream reader for a given input stream `stream` - #[deprecated(since="0.28.0", note="wrap you stream into a buffered reader if necessary and use consensus_encode directly")] + #[deprecated(since="0.28.0", note="wrap your stream into a buffered reader if necessary and use consensus_encode directly")] pub fn new(stream: R, _buffer_size: Option) -> StreamReader { StreamReader { stream: BufReader::new(stream), @@ -49,7 +49,7 @@ impl StreamReader { } /// Reads stream and parses next message from its current input - #[deprecated(since="0.28.0", note="wrap you stream into a buffered reader if necessary and use consensus_encode directly")] + #[deprecated(since="0.28.0", note="wrap your stream into a buffered reader if necessary and use consensus_encode directly")] pub fn read_next(&mut self) -> Result { Decodable::consensus_decode(&mut self.stream) }