diff --git a/Cargo.lock b/Cargo.lock index 583bea1..1c0aa95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + [[package]] name = "bit-vec" version = "0.6.3" @@ -179,6 +185,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.4.0" @@ -326,6 +338,40 @@ dependencies = [ "typenum", ] +[[package]] +name = "deadpool" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" +dependencies = [ + "async-trait", + "deadpool-runtime", + "num_cpus", + "retain_mut", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a24a9d49deefe610b8b60c767a7412e9a931d79a89415cd2d2d71630ca8d7" +dependencies = [ + "deadpool", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" +dependencies = [ + "tokio", +] + [[package]] name = "digest" version = "0.10.7" @@ -334,6 +380,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -373,6 +420,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" @@ -395,6 +448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -403,6 +457,23 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + [[package]] name = "futures-task" version = "0.3.28" @@ -416,6 +487,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", + "futures-macro", + "futures-sink", "futures-task", "pin-project-lite", "pin-utils", @@ -455,7 +528,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ - "base64", + "base64 0.13.1", "bitflags 1.3.2", "bytes", "headers-core", @@ -492,6 +565,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "0.2.9" @@ -548,7 +630,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -627,6 +709,15 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + [[package]] name = "memchr" version = "2.5.0" @@ -676,8 +767,10 @@ dependencies = [ "bloomfilter", "clap", "color-eyre", + "deadpool-postgres", "serde", "tokio", + "tokio-postgres", "tower-http", "tracing", "tracing-error", @@ -766,6 +859,24 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -798,6 +909,41 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "postgres-protocol" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" +dependencies = [ + "base64 0.21.2", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -840,6 +986,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.7.0" @@ -915,6 +1091,12 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +[[package]] +name = "retain_mut" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1016,6 +1198,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -1065,12 +1258,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stringprep" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -1118,6 +1337,21 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.29.1" @@ -1133,7 +1367,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "tracing", "windows-sys", @@ -1150,6 +1384,44 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "tokio-postgres" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "socket2 0.5.3", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -1295,12 +1567,27 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + [[package]] name = "unicode-ident" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + [[package]] name = "valuable" version = "0.1.0" diff --git a/mnemonic-hash-checker/Cargo.toml b/mnemonic-hash-checker/Cargo.toml index bab549f..77722b4 100644 --- a/mnemonic-hash-checker/Cargo.toml +++ b/mnemonic-hash-checker/Cargo.toml @@ -10,8 +10,10 @@ axum = { version = "0.6.20", features = ["headers", "macros", "tracing"] } bloomfilter = "1.0.9" clap = { version = "=4.0.32", features = ["derive", "env"] } color-eyre = "0.6.2" +deadpool-postgres = "0.10.5" serde = { version = "1.0.181", features = ["serde_derive"] } tokio = { version = "1.29.1", features = ["full", "fs", "tracing"] } +tokio-postgres = "0.7.8" tower-http = { version = "0.4.3", features = ["trace", "catch-panic"] } tracing = "0.1.37" tracing-error = "0.2.0" diff --git a/mnemonic-hash-checker/src/main.rs b/mnemonic-hash-checker/src/main.rs index b57ca3c..c141be1 100644 --- a/mnemonic-hash-checker/src/main.rs +++ b/mnemonic-hash-checker/src/main.rs @@ -16,6 +16,9 @@ use color_eyre::eyre::Result; use clap::Parser; +use deadpool_postgres::{Config, ManagerConfig, Pool, RecyclingMethod, Runtime}; +use tokio_postgres::NoTls; + use serde::Deserialize; use tower_http::catch_panic::CatchPanicLayer; @@ -25,7 +28,7 @@ use tracing_subscriber::filter::{EnvFilter, LevelFilter}; use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::prelude::*; -use tracing::info; +use tracing::{error, info}; mod bloom; @@ -34,12 +37,35 @@ struct CliConfig { #[clap(long, env)] bloom_filter_path: PathBuf, + #[clap(long, env, default_value = "lookup")] + postgres_dbname: String, + + #[clap(long, env, default_value = "localhost")] + postgres_host: String, + + #[clap(long, env, default_value_t = 5432)] + postgres_port: u16, + + #[clap(long, env = "USER", default_value = "postgres")] + postgres_user: String, + + #[clap(long, env)] + postgres_password: Option, + + #[clap(long, env, default_value = "hashes")] + database_table: String, + + #[clap(long, env, default_value = "hash")] + database_column: String, + #[clap(long, env)] bind_address: Option, } struct AppState { - bloom_filter: Bloom + pool: Pool, + database_query: String, + bloom_filter: Bloom, } fn setup_registry() { @@ -61,9 +87,17 @@ async fn index() -> impl IntoResponse { } #[tracing::instrument(skip(state))] -async fn check_hash(hash: &str, state: &AppState) -> bool { - let bloom_filter = &state.bloom_filter; - bloom_filter.check(&hash.to_string()) +async fn check_hash_in_db(hash: &str, state: &AppState) -> Result { + let client = state.pool.get().await?; + let query = client.prepare_cached(&state.database_query).await?; + let rows = client.query(&query, &[&hash]).await?; + + if let Some(row) = rows.get(0) { + let retrieved_hash: String = row.try_get(0)?; + Ok(hash == retrieved_hash) + } else { + Ok(false) + } } #[derive(Deserialize)] @@ -71,18 +105,37 @@ struct CheckQuery { sha256: String, } +const VULNERABLE: &'static str = "vulnerable"; +const NOT_FOUND: &'static str = "match not found"; + // Note: Exposes *zero* information of potential errors to clients. #[tracing::instrument(skip(query, state))] async fn check_hash_slug( query: Query, State(state): State>, ) -> (StatusCode, &'static str) { - match state.bloom_filter.check(&query.sha256.to_string()) { - true => (StatusCode::OK, "vulnerable"), - false => (StatusCode::NOT_FOUND, "match not found") + let hash = query.sha256.to_string(); + match state.bloom_filter.check(&hash) { + true => { + info!("Found hash in bloom filter, looking in db"); + match check_hash_in_db(&hash, &state).await { + Ok(true) => (StatusCode::OK, VULNERABLE), + Ok(false) => (StatusCode::NOT_FOUND, NOT_FOUND), + Err(e) => { + error!(%e, "Error while handling postgres lookup"); + (StatusCode::NOT_FOUND, NOT_FOUND) + } + } + } + false => (StatusCode::NOT_FOUND, NOT_FOUND), } } +const MILK_SAD: &str = concat!( + "1f76da7a36fcacfe22b6c68d1acf6204", + "73744e78aa04242a40374953783706f0", +); + #[tokio::main] async fn main() -> Result<()> { setup_registry(); @@ -93,10 +146,42 @@ async fn main() -> Result<()> { .bind_address .unwrap_or_else(|| (std::net::Ipv4Addr::new(127, 0, 0, 1), 8000).into()); + info!("configuring database pool"); + + let mut db_config = Config::new(); + db_config.dbname = Some(cli_config.postgres_dbname); + db_config.host = Some(cli_config.postgres_host); + db_config.password = cli_config.postgres_password; + db_config.port = Some(cli_config.postgres_port); + db_config.user = Some(cli_config.postgres_user); + db_config.manager = Some(ManagerConfig { + recycling_method: RecyclingMethod::Fast, + }); + let pool = db_config.create_pool(Some(Runtime::Tokio1), NoTls)?; + + info!("testing db connection"); + + let database_query = format!( + "SELECT encode({column}, 'hex') FROM {table} WHERE hash = decode($1, 'hex')", + column = cli_config.database_column, + table = cli_config.database_table + ); + + { + let client = pool.get().await?; + let query = client.prepare_cached(&database_query).await?; + let rows = client.query(&query, &[&MILK_SAD]).await?; + let row = rows.get(0).expect("db row"); + let column: String = row.try_get(0).expect("db column"); + assert_eq!(column, MILK_SAD); + } + info!("loading bloom filter"); let state = AppState { - bloom_filter: bloom::load(cli_config.bloom_filter_path.as_ref())? + bloom_filter: bloom::load(cli_config.bloom_filter_path.as_ref())?, + pool, + database_query, }; let app = Router::new()