From 57ab48499575cdaa3db6ccbdcd13565b0b71e17a Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 19 Dec 2024 21:26:24 -0500 Subject: [PATCH] icepick workflow: add workflow simulator --- crates/by-chain/icepick-solana/src/lib.rs | 24 +++---- crates/icepick/src/cli/mod.rs | 62 +++++++++--------- crates/icepick/src/cli/workflow.rs | 80 +++++++++++++++++++++-- icepick.toml | 59 +++++++++-------- 4 files changed, 148 insertions(+), 77 deletions(-) diff --git a/crates/by-chain/icepick-solana/src/lib.rs b/crates/by-chain/icepick-solana/src/lib.rs index 12c7097..6e37124 100644 --- a/crates/by-chain/icepick-solana/src/lib.rs +++ b/crates/by-chain/icepick-solana/src/lib.rs @@ -178,7 +178,7 @@ pub struct Request { } #[derive(Serialize, Deserialize, Debug)] -#[serde(tag = "operation", content = "values")] +#[serde(tag = "operation", content = "values", rename_all = "kebab-case")] pub enum Operation { GetBlockhash(GetBlockhash), GenerateWallet(GenerateWallet), @@ -217,7 +217,7 @@ impl Module for Solana { r#type: ArgumentType::Optional, }; let account = Argument { - name: "from-account".to_string(), + name: "from_account".to_string(), description: "The derivation account used for the transaction.".to_string(), r#type: ArgumentType::Optional, }; @@ -227,17 +227,17 @@ impl Module for Solana { r#type: ArgumentType::Optional, }; let fee_payer_address = Argument { - name: "fee-payer-address".to_string(), + name: "fee_payer_address".to_string(), description: "The address used to pay the fee.".to_string(), r#type: ArgumentType::Optional, }; let fee_payer = Argument { - name: "fee-payer".to_string(), + name: "fee_payer".to_string(), description: "The derivation account used to pay the fee.".to_string(), r#type: ArgumentType::Optional, }; let from_address = Argument { - name: "from-address".to_string(), + name: "from_address".to_string(), description: concat!( "The address to send SOL from; will be used to verify ", "the derivation account." @@ -287,7 +287,7 @@ impl Module for Solana { }, account.clone(), Argument { - name: "to-address".to_string(), + name: "to_address".to_string(), description: "The address to send SOL to.".to_string(), r#type: ArgumentType::Required, }, @@ -319,17 +319,17 @@ impl Module for Solana { description: "Create an account for a given token".to_string(), arguments: vec![ Argument { - name: "wallet-address".to_string(), + name: "wallet_address".to_string(), description: "The address of the token.".to_string(), r#type: ArgumentType::Required, }, Argument { - name: "token-address".to_string(), + name: "token_address".to_string(), description: "The address of the token.".to_string(), r#type: ArgumentType::Required, }, Argument { - name: "funder-address".to_string(), + name: "funder_address".to_string(), description: "The address of the funder (signer).".to_string(), r#type: ArgumentType::Optional, }, @@ -346,18 +346,18 @@ impl Module for Solana { r#type: ArgumentType::Required, }, Argument { - name: "token-address".to_string(), + name: "token_address".to_string(), description: "The address of the token.".to_string(), r#type: ArgumentType::Required, }, account.clone(), Argument { - name: "to-address".to_string(), + name: "to_address".to_string(), description: "The address to send the tokens to.".to_string(), r#type: ArgumentType::Required, }, Argument { - name: "from-address".to_string(), + name: "from_address".to_string(), description: "The address to send the tokens from; will be used to verify the derivation account.".to_string(), r#type: ArgumentType::Required, }, diff --git a/crates/icepick/src/cli/mod.rs b/crates/icepick/src/cli/mod.rs index a324512..42894de 100644 --- a/crates/icepick/src/cli/mod.rs +++ b/crates/icepick/src/cli/mod.rs @@ -58,6 +58,9 @@ struct Config { modules: Vec, } +// command name, invocable binary, operations +type Commands<'a> = &'a [(String, String, Vec)]; + pub fn do_cli_thing() { /* parse config file to get module names */ let config_file = std::env::vars().find_map(|(k, v)| { @@ -72,8 +75,6 @@ pub fn do_cli_thing() { let mut commands = vec![]; let mut icepick_command = command!(); - // NOTE: this needs to be .cloned(), since commands is leaked to be 'static - // and coin_bin otherwise wouldn't live long enough for module in &config.modules { let module_name = &module.name; let bin = module @@ -100,6 +101,7 @@ pub fn do_cli_thing() { commands.push((module_name.clone(), bin, operations)); } + // Add workflow subcommands let mut workflows = vec![]; for module in &config.modules { workflows.push((module.name.clone(), module.workflows.clone())); @@ -117,16 +119,17 @@ pub fn do_cli_thing() { icepick_command = icepick_command.subcommand(workflow_command); + // Add per-module subcommands let commands = commands.leak(); for command in commands.iter() { let mut subcommand = clap::Command::new(command.0.as_str()); for op in &command.2 { - let mut op_command = clap::Command::new(&op.name).about(&op.description); + let mut op_command = clap::Command::new(op.name.replace('_', "-")).about(&op.description); for arg in &op.arguments { - let mut op_arg = clap::Arg::new(&arg.name).help(arg.description.as_str()); + let mut op_arg = clap::Arg::new(arg.name.replace('_', "-")).help(arg.description.as_str()); op_arg = match arg.r#type { ArgumentType::Required => op_arg.required(true), - ArgumentType::Optional => op_arg.required(false).long(&arg.name), + ArgumentType::Optional => op_arg.required(false).long(arg.name.replace('_', "-")), }; op_command = op_command.arg(op_arg); } @@ -135,11 +138,28 @@ pub fn do_cli_thing() { icepick_command = icepick_command.subcommand(subcommand); } + let matches = icepick_command.get_matches(); + + // If we have a Workflow command, run the workflow and exit. + if let Some(("workflow", matches)) = matches.subcommand() { + let (module_name, matches) = matches + .subcommand() + .expect("icepick workflow: missing module"); + let (workflow_name, matches) = matches + .subcommand() + .expect("icepick workflow: missing workflow"); + let workflow = workflows + .iter() + .find(|(module, _)| module == module_name) + .and_then(|(_, workflows)| workflows.iter().find(|x| x.name == workflow_name)) + .expect("workflow from CLI should match config"); + workflow.handle(matches, commands); + return; + } + + // Load the input for modules if we don't have a terminal input. let stdin = std::io::stdin(); let mut cli_input: Option = None; - // HACK: Allow good UX when running from CLI without piping in a blob. - // This is because we have CLI arguments _only_ for the module. We _could_ add a global - // argument `--input` instead of reading input. if !stdin.is_terminal() { cli_input = serde_json::from_reader(stdin).ok(); } @@ -149,7 +169,7 @@ pub fn do_cli_thing() { .as_ref() .and_then(|json| json.get("derivation_accounts")); - let matches = icepick_command.get_matches(); + // We don't have a Workflow command, so let's find the matching Module command if let Some((module, matches)) = matches.subcommand() { if let Some((subcommand, matches)) = matches.subcommand() { if let Some(operation) = commands @@ -162,8 +182,8 @@ pub fn do_cli_thing() { ); for arg in &operation.arguments { args.insert( - arg.name.replace('-', "_"), - matches.get_one::(&arg.name), + arg.name.clone(), + matches.get_one::(&arg.name.replace('_', "-")), ); } @@ -206,24 +226,6 @@ pub fn do_cli_thing() { } } - // in the event this is not PascalCase, this would be false. - // we set this to true to capitalize the first character. - let mut last_char_was_dash = true; - let subcommand = subcommand - .chars() - .filter_map(|c| { - if last_char_was_dash { - last_char_was_dash = false; - return Some(c.to_ascii_uppercase()); - } - if c == '-' { - last_char_was_dash = true; - None - } else { - Some(c) - } - }) - .collect::(); let json = serde_json::json!({ "operation": subcommand, "values": args, @@ -233,7 +235,7 @@ pub fn do_cli_thing() { let bin = commands .iter() .find_map(|(fmodule, fcommand, _)| { - if fmodule == module { + if *fmodule == module { Some(fcommand) } else { None diff --git a/crates/icepick/src/cli/workflow.rs b/crates/icepick/src/cli/workflow.rs index f1d5516..97dd29e 100644 --- a/crates/icepick/src/cli/workflow.rs +++ b/crates/icepick/src/cli/workflow.rs @@ -1,6 +1,8 @@ -use super::get_command; use serde::{Deserialize, Serialize}; -use std::process::{Command, Stdio}; +use serde_json::Value; +use std::collections::{HashMap, HashSet}; + +use super::{Commands, Operation}; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Workflow { @@ -17,6 +19,9 @@ pub type StringMap = std::collections::HashMap; pub struct WorkflowStep { r#type: String, + #[serde(default)] + blob: StringMap, + #[serde(default)] values: StringMap, @@ -27,6 +32,13 @@ pub struct WorkflowStep { outputs: StringMap, } +#[derive(Clone, Debug)] +struct InvocableOperation { + name: String, + binary: String, + operation: Operation, +} + impl Workflow { /// Generate a [`clap::Command`] for a [`Workflow`], where the inputs can be defined either by /// command-line arguments or via a JSON input file. @@ -38,7 +50,7 @@ impl Workflow { for input in &self.inputs { let arg = clap::Arg::new(input) .required(false) - .long(input) + .long(input.replace('_', "-")) .value_name(input.to_uppercase()); command = command.arg(arg); } @@ -52,7 +64,7 @@ impl Workflow { .and_then(|p| std::fs::File::open(p).ok()) .and_then(|f| serde_json::from_reader(f).ok()); for input in &self.inputs { - match matches.get_one::(&input.replace('_', "-")) { + match matches.get_one::(input) { Some(value) => { map.insert(input.clone(), value.clone()); continue; @@ -70,8 +82,64 @@ impl Workflow { map } - pub fn handle(self, matches: &clap::ArgMatches) { + pub fn handle(&self, matches: &clap::ArgMatches, modules: Commands) { let inputs = self.load_inputs(matches); - // step 2: run through commands + let data: HashMap = inputs + .into_iter() + .map(|(k, v)| (k, Value::String(v))) + .collect(); + + let mut operations = vec![]; + + for (module_name, module_binary, module_operations) in modules { + for operation in module_operations { + let operation_name = &operation.name; + let io = InvocableOperation { + name: format!("{module_name}-{operation_name}"), + binary: module_binary.clone(), + operation: operation.clone(), + }; + operations.push(io); + } + } + + // simulate the steps by using a HashSet to traverse the inputs and outputs and ensure + // there's no inconsistencies + let mut simulated_values = data.keys().collect::>(); + for (i, step) in self.steps.iter().enumerate() { + // NOTE: overflow possible but unlikely + let step_index = i + 1; + + // Find the relevant Operation + let Some(invocable) = operations.iter().find(|op| op.name == step.r#type) else { + panic!("Could not find operation: {}", step.r#type); + }; + + // Check if we have the keys we want to pass into the module. + for in_memory_name in step.inputs.values() { + if !simulated_values.contains(in_memory_name) + && !step.values.contains_key(in_memory_name) + { + panic!("Failed simulation: step #{step_index}: missing value {in_memory_name}"); + } + } + + // Check that the module accepts those keys. + for module_input_name in step.inputs.keys() { + if !invocable + .operation + .arguments + .iter() + .any(|arg| *module_input_name == arg.name) + { + eprintln!("Simulation: step #{step_index}: input value {module_input_name} will be passed through as JSON input"); + } + } + + // Add the keys we get from the module. + for in_memory_name in step.outputs.values() { + simulated_values.insert(in_memory_name); + } + } } } diff --git a/icepick.toml b/icepick.toml index 7e5c439..56c3555 100644 --- a/icepick.toml +++ b/icepick.toml @@ -15,16 +15,16 @@ name = "transfer-token" # of later-defined signature validation. inputs = ["from_address", "to_address", "token_name", "token_amount"] -# Load the Blockhash from the SD card -[[module.workflow.step]] -type = "internal-load-file" - -# Pre-defined values to be passed to the module -values = { filename = "blockhash.json" } - -# This value is marked to be saved in-memory, and can be used as an input for -# later steps. -outputs = { blockhash = "blockhash" } +## Load the Blockhash from the SD card +#[[module.workflow.step]] +#type = "internal-load-file" +# +## Pre-defined values to be passed to the module +#values = { filename = "blockhash.json" } +# +## This value is marked to be saved in-memory, and can be used as an input for +## later steps. +#outputs = { blockhash = "blockhash" } # Get the token address and token decimals for the given token [[module.workflow.step]] @@ -32,10 +32,8 @@ type = "sol-get-token-info" # The key is the key that is passed to the program in the # `values` field. The value is the item in storage. In this case, -# they are the same, because we read a `token-name` from our input, -# store it in our storage as `token-name`, and `sol-token-info` will -# expect a `token-name`. -inputs = { token_name = "token_name" } +# we read a `token-name` from our input, but the operation expects `token`. +inputs = { token= "token_name" } # Because these two fields are currently unused in our storage, we can grab # them from the outputs of our module. The key is the key of the output value @@ -54,9 +52,9 @@ type = "sol-transfer-token" [module.workflow.step.inputs] amount = "token_amount" token_address = "token_address" -token_decimals = "token_decimals" +decimals = "token_decimals" to_address = "to_address" -from_address = "from-address" +from_address = "from_address" [module.workflow.step.outputs] transaction = "unsigned_transaction" @@ -65,18 +63,21 @@ transaction = "unsigned_transaction" [[module.workflow.step]] type = "sol-sign" -inputs = { transaction = "unsigned_transaction", blockhash = "blockhash" } +[module.workflow.step.inputs] +transaction = "unsigned_transaction" +# blockhash = "blockhash" -outputs = { transaction = "signed_transaction" } +[module.workflow.step.outputs] +transaction = "signed_transaction" -# Write the signed transaction to a file -[[module.workflow.step]] -type = "internal-save-file" - -# We are using a static filename here, so we use `values` instead of `inputs`. -values = { filename = "transaction.json" } - -# All fields in both `inputs` and `values`, other than `filename`, will be -# persisted to the file. In this case, the `transaction` field of the file will -# contain the signed transaction. -inputs = { transaction = "signed_transaction" } +## Write the signed transaction to a file +#[[module.workflow.step]] +#type = "internal-save-file" +# +## We are using a static filename here, so we use `values` instead of `inputs`. +#values = { filename = "transaction.json" } +# +## All fields in both `inputs` and `values`, other than `filename`, will be +## persisted to the file. In this case, the `transaction` field of the file will +## contain the signed transaction. +#inputs = { transaction = "signed_transaction" }