icepick workflow: add workflow simulator

crates/by-chain/icepick-solana/src/lib.rs

@@ -178,7 +178,7 @@ pub struct Request {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-#[serde(tag = "operation", content = "values")]
+#[serde(tag = "operation", content = "values", rename_all = "kebab-case")]
 pub enum Operation {
     GetBlockhash(GetBlockhash),
     GenerateWallet(GenerateWallet),
@@ -217,7 +217,7 @@ impl Module for Solana {
             r#type: ArgumentType::Optional,
         };
         let account = Argument {
-            name: "from-account".to_string(),
+            name: "from_account".to_string(),
             description: "The derivation account used for the transaction.".to_string(),
             r#type: ArgumentType::Optional,
         };
@@ -227,17 +227,17 @@ impl Module for Solana {
             r#type: ArgumentType::Optional,
         };
         let fee_payer_address = Argument {
-            name: "fee-payer-address".to_string(),
+            name: "fee_payer_address".to_string(),
             description: "The address used to pay the fee.".to_string(),
             r#type: ArgumentType::Optional,
         };
         let fee_payer = Argument {
-            name: "fee-payer".to_string(),
+            name: "fee_payer".to_string(),
             description: "The derivation account used to pay the fee.".to_string(),
             r#type: ArgumentType::Optional,
         };
         let from_address = Argument {
-            name: "from-address".to_string(),
+            name: "from_address".to_string(),
             description: concat!(
                 "The address to send SOL from; will be used to verify ",
                 "the derivation account."
@@ -287,7 +287,7 @@ impl Module for Solana {
                     },
                     account.clone(),
                     Argument {
-                        name: "to-address".to_string(),
+                        name: "to_address".to_string(),
                         description: "The address to send SOL to.".to_string(),
                         r#type: ArgumentType::Required,
                     },
@@ -319,17 +319,17 @@ impl Module for Solana {
                 description: "Create an account for a given token".to_string(),
                 arguments: vec![
                     Argument {
-                        name: "wallet-address".to_string(),
+                        name: "wallet_address".to_string(),
                         description: "The address of the token.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "token-address".to_string(),
+                        name: "token_address".to_string(),
                         description: "The address of the token.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "funder-address".to_string(),
+                        name: "funder_address".to_string(),
                         description: "The address of the funder (signer).".to_string(),
                         r#type: ArgumentType::Optional,
                     },
@@ -346,18 +346,18 @@ impl Module for Solana {
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "token-address".to_string(),
+                        name: "token_address".to_string(),
                         description: "The address of the token.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     account.clone(),
                     Argument {
-                        name: "to-address".to_string(),
+                        name: "to_address".to_string(),
                         description: "The address to send the tokens to.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "from-address".to_string(),
+                        name: "from_address".to_string(),
                         description: "The address to send the tokens from; will be used to verify the derivation account.".to_string(),
                         r#type: ArgumentType::Required,
                     },

crates/icepick/src/cli/mod.rs

@@ -58,6 +58,9 @@ struct Config {
     modules: Vec<ModuleConfig>,
 }
 
+// command name, invocable binary, operations
+type Commands<'a> = &'a [(String, String, Vec<Operation>)];
+
 pub fn do_cli_thing() {
     /* parse config file to get module names */
     let config_file = std::env::vars().find_map(|(k, v)| {
@@ -72,8 +75,6 @@ pub fn do_cli_thing() {
 
     let mut commands = vec![];
     let mut icepick_command = command!();
-    // NOTE: this needs to be .cloned(), since commands is leaked to be 'static
-    // and coin_bin otherwise wouldn't live long enough
     for module in &config.modules {
         let module_name = &module.name;
         let bin = module
@@ -100,6 +101,7 @@ pub fn do_cli_thing() {
         commands.push((module_name.clone(), bin, operations));
     }
 
+    // Add workflow subcommands
     let mut workflows = vec![];
     for module in &config.modules {
         workflows.push((module.name.clone(), module.workflows.clone()));
@@ -117,16 +119,17 @@ pub fn do_cli_thing() {
 
     icepick_command = icepick_command.subcommand(workflow_command);
 
+    // Add per-module subcommands
     let commands = commands.leak();
     for command in commands.iter() {
         let mut subcommand = clap::Command::new(command.0.as_str());
         for op in &command.2 {
-            let mut op_command = clap::Command::new(&op.name).about(&op.description);
+            let mut op_command = clap::Command::new(op.name.replace('_', "-")).about(&op.description);
             for arg in &op.arguments {
-                let mut op_arg = clap::Arg::new(&arg.name).help(arg.description.as_str());
+                let mut op_arg = clap::Arg::new(arg.name.replace('_', "-")).help(arg.description.as_str());
                 op_arg = match arg.r#type {
                     ArgumentType::Required => op_arg.required(true),
-                    ArgumentType::Optional => op_arg.required(false).long(&arg.name),
+                    ArgumentType::Optional => op_arg.required(false).long(arg.name.replace('_', "-")),
                 };
                 op_command = op_command.arg(op_arg);
             }
@@ -135,11 +138,28 @@ pub fn do_cli_thing() {
         icepick_command = icepick_command.subcommand(subcommand);
     }
 
+    let matches = icepick_command.get_matches();
+
+    // If we have a Workflow command, run the workflow and exit.
+    if let Some(("workflow", matches)) = matches.subcommand() {
+        let (module_name, matches) = matches
+            .subcommand()
+            .expect("icepick workflow: missing module");
+        let (workflow_name, matches) = matches
+            .subcommand()
+            .expect("icepick workflow: missing workflow");
+        let workflow = workflows
+            .iter()
+            .find(|(module, _)| module == module_name)
+            .and_then(|(_, workflows)| workflows.iter().find(|x| x.name == workflow_name))
+            .expect("workflow from CLI should match config");
+        workflow.handle(matches, commands);
+        return;
+    }
+
+    // Load the input for modules if we don't have a terminal input.
     let stdin = std::io::stdin();
     let mut cli_input: Option<serde_json::Value> = None;
-    // HACK: Allow good UX when running from CLI without piping in a blob.
-    // This is because we have CLI arguments _only_ for the module. We _could_ add a global
-    // argument `--input` instead of reading input.
     if !stdin.is_terminal() {
         cli_input = serde_json::from_reader(stdin).ok();
     }
@@ -149,7 +169,7 @@ pub fn do_cli_thing() {
         .as_ref()
         .and_then(|json| json.get("derivation_accounts"));
 
-    let matches = icepick_command.get_matches();
+    // We don't have a Workflow command, so let's find the matching Module command
     if let Some((module, matches)) = matches.subcommand() {
         if let Some((subcommand, matches)) = matches.subcommand() {
             if let Some(operation) = commands
@@ -162,8 +182,8 @@ pub fn do_cli_thing() {
                 );
                 for arg in &operation.arguments {
                     args.insert(
-                        arg.name.replace('-', "_"),
+                        arg.name.clone(),
-                        matches.get_one::<String>(&arg.name),
+                        matches.get_one::<String>(&arg.name.replace('_', "-")),
                     );
                 }
 
@@ -206,24 +226,6 @@ pub fn do_cli_thing() {
                     }
                 }
 
-                // in the event this is not PascalCase, this would be false.
-                // we set this to true to capitalize the first character.
-                let mut last_char_was_dash = true;
-                let subcommand = subcommand
-                    .chars()
-                    .filter_map(|c| {
-                        if last_char_was_dash {
-                            last_char_was_dash = false;
-                            return Some(c.to_ascii_uppercase());
-                        }
-                        if c == '-' {
-                            last_char_was_dash = true;
-                            None
-                        } else {
-                            Some(c)
-                        }
-                    })
-                    .collect::<String>();
                 let json = serde_json::json!({
                     "operation": subcommand,
                     "values": args,
@@ -233,7 +235,7 @@ pub fn do_cli_thing() {
                 let bin = commands
                     .iter()
                     .find_map(|(fmodule, fcommand, _)| {
-                        if fmodule == module {
+                        if *fmodule == module {
                             Some(fcommand)
                         } else {
                             None

crates/icepick/src/cli/workflow.rs

@@ -1,6 +1,8 @@
-use super::get_command;
 use serde::{Deserialize, Serialize};
-use std::process::{Command, Stdio};
+use serde_json::Value;
+use std::collections::{HashMap, HashSet};
+
+use super::{Commands, Operation};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Workflow {
@@ -17,6 +19,9 @@ pub type StringMap = std::collections::HashMap<String, String>;
 pub struct WorkflowStep {
     r#type: String,
 
+    #[serde(default)]
+    blob: StringMap,
+
     #[serde(default)]
     values: StringMap,
 
@@ -27,6 +32,13 @@ pub struct WorkflowStep {
     outputs: StringMap,
 }
 
+#[derive(Clone, Debug)]
+struct InvocableOperation {
+    name: String,
+    binary: String,
+    operation: Operation,
+}
+
 impl Workflow {
     /// Generate a [`clap::Command`] for a [`Workflow`], where the inputs can be defined either by
     /// command-line arguments or via a JSON input file.
@@ -38,7 +50,7 @@ impl Workflow {
         for input in &self.inputs {
             let arg = clap::Arg::new(input)
                 .required(false)
-                .long(input)
+                .long(input.replace('_', "-"))
                 .value_name(input.to_uppercase());
             command = command.arg(arg);
         }
@@ -52,7 +64,7 @@ impl Workflow {
             .and_then(|p| std::fs::File::open(p).ok())
             .and_then(|f| serde_json::from_reader(f).ok());
         for input in &self.inputs {
-            match matches.get_one::<String>(&input.replace('_', "-")) {
+            match matches.get_one::<String>(input) {
                 Some(value) => {
                     map.insert(input.clone(), value.clone());
                     continue;
@@ -70,8 +82,64 @@ impl Workflow {
         map
     }
 
-    pub fn handle(self, matches: &clap::ArgMatches) {
+    pub fn handle(&self, matches: &clap::ArgMatches, modules: Commands) {
         let inputs = self.load_inputs(matches);
-        // step 2: run through commands
+        let data: HashMap<String, Value> = inputs
+            .into_iter()
+            .map(|(k, v)| (k, Value::String(v)))
+            .collect();
+
+        let mut operations = vec![];
+
+        for (module_name, module_binary, module_operations) in modules {
+            for operation in module_operations {
+                let operation_name = &operation.name;
+                let io = InvocableOperation {
+                    name: format!("{module_name}-{operation_name}"),
+                    binary: module_binary.clone(),
+                    operation: operation.clone(),
+                };
+                operations.push(io);
+            }
+        }
+
+        // simulate the steps by using a HashSet to traverse the inputs and outputs and ensure
+        // there's no inconsistencies
+        let mut simulated_values = data.keys().collect::<HashSet<_>>();
+        for (i, step) in self.steps.iter().enumerate() {
+            // NOTE: overflow possible but unlikely
+            let step_index = i + 1;
+
+            // Find the relevant Operation
+            let Some(invocable) = operations.iter().find(|op| op.name == step.r#type) else {
+                panic!("Could not find operation: {}", step.r#type);
+            };
+
+            // Check if we have the keys we want to pass into the module.
+            for in_memory_name in step.inputs.values() {
+                if !simulated_values.contains(in_memory_name)
+                    && !step.values.contains_key(in_memory_name)
+                {
+                    panic!("Failed simulation: step #{step_index}: missing value {in_memory_name}");
+                }
+            }
+
+            // Check that the module accepts those keys.
+            for module_input_name in step.inputs.keys() {
+                if !invocable
+                    .operation
+                    .arguments
+                    .iter()
+                    .any(|arg| *module_input_name == arg.name)
+                {
+                    eprintln!("Simulation: step #{step_index}: input value {module_input_name} will be passed through as JSON input");
+                }
+            }
+
+            // Add the keys we get from the module.
+            for in_memory_name in step.outputs.values() {
+                simulated_values.insert(in_memory_name);
+            }
+        }
     }
 }

icepick.toml

@@ -15,16 +15,16 @@ name = "transfer-token"
 # of later-defined signature validation.
 inputs = ["from_address", "to_address", "token_name", "token_amount"]
 
-# Load the Blockhash from the SD card
+## Load the Blockhash from the SD card
-[[module.workflow.step]]
+#[[module.workflow.step]]
-type = "internal-load-file"
+#type = "internal-load-file"
-
+#
-# Pre-defined values to be passed to the module
+## Pre-defined values to be passed to the module
-values = { filename = "blockhash.json" }
+#values = { filename = "blockhash.json" }
-
+#
-# This value is marked to be saved in-memory, and can be used as an input for
+## This value is marked to be saved in-memory, and can be used as an input for
-# later steps.
+## later steps.
-outputs = { blockhash = "blockhash" }
+#outputs = { blockhash = "blockhash" }
 
 # Get the token address and token decimals for the given token
 [[module.workflow.step]]
@@ -32,10 +32,8 @@ type = "sol-get-token-info"
 
 # The key is the key that is passed to the program in the
 # `values` field. The value is the item in storage. In this case,
-# they are the same, because we read a `token-name` from our input,
+# we read a `token-name` from our input, but the operation expects `token`.
-# store it in our storage as `token-name`, and `sol-token-info` will
+inputs = { token= "token_name" }
-# expect a `token-name`.
-inputs = { token_name = "token_name" }
 
 # Because these two fields are currently unused in our storage, we can grab
 # them from the outputs of our module. The key is the key of the output value
@@ -54,9 +52,9 @@ type = "sol-transfer-token"
 [module.workflow.step.inputs]
 amount = "token_amount"
 token_address = "token_address"
-token_decimals = "token_decimals"
+decimals = "token_decimals"
 to_address = "to_address"
-from_address = "from-address"
+from_address = "from_address"
 
 [module.workflow.step.outputs]
 transaction = "unsigned_transaction"
@@ -65,18 +63,21 @@ transaction = "unsigned_transaction"
 [[module.workflow.step]]
 type = "sol-sign"
 
-inputs = { transaction = "unsigned_transaction", blockhash = "blockhash" }
+[module.workflow.step.inputs]
+transaction = "unsigned_transaction"
+# blockhash = "blockhash"
 
-outputs = { transaction = "signed_transaction" }
+[module.workflow.step.outputs]
+transaction = "signed_transaction"
 
-# Write the signed transaction to a file
+## Write the signed transaction to a file
-[[module.workflow.step]]
+#[[module.workflow.step]]
-type = "internal-save-file"
+#type = "internal-save-file"
-
+#
-# We are using a static filename here, so we use `values` instead of `inputs`.
+## We are using a static filename here, so we use `values` instead of `inputs`.
-values = { filename = "transaction.json" }
+#values = { filename = "transaction.json" }
-
+#
-# All fields in both `inputs` and `values`, other than `filename`, will be
+## All fields in both `inputs` and `values`, other than `filename`, will be
-# persisted to the file. In this case, the `transaction` field of the file will
+## persisted to the file. In this case, the `transaction` field of the file will
-# contain the signed transaction.
+## contain the signed transaction.
-inputs = { transaction = "signed_transaction" }
+#inputs = { transaction = "signed_transaction" }