icepick workflow: detach simulator from normal flow

crates/by-chain/icepick-solana/src/lib.rs

@@ -178,7 +178,7 @@ pub struct Request {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-#[serde(tag = "operation", content = "values")]
+#[serde(tag = "operation", content = "values", rename_all = "kebab-case")]
 pub enum Operation {
     GetBlockhash(GetBlockhash),
     GenerateWallet(GenerateWallet),
@@ -217,7 +217,7 @@ impl Module for Solana {
             r#type: ArgumentType::Optional,
         };
         let account = Argument {
-            name: "from-account".to_string(),
+            name: "from_account".to_string(),
             description: "The derivation account used for the transaction.".to_string(),
             r#type: ArgumentType::Optional,
         };
@@ -227,17 +227,17 @@ impl Module for Solana {
             r#type: ArgumentType::Optional,
         };
         let fee_payer_address = Argument {
-            name: "fee-payer-address".to_string(),
+            name: "fee_payer_address".to_string(),
             description: "The address used to pay the fee.".to_string(),
             r#type: ArgumentType::Optional,
         };
         let fee_payer = Argument {
-            name: "fee-payer".to_string(),
+            name: "fee_payer".to_string(),
             description: "The derivation account used to pay the fee.".to_string(),
             r#type: ArgumentType::Optional,
         };
         let from_address = Argument {
-            name: "from-address".to_string(),
+            name: "from_address".to_string(),
             description: concat!(
                 "The address to send SOL from; will be used to verify ",
                 "the derivation account."
@@ -287,7 +287,7 @@ impl Module for Solana {
                     },
                     account.clone(),
                     Argument {
-                        name: "to-address".to_string(),
+                        name: "to_address".to_string(),
                         description: "The address to send SOL to.".to_string(),
                         r#type: ArgumentType::Required,
                     },
@@ -319,17 +319,17 @@ impl Module for Solana {
                 description: "Create an account for a given token".to_string(),
                 arguments: vec![
                     Argument {
-                        name: "wallet-address".to_string(),
+                        name: "wallet_address".to_string(),
                         description: "The address of the token.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "token-address".to_string(),
+                        name: "token_address".to_string(),
                         description: "The address of the token.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "funder-address".to_string(),
+                        name: "funder_address".to_string(),
                         description: "The address of the funder (signer).".to_string(),
                         r#type: ArgumentType::Optional,
                     },
@@ -346,18 +346,18 @@ impl Module for Solana {
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "token-address".to_string(),
+                        name: "token_address".to_string(),
                         description: "The address of the token.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     account.clone(),
                     Argument {
-                        name: "to-address".to_string(),
+                        name: "to_address".to_string(),
                         description: "The address to send the tokens to.".to_string(),
                         r#type: ArgumentType::Required,
                     },
                     Argument {
-                        name: "from-address".to_string(),
+                        name: "from_address".to_string(),
                         description: "The address to send the tokens from; will be used to verify the derivation account.".to_string(),
                         r#type: ArgumentType::Required,
                     },

crates/icepick/src/cli/mod.rs

@@ -6,6 +6,8 @@ use std::{
     process::{Command, Stdio},
 };
 
+mod workflow;
+
 pub fn get_command(bin_name: &str) -> (&str, Vec<&str>) {
     if std::env::vars().any(|(k, _)| &k == "ICEPICK_USE_CARGO") {
         ("cargo", vec!["run", "-q", "--bin", bin_name, "--"])
@@ -19,8 +21,12 @@ struct ModuleConfig {
     name: String,
     command_name: Option<String>,
     algorithm: keyfork_derive_util::request::DerivationAlgorithm,
+
     #[serde(with = "serde_derivation")]
     derivation_prefix: keyfork_derive_util::DerivationPath,
+
+    #[serde(rename = "workflow", default)]
+    workflows: Vec<workflow::Workflow>,
 }
 
 mod serde_derivation {
@@ -41,9 +47,8 @@ mod serde_derivation {
         D: Deserializer<'de>,
     {
         use serde::de::Error;
-        String::deserialize(deserializer).and_then(|string| {
-            DerivationPath::from_str(&string).map_err(|e| Error::custom(e.to_string()))
-        })
+        String::deserialize(deserializer)
+            .and_then(|string| DerivationPath::from_str(&string).map_err(Error::custom))
     }
 }
 
@@ -53,6 +58,9 @@ struct Config {
     modules: Vec<ModuleConfig>,
 }
 
+// command name, invocable binary, operations
+type Commands<'a> = &'a [(String, String, Vec<Operation>)];
+
 pub fn do_cli_thing() {
     /* parse config file to get module names */
     let config_file = std::env::vars().find_map(|(k, v)| {
@@ -67,8 +75,6 @@ pub fn do_cli_thing() {
 
     let mut commands = vec![];
     let mut icepick_command = command!();
-    // NOTE: this needs to be .cloned(), since commands is leaked to be 'static
-    // and coin_bin otherwise wouldn't live long enough
     for module in &config.modules {
         let module_name = &module.name;
         let bin = module
@@ -94,16 +100,41 @@ pub fn do_cli_thing() {
             .expect("successful deserialization of operation");
         commands.push((module_name.clone(), bin, operations));
     }
+
+    // Add workflow subcommands
+    let mut workflows = vec![];
+    for module in &config.modules {
+        workflows.push((module.name.clone(), module.workflows.clone()));
+    }
+    let workflows = workflows.leak();
+    let mut workflow_command = clap::Command::new("workflow")
+        .about("Run a pre-defined Icepick workflow")
+        .arg(clap::arg!(--"simulate-workflow").global(true));
+    for module in workflows.iter() {
+        let mut module_subcommand = clap::Command::new(module.0.as_str());
+        for workflow in &module.1 {
+            module_subcommand = module_subcommand.subcommand(workflow.generate_command());
+        }
+        workflow_command = workflow_command.subcommand(module_subcommand);
+    }
+
+    icepick_command = icepick_command.subcommand(workflow_command);
+
+    // Add per-module subcommands
     let commands = commands.leak();
     for command in commands.iter() {
         let mut subcommand = clap::Command::new(command.0.as_str());
         for op in &command.2 {
-            let mut op_command = clap::Command::new(&op.name).about(&op.description);
+            let mut op_command =
+                clap::Command::new(op.name.replace('_', "-")).about(&op.description);
             for arg in &op.arguments {
-                let mut op_arg = clap::Arg::new(&arg.name).help(arg.description.as_str());
+                let mut op_arg =
+                    clap::Arg::new(arg.name.replace('_', "-")).help(arg.description.as_str());
                 op_arg = match arg.r#type {
                     ArgumentType::Required => op_arg.required(true),
-                    ArgumentType::Optional => op_arg.required(false).long(&arg.name),
+                    ArgumentType::Optional => {
+                        op_arg.required(false).long(arg.name.replace('_', "-"))
+                    }
                 };
                 op_command = op_command.arg(op_arg);
             }
@@ -112,11 +143,28 @@ pub fn do_cli_thing() {
         icepick_command = icepick_command.subcommand(subcommand);
     }
 
+    let matches = icepick_command.get_matches();
+
+    // If we have a Workflow command, run the workflow and exit.
+    if let Some(("workflow", matches)) = matches.subcommand() {
+        let (module_name, matches) = matches
+            .subcommand()
+            .expect("icepick workflow: missing module");
+        let (workflow_name, matches) = matches
+            .subcommand()
+            .expect("icepick workflow: missing workflow");
+        let workflow = workflows
+            .iter()
+            .find(|(module, _)| module == module_name)
+            .and_then(|(_, workflows)| workflows.iter().find(|x| x.name == workflow_name))
+            .expect("workflow from CLI should match config");
+        workflow.handle(matches, commands);
+        return;
+    }
+
+    // Load the input for modules if we don't have a terminal input.
     let stdin = std::io::stdin();
     let mut cli_input: Option<serde_json::Value> = None;
-    // HACK: Allow good UX when running from CLI without piping in a blob.
-    // This is because we have CLI arguments _only_ for the module. We _could_ add a global
-    // argument `--input` instead of reading input.
     if !stdin.is_terminal() {
         cli_input = serde_json::from_reader(stdin).ok();
     }
@@ -126,7 +174,7 @@ pub fn do_cli_thing() {
         .as_ref()
         .and_then(|json| json.get("derivation_accounts"));
 
-    let matches = icepick_command.get_matches();
+    // We don't have a Workflow command, so let's find the matching Module command
     if let Some((module, matches)) = matches.subcommand() {
         if let Some((subcommand, matches)) = matches.subcommand() {
             if let Some(operation) = commands
@@ -139,8 +187,8 @@ pub fn do_cli_thing() {
                 );
                 for arg in &operation.arguments {
                     args.insert(
-                        arg.name.replace('-', "_"),
-                        matches.get_one::<String>(&arg.name),
+                        arg.name.clone(),
+                        matches.get_one::<String>(&arg.name.replace('_', "-")),
                     );
                 }
 
@@ -183,24 +231,6 @@ pub fn do_cli_thing() {
                     }
                 }
 
-                // in the event this is not PascalCase, this would be false.
-                // we set this to true to capitalize the first character.
-                let mut last_char_was_dash = true;
-                let subcommand = subcommand
-                    .chars()
-                    .filter_map(|c| {
-                        if last_char_was_dash {
-                            last_char_was_dash = false;
-                            return Some(c.to_ascii_uppercase());
-                        }
-                        if c == '-' {
-                            last_char_was_dash = true;
-                            None
-                        } else {
-                            Some(c)
-                        }
-                    })
-                    .collect::<String>();
                 let json = serde_json::json!({
                     "operation": subcommand,
                     "values": args,
@@ -210,7 +240,7 @@ pub fn do_cli_thing() {
                 let bin = commands
                     .iter()
                     .find_map(|(fmodule, fcommand, _)| {
-                        if fmodule == module {
+                        if *fmodule == module {
                             Some(fcommand)
                         } else {
                             None

crates/icepick/src/cli/workflow.rs

@@ -0,0 +1,151 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::{HashMap, HashSet};
+
+use super::{Commands, Operation};
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct Workflow {
+    pub name: String,
+    pub inputs: Vec<String>,
+
+    #[serde(rename = "step")]
+    steps: Vec<WorkflowStep>,
+}
+
+pub type StringMap = std::collections::HashMap<String, String>;
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct WorkflowStep {
+    r#type: String,
+
+    #[serde(default)]
+    blob: StringMap,
+
+    #[serde(default)]
+    values: StringMap,
+
+    #[serde(default)]
+    inputs: StringMap,
+
+    #[serde(default)]
+    outputs: StringMap,
+}
+
+#[derive(Clone, Debug)]
+struct InvocableOperation {
+    name: String,
+    binary: String,
+    operation: Operation,
+}
+
+impl Workflow {
+    /// Generate a [`clap::Command`] for a [`Workflow`], where the inputs can be defined either by
+    /// command-line arguments or via a JSON input file.
+    pub fn generate_command(&self) -> clap::Command {
+        let mut command = clap::Command::new(&self.name).arg(clap::arg!(
+                --"input-file" [FILE]
+                "A file containing any inputs not passed on the command line"
+        ));
+        for input in &self.inputs {
+            let arg = clap::Arg::new(input)
+                .required(false)
+                .long(input.replace('_', "-"))
+                .value_name(input.to_uppercase());
+            command = command.arg(arg);
+        }
+        command
+    }
+
+    fn load_inputs(&self, matches: &clap::ArgMatches) -> StringMap {
+        let mut map = StringMap::default();
+        let input_file: Option<StringMap> = matches
+            .get_one::<std::path::PathBuf>("input-file")
+            .and_then(|p| std::fs::File::open(p).ok())
+            .and_then(|f| serde_json::from_reader(f).ok());
+        for input in &self.inputs {
+            match matches.get_one::<String>(input) {
+                Some(value) => {
+                    map.insert(input.clone(), value.clone());
+                    continue;
+                }
+                None => {
+                    if let Some(value) = input_file.as_ref().and_then(|f| f.get(input)) {
+                        map.insert(input.clone(), value.clone());
+                        continue;
+                    }
+                }
+            }
+            panic!("Key was not found: {input}");
+        }
+
+        map
+    }
+
+    pub fn simulate_workflow(&self, mut data: HashSet<String>, operations: &[InvocableOperation]) {
+        // simulate the steps by using a HashSet to traverse the inputs and outputs and ensure
+        // there's no inconsistencies
+        for (i, step) in self.steps.iter().enumerate() {
+            // NOTE: overflow possible but unlikely
+            let step_index = i + 1;
+
+            // Find the relevant Operation
+            let Some(invocable) = operations.iter().find(|op| op.name == step.r#type) else {
+                panic!("Could not find operation: {}", step.r#type);
+            };
+
+            // Check if we have the keys we want to pass into the module.
+            for in_memory_name in step.inputs.values() {
+                if !data.contains(in_memory_name) && !step.values.contains_key(in_memory_name) {
+                    panic!("Failed simulation: step #{step_index}: missing value {in_memory_name}");
+                }
+            }
+
+            // Check that the module accepts those keys.
+            for module_input_name in step.inputs.keys() {
+                if !invocable
+                    .operation
+                    .arguments
+                    .iter()
+                    .any(|arg| *module_input_name == arg.name)
+                {
+                    eprintln!("Simulation: step #{step_index}: input value {module_input_name} will be passed through as JSON input");
+                }
+            }
+
+            // Add the keys we get from the module.
+            for in_memory_name in step.outputs.values() {
+                data.insert(in_memory_name.clone());
+            }
+        }
+    }
+
+    pub fn handle(&self, matches: &clap::ArgMatches, modules: Commands) {
+        let inputs = self.load_inputs(matches);
+        let data: HashMap<String, Value> = inputs
+            .into_iter()
+            .map(|(k, v)| (k, Value::String(v)))
+            .collect();
+
+        let mut operations = vec![];
+
+        for (module_name, module_binary, module_operations) in modules {
+            for operation in module_operations {
+                let operation_name = &operation.name;
+                let io = InvocableOperation {
+                    name: format!("{module_name}-{operation_name}"),
+                    binary: module_binary.clone(),
+                    operation: operation.clone(),
+                };
+                operations.push(io);
+            }
+        }
+
+        if matches.get_flag("simulate-workflow") {
+            self.simulate_workflow(data.into_keys().collect(), &operations);
+            return;
+        }
+
+        todo!("Unsimulated transaction!");
+    }
+}

icepick.toml

@@ -2,3 +2,82 @@
 name = "sol"
 derivation_prefix = "m/44'/501'/0'"
 algorithm = "Ed25519"
+
+[[module.workflow]]
+# The name of the workflow, which can be called by:
+# `icepick workflow sol transfer-token`
+name = "transfer-token"
+
+# These values are used as inputs for other workflows, acquired from the CLI.
+# These values can only be strings, but other values can be any value that can
+# be serialized by serde_json::Value.
+# These values can also be loaded using "internal-load-file", using some form
+# of later-defined signature validation.
+inputs = ["from_address", "to_address", "token_name", "token_amount"]
+
+## Load the Blockhash from the SD card
+#[[module.workflow.step]]
+#type = "internal-load-file"
+#
+## Pre-defined values to be passed to the module
+#values = { filename = "blockhash.json" }
+#
+## This value is marked to be saved in-memory, and can be used as an input for
+## later steps.
+#outputs = { blockhash = "blockhash" }
+
+# Get the token address and token decimals for the given token
+[[module.workflow.step]]
+type = "sol-get-token-info"
+
+# The key is the key that is passed to the program in the
+# `values` field. The value is the item in storage. In this case,
+# we read a `token-name` from our input, but the operation expects `token`.
+inputs = { token= "token_name" }
+
+# Because these two fields are currently unused in our storage, we can grab
+# them from the outputs of our module. The key is the key of the output value
+# we want to store, and the value is the name to be assigned in storage.
+outputs = { token_address = "token_address", token_decimals = "token_decimals" }
+
+[[module.workflow.step]]
+# Generate an unsigned Transaction
+type = "sol-transfer-token"
+
+# If using a lot of inputs, it may be best to use a non-inline table.
+# Non-inline tables _must_ be the last step, as otherwise, `outputs` for
+# example would be considered a member of `inputs`. In this case, we use a
+# non-inline table for `outputs` even though it would fit on one line, to avoid
+# the ambiguity.
+[module.workflow.step.inputs]
+amount = "token_amount"
+token_address = "token_address"
+decimals = "token_decimals"
+to_address = "to_address"
+from_address = "from_address"
+
+[module.workflow.step.outputs]
+transaction = "unsigned_transaction"
+
+# Sign the transaction
+[[module.workflow.step]]
+type = "sol-sign"
+
+[module.workflow.step.inputs]
+transaction = "unsigned_transaction"
+# blockhash = "blockhash"
+
+[module.workflow.step.outputs]
+transaction = "signed_transaction"
+
+## Write the signed transaction to a file
+#[[module.workflow.step]]
+#type = "internal-save-file"
+#
+## We are using a static filename here, so we use `values` instead of `inputs`.
+#values = { filename = "transaction.json" }
+#
+## All fields in both `inputs` and `values`, other than `filename`, will be
+## persisted to the file. In this case, the `transaction` field of the file will
+## contain the signed transaction.
+#inputs = { transaction = "signed_transaction" }