icepick-internal: initial commit

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "addr2line"
@@ -1460,10 +1460,20 @@ dependencies = [
  "keyforkd-models",
  "serde",
  "serde_json",
- "thiserror 2.0.3",
+ "thiserror 2.0.9",
  "toml 0.8.19",
 ]
 
+[[package]]
+name = "icepick-internal"
+version = "0.1.0"
+dependencies = [
+ "icepick-module",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.9",
+]
+
 [[package]]
 name = "icepick-module"
 version = "0.1.0"
@@ -1486,7 +1496,7 @@ dependencies = [
  "spl-associated-token-account",
  "spl-token",
  "spl-token-2022",
- "thiserror 2.0.3",
+ "thiserror 2.0.9",
 ]
 
 [[package]]
@@ -4116,11 +4126,11 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.3"
+version = "2.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa"
+checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc"
 dependencies = [
- "thiserror-impl 2.0.3",
+ "thiserror-impl 2.0.9",
 ]
 
 [[package]]
@@ -4136,9 +4146,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.3"
+version = "2.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568"
+checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -4,6 +4,7 @@ resolver = "2"
 members = [
     "crates/icepick",
     "crates/icepick-module",
+    "crates/builtins/icepick-internal",
     "crates/by-chain/icepick-solana",
 ]
 

crates/builtins/icepick-internal/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "icepick-internal"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+icepick-module = { version = "0.1.0", path = "../../icepick-module" }
+serde.workspace = true
+serde_json.workspace = true
+thiserror = "2.0.9"

crates/builtins/icepick-internal/src/lib.rs

@@ -0,0 +1,105 @@
+use icepick_module::{
+    help::{Argument, ArgumentType},
+    Module,
+};
+use serde::{Deserialize, Serialize};
+use std::path::{Path, PathBuf};
+
+fn path_for_filename(filename: &Path) -> PathBuf {
+    PathBuf::from(
+        std::env::vars()
+            .find(|(k, _)| k == "ICEPICK_DATA_DIRECTORY")
+            .map(|(_, v)| v)
+            .as_deref()
+            .unwrap_or("/media/external"),
+    )
+    .join(filename)
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(tag = "operation", content = "values", rename_all = "kebab-case")]
+pub enum Request {
+    LoadFile {
+        filename: PathBuf,
+    },
+
+    SaveFile {
+        filename: PathBuf,
+
+        #[serde(flatten)]
+        values: serde_json::Value,
+    },
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {}
+
+pub struct Internal;
+
+impl Module for Internal {
+    type Error = Error;
+
+    type Request = Request;
+
+    fn describe_operations() -> Vec<icepick_module::help::Operation> {
+        let filename = Argument {
+            name: "filename".to_string(),
+            description: "The file to load or save data to.".to_string(),
+            r#type: ArgumentType::Required,
+        };
+        vec![
+            icepick_module::help::Operation {
+                name: "load-file".to_string(),
+                description: "Load data from a JSON file.".to_string(),
+                arguments: vec![filename.clone()],
+            },
+            icepick_module::help::Operation {
+                name: "save-file".to_string(),
+                description: "Save data from a JSON file.".to_string(),
+                arguments: vec![filename.clone()],
+            },
+        ]
+    }
+
+    fn handle_request(request: Self::Request) -> Result<serde_json::Value, Self::Error> {
+        match request {
+            Request::LoadFile { filename } => {
+                let path = path_for_filename(&filename);
+
+                let mut attempt = 0;
+                while !std::fs::exists(&path).is_ok_and(|v| v) {
+                    if attempt % 10 == 0 {
+                        eprintln!(
+                            "Waiting for {path} to be populated...",
+                            path = path.to_string_lossy()
+                        );
+                    }
+                    attempt += 1;
+                    std::thread::sleep(std::time::Duration::from_secs(1));
+                }
+
+                // if we ran at least once, we should have previously printed a message. write a
+                // confirmation that we are no longer waiting. if we haven't, we've never printed
+                // a message, therefore we don't need to confirm the prior reading.
+                if attempt > 0 {
+                    eprintln!("File contents loaded.");
+                }
+
+                let file = std::fs::File::open(path).unwrap();
+                let json: serde_json::Value = serde_json::from_reader(file).unwrap();
+                Ok(serde_json::json!({
+                    "blob": json,
+                }))
+            }
+            Request::SaveFile { filename, values } => {
+                let path = path_for_filename(&filename);
+                let file = std::fs::File::create(path).unwrap();
+                serde_json::to_writer(file, &values).unwrap();
+
+                Ok(serde_json::json!({
+                    "blob": {},
+                }))
+            }
+        }
+    }
+}

crates/builtins/icepick-internal/src/main.rs

@@ -0,0 +1,6 @@
+use icepick_module::Module;
+use icepick_internal::Internal;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    Internal::run_responder()
+}

crates/icepick/src/cli/mod.rs

@@ -59,12 +59,13 @@ struct ModuleConfig {
     /// The bip32 derivation algorithm. This is currently used for deriving keys from Keyfork, but
     /// may be passed to modules within the workflow to provide additional context, such as the
     /// algorithm for a generic signer.
+    #[serde(default)]
     algorithm: Option<DerivationAlgorithm>,
 
     /// The bip44 derivation prefix. This is currently used for deriving keys from Keyfork directly
     /// within Icepick, but may be passed to modules within the workflow to provide additional
     /// context, such as a module for deriving keys.
-    #[serde(with = "serde_derivation")]
+    #[serde(with = "serde_derivation", default)]
     derivation_prefix: Option<DerivationPath>,
 
     /// All workflows for a module.
@@ -121,7 +122,14 @@ pub fn do_cli_thing() {
     });
     let config_path = config_file.unwrap_or_else(|| "icepick.toml".to_string());
     let config_content = std::fs::read_to_string(config_path).expect("can't read config file");
-    let config: Config = toml::from_str(&config_content).expect("config file had invalid toml");
+    let mut config: Config = toml::from_str(&config_content).expect("config file had invalid toml");
+    config.modules.push(ModuleConfig {
+        name: "internal".to_string(),
+        command_name: Default::default(),
+        algorithm: Default::default(),
+        derivation_prefix: Default::default(),
+        workflows: Default::default(),
+    });
 
     let mut commands = vec![];
     let mut icepick_command = command!();

crates/icepick/src/cli/workflow.rs

@@ -198,14 +198,16 @@ impl Workflow {
                 .expect("could not find module config");
             let algo = &config.algorithm;
             let path_prefix = &config.derivation_prefix;
-            derived_keys.extend(derive_keys(
+            if !derivation_accounts.is_empty() {
-                algo.as_ref()
+                derived_keys.extend(derive_keys(
-                    .expect("a module requested keys but didn't provide algorithm"),
+                    algo.as_ref()
-                path_prefix
+                        .expect("a module requested keys but didn't provide algorithm"),
-                    .as_ref()
+                    path_prefix
-                    .expect("a module requested keys but didn't provide prefix"),
+                        .as_ref()
-                &derivation_accounts,
+                        .expect("a module requested keys but didn't provide prefix"),
-            ));
+                    &derivation_accounts,
+                ));
+            }
             derivation_accounts.clear();
 
             // Prepare all inputs for the operation invocation

e2e-tests/solana/base.Containerfile

@@ -44,8 +44,12 @@ RUN <<EOF
 cargo fetch --locked
 cargo build --frozen --release --target x86_64-unknown-linux-musl --bin icepick
 cargo build --frozen --release --target x86_64-unknown-linux-musl --bin icepick-sol
+cargo build --frozen --release --target x86_64-unknown-linux-musl --bin icepick-internal
 cp /app/target/x86_64-unknown-linux-musl/release/icepick /usr/bin
+cp /app/target/x86_64-unknown-linux-musl/release/icepick-internal /usr/bin
 cp /app/target/x86_64-unknown-linux-musl/release/icepick-sol /usr/bin
 EOF
 
+ENV ICEPICK_DATA_DIRECTORY=/data
+
 WORKDIR /

e2e-tests/solana/offline.sh

@@ -13,15 +13,11 @@ from_address="$(jq -r .from_address /data/input.json)"
 to_address="$(jq -r .to_address /data/input.json)"
 token_name="$(jq -r .token_name /data/input.json)"
 token_amount="$(jq -r .token_amount /data/input.json)"
-blockhash="$(jq -r .blockhash /data/input.json)"
-token_address="$(icepick sol get-token-info "$token_name" | jq -r .blob.token_address)"
-token_decimals="$(icepick sol get-token-info "$token_name" | jq -r .blob.token_decimals)"
 
 jq . /data/input.json
 echo "Do these values look correct? If not, press ctrl-c. Otherwise, press Enter."
-read _
+read -r _
 
 echo "Creating and signing transaction"
 
-icepick sol transfer-token "$token_amount" "$token_address" "$to_address" "$from_address" "$token_decimals" | icepick sol sign "$blockhash" > /data/output.json.tmp
+icepick workflow sol transfer-token --from-address "$from_address" --to-address "$to_address" --token-name "$token_name" --token-amount "$token_amount"
-mv /data/output.json.tmp /data/output.json

e2e-tests/solana/online.sh

@@ -1,32 +1,24 @@
 printf "%s" "Public key of the sender address: "
-read from_address
+read -r from_address
 
 printf "%s" "Public key of the recipient address: "
-read to_address
+read -r to_address
 
 printf "%s" "Name of the token to transfer: "
-read token_name
+read -r token_name
 
 printf "%s" "Amount of token to transfer: "
-read token_amount
+read -r token_amount
 
-echo "Acquiring blockhash..."
+echo "Saving inputs to file"
-blockhash="$(icepick sol get-blockhash --cluster devnet | jq -r .blob.blockhash)"
-
-echo "Saving information to file"
 
 cat <<EOF > /data/input.json
 {
 	"from_address": "$from_address",
 	"to_address": "$to_address",
 	"token_name": "$token_name",
-	"token_amount": "$token_amount",
+	"token_amount": "$token_amount"
-	"blockhash": "$blockhash"
 }
 EOF
 
-echo "Waiting for signed transaction..."
+icepick workflow sol broadcast --cluster devnet
-while test ! -f /data/output.json; do sleep 1; done
-
-echo "Broadcasting transaction"
-icepick sol broadcast --cluster devnet < /data/output.json | jq .

icepick.toml

@@ -13,18 +13,7 @@ name = "transfer-token"
 # be serialized by serde_json::Value.
 # These values can also be loaded using "internal-load-file", using some form
 # of later-defined signature validation.
-inputs = ["from_address", "to_address", "token_name", "token_amount", "cluster"]
+inputs = ["from_address", "to_address", "token_name", "token_amount"]
-
-## Load the Blockhash from the SD card
-#[[module.workflow.step]]
-#type = "internal-load-file"
-#
-## Pre-defined values to be passed to the module
-#values = { filename = "blockhash.json" }
-#
-## This value is marked to be saved in-memory, and can be used as an input for
-## later steps.
-#outputs = { blockhash = "blockhash" }
 
 # Get the token address and token decimals for the given token
 [[module.workflow.step]]
@@ -40,12 +29,15 @@ inputs = { token = "token_name" }
 # we want to store, and the value is the name to be assigned in storage.
 outputs = { token_address = "token_address", token_decimals = "token_decimals" }
 
-# Get a blockhash
+# Load the Blockhash from the SD card
 [[module.workflow.step]]
-type = "sol-get-blockhash"
+type = "internal-load-file"
 
-inputs = { cluster = "cluster" }
+# Pre-defined values to be passed to the module
+values = { filename = "blockhash.json" }
 
+# This value is marked to be saved in-memory, and can be used as an input for
+# later steps.
 outputs = { blockhash = "blockhash" }
 
 [[module.workflow.step]]
@@ -80,21 +72,38 @@ blockhash = "blockhash"
 [module.workflow.step.outputs]
 transaction = "signed_transaction"
 
-# Broadcast the transaction
+# Write the signed transaction to a file
+[[module.workflow.step]]
+type = "internal-save-file"
+
+# We are using a static filename here, so we use `values` instead of `inputs`.
+values = { filename = "transaction.json" }
+
+# All fields in both `inputs` and `values`, other than `filename`, will be
+# persisted to the file. In this case, the `transaction` field of the file will
+# contain the signed transaction.
+inputs = { transaction = "signed_transaction" }
+
+[[module.workflow]]
+name = "broadcast"
+inputs = ["cluster"]
+
+[[module.workflow.step]]
+type = "sol-get-blockhash"
+inputs = { cluster = "cluster" }
+outputs = { blockhash = "blockhash" }
+
+[[module.workflow.step]]
+type = "internal-save-file"
+values = { filename = "blockhash.json" }
+inputs = { blockhash = "blockhash" }
+
+[[module.workflow.step]]
+type = "internal-load-file"
+values = { filename = "transaction.json" }
+outputs = { transaction = "transaction" }
+
 [[module.workflow.step]]
 type = "sol-broadcast"
-
+inputs = { cluster = "cluster", transaction = "transaction" }
-inputs = { cluster = "cluster", transaction = "signed_transaction" }
 outputs = { status = "status", url = "url" }
-
-## Write the signed transaction to a file
-#[[module.workflow.step]]
-#type = "internal-save-file"
-#
-## We are using a static filename here, so we use `values` instead of `inputs`.
-#values = { filename = "transaction.json" }
-#
-## All fields in both `inputs` and `values`, other than `filename`, will be
-## persisted to the file. In this case, the `transaction` field of the file will
-## contain the signed transaction.
-#inputs = { transaction = "signed_transaction" }