From e64e466e521cc9d463af954752cb2fa38b05aabc Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Tue, 17 Dec 2024 17:51:00 -0600
Subject: [PATCH 01/91] auth update

---
 rust/clients/orchestrator/src/endpoints.rs |  40 ++++
 rust/clients/orchestrator/src/main.rs      | 135 +++++++++++
 rust/services/auth/Cargo.toml              |  23 ++
 rust/services/auth/src/endpoints.rs        | 251 +++++++++++++++++++++
 rust/services/auth/src/main.rs             | 110 +++++++++
 rust/services/auth/src/types.rs            |  13 ++
 6 files changed, 572 insertions(+)
 create mode 100644 rust/clients/orchestrator/src/endpoints.rs
 create mode 100644 rust/clients/orchestrator/src/main.rs
 create mode 100644 rust/services/auth/Cargo.toml
 create mode 100644 rust/services/auth/src/endpoints.rs
 create mode 100644 rust/services/auth/src/main.rs
 create mode 100644 rust/services/auth/src/types.rs

diff --git a/rust/clients/orchestrator/src/endpoints.rs b/rust/clients/orchestrator/src/endpoints.rs
new file mode 100644
index 0000000..c54b32e
--- /dev/null
+++ b/rust/clients/orchestrator/src/endpoints.rs
@@ -0,0 +1,40 @@
+use anyhow::Result;
+use async_nats::Message;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use util_libs::nats_client;
+use workload::api::WorkloadApi;
+
+/// TODO:
+pub async fn add_workload(workload_api: WorkloadApi) -> nats_client::AsyncEndpointHandler {
+    Arc::new(
+            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+                log::warn!("INCOMING Message for 'WORKLOAD.add' : {:?}", msg);
+                let db_api = workload_api.clone();
+                Box::pin(async move {
+                    db_api.add_workload(msg).await
+                })
+            },
+        )
+}
+
+/// TODO:
+pub async fn handle_db_change() ->  nats_client::AsyncEndpointHandler {
+    Arc::new(
+        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+            log::warn!("INCOMING Message for 'WORKLOAD.handle_change' : {:?}", msg);
+            Box::pin(async move {
+                // 1. Map over workload items in message and grab capacity requirements
+
+                // 2. Call mongodb to get host/node info and filter by capacity availability
+
+                // 3. Randomly choose host/node
+
+                // 4. Respond to endpoint request
+                let response = b"Successfully handled updated workload!".to_vec();
+                Ok(response)
+            })
+        },
+    )
+}
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
new file mode 100644
index 0000000..5a06298
--- /dev/null
+++ b/rust/clients/orchestrator/src/main.rs
@@ -0,0 +1,135 @@
+/*
+ This client is associated with the:
+- WORKLOAD account
+- orchestrator user
+
+// This client is responsible for:
+*/
+
+mod api;
+mod endpoints;
+use anyhow::Result;
+use dotenv::dotenv;
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use util_libs::{
+    db::{mongodb::get_mongodb_url, schemas},
+    js_microservice::JsStreamService,
+    nats_client::{self, EventListener},
+};
+
+const ORCHESTRATOR_CLIENT_NAME: &str = "Orchestrator Agent";
+const ORCHESTRATOR_CLIENT_INBOX_PREFIX: &str = "_orchestrator_inbox";
+
+#[tokio::main]
+async fn main() -> Result<(), async_nats::Error> {
+    dotenv().ok();
+    env_logger::init();
+
+    // ==================== NATS Setup ====================
+    let nats_url = nats_client::get_nats_url();
+    let creds_path = nats_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
+    let event_listeners = endpoints::get_orchestrator_workload_event_listeners();
+
+    let workload_service_inbox_prefix: &str = "_workload";
+
+    let workload_service = nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
+        nats_url,
+        name: WORKLOAD_SRV_OWNER_NAME.to_string(),
+        inbox_prefix: workload_service_inbox_prefix.to_string(),
+        opts: vec![nats_client::with_event_listeners(event_listeners)],
+        credentials_path: Some(creds_path),
+        ..Default::default()
+    })
+    .await?;
+
+    // Create a new Jetstream Microservice
+    let js_context = JsStreamService::get_context(workload_service.client.clone());
+    let js_service = JsStreamService::new(
+        js_context,
+        WORKLOAD_SRV_NAME,
+        WORKLOAD_SRV_DESC,
+        WORKLOAD_SRV_VERSION,
+        WORKLOAD_SRV_SUBJ,
+    )
+    .await?;
+
+    // ==================== DB Setup ====================
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
+
+    // // Create a typed collection for User
+    // let mut user_api = MongoCollection::<schemas::User>::new(
+    //     &client,
+    //     schemas::DATABASE_NAME,
+    //     schemas::HOST_COLLECTION_NAME,
+    // )
+    // .await?;
+
+    // // Create a typed collection for Host
+    // let mut host_api = MongoCollection::<schemas::Host>::new(
+    //     &client,
+    //     schemas::DATABASE_NAME,
+    //     schemas::HOST_COLLECTION_NAME,
+    // )
+    // .await?;
+
+    // Create a typed collection for Workload
+    let workload_api = api::WorkloadApi::new(&client).await?;
+
+    // ==================== API ENDPOINTS ====================
+
+    // For ORCHESTRATOR to consume
+    // (subjects should be published by developer)
+    js_service
+        .add_local_consumer(
+            "add_workload",
+            "add",
+            nats_client::EndpointType::Async(endpoints::add_workload(workload_api).await),
+            None,
+        )
+        .await?;
+
+    js_service
+        .add_local_consumer(
+            "handle_changed_db_workload",
+            "handle_change",
+            nats_client::EndpointType::Async(endpoints::handle_db_change().await),
+            None,
+        )
+        .await?;
+
+ 
+    log::trace!(
+        "{} Service is running. Waiting for requests...",
+        WORKLOAD_SRV_NAME
+    );
+
+    Ok(())
+}
+
+pub fn get_orchestrator_workload_event_listeners() -> Vec<EventListener> {
+    // TODO: Use duration in handlers..
+    let published_msg_handler = |msg: &str, _duration: Duration| {
+        log::info!(
+            "Successfully published message for {} client: {:?}",
+            WORKLOAD_SRV_OWNER_NAME,
+            msg
+        );
+    };
+    let failure_handler = |err: &str, _duration: Duration| {
+        log::error!(
+            "Failed to publish message for {} client: {:?}",
+            WORKLOAD_SRV_OWNER_NAME,
+            err
+        );
+    };
+
+    let event_listeners = vec![
+        nats_client::on_msg_published_event(published_msg_handler),
+        nats_client::on_msg_failed_event(failure_handler),
+    ];
+
+    event_listeners
+}
diff --git a/rust/services/auth/Cargo.toml b/rust/services/auth/Cargo.toml
new file mode 100644
index 0000000..bec8e4d
--- /dev/null
+++ b/rust/services/auth/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "auth"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+async-nats = { workspace = true }
+anyhow = { workspace = true }
+futures = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true } 
+env_logger = { workspace = true }
+log = { workspace = true }
+dotenv = { workspace = true }
+thiserror = "2.0"
+mongodb = "3.1"
+bson = { version = "2.6.1", features = ["chrono-0_4"] }
+url = { version = "2", features = ["serde"] }
+nkeys = "=0.4.4"
+bytes = "1.8.0"
+chrono = "0.4.0"
+util_libs = { path = "../../util_libs" }
\ No newline at end of file
diff --git a/rust/services/auth/src/endpoints.rs b/rust/services/auth/src/endpoints.rs
new file mode 100644
index 0000000..95f97ee
--- /dev/null
+++ b/rust/services/auth/src/endpoints.rs
@@ -0,0 +1,251 @@
+// use super::types;
+use super::nats_client::{self, EventListener};
+use super::AUTH_SRV_OWNER_NAME;
+
+use anyhow::{anyhow, Result};
+// use async_nats::HeaderValue;
+use async_nats::Message;
+use std::sync::Arc;
+use std::future::Future;
+use std::pin::Pin;
+use std::time::Duration;
+// use async_nats::jetstream::{Context};
+// use async_nats::jetstream::ErrorCode;
+// use async_nats::jetstream::consumer::Consumer;
+// use async_nats::jetstream::consumer::PullConsumer;
+// use async_nats::jetstream::consumer::pull::Stream;
+// // use std::io::Read;
+// use tokio::fs::OpenOptions;
+// use tokio::{fs::File, io::AsyncWriteExt};
+// use tokio::io;
+// use futures::future;
+// use futures::stream::{self, StreamExt};
+
+
+// NB: Message { subject, reply, payload, headers, status, description, length }
+
+// const CHUNK_SIZE: usize = 1024; // 1 KB chunks
+
+pub async fn start_handshake() ->  nats_client::AsyncEndpointHandler {
+    Arc::new(
+        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+            log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
+            let msg_clone = msg.clone();
+            Box::pin(async move {
+                // 1. Verify expected data was received
+                if msg_clone.headers.is_none() {
+                    log::error!("Error: Missing headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'");
+                    // anyhow!(ErrorCode::BAD_REQUEST)
+                }
+
+                // let signature = msg_clone.headers.unwrap().get("Signature").unwrap_or(&HeaderValue::new());
+
+                // match  serde_json::from_str::<types::AuthHeaders>(signature.as_str()) {
+                //     Ok(r) => {}
+                //     Err(e) => {
+                //         log::error!("Error: Failed to deserialize headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
+                //         // anyhow!(ErrorCode::BAD_REQUEST)
+                //     }
+                // }
+
+                // match serde_json::from_slice::<types::AuthPayload>(msg.payload.as_ref()) {
+                //     Ok(r) => {}
+                //     Err(e) => {
+                //         log::error!("Error: Failed to deserialize payload. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
+                //         // anyhow!(ErrorCode::BAD_REQUEST)
+                //     }
+                // }
+
+                // 2. Authenticate the HPOS client(?via email and host id info?)
+
+                // 3. Publish operator and sys account jwts for orchestrator
+                // let hub_operator_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
+                // let hub_sys_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
+
+                let response = serde_json::to_vec(&"OK")?;
+                Ok(response)
+            })
+        },
+    )
+}
+
+pub async fn save_hub_auth() ->  nats_client::AsyncEndpointHandler {
+    Arc::new(
+        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+            log::warn!("INCOMING Message for 'AUTH.save_hub_auth' : {:?}", msg);
+            Box::pin(async move {
+                // receive_and_write_file();
+
+                // Respond to endpoint request
+                // let response = b"Hello, NATS!".to_vec();
+                // Ok(response)
+                
+                todo!();
+            })
+        },
+    )
+}
+
+pub async fn send_user_pubkey() ->  nats_client::AsyncEndpointHandler {
+    Arc::new(
+        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+            log::warn!("INCOMING Message for 'AUTH.send_user_pubkey' : {:?}", msg);
+            Box::pin(async move {
+                // 1. validate nk key...
+                // let auth_endpoint_subject =
+                // format!("AUTH.{}.file.transfer.JWT-operator", "host_id_placeholder"); // endpoint_subject
+            
+                // 2. Update the hub nsc with user pubkey
+                
+                // 3. create signed jwt
+            
+                // 4. `Ack last request and publish the new jwt to for hpos
+            
+                // 5. Respond to endpoint request
+                // let response = b"Hello, NATS!".to_vec();
+                // Ok(response)
+
+                todo!()
+            })
+        },
+    )
+}
+
+
+    // let auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", host_id); // endpoint_subject
+    // let c = js_service
+    //     .add_local_consumer(
+    //         "save_user_auth", // called from orchestrator (no auth service)
+    //         "save_user_auth",
+    //         EndpointType::Async(Arc::new(
+    //             async |msg: &Message| -> Result<Vec<u8>, anyhow::Error> {
+    //                 log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+    //                 receive_and_write_file()
+
+    //                 // 2. Respond to endpoint request
+    //                 let response = b"Hello, NATS!".to_vec();
+    //                 Ok(response)
+    //             },
+    //         )),
+    //         None,
+    //     )
+    //     .await?;
+
+    // let c = js_service
+    //     .add_local_consumer(
+    //         "send_user_file", //  called from orchestrator (no auth service)
+    //         "send_user_file",
+    //         EndpointType::Async(Arc::new(
+    //             async |msg: &Message| -> Result<Vec<u8>, anyhow::Error> {
+    //                 log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+    //                 receive_and_write_file()
+
+    //                 // 2. Respond to endpoint request
+    //                 let response = b"Hello, NATS!".to_vec();
+    //                 Ok(response)
+    //             },
+    //         )),
+    //         None,
+    //     )
+    //     .await?;
+
+    //     let c = js_service
+    //     .add_local_consumer(
+    //         "save_user_file", // called from hpos
+    //         "end_hub_handshake",
+    //         EndpointType::Async(Arc::new(
+    //             async |msg: &Message| -> Result<Vec<u8>, anyhow::Error> {
+    //                 log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+    //                 receive_and_write_file()
+
+    //                 // 2. Respond to endpoint request
+    //                 let response = b"Hello, NATS!".to_vec();
+    //                 Ok(response)
+    //             },
+    //         )),
+    //         None,
+    //     )
+    //     .await?;
+
+
+
+// ==================== Helpers ====================
+
+pub fn get_event_listeners() -> Vec<EventListener> {
+    // TODO: Use duration in handlers..
+    let published_msg_handler = |msg: &str, _duration: Duration| {
+        log::info!(
+            "Successfully published message for {} client: {:?}",
+            AUTH_SRV_OWNER_NAME,
+            msg
+        );
+    };
+    let failure_handler = |err: &str, _duration: Duration| {
+        log::error!(
+            "Failed to publish message for {} client: {:?}",
+            AUTH_SRV_OWNER_NAME,
+            err
+        );
+    };
+
+    let event_listeners = vec![
+        nats_client::on_msg_published_event(published_msg_handler),
+        nats_client::on_msg_failed_event(failure_handler),
+    ];
+
+    event_listeners
+}
+
+
+// async fn chunk_file_and_publish(js: &Context, subject: &str, file_path: &str) -> io::Result<()> {
+//     let mut file = std::fs::File::open(file_path)?;
+//     let mut buffer = vec![0; CHUNK_SIZE];
+//     let mut chunk_id = 0;
+
+//     while let Ok(bytes_read) = file.read(mut buffer) {
+//         if bytes_read == 0 {
+//             break;
+//         }
+//         let chunk_data = &buffer[..bytes_read];
+//         js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
+//         chunk_id += 1;
+//     }
+
+//     // Send an EOF marker
+//     js.publish(subject.to_string(), "EOF".into()).await.unwrap();
+
+//     Ok(())
+// }
+
+// async fn receive_and_write_file(stream: Stream, consumer: PullConsumer, header_subject: String, output_dir: &str, file_name: &str) -> Result<(), std::io::Error> {
+//     let output_path = format!("{}/{}", output_dir, file_name);
+//     let mut file = OpenOptions::new().create(true).append(true).open(&output_path)?;
+
+//     let mut messages = consumer
+//         .stream()
+//         .max_messages_per_batch(100)
+//         .max_bytes_per_batch(1024)
+//         .heartbeat(std::time::Duration::from_secs(10))
+//         .messages()
+//         .await?;
+
+//     // while let Some(Ok(message)) = message.next().await {}
+//     let file_msgs = messages().take_while(|msg| future::ready(*msg.subject.contains(file_name)));
+//     while let Some(Ok(msg)) = file_msgs.next().await {
+//         if msg.payload.to_string().contains("EOF") {
+//             // if msg.payload == b"EOF" {
+//             msg.ack().await?;
+//             println!("File transfer complete.");
+//             return Ok(());
+//         }
+
+//         file.write_all(&msg.payload).await?;
+//         file.flush().await?;
+//         msg.ack().await?;
+//     }
+
+//     Ok(())
+// }
\ No newline at end of file
diff --git a/rust/services/auth/src/main.rs b/rust/services/auth/src/main.rs
new file mode 100644
index 0000000..40c1d0c
--- /dev/null
+++ b/rust/services/auth/src/main.rs
@@ -0,0 +1,110 @@
+/*
+Service Name: AUTH
+Subject: "AUTH.>"
+Provisioning Account: AUTH Account
+Importing Account: Auth/NoAuth Account
+
+This service should be run on the ORCHESTRATOR side and called from the HPOS side.
+The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
+NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
+This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
+
+Endpoints & Managed Subjects:
+    - start_hub_handshake
+    - end_hub_handshake
+    - save_hub_auth
+    - save_user_auth
+
+*/
+
+mod endpoints;
+mod types;
+use anyhow::Result;
+use async_nats::Message;
+use bytes::Bytes;
+use dotenv::dotenv;
+use futures::StreamExt;
+use std::sync::Arc;
+use std::time::Duration;
+use util_libs::{
+    js_microservice::JsStreamService,
+    nats_client::{self, EndpointType},
+};
+
+const AUTH_SRV_OWNER_NAME: &str = "AUTH_OWNER";
+const AUTH_SRV_NAME: &str = "AUTH";
+const AUTH_SRV_SUBJ: &str = "AUTH";
+const AUTH_SRV_VERSION: &str = "0.0.1";
+const AUTH_SRV_DESC: &str =
+    "This service handles the Authentication flow the HPOS and the Orchestrator.";
+
+#[tokio::main]
+async fn main() -> Result<(), async_nats::Error> {
+    dotenv().ok();
+    env_logger::init();
+
+    // ==================== NATS Setup ====================
+
+    let nats_url = nats_client::get_nats_url();
+    let creds_path = nats_client::get_nats_client_creds("HOLO", "ADMIN", "orchestrator");
+    let event_listeners = endpoints::get_event_listeners();
+
+    let auth_service_inbox_prefix: &str = "_auth";
+
+    let auth_service = nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
+        nats_url,
+        name: AUTH_SRV_OWNER_NAME.to_string(),
+        inbox_prefix: auth_service_inbox_prefix.to_string(),
+        opts: vec![nats_client::with_event_listeners(event_listeners)],
+        credentials_path: Some(creds_path),
+        ..Default::default()
+    })
+    .await?;
+
+    // Create a new Jetstream Microservice
+    let js_context = JsStreamService::get_context(auth_service.client.clone());
+    let js_service = JsStreamService::new(
+        js_context,
+        AUTH_SRV_NAME,
+        AUTH_SRV_DESC,
+        AUTH_SRV_VERSION,
+        AUTH_SRV_SUBJ,
+    )
+    .await?;
+
+    // ==================== API ENDPOINTS ====================
+
+    js_service
+        .add_local_consumer(
+            "publish_hub_files", // called from hpos
+            "start_hub_handshake",
+            nats_client::EndpointType::Async(endpoints::start_handshake().await),
+            None,
+        )
+        .await?;
+
+    js_service
+        .add_local_consumer(
+            "save_hub_auth", // called from hpos
+            "save_hub_auth",
+            nats_client::EndpointType::Async(endpoints::save_hub_auth().await),
+            None,
+        )
+        .await?;
+
+    js_service
+        .add_local_consumer(
+            "send_user_pubkey", // called from hpos
+            "send_user_pubkey",
+            nats_client::EndpointType::Async(endpoints::send_user_pubkey().await),
+            None,
+        )
+        .await?;
+
+    log::trace!(
+        "{} Service is running. Waiting for requests...",
+        AUTH_SRV_NAME
+    );
+
+    Ok(())
+}
diff --git a/rust/services/auth/src/types.rs b/rust/services/auth/src/types.rs
new file mode 100644
index 0000000..266b02c
--- /dev/null
+++ b/rust/services/auth/src/types.rs
@@ -0,0 +1,13 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default)]
+pub struct AuthHeaders {
+    signature: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default)]
+pub struct AuthPayload {
+    email: String,
+    host_id: String,
+    pubkey: String,
+}

From 271432b3b2331f883095bfb19adbf8f8b8045748 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Tue, 17 Dec 2024 22:16:23 -0600
Subject: [PATCH 02/91] add client auth

---
 .../host_agent/src/auth/initializer.rs        | 108 +++++++++++++++++
 rust/clients/host_agent/src/auth/key_utils.rs | 111 ++++++++++++++++++
 rust/clients/host_agent/src/auth/mod.rs       |   2 +
 3 files changed, 221 insertions(+)
 create mode 100644 rust/clients/host_agent/src/auth/initializer.rs
 create mode 100644 rust/clients/host_agent/src/auth/key_utils.rs
 create mode 100644 rust/clients/host_agent/src/auth/mod.rs

diff --git a/rust/clients/host_agent/src/auth/initializer.rs b/rust/clients/host_agent/src/auth/initializer.rs
new file mode 100644
index 0000000..eb12877
--- /dev/null
+++ b/rust/clients/host_agent/src/auth/initializer.rs
@@ -0,0 +1,108 @@
+/*
+ This client is associated with the:
+- ADMIN account
+- noauth user
+
+...once this the host and hoster are validated, this client should close and the hpos manager should spin up.
+
+// This client is responsible for:
+1. generating new key / re-using the user key from provided file
+2. calling the auth service to:
+  - verify host/hoster via `auth/start_hub_handshake` call
+  - get hub operator jwt and hub sys account jwt via `auth/start_hub_handshake`
+  - send "nkey" version of pubkey as file to hub via via `auth/end_hub_handshake`
+  - get user jwt from hub via `auth/save_`
+3. create user creds file with file path
+4. instantiate the leaf server via the leaf-server struct/service
+*/
+
+use anyhow::Result;
+// use auth::AUTH_SRV_NAME;
+use std::time::Duration;
+use util_libs::nats_client::{self, Client as NatClient, EventListener};
+
+const HOST_INIT_CLIENT_NAME: &str = "Host Initializer";
+const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_init_inbox";
+
+pub async fn run() -> Result<String, async_nats::Error> {
+    log::info!("Host Initializer Client: Connecting to server...");
+    // 1. Connect to Nats server
+    let nats_url = nats_client::get_nats_url();
+    let event_listeners = get_init_host_event_listeners();
+
+    let init_host_client = nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
+        nats_url,
+        name: HOST_INIT_CLIENT_NAME.to_string(),
+        inbox_prefix: HOST_INIT_CLIENT_INBOX_PREFIX.to_string(),
+        credentials_path: None,
+        opts: vec![nats_client::with_event_listeners(event_listeners)],
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(5)),
+    })
+    .await?;
+
+    // Discover the server Node ID via INFO response
+    let server_node_id = init_host_client.client.server_info().server_id;
+    log::trace!(
+        "Host Initializer Client: Retrieved Node ID: {}",
+        server_node_id
+    );
+
+    // Publish a message with the Node ID as part of the subject
+    let publish_options = nats_client::PublishOptions {
+        subject: format!("HPOS.init.{}", server_node_id),
+        msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
+        data: b"Host Initializer Connected!".to_vec(),
+    };
+
+    match init_host_client
+        .publish_with_retry(&publish_options, 3)
+        .await
+    {
+        Ok(_r) => {
+            log::trace!("Host Initializer Client: Node ID published.");
+        }
+        Err(_e) => {}
+    };
+
+    // Call auth service and preform handshake
+    // let auth_service = init_host_client.get_stream(AUTH_SRV_NAME).await?;
+    // i. call `save_hub_auth`
+    // ii. call `start_hub_handshake`
+    // iii. THEN (once get resp from start_handshake) `send_user_pubkey`
+    // iv. call`end_hub_handshake`
+    // v. call save_user_file``
+
+    // 5. Create creds
+    let user_creds_path = "/path/to/host/user.creds".to_string();
+
+    // 6. Close and drain internal buffer before exiting to make sure all messages are sent
+    init_host_client.close().await?;
+
+    Ok(user_creds_path)
+}
+
+pub fn get_init_host_event_listeners() -> Vec<EventListener> {
+    // TODO: Use duration in handlers..
+    let published_msg_handler = |msg: &str, _duration: Duration| {
+        log::info!(
+            "Successfully published message for {} client: {:?}",
+            HOST_INIT_CLIENT_NAME,
+            msg
+        );
+    };
+    let failure_handler = |err: &str, _duration: Duration| {
+        log::error!(
+            "Failed to publish message for {} client: {:?}",
+            HOST_INIT_CLIENT_NAME,
+            err
+        );
+    };
+
+    let event_listeners = vec![
+        nats_client::on_msg_published_event(published_msg_handler),
+        nats_client::on_msg_failed_event(failure_handler),
+    ];
+
+    event_listeners
+}
diff --git a/rust/clients/host_agent/src/auth/key_utils.rs b/rust/clients/host_agent/src/auth/key_utils.rs
new file mode 100644
index 0000000..469cbb8
--- /dev/null
+++ b/rust/clients/host_agent/src/auth/key_utils.rs
@@ -0,0 +1,111 @@
+// use std::sync::mpsc::{self, Receiver, Sender};
+// use std::thread;
+// use std::time::Duration;
+
+// fn obtain_authorization_token() -> String {
+//     // whatever you want, 3rd party token/username&password
+//     String::new()
+// }
+
+// fn is_token_authorized(token: &str) -> bool {
+//     // whatever logic to determine if the input authorizes the requester to obtain a user jwt
+//     token.is_empty()
+// }
+
+// // request struct to exchange data
+// struct UserRequest {
+//     user_jwt_response_chan: Sender<String>,
+//     user_public_key: String,
+//     auth_info: String,
+// }
+
+// fn start_user_provisioning_service(is_authorized_cb: fn(&str) -> bool) -> Receiver<UserRequest> {
+//     let (user_request_chan, user_request_receiver): (Sender<UserRequest>, Receiver<UserRequest>) =
+//         mpsc::channel();
+
+//     thread::spawn(move || {
+//         let account_signing_key = get_account_signing_key(); // Setup, obtain account signing key
+//         loop {
+//             if let Ok(req) = user_request_receiver.recv() {
+//                 // receive request
+//                 if !is_authorized_cb(&req.auth_info) {
+//                     println!("Request is not authorized to receive a JWT, timeout on purpose");
+//                 } else if let Some(user_jwt) =
+//                     generate_user_jwt(&req.user_public_key, &account_signing_key)
+//                 {
+//                     let _ = req.user_jwt_response_chan.send(user_jwt); // respond with jwt
+//                 }
+//             }
+//         }
+//     });
+
+//     user_request_chan
+// }
+
+// fn start_user_process(
+//     user_request_chan: Receiver<UserRequest>,
+//     obtain_authorization_cb: fn() -> String,
+// ) {
+//     let request_user = |user_request_chan: Receiver<UserRequest>, auth_info: String| {
+//         let (resp_chan, resp_receiver): (Sender<String>, Receiver<String>) = mpsc::channel();
+//         let (user_public_key, _, user_key_pair) = generate_user_key();
+
+//         // request jwt
+//         let _ = user_request_chan.send(UserRequest {
+//             user_jwt_response_chan: resp_chan,
+//             user_public_key,
+//             auth_info,
+//         });
+
+//         let user_jwt = resp_receiver.recv().unwrap(); // wait for response
+//                                                       // user_jwt and user_key_pair can be used in conjunction with this nats.Option
+//         let jwt_auth_option = nats::UserJWT::new(
+//             move || Ok(user_jwt.clone()),
+//             move |bytes| user_key_pair.sign(bytes),
+//         );
+
+//         // Alternatively you can create a creds file and use it as nats.Option
+//         jwt_auth_option
+//     };
+
+//     thread::spawn(move || {
+//         let jwt_auth_option = request_user(user_request_chan, obtain_authorization_cb());
+//         let nc = nats::connect("nats://localhost:4111", jwt_auth_option).unwrap();
+//         // simulate work one would want to do
+//         thread::sleep(Duration::from_secs(1));
+//     });
+// }
+
+// fn request_user_distributed() {
+//     let req_chan = start_user_provisioning_service(is_token_authorized);
+//     // start multiple user processes
+//     for _ in 0..4 {
+//         start_user_process(req_chan.clone(), obtain_authorization_token);
+//     }
+//     thread::sleep(Duration::from_secs(5));
+// }
+
+// // Placeholder functions for the missing implementations
+// fn get_account_signing_key() -> String {
+//     // Implementation here
+//     String::new()
+// }
+
+// fn generate_user_jwt(user_public_key: &str, account_signing_key: &str) -> Option<String> {
+//     // Implementation here
+//     Some(String::new())
+// }
+
+// fn generate_user_key() -> (String, String, UserKeyPair) {
+//     // Implementation here
+//     (String::new(), String::new(), UserKeyPair {})
+// }
+
+// struct UserKeyPair;
+
+// impl UserKeyPair {
+//     fn sign(&self, _bytes: &[u8]) -> Result<Vec<u8>, ()> {
+//         // Implementation here
+//         Ok(vec![])
+//     }
+// }
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
new file mode 100644
index 0000000..1551fc5
--- /dev/null
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -0,0 +1,2 @@
+pub mod initializer;
+pub mod key_utils;

From 23f981d08103160af93fd9eb56815da2b5af8abb Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Tue, 17 Dec 2024 22:44:46 -0600
Subject: [PATCH 03/91] remove node type name alias

---
 rust/util_libs/src/db/schemas.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index fbe857c..f848e5a 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -101,9 +101,6 @@ pub struct VM {
 // Provide type Alias for HosterPubKey
 pub use String as HosterPubKey;
 
-// Provide type Alias for Host
-pub use Host as Node;
-
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct Host {
     pub _id: String,       // Mongodb ID (automated default)

From e8ad6e957d2ac5525a79496bd1c07c63d408b9d1 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Thu, 19 Dec 2024 00:54:42 -0600
Subject: [PATCH 04/91] add auth structure

---
 Cargo.lock                                    |  54 +++-
 Cargo.toml                                    |   6 +-
 rust/clients/host_agent/Cargo.toml            |   5 +-
 .../src/auth/{key_utils.rs => agent_key.rs}   |   0
 rust/clients/host_agent/src/auth/endpoints.rs |  34 +++
 .../host_agent/src/auth/initializer.rs        | 183 +++++++----
 rust/clients/host_agent/src/auth/mod.rs       |   3 +-
 rust/clients/host_agent/src/main.rs           |   8 +-
 rust/clients/host_agent/src/utils.rs          |  40 +++
 .../host_agent/src/workloads/endpoints.rs     |  10 +-
 .../host_agent/src/workloads/manager.rs       |  92 +++---
 rust/clients/orchestrator/Cargo.toml          |  26 ++
 .../orchestrator/src/auth/controller.rs       | 115 +++++++
 .../orchestrator/src/auth/endpoints.rs        |  19 ++
 rust/clients/orchestrator/src/auth/mod.rs     |   2 +
 rust/clients/orchestrator/src/main.rs         | 120 +-------
 rust/clients/orchestrator/src/utils.rs        |  51 ++++
 .../orchestrator/src/workloads/controller.rs  |  89 ++++++
 .../src/{ => workloads}/endpoints.rs          |   8 +-
 .../clients/orchestrator/src/workloads/mod.rs |   2 +
 rust/services/auth/src/endpoints.rs           | 251 ---------------
 rust/services/auth/src/main.rs                | 110 -------
 .../{auth => authentication}/Cargo.toml       |   4 +-
 rust/services/authentication/src/lib.rs       | 212 +++++++++++++
 .../{auth => authentication}/src/types.rs     |   0
 rust/services/authentication/src/utils.rs     |  30 ++
 rust/services/workload/Cargo.toml             |   2 +-
 rust/services/workload/src/lib.rs             |  57 ++--
 ...s_microservice.rs => js_stream_service.rs} |  61 ++--
 rust/util_libs/src/lib.rs                     |   4 +-
 .../src/{nats_client.rs => nats_js_client.rs} | 285 ++++++++----------
 31 files changed, 1079 insertions(+), 804 deletions(-)
 rename rust/clients/host_agent/src/auth/{key_utils.rs => agent_key.rs} (100%)
 create mode 100644 rust/clients/host_agent/src/auth/endpoints.rs
 create mode 100644 rust/clients/host_agent/src/utils.rs
 create mode 100644 rust/clients/orchestrator/Cargo.toml
 create mode 100644 rust/clients/orchestrator/src/auth/controller.rs
 create mode 100644 rust/clients/orchestrator/src/auth/endpoints.rs
 create mode 100644 rust/clients/orchestrator/src/auth/mod.rs
 create mode 100644 rust/clients/orchestrator/src/utils.rs
 create mode 100644 rust/clients/orchestrator/src/workloads/controller.rs
 rename rust/clients/orchestrator/src/{ => workloads}/endpoints.rs (83%)
 create mode 100644 rust/clients/orchestrator/src/workloads/mod.rs
 delete mode 100644 rust/services/auth/src/endpoints.rs
 delete mode 100644 rust/services/auth/src/main.rs
 rename rust/services/{auth => authentication}/Cargo.toml (92%)
 create mode 100644 rust/services/authentication/src/lib.rs
 rename rust/services/{auth => authentication}/src/types.rs (100%)
 create mode 100644 rust/services/authentication/src/utils.rs
 rename rust/util_libs/src/{js_microservice.rs => js_stream_service.rs} (94%)
 rename rust/util_libs/src/{nats_client.rs => nats_js_client.rs} (61%)

diff --git a/Cargo.lock b/Cargo.lock
index f09a986..2934ddc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -156,6 +156,29 @@ dependencies = [
  "syn 2.0.90",
 ]
 
+[[package]]
+name = "authentication"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "async-nats",
+ "bson",
+ "bytes",
+ "chrono",
+ "dotenv",
+ "env_logger",
+ "futures",
+ "log",
+ "mongodb",
+ "nkeys",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.7",
+ "tokio",
+ "url",
+ "util_libs",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.4.0"
@@ -805,10 +828,11 @@ dependencies = [
 
 [[package]]
 name = "host_agent"
-version = "0.1.0"
+version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
+ "authentication",
  "bson",
  "bytes",
  "chrono",
@@ -1309,6 +1333,32 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "orchestrator"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "async-nats",
+ "authentication",
+ "bson",
+ "bytes",
+ "chrono",
+ "dotenv",
+ "env_logger",
+ "futures",
+ "log",
+ "mongodb",
+ "nkeys",
+ "rand",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.7",
+ "tokio",
+ "url",
+ "util_libs",
+ "workload",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.3"
@@ -2652,7 +2702,7 @@ dependencies = [
 
 [[package]]
 name = "workload"
-version = "0.1.0"
+version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
diff --git a/Cargo.toml b/Cargo.toml
index 9f712bf..db731bc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,12 +1,14 @@
 [workspace]
-version = "0.1.0"
+version = "0.0.1"
 edition = "2021"
 resolver = "2"
 
 members = [
-    "rust/util_libs",
     "rust/clients/host_agent",
+    "rust/clients/orchestrator",
+    "rust/services/authentication",
     "rust/services/workload",
+    "rust/util_libs",
 ]
 
 [workspace.dependencies]
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index ed50e49..54d160a 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "host_agent"
-version = "0.1.0"
+version = "0.0.1"
 edition = "2021"
 
 [dependencies]
@@ -22,4 +22,5 @@ bytes = "1.8.0"
 nkeys = "=0.4.4"
 rand = "0.8.5"
 util_libs = { path = "../../util_libs" }
-workload = { path = "../../services/workload" }
\ No newline at end of file
+workload = { path = "../../services/workload" }
+authentication = { path = "../../services/authentication" }
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/auth/key_utils.rs b/rust/clients/host_agent/src/auth/agent_key.rs
similarity index 100%
rename from rust/clients/host_agent/src/auth/key_utils.rs
rename to rust/clients/host_agent/src/auth/agent_key.rs
diff --git a/rust/clients/host_agent/src/auth/endpoints.rs b/rust/clients/host_agent/src/auth/endpoints.rs
new file mode 100644
index 0000000..1124ec6
--- /dev/null
+++ b/rust/clients/host_agent/src/auth/endpoints.rs
@@ -0,0 +1,34 @@
+use anyhow::Result;
+use async_nats::Message;
+use authentication::AuthApi;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use util_libs::nats_js_client;
+
+const USER_CREDENTIALS_PATH: &str = "./user_creds";
+
+pub async fn save_user_jwt(auth_api: &AuthApi) -> nats_js_client::AsyncEndpointHandler {
+    let api = auth_api.to_owned();
+    // let user_name_clone = user_name.clone();
+    Arc::new(
+            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+                let api_clone = api.clone();
+                Box::pin(async move {
+                    api_clone.save_user_jwt(msg, USER_CREDENTIALS_PATH).await
+                })
+            },
+        )
+}
+
+pub async fn save_hub_jwts(auth_api: &AuthApi) -> nats_js_client::AsyncEndpointHandler {
+    let api = auth_api.to_owned();
+    Arc::new(
+            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+                let api_clone = api.clone();
+                Box::pin(async move {
+                    api_clone.save_hub_jwts(msg).await
+                })
+            },
+        )
+}
diff --git a/rust/clients/host_agent/src/auth/initializer.rs b/rust/clients/host_agent/src/auth/initializer.rs
index eb12877..3ffc1c5 100644
--- a/rust/clients/host_agent/src/auth/initializer.rs
+++ b/rust/clients/host_agent/src/auth/initializer.rs
@@ -16,46 +16,77 @@
 4. instantiate the leaf server via the leaf-server struct/service
 */
 
-use anyhow::Result;
-// use auth::AUTH_SRV_NAME;
+use super::endpoints;
+use crate::utils;
+use anyhow::{anyhow, Result};
+use authentication::{AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::time::Duration;
-use util_libs::nats_client::{self, Client as NatClient, EventListener};
+use util_libs::{
+    db::mongodb::get_mongodb_url,
+    js_stream_service::{JsServiceParamsPartial, JsStreamService},
+    nats_js_client::{self, EndpointType, EventListener, JsClient},
+};
 
-const HOST_INIT_CLIENT_NAME: &str = "Host Initializer";
-const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_init_inbox";
+pub const HOST_INIT_CLIENT_NAME: &str = "Host Initializer";
+pub const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_init_inbox";
 
 pub async fn run() -> Result<String, async_nats::Error> {
     log::info!("Host Initializer Client: Connecting to server...");
-    // 1. Connect to Nats server
-    let nats_url = nats_client::get_nats_url();
-    let event_listeners = get_init_host_event_listeners();
-
-    let init_host_client = nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
-        nats_url,
-        name: HOST_INIT_CLIENT_NAME.to_string(),
-        inbox_prefix: HOST_INIT_CLIENT_INBOX_PREFIX.to_string(),
-        credentials_path: None,
-        opts: vec![nats_client::with_event_listeners(event_listeners)],
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(5)),
-    })
-    .await?;
+
+    // ==================== NATS Setup ====================
+    // Connect to Nats server
+    let nats_url = nats_js_client::get_nats_url();
+    let event_listeners = nats_js_client::get_event_listeners();
+
+    // Setup JS Stream Service
+    let auth_stream_service_params = JsServiceParamsPartial {
+        name: AUTH_SRV_NAME.to_string(),
+        description: AUTH_SRV_DESC.to_string(),
+        version: AUTH_SRV_VERSION.to_string(),
+        service_subject: AUTH_SRV_SUBJ.to_string(),
+    };
+
+    let initializer_client =
+        nats_js_client::DefaultJsClient::new(nats_js_client::NewDefaultJsClientParams {
+            nats_url,
+            name: HOST_INIT_CLIENT_NAME.to_string(),
+            inbox_prefix: HOST_INIT_CLIENT_INBOX_PREFIX.to_string(),
+            credentials_path: None,
+            service_params: vec![auth_stream_service_params],
+            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            ping_interval: Some(Duration::from_secs(10)),
+            request_timeout: Some(Duration::from_secs(5)),
+        })
+        .await?;
+
+    // ==================== DB Setup ====================
+
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
+
+    // Generate the Auth API with access to db
+    let auth_api = AuthApi::new(&client).await?;
+
+    // ==================== Report Host to Orchestator ====================
 
     // Discover the server Node ID via INFO response
-    let server_node_id = init_host_client.client.server_info().server_id;
+    let server_node_id = initializer_client.get_server_info().server_id;
     log::trace!(
         "Host Initializer Client: Retrieved Node ID: {}",
         server_node_id
     );
 
     // Publish a message with the Node ID as part of the subject
-    let publish_options = nats_client::PublishOptions {
+    let publish_options = nats_js_client::PublishOptions {
         subject: format!("HPOS.init.{}", server_node_id),
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
         data: b"Host Initializer Connected!".to_vec(),
     };
 
-    match init_host_client
+    match initializer_client
         .publish_with_retry(&publish_options, 3)
         .await
     {
@@ -65,44 +96,82 @@ pub async fn run() -> Result<String, async_nats::Error> {
         Err(_e) => {}
     };
 
-    // Call auth service and preform handshake
-    // let auth_service = init_host_client.get_stream(AUTH_SRV_NAME).await?;
-    // i. call `save_hub_auth`
-    // ii. call `start_hub_handshake`
-    // iii. THEN (once get resp from start_handshake) `send_user_pubkey`
-    // iv. call`end_hub_handshake`
-    // v. call save_user_file``
-
-    // 5. Create creds
-    let user_creds_path = "/path/to/host/user.creds".to_string();
-
-    // 6. Close and drain internal buffer before exiting to make sure all messages are sent
-    init_host_client.close().await?;
-
-    Ok(user_creds_path)
-}
+    // ==================== API ENDPOINTS ====================
+    // Register Workload Streams for Host Agent to consume
+    // (subjects should be published by orchestrator or nats-db-connector)
 
-pub fn get_init_host_event_listeners() -> Vec<EventListener> {
-    // TODO: Use duration in handlers..
-    let published_msg_handler = |msg: &str, _duration: Duration| {
-        log::info!(
-            "Successfully published message for {} client: {:?}",
-            HOST_INIT_CLIENT_NAME,
-            msg
-        );
+    // Call auth service and perform auth handshake
+    let auth_service = initializer_client
+        .get_js_service(AUTH_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate workload service. Unable to spin up Host Agent."
+        ))?;
+
+    // i. register `save_hub_auth` consumer
+    // ii. register `save_user_file` consumer
+    // iii. send req for `` /eg:`start_hub_handshake`
+    // iv. THEN (on get resp from start_handshake) `send_user_pubkey`
+
+    // 1. make req (with agent key & email & nonce in payload, & sig in headers)
+    // to receive_handhake_request
+    // then await the reply (which should include the hub jwts)
+
+    // 2. make req (with agent key as payload)
+    // to add_user_pubkey
+    // then await the reply (which should include the user jwt)
+
+    // register save service for hub auth files (operator and sys)
+    auth_service
+        .add_local_consumer(
+            "save_hub_auth",
+            "save_hub_auth",
+            nats_js_client::EndpointType::Async(endpoints::save_hub_jwts(&auth_api).await),
+            None,
+        )
+        .await?;
+
+    // register save service for signed user jwt file
+    auth_service
+        .add_local_consumer(
+            "save_user_file",
+            "end_hub_handshake",
+            EndpointType::Async(endpoints::save_user_jwt(&auth_api).await),
+            None,
+        )
+        .await?;
+
+    // Send the request (with payload) for the hub auth files and await a response
+    // match client.request(subject, payload.into()).await {
+    //     Ok(response) => {
+    //         let response_str = String::from_utf8_lossy(&response.payload);
+    //         println!("Received response: {}", response_str);
+    //     }
+    //     Err(e) => {
+    //         println!("Failed to get a response: {}", e);
+    //     }
+    // }
+    let req_hub_files_options = nats_js_client::PublishOptions {
+        subject: format!("HPOS.init.{}", server_node_id),
+        msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
+        data: b"Host Initializer Connected!".to_vec(),
     };
-    let failure_handler = |err: &str, _duration: Duration| {
-        log::error!(
-            "Failed to publish message for {} client: {:?}",
-            HOST_INIT_CLIENT_NAME,
-            err
-        );
+    initializer_client.publish(&req_hub_files_options);
+
+    // ...upon the reply to the above, do the following: publish user pubkey file
+    let send_user_pubkey_publish_options = nats_js_client::PublishOptions {
+        subject: format!("HPOS.init.{}", server_node_id),
+        msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
+        data: b"Host Initializer Connected!".to_vec(),
     };
+    // initializer_client.publish(send_user_pubkey_publish_options);
+    utils::chunk_file_and_publish(&initializer_client.js, "subject", "file_path");
 
-    let event_listeners = vec![
-        nats_client::on_msg_published_event(published_msg_handler),
-        nats_client::on_msg_failed_event(failure_handler),
-    ];
+    // 5. Generate user creds file
+    let user_creds_path = utils::generate_creds_file();
 
-    event_listeners
+    // 6. Close and drain internal buffer before exiting to make sure all messages are sent
+    initializer_client.close().await?;
+
+    Ok(user_creds_path)
 }
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
index 1551fc5..9c3b3b1 100644
--- a/rust/clients/host_agent/src/auth/mod.rs
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -1,2 +1,3 @@
+pub mod agent_key;
+pub mod endpoints;
 pub mod initializer;
-pub mod key_utils;
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 016592c..3015162 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -11,7 +11,8 @@
   - sending active periodic workload reports
 */
 
-// mod auth;
+mod auth;
+mod utils;
 mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
@@ -22,8 +23,9 @@ async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
 
-    // let user_creds_path = auth::initializer::run().await?;
-    // gen_leaf_server::run(&user_creds_path).await;
+    let user_creds_path = auth::initializer::run().await?;
+
+    gen_leaf_server::run(&user_creds_path).await;
 
     let user_creds_path = "placeholder_while_we_use_pw_auth";
     workloads::manager::run(user_creds_path).await?;
diff --git a/rust/clients/host_agent/src/utils.rs b/rust/clients/host_agent/src/utils.rs
new file mode 100644
index 0000000..2e5fb41
--- /dev/null
+++ b/rust/clients/host_agent/src/utils.rs
@@ -0,0 +1,40 @@
+use super::auth::initializer::HOST_INIT_CLIENT_NAME;
+use anyhow::Result;
+use async_nats::jetstream::Context;
+use std::process::Command;
+use std::time::Duration;
+use util_libs::{
+    js_stream_service::JsStreamService,
+    nats_js_client::{self, EventListener},
+};
+
+pub async fn chunk_file_and_publish(js: &Context, subject: &str, file_path: &str) -> Result<()> {
+    // let mut file = std::fs::File::open(file_path)?;
+    // let mut buffer = vec![0; CHUNK_SIZE];
+    // let mut chunk_id = 0;
+
+    // while let Ok(bytes_read) = file.read(mut buffer) {
+    //     if bytes_read == 0 {
+    //         break;
+    //     }
+    //     let chunk_data = &buffer[..bytes_read];
+    //     js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
+    //     chunk_id += 1;
+    // }
+
+    // // Send an EOF marker
+    // js.publish(subject.to_string(), "EOF".into()).await.unwrap();
+
+    Ok(())
+}
+
+pub fn generate_creds_file() -> String {
+    let user_creds_path = "/path/to/host/user.creds".to_string();
+    Command::new("nsc")
+        .arg(format!("... > {}", user_creds_path))
+        .output()
+        .expect("Failed to add user with provided keys")
+        .stdout;
+
+    "placeholder_user.creds".to_string()
+}
diff --git a/rust/clients/host_agent/src/workloads/endpoints.rs b/rust/clients/host_agent/src/workloads/endpoints.rs
index 90b910a..e13b6d7 100644
--- a/rust/clients/host_agent/src/workloads/endpoints.rs
+++ b/rust/clients/host_agent/src/workloads/endpoints.rs
@@ -3,10 +3,10 @@ use async_nats::Message;
 use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
-use util_libs::nats_client;
+use util_libs::nats_js_client;
 use workload::WorkloadApi;
 
-pub async fn start_workload(workload_api: &WorkloadApi) -> nats_client::AsyncEndpointHandler {
+pub async fn start_workload(workload_api: &WorkloadApi) -> nats_js_client::AsyncEndpointHandler {
     let api = workload_api.to_owned();
     Arc::new(
             move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
@@ -18,7 +18,9 @@ pub async fn start_workload(workload_api: &WorkloadApi) -> nats_client::AsyncEnd
         )
 }
 
-pub async fn signal_status_update(workload_api: &WorkloadApi) -> nats_client::AsyncEndpointHandler {
+pub async fn signal_status_update(
+    workload_api: &WorkloadApi,
+) -> nats_js_client::AsyncEndpointHandler {
     let api = workload_api.to_owned();
     Arc::new(
             move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
@@ -30,7 +32,7 @@ pub async fn signal_status_update(workload_api: &WorkloadApi) -> nats_client::As
         )
 }
 
-pub async fn remove_workload(workload_api: &WorkloadApi) -> nats_client::AsyncEndpointHandler {
+pub async fn remove_workload(workload_api: &WorkloadApi) -> nats_js_client::AsyncEndpointHandler {
     let api = workload_api.to_owned();
     Arc::new(
             move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
diff --git a/rust/clients/host_agent/src/workloads/manager.rs b/rust/clients/host_agent/src/workloads/manager.rs
index 539ebbf..9553ad2 100644
--- a/rust/clients/host_agent/src/workloads/manager.rs
+++ b/rust/clients/host_agent/src/workloads/manager.rs
@@ -12,10 +12,14 @@
 */
 
 use super::endpoints;
-use anyhow::Result;
+use anyhow::{anyhow, Result};
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::time::Duration;
-use util_libs::js_microservice::JsStreamService;
-use util_libs::nats_client::{self, Client as NatClient, EndpointType, EventListener};
+use util_libs::{
+    db::mongodb::get_mongodb_url,
+    js_stream_service::{JsServiceParamsPartial, JsStreamService},
+    nats_js_client::{self, EndpointType, EventListener, JsClient},
+};
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
@@ -24,43 +28,60 @@ const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
 const HOST_AGENT_CLIENT_INBOX_PREFIX: &str = "_host_inbox";
 
 // TODO: Use _user_creds_path for auth once we add in the more resilient auth pattern.
-pub async fn run(_user_creds_path: &str) -> Result<(), async_nats::Error> {
+pub async fn run(user_creds_path: &str) -> Result<(), async_nats::Error> {
     log::info!("HPOS Agent Client: Connecting to server...");
+
+    // ==================== NATS Setup ====================
     // Connect to Nats server
-    let nats_url = nats_client::get_nats_url();
-    let event_listeners = get_host_workload_event_listeners();
+    let nats_url = nats_js_client::get_nats_url();
+    let event_listeners = nats_js_client::get_event_listeners();
+
+    // Setup JS Stream Service
+    let workload_stream_service_params = JsServiceParamsPartial {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
 
+    // Spin up Nats Client and loaded in the Js Stream Service
     let host_workload_client =
-        nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
+        nats_js_client::DefaultJsClient::new(nats_js_client::NewDefaultJsClientParams {
             nats_url,
             name: HOST_AGENT_CLIENT_NAME.to_string(),
             inbox_prefix: format!(
                 "{}_{}",
                 HOST_AGENT_CLIENT_INBOX_PREFIX, "host_id_placeholder"
             ),
-            credentials_path: None, // TEMP: Some(user_creds_path),
-            opts: vec![nats_client::with_event_listeners(event_listeners)],
+            service_params: vec![workload_stream_service_params],
+            credentials_path: Some(user_creds_path.to_string()),
+            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
             ping_interval: Some(Duration::from_secs(10)),
             request_timeout: Some(Duration::from_secs(5)),
         })
         .await?;
 
-    // Call workload service and call relevant endpoints
-    let js_context = JsStreamService::get_context(host_workload_client.client.clone());
-    let workload_stream = JsStreamService::new(
-        js_context,
-        WORKLOAD_SRV_NAME,
-        WORKLOAD_SRV_DESC,
-        WORKLOAD_SRV_VERSION,
-        WORKLOAD_SRV_SUBJ,
-    )
-    .await?;
+    // ==================== DB Setup ====================
 
-    let workload_api = WorkloadApi::new().await?;
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
 
+    // Generate the Workload API with access to db
+    let workload_api = WorkloadApi::new(&client).await?;
+
+    // ==================== API ENDPOINTS ====================
     // Register Workload Streams for Host Agent to consume
     // (subjects should be published by orchestrator or nats-db-connector)
-    workload_stream
+    let workload_service = host_workload_client
+        .get_js_service(WORKLOAD_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate workload service. Unable to spin up Host Agent."
+        ))?;
+
+    workload_service
         .add_local_consumer(
             "start_workload",
             "start",
@@ -69,7 +90,7 @@ pub async fn run(_user_creds_path: &str) -> Result<(), async_nats::Error> {
         )
         .await?;
 
-    workload_stream
+    workload_service
         .add_local_consumer(
             "signal_status_update",
             "signal_status_update",
@@ -78,7 +99,7 @@ pub async fn run(_user_creds_path: &str) -> Result<(), async_nats::Error> {
         )
         .await?;
 
-    workload_stream
+    workload_service
         .add_local_consumer(
             "remove_workload",
             "remove",
@@ -100,28 +121,3 @@ pub async fn run(_user_creds_path: &str) -> Result<(), async_nats::Error> {
 
     Ok(())
 }
-
-pub fn get_host_workload_event_listeners() -> Vec<EventListener> {
-    // TODO: Use duration in handlers..
-    let published_msg_handler = |msg: &str, _duration: Duration| {
-        log::info!(
-            "Successfully published message for {} client: {:?}",
-            HOST_AGENT_CLIENT_NAME,
-            msg
-        );
-    };
-    let failure_handler = |err: &str, _duration: Duration| {
-        log::error!(
-            "Failed to publish message for {} client: {:?}",
-            HOST_AGENT_CLIENT_NAME,
-            err
-        );
-    };
-
-    let event_listeners = vec![
-        nats_client::on_msg_published_event(published_msg_handler),
-        nats_client::on_msg_failed_event(failure_handler),
-    ];
-
-    event_listeners
-}
diff --git a/rust/clients/orchestrator/Cargo.toml b/rust/clients/orchestrator/Cargo.toml
new file mode 100644
index 0000000..d134605
--- /dev/null
+++ b/rust/clients/orchestrator/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "orchestrator"
+version = "0.0.1"
+edition = "2021"
+
+[dependencies]
+async-nats = { workspace = true }
+anyhow = { workspace = true }
+tokio = { workspace = true }
+futures = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+log = { workspace = true }
+dotenv = { workspace = true }
+thiserror = "2.0"
+url = { version = "2", features = ["serde"] }
+bson = { version = "2.6.1", features = ["chrono-0_4"] }
+env_logger = { workspace = true }
+mongodb = "3.1"
+chrono = "0.4.0"
+bytes = "1.8.0"
+nkeys = "=0.4.4"
+rand = "0.8.5"
+util_libs = { path = "../../util_libs" }
+workload = { path = "../../services/workload" }
+authentication = { path = "../../services/authentication" }
\ No newline at end of file
diff --git a/rust/clients/orchestrator/src/auth/controller.rs b/rust/clients/orchestrator/src/auth/controller.rs
new file mode 100644
index 0000000..de3b07e
--- /dev/null
+++ b/rust/clients/orchestrator/src/auth/controller.rs
@@ -0,0 +1,115 @@
+use super::endpoints;
+use crate::utils;
+use anyhow::Result;
+use authentication::{AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use std::process::Command;
+use util_libs::{
+    db::mongodb::get_mongodb_url,
+    js_stream_service::JsStreamService,
+    nats_js_client::{self, EndpointType, JsClient},
+};
+
+pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Agent";
+pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_orchestrator_auth_inbox";
+
+pub async fn run() -> Result<(), async_nats::Error> {
+    // ==================== NATS Setup ====================
+    let nats_url = nats_js_client::get_nats_url();
+    let creds_path = nats_js_client::get_nats_client_creds("HOLO", "ADMIN", "orchestrator");
+    let event_listeners = nats_js_client::get_event_listeners();
+
+    let orchestrator_auth_client =
+        nats_js_client::DefaultJsClient::new(nats_js_client::NewDefaultJsClientParams {
+            nats_url,
+            name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
+            inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
+            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            credentials_path: Some(creds_path),
+            ..Default::default()
+        })
+        .await?;
+
+    // Create a new Jetstream Microservice
+    let js_service = JsStreamService::new(
+        orchestrator_auth_client.js.clone(),
+        AUTH_SRV_NAME,
+        AUTH_SRV_DESC,
+        AUTH_SRV_VERSION,
+        AUTH_SRV_SUBJ,
+    )
+    .await?;
+
+    // ==================== DB Setup ====================
+
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
+
+    // Generate the Workload API with access to db
+    let auth_api = AuthApi::new(&client).await?;
+
+    // ==================== API ENDPOINTS ====================
+    // Register Workload Streams for Host Agent to consume
+    // (subjects should be published by orchestrator or nats-db-connector)
+
+    let auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", "host_id_placeholder"); // endpoint_subject
+
+    js_service
+        .add_local_consumer(
+            "add_user_pubkey", // called from orchestrator (no -auth service)
+            "add_user_pubkey",
+            EndpointType::Async(endpoints::add_user_pubkey(&auth_api).await),
+            None,
+        )
+        .await?;
+
+    log::trace!(
+        "{} Service is running. Waiting for requests...",
+        AUTH_SRV_NAME
+    );
+
+    let resolver_path = utils::get_resolver_path();
+
+    // Generate resolver file and create resolver file
+    Command::new("nsc")
+        .arg("generate")
+        .arg("config")
+        .arg("--nats-resolver")
+        .arg("sys-account SYS")
+        .arg("--force")
+        .arg(format!("--config-file {}", resolver_path))
+        .output()
+        .expect("Failed to create resolver config file")
+        .stdout;
+
+    // Push auth updates to hub server
+    Command::new("nsc")
+        .arg("push -A")
+        .output()
+        .expect("Failed to create resolver config file")
+        .stdout;
+
+    // publish user jwt file
+    let server_node_id = "server_node_id_placeholder";
+    utils::chunk_file_and_publish(
+        &orchestrator_auth_client,
+        &format!("HPOS.init.{}", server_node_id),
+        "placeholder_user_id / pubkey",
+    )
+    .await?;
+
+    // Only exit program when explicitly requested
+    tokio::signal::ctrl_c().await?;
+    log::warn!("CTRL+C detected. Please press CTRL+C again within 5 seconds to confirm exit...");
+    tokio::select! {
+        _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => log::warn!("Resuming service."),
+        _ = tokio::signal::ctrl_c() => log::error!("Shutting down."),
+    }
+
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    orchestrator_auth_client.close().await?;
+
+    Ok(())
+}
diff --git a/rust/clients/orchestrator/src/auth/endpoints.rs b/rust/clients/orchestrator/src/auth/endpoints.rs
new file mode 100644
index 0000000..6f6e755
--- /dev/null
+++ b/rust/clients/orchestrator/src/auth/endpoints.rs
@@ -0,0 +1,19 @@
+use anyhow::Result;
+use async_nats::Message;
+use authentication::AuthApi;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use util_libs::nats_js_client::AsyncEndpointHandler;
+
+pub async fn add_user_pubkey(auth_api: &AuthApi) -> AsyncEndpointHandler {
+    let api = auth_api.to_owned();
+    Arc::new(
+            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
+                let api_clone = api.clone();
+                Box::pin(async move {
+                    api_clone.add_user_pubkey(msg).await
+                })
+            },
+        )
+}
diff --git a/rust/clients/orchestrator/src/auth/mod.rs b/rust/clients/orchestrator/src/auth/mod.rs
new file mode 100644
index 0000000..32ba67b
--- /dev/null
+++ b/rust/clients/orchestrator/src/auth/mod.rs
@@ -0,0 +1,2 @@
+pub mod controller;
+pub mod endpoints;
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index 5a06298..d541dc8 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -6,130 +6,20 @@
 // This client is responsible for:
 */
 
-mod api;
-mod endpoints;
+mod auth;
+mod utils;
+mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use util_libs::{
-    db::{mongodb::get_mongodb_url, schemas},
-    js_microservice::JsStreamService,
-    nats_client::{self, EventListener},
-};
-
-const ORCHESTRATOR_CLIENT_NAME: &str = "Orchestrator Agent";
-const ORCHESTRATOR_CLIENT_INBOX_PREFIX: &str = "_orchestrator_inbox";
 
 #[tokio::main]
 async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
 
-    // ==================== NATS Setup ====================
-    let nats_url = nats_client::get_nats_url();
-    let creds_path = nats_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
-    let event_listeners = endpoints::get_orchestrator_workload_event_listeners();
-
-    let workload_service_inbox_prefix: &str = "_workload";
-
-    let workload_service = nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
-        nats_url,
-        name: WORKLOAD_SRV_OWNER_NAME.to_string(),
-        inbox_prefix: workload_service_inbox_prefix.to_string(),
-        opts: vec![nats_client::with_event_listeners(event_listeners)],
-        credentials_path: Some(creds_path),
-        ..Default::default()
-    })
-    .await?;
-
-    // Create a new Jetstream Microservice
-    let js_context = JsStreamService::get_context(workload_service.client.clone());
-    let js_service = JsStreamService::new(
-        js_context,
-        WORKLOAD_SRV_NAME,
-        WORKLOAD_SRV_DESC,
-        WORKLOAD_SRV_VERSION,
-        WORKLOAD_SRV_SUBJ,
-    )
-    .await?;
-
-    // ==================== DB Setup ====================
-    // Create a new MongoDB Client and connect it to the cluster
-    let mongo_uri = get_mongodb_url();
-    let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-
-    // // Create a typed collection for User
-    // let mut user_api = MongoCollection::<schemas::User>::new(
-    //     &client,
-    //     schemas::DATABASE_NAME,
-    //     schemas::HOST_COLLECTION_NAME,
-    // )
-    // .await?;
-
-    // // Create a typed collection for Host
-    // let mut host_api = MongoCollection::<schemas::Host>::new(
-    //     &client,
-    //     schemas::DATABASE_NAME,
-    //     schemas::HOST_COLLECTION_NAME,
-    // )
-    // .await?;
-
-    // Create a typed collection for Workload
-    let workload_api = api::WorkloadApi::new(&client).await?;
+    auth::controller::run().await;
 
-    // ==================== API ENDPOINTS ====================
-
-    // For ORCHESTRATOR to consume
-    // (subjects should be published by developer)
-    js_service
-        .add_local_consumer(
-            "add_workload",
-            "add",
-            nats_client::EndpointType::Async(endpoints::add_workload(workload_api).await),
-            None,
-        )
-        .await?;
-
-    js_service
-        .add_local_consumer(
-            "handle_changed_db_workload",
-            "handle_change",
-            nats_client::EndpointType::Async(endpoints::handle_db_change().await),
-            None,
-        )
-        .await?;
-
- 
-    log::trace!(
-        "{} Service is running. Waiting for requests...",
-        WORKLOAD_SRV_NAME
-    );
+    workloads::controller::run().await;
 
     Ok(())
 }
-
-pub fn get_orchestrator_workload_event_listeners() -> Vec<EventListener> {
-    // TODO: Use duration in handlers..
-    let published_msg_handler = |msg: &str, _duration: Duration| {
-        log::info!(
-            "Successfully published message for {} client: {:?}",
-            WORKLOAD_SRV_OWNER_NAME,
-            msg
-        );
-    };
-    let failure_handler = |err: &str, _duration: Duration| {
-        log::error!(
-            "Failed to publish message for {} client: {:?}",
-            WORKLOAD_SRV_OWNER_NAME,
-            err
-        );
-    };
-
-    let event_listeners = vec![
-        nats_client::on_msg_published_event(published_msg_handler),
-        nats_client::on_msg_failed_event(failure_handler),
-    ];
-
-    event_listeners
-}
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
new file mode 100644
index 0000000..cb463ec
--- /dev/null
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -0,0 +1,51 @@
+use super::auth::controller::ORCHESTRATOR_AUTH_CLIENT_NAME;
+use anyhow::Result;
+use std::io::Read;
+use tokio::time::Duration;
+use util_libs::nats_js_client::{self, DefaultJsClient, EventListener, JsClient, PublishOptions};
+
+const CHUNK_SIZE: usize = 1024; // 1 KB chunk size
+
+pub fn get_hpos_users_pubkey_path() -> String {
+    std::env::var("RESOLVER_FILE_PATH").unwrap_or_else(|_| "./resolver.conf".to_string())
+}
+
+pub fn get_resolver_path() -> String {
+    std::env::var("RESOLVER_FILE_PATH").unwrap_or_else(|_| "./resolver.conf".to_string())
+}
+
+pub async fn chunk_file_and_publish(
+    auth_client: &DefaultJsClient,
+    subject: &str,
+    host_id: &str,
+) -> Result<()> {
+    let file_path = format!("{}/{}.jwt", get_hpos_users_pubkey_path(), host_id);
+    let mut file = std::fs::File::open(file_path)?;
+    let mut buffer = vec![0; CHUNK_SIZE];
+    let mut chunk_id = 0;
+
+    while let Ok(bytes_read) = file.read(&mut buffer) {
+        if bytes_read == 0 {
+            break;
+        }
+        let chunk_data = &buffer[..bytes_read];
+
+        let send_user_jwt_publish_options = PublishOptions {
+            subject: subject.to_string(),
+            msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
+            data: chunk_data.into(),
+        };
+        auth_client.publish(&send_user_jwt_publish_options).await;
+        chunk_id += 1;
+    }
+
+    // Send an EOF marker
+    let send_user_jwt_publish_options = PublishOptions {
+        subject: subject.to_string(),
+        msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
+        data: "EOF".into(),
+    };
+    auth_client.publish(&send_user_jwt_publish_options);
+
+    Ok(())
+}
diff --git a/rust/clients/orchestrator/src/workloads/controller.rs b/rust/clients/orchestrator/src/workloads/controller.rs
new file mode 100644
index 0000000..7b32f1b
--- /dev/null
+++ b/rust/clients/orchestrator/src/workloads/controller.rs
@@ -0,0 +1,89 @@
+/*
+ This client is associated with the:
+- WORKLOAD account
+- orchestrator user
+
+// This client is responsible for:
+*/
+
+use super::endpoints;
+use anyhow::Result;
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use tokio::time::Duration;
+use util_libs::{
+    db::mongodb::get_mongodb_url,
+    js_stream_service::JsStreamService,
+    nats_js_client::{self, EventListener},
+};
+use workload::{
+    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+};
+
+const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_orchestrator_workload_inbox";
+
+pub async fn run() -> Result<(), async_nats::Error> {
+    // ==================== NATS Setup ====================
+    let nats_url = nats_js_client::get_nats_url();
+    let creds_path = nats_js_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
+    let event_listeners = nats_js_client::get_event_listeners();
+
+    let workload_service =
+        nats_js_client::DefaultJsClient::new(nats_js_client::NewDefaultJsClientParams {
+            nats_url,
+            name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
+            inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
+            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            credentials_path: Some(creds_path),
+            ..Default::default()
+        })
+        .await?;
+
+    // Create a new Jetstream Microservice
+    let js_service = JsStreamService::new(
+        workload_service.js.clone(),
+        WORKLOAD_SRV_NAME,
+        WORKLOAD_SRV_DESC,
+        WORKLOAD_SRV_VERSION,
+        WORKLOAD_SRV_SUBJ,
+    )
+    .await?;
+
+    // ==================== DB Setup ====================
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
+
+    // Generate the Workload API with access to db
+    let workload_api = WorkloadApi::new(&client).await?;
+
+    // ==================== API ENDPOINTS ====================
+
+    // For ORCHESTRATOR to consume
+    // (subjects should be published by developer)
+    js_service
+        .add_local_consumer(
+            "add_workload",
+            "add",
+            nats_js_client::EndpointType::Async(endpoints::add_workload(workload_api).await),
+            None,
+        )
+        .await?;
+
+    js_service
+        .add_local_consumer(
+            "handle_changed_db_workload",
+            "handle_change",
+            nats_js_client::EndpointType::Async(endpoints::handle_db_change().await),
+            None,
+        )
+        .await?;
+
+    log::trace!(
+        "{} Service is running. Waiting for requests...",
+        WORKLOAD_SRV_NAME
+    );
+
+    Ok(())
+}
diff --git a/rust/clients/orchestrator/src/endpoints.rs b/rust/clients/orchestrator/src/workloads/endpoints.rs
similarity index 83%
rename from rust/clients/orchestrator/src/endpoints.rs
rename to rust/clients/orchestrator/src/workloads/endpoints.rs
index c54b32e..1aa0402 100644
--- a/rust/clients/orchestrator/src/endpoints.rs
+++ b/rust/clients/orchestrator/src/workloads/endpoints.rs
@@ -3,11 +3,11 @@ use async_nats::Message;
 use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
-use util_libs::nats_client;
-use workload::api::WorkloadApi;
+use util_libs::nats_js_client;
+use workload::WorkloadApi;
 
 /// TODO:
-pub async fn add_workload(workload_api: WorkloadApi) -> nats_client::AsyncEndpointHandler {
+pub async fn add_workload(workload_api: WorkloadApi) -> nats_js_client::AsyncEndpointHandler {
     Arc::new(
             move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
                 log::warn!("INCOMING Message for 'WORKLOAD.add' : {:?}", msg);
@@ -20,7 +20,7 @@ pub async fn add_workload(workload_api: WorkloadApi) -> nats_client::AsyncEndpoi
 }
 
 /// TODO:
-pub async fn handle_db_change() ->  nats_client::AsyncEndpointHandler {
+pub async fn handle_db_change() -> nats_js_client::AsyncEndpointHandler {
     Arc::new(
         move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
             log::warn!("INCOMING Message for 'WORKLOAD.handle_change' : {:?}", msg);
diff --git a/rust/clients/orchestrator/src/workloads/mod.rs b/rust/clients/orchestrator/src/workloads/mod.rs
new file mode 100644
index 0000000..32ba67b
--- /dev/null
+++ b/rust/clients/orchestrator/src/workloads/mod.rs
@@ -0,0 +1,2 @@
+pub mod controller;
+pub mod endpoints;
diff --git a/rust/services/auth/src/endpoints.rs b/rust/services/auth/src/endpoints.rs
deleted file mode 100644
index 95f97ee..0000000
--- a/rust/services/auth/src/endpoints.rs
+++ /dev/null
@@ -1,251 +0,0 @@
-// use super::types;
-use super::nats_client::{self, EventListener};
-use super::AUTH_SRV_OWNER_NAME;
-
-use anyhow::{anyhow, Result};
-// use async_nats::HeaderValue;
-use async_nats::Message;
-use std::sync::Arc;
-use std::future::Future;
-use std::pin::Pin;
-use std::time::Duration;
-// use async_nats::jetstream::{Context};
-// use async_nats::jetstream::ErrorCode;
-// use async_nats::jetstream::consumer::Consumer;
-// use async_nats::jetstream::consumer::PullConsumer;
-// use async_nats::jetstream::consumer::pull::Stream;
-// // use std::io::Read;
-// use tokio::fs::OpenOptions;
-// use tokio::{fs::File, io::AsyncWriteExt};
-// use tokio::io;
-// use futures::future;
-// use futures::stream::{self, StreamExt};
-
-
-// NB: Message { subject, reply, payload, headers, status, description, length }
-
-// const CHUNK_SIZE: usize = 1024; // 1 KB chunks
-
-pub async fn start_handshake() ->  nats_client::AsyncEndpointHandler {
-    Arc::new(
-        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-            log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
-            let msg_clone = msg.clone();
-            Box::pin(async move {
-                // 1. Verify expected data was received
-                if msg_clone.headers.is_none() {
-                    log::error!("Error: Missing headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'");
-                    // anyhow!(ErrorCode::BAD_REQUEST)
-                }
-
-                // let signature = msg_clone.headers.unwrap().get("Signature").unwrap_or(&HeaderValue::new());
-
-                // match  serde_json::from_str::<types::AuthHeaders>(signature.as_str()) {
-                //     Ok(r) => {}
-                //     Err(e) => {
-                //         log::error!("Error: Failed to deserialize headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
-                //         // anyhow!(ErrorCode::BAD_REQUEST)
-                //     }
-                // }
-
-                // match serde_json::from_slice::<types::AuthPayload>(msg.payload.as_ref()) {
-                //     Ok(r) => {}
-                //     Err(e) => {
-                //         log::error!("Error: Failed to deserialize payload. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
-                //         // anyhow!(ErrorCode::BAD_REQUEST)
-                //     }
-                // }
-
-                // 2. Authenticate the HPOS client(?via email and host id info?)
-
-                // 3. Publish operator and sys account jwts for orchestrator
-                // let hub_operator_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
-                // let hub_sys_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
-
-                let response = serde_json::to_vec(&"OK")?;
-                Ok(response)
-            })
-        },
-    )
-}
-
-pub async fn save_hub_auth() ->  nats_client::AsyncEndpointHandler {
-    Arc::new(
-        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-            log::warn!("INCOMING Message for 'AUTH.save_hub_auth' : {:?}", msg);
-            Box::pin(async move {
-                // receive_and_write_file();
-
-                // Respond to endpoint request
-                // let response = b"Hello, NATS!".to_vec();
-                // Ok(response)
-                
-                todo!();
-            })
-        },
-    )
-}
-
-pub async fn send_user_pubkey() ->  nats_client::AsyncEndpointHandler {
-    Arc::new(
-        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-            log::warn!("INCOMING Message for 'AUTH.send_user_pubkey' : {:?}", msg);
-            Box::pin(async move {
-                // 1. validate nk key...
-                // let auth_endpoint_subject =
-                // format!("AUTH.{}.file.transfer.JWT-operator", "host_id_placeholder"); // endpoint_subject
-            
-                // 2. Update the hub nsc with user pubkey
-                
-                // 3. create signed jwt
-            
-                // 4. `Ack last request and publish the new jwt to for hpos
-            
-                // 5. Respond to endpoint request
-                // let response = b"Hello, NATS!".to_vec();
-                // Ok(response)
-
-                todo!()
-            })
-        },
-    )
-}
-
-
-    // let auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", host_id); // endpoint_subject
-    // let c = js_service
-    //     .add_local_consumer(
-    //         "save_user_auth", // called from orchestrator (no auth service)
-    //         "save_user_auth",
-    //         EndpointType::Async(Arc::new(
-    //             async |msg: &Message| -> Result<Vec<u8>, anyhow::Error> {
-    //                 log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
-
-    //                 receive_and_write_file()
-
-    //                 // 2. Respond to endpoint request
-    //                 let response = b"Hello, NATS!".to_vec();
-    //                 Ok(response)
-    //             },
-    //         )),
-    //         None,
-    //     )
-    //     .await?;
-
-    // let c = js_service
-    //     .add_local_consumer(
-    //         "send_user_file", //  called from orchestrator (no auth service)
-    //         "send_user_file",
-    //         EndpointType::Async(Arc::new(
-    //             async |msg: &Message| -> Result<Vec<u8>, anyhow::Error> {
-    //                 log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
-
-    //                 receive_and_write_file()
-
-    //                 // 2. Respond to endpoint request
-    //                 let response = b"Hello, NATS!".to_vec();
-    //                 Ok(response)
-    //             },
-    //         )),
-    //         None,
-    //     )
-    //     .await?;
-
-    //     let c = js_service
-    //     .add_local_consumer(
-    //         "save_user_file", // called from hpos
-    //         "end_hub_handshake",
-    //         EndpointType::Async(Arc::new(
-    //             async |msg: &Message| -> Result<Vec<u8>, anyhow::Error> {
-    //                 log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
-
-    //                 receive_and_write_file()
-
-    //                 // 2. Respond to endpoint request
-    //                 let response = b"Hello, NATS!".to_vec();
-    //                 Ok(response)
-    //             },
-    //         )),
-    //         None,
-    //     )
-    //     .await?;
-
-
-
-// ==================== Helpers ====================
-
-pub fn get_event_listeners() -> Vec<EventListener> {
-    // TODO: Use duration in handlers..
-    let published_msg_handler = |msg: &str, _duration: Duration| {
-        log::info!(
-            "Successfully published message for {} client: {:?}",
-            AUTH_SRV_OWNER_NAME,
-            msg
-        );
-    };
-    let failure_handler = |err: &str, _duration: Duration| {
-        log::error!(
-            "Failed to publish message for {} client: {:?}",
-            AUTH_SRV_OWNER_NAME,
-            err
-        );
-    };
-
-    let event_listeners = vec![
-        nats_client::on_msg_published_event(published_msg_handler),
-        nats_client::on_msg_failed_event(failure_handler),
-    ];
-
-    event_listeners
-}
-
-
-// async fn chunk_file_and_publish(js: &Context, subject: &str, file_path: &str) -> io::Result<()> {
-//     let mut file = std::fs::File::open(file_path)?;
-//     let mut buffer = vec![0; CHUNK_SIZE];
-//     let mut chunk_id = 0;
-
-//     while let Ok(bytes_read) = file.read(mut buffer) {
-//         if bytes_read == 0 {
-//             break;
-//         }
-//         let chunk_data = &buffer[..bytes_read];
-//         js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
-//         chunk_id += 1;
-//     }
-
-//     // Send an EOF marker
-//     js.publish(subject.to_string(), "EOF".into()).await.unwrap();
-
-//     Ok(())
-// }
-
-// async fn receive_and_write_file(stream: Stream, consumer: PullConsumer, header_subject: String, output_dir: &str, file_name: &str) -> Result<(), std::io::Error> {
-//     let output_path = format!("{}/{}", output_dir, file_name);
-//     let mut file = OpenOptions::new().create(true).append(true).open(&output_path)?;
-
-//     let mut messages = consumer
-//         .stream()
-//         .max_messages_per_batch(100)
-//         .max_bytes_per_batch(1024)
-//         .heartbeat(std::time::Duration::from_secs(10))
-//         .messages()
-//         .await?;
-
-//     // while let Some(Ok(message)) = message.next().await {}
-//     let file_msgs = messages().take_while(|msg| future::ready(*msg.subject.contains(file_name)));
-//     while let Some(Ok(msg)) = file_msgs.next().await {
-//         if msg.payload.to_string().contains("EOF") {
-//             // if msg.payload == b"EOF" {
-//             msg.ack().await?;
-//             println!("File transfer complete.");
-//             return Ok(());
-//         }
-
-//         file.write_all(&msg.payload).await?;
-//         file.flush().await?;
-//         msg.ack().await?;
-//     }
-
-//     Ok(())
-// }
\ No newline at end of file
diff --git a/rust/services/auth/src/main.rs b/rust/services/auth/src/main.rs
deleted file mode 100644
index 40c1d0c..0000000
--- a/rust/services/auth/src/main.rs
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
-Service Name: AUTH
-Subject: "AUTH.>"
-Provisioning Account: AUTH Account
-Importing Account: Auth/NoAuth Account
-
-This service should be run on the ORCHESTRATOR side and called from the HPOS side.
-The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
-NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
-This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
-
-Endpoints & Managed Subjects:
-    - start_hub_handshake
-    - end_hub_handshake
-    - save_hub_auth
-    - save_user_auth
-
-*/
-
-mod endpoints;
-mod types;
-use anyhow::Result;
-use async_nats::Message;
-use bytes::Bytes;
-use dotenv::dotenv;
-use futures::StreamExt;
-use std::sync::Arc;
-use std::time::Duration;
-use util_libs::{
-    js_microservice::JsStreamService,
-    nats_client::{self, EndpointType},
-};
-
-const AUTH_SRV_OWNER_NAME: &str = "AUTH_OWNER";
-const AUTH_SRV_NAME: &str = "AUTH";
-const AUTH_SRV_SUBJ: &str = "AUTH";
-const AUTH_SRV_VERSION: &str = "0.0.1";
-const AUTH_SRV_DESC: &str =
-    "This service handles the Authentication flow the HPOS and the Orchestrator.";
-
-#[tokio::main]
-async fn main() -> Result<(), async_nats::Error> {
-    dotenv().ok();
-    env_logger::init();
-
-    // ==================== NATS Setup ====================
-
-    let nats_url = nats_client::get_nats_url();
-    let creds_path = nats_client::get_nats_client_creds("HOLO", "ADMIN", "orchestrator");
-    let event_listeners = endpoints::get_event_listeners();
-
-    let auth_service_inbox_prefix: &str = "_auth";
-
-    let auth_service = nats_client::DefaultClient::new(nats_client::NewDefaultClientParams {
-        nats_url,
-        name: AUTH_SRV_OWNER_NAME.to_string(),
-        inbox_prefix: auth_service_inbox_prefix.to_string(),
-        opts: vec![nats_client::with_event_listeners(event_listeners)],
-        credentials_path: Some(creds_path),
-        ..Default::default()
-    })
-    .await?;
-
-    // Create a new Jetstream Microservice
-    let js_context = JsStreamService::get_context(auth_service.client.clone());
-    let js_service = JsStreamService::new(
-        js_context,
-        AUTH_SRV_NAME,
-        AUTH_SRV_DESC,
-        AUTH_SRV_VERSION,
-        AUTH_SRV_SUBJ,
-    )
-    .await?;
-
-    // ==================== API ENDPOINTS ====================
-
-    js_service
-        .add_local_consumer(
-            "publish_hub_files", // called from hpos
-            "start_hub_handshake",
-            nats_client::EndpointType::Async(endpoints::start_handshake().await),
-            None,
-        )
-        .await?;
-
-    js_service
-        .add_local_consumer(
-            "save_hub_auth", // called from hpos
-            "save_hub_auth",
-            nats_client::EndpointType::Async(endpoints::save_hub_auth().await),
-            None,
-        )
-        .await?;
-
-    js_service
-        .add_local_consumer(
-            "send_user_pubkey", // called from hpos
-            "send_user_pubkey",
-            nats_client::EndpointType::Async(endpoints::send_user_pubkey().await),
-            None,
-        )
-        .await?;
-
-    log::trace!(
-        "{} Service is running. Waiting for requests...",
-        AUTH_SRV_NAME
-    );
-
-    Ok(())
-}
diff --git a/rust/services/auth/Cargo.toml b/rust/services/authentication/Cargo.toml
similarity index 92%
rename from rust/services/auth/Cargo.toml
rename to rust/services/authentication/Cargo.toml
index bec8e4d..8b0ed90 100644
--- a/rust/services/auth/Cargo.toml
+++ b/rust/services/authentication/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
-name = "auth"
-version = "0.1.0"
+name = "authentication"
+version = "0.0.1"
 edition = "2021"
 
 [dependencies]
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
new file mode 100644
index 0000000..7370a75
--- /dev/null
+++ b/rust/services/authentication/src/lib.rs
@@ -0,0 +1,212 @@
+/*
+Service Name: AUTH
+Subject: "AUTH.>"
+Provisioning Account: AUTH Account
+Importing Account: Auth/NoAuth Account
+
+This service should be run on the ORCHESTRATOR side and called from the HPOS side.
+The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
+NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
+This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
+
+Endpoints & Managed Subjects:
+    - start_hub_handshake
+    - end_hub_handshake
+    - save_hub_auth
+    - save_user_auth
+
+*/
+
+use anyhow::Result;
+use async_nats::Message;
+use mongodb::Client as MongoDBClient;
+use std::process::Command;
+use std::sync::Arc;
+use util_libs::db::{mongodb::MongoCollection, schemas};
+
+pub const AUTH_SRV_OWNER_NAME: &str = "AUTH_OWNER";
+pub const AUTH_SRV_NAME: &str = "AUTH";
+pub const AUTH_SRV_SUBJ: &str = "AUTH";
+pub const AUTH_SRV_VERSION: &str = "0.0.1";
+pub const AUTH_SRV_DESC: &str =
+    "This service handles the Authentication flow the HPOS and the Orchestrator.";
+
+#[derive(Debug, Clone)]
+pub struct AuthApi {
+    pub user_collection: MongoCollection<schemas::User>,
+    pub hoster_collection: MongoCollection<schemas::Hoster>,
+    pub host_collection: MongoCollection<schemas::Host>,
+}
+
+impl AuthApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        // Create a typed collection for User
+        let user_api: MongoCollection<schemas::User> = MongoCollection::<schemas::User>::new(
+            client,
+            schemas::DATABASE_NAME,
+            schemas::HOST_COLLECTION_NAME,
+        )
+        .await?;
+
+        // Create a typed collection for Hoster
+        let hoster_api = MongoCollection::<schemas::Hoster>::new(
+            &client,
+            schemas::DATABASE_NAME,
+            schemas::HOST_COLLECTION_NAME,
+        )
+        .await?;
+
+        // Create a typed collection for Host
+        let host_api = MongoCollection::<schemas::Host>::new(
+            &client,
+            schemas::DATABASE_NAME,
+            schemas::HOST_COLLECTION_NAME,
+        )
+        .await?;
+
+        Ok(Self {
+            user_collection: user_api,
+            hoster_collection: hoster_api,
+            host_collection: host_api,
+        })
+    }
+
+    // For orchestrator
+    pub async fn receive_handshake_request(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<Vec<u8>, anyhow::Error> {
+        // 1. Verify expected data was received
+        if msg.headers.is_none() {
+            log::error!(
+                "Error: Missing headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'"
+            );
+            // anyhow!(ErrorCode::BAD_REQUEST)
+        }
+
+        // let signature = msg_clone.headers.unwrap().get("Signature").unwrap_or(&HeaderValue::new());
+
+        // match  serde_json::from_str::<types::AuthHeaders>(signature.as_str()) {
+        //     Ok(r) => {}
+        //     Err(e) => {
+        //         log::error!("Error: Failed to deserialize headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
+        //         // anyhow!(ErrorCode::BAD_REQUEST)
+        //     }
+        // }
+
+        // match serde_json::from_slice::<types::AuthPayload>(msg.payload.as_ref()) {
+        //     Ok(r) => {}
+        //     Err(e) => {
+        //         log::error!("Error: Failed to deserialize payload. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
+        //         // anyhow!(ErrorCode::BAD_REQUEST)
+        //     }
+        // }
+
+        // 2. Authenticate the HPOS client(?via email and host id info?)
+
+        // 3. Publish operator and sys account jwts for orchestrator
+        // let hub_operator_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
+        // let hub_sys_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
+
+        let response = serde_json::to_vec(&"OK")?;
+        Ok(response)
+    }
+
+    // For hpos
+    pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
+        // receive_and_write_file();
+
+        // Respond to endpoint request
+        // let response = b"Hello, NATS!".to_vec();
+        // Ok(response)
+
+        todo!();
+    }
+
+    // For orchestrator
+    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
+        log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+        // Add user with Keys and create jwt
+        Command::new("nsc")
+            .arg("...")
+            .output()
+            .expect("Failed to add user with provided keys")
+            .stdout;
+
+        // Output jwt
+        let user_jwt_path = Command::new("nsc")
+            .arg("...")
+            // .arg(format!("> {}", output_dir))
+            .output()
+            .expect("Failed to output user jwt to file")
+            .stdout;
+
+        // 2. Respond to endpoint request
+        // let resposne = user_jwt_path;
+        let response = b"Hello, NATS!".to_vec();
+        Ok(response)
+    }
+
+    // For hpos
+    pub async fn save_user_jwt(
+        &self,
+        msg: Arc<Message>,
+        output_dir: &str,
+    ) -> Result<Vec<u8>, anyhow::Error> {
+        log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+        // utils::receive_and_write_file(msg, output_dir, file_name).await?;
+
+        // 2. Respond to endpoint request
+        let response = b"Hello, NATS!".to_vec();
+        Ok(response)
+    }
+}
+
+// In orchestrator
+// pub async fn send_hub_jwts(
+//     &self,
+//     msg: Arc<Message>,
+// ) -> Result<Vec<u8>, anyhow::Error> {
+//     log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+//     utils::chunk_file_and_publish(msg, output_dir, file_name).await?;
+
+//     // 2. Respond to endpoint request
+//     let response = b"Hello, NATS!".to_vec();
+//     Ok(response)
+// }
+
+// In hpos
+// pub async fn send_user_pubkey(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
+//     // 1. validate nk key...
+//     // let auth_endpoint_subject =
+//     // format!("AUTH.{}.file.transfer.JWT-operator", "host_id_placeholder"); // endpoint_subject
+
+//     // 2. Update the hub nsc with user pubkey
+
+//     // 3. create signed jwt
+
+//     // 4. `Ack last request and publish the new jwt to for hpos
+
+//     // 5. Respond to endpoint request
+//     // let response = b"Hello, NATS!".to_vec();
+//     // Ok(response)
+
+//     todo!()
+// }
+
+// In orchestrator
+// pub async fn send_user_file(
+//     &self,
+//     msg: Arc<Message>,
+// ) -> Result<Vec<u8>, anyhow::Error> {
+//     log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+
+//     utils::chunk_file_and_publish(msg, output_dir, file_name).await?;
+
+//     // 2. Respond to endpoint request
+//     let response = b"Hello, NATS!".to_vec();
+//     Ok(response)
+// }
diff --git a/rust/services/auth/src/types.rs b/rust/services/authentication/src/types.rs
similarity index 100%
rename from rust/services/auth/src/types.rs
rename to rust/services/authentication/src/types.rs
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
new file mode 100644
index 0000000..8f9c3a3
--- /dev/null
+++ b/rust/services/authentication/src/utils.rs
@@ -0,0 +1,30 @@
+use anyhow::Result;
+use async_nats::jetstream::Context;
+use async_nats::{jetstream::consumer::PullConsumer, Message};
+use std::sync::Arc;
+use tokio::{fs::File, io::AsyncWriteExt};
+
+pub async fn receive_and_write_file(
+    msg: Arc<Message>,
+    output_dir: &str,
+    file_name: &str,
+) -> Result<()> {
+    let output_path = format!("{}/{}", output_dir, file_name);
+    let mut file = tokio::fs::OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&output_path)
+        .await?;
+
+    let payload_buf = msg.payload.to_vec();
+    let payload = serde_json::from_slice::<String>(&payload_buf)?;
+    if payload.to_string().contains("EOF") {
+        log::info!("File transfer complete.");
+        return Ok(());
+    }
+
+    file.write_all(&msg.payload).await?;
+    file.flush().await?;
+
+    Ok(())
+}
diff --git a/rust/services/workload/Cargo.toml b/rust/services/workload/Cargo.toml
index 34264a1..921c8e8 100644
--- a/rust/services/workload/Cargo.toml
+++ b/rust/services/workload/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "workload"
-version = "0.1.0"
+version = "0.0.1"
 edition = "2021"
 
 [dependencies]
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 455caa9..de5b28d 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -13,13 +13,10 @@ Endpoints & Managed Subjects:
 
 use anyhow::Result;
 use async_nats::Message;
-// use mongodb::Client as MongoDBClient;
+use mongodb::Client as MongoDBClient;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
-use util_libs::db::{
-    // mongodb::{MongoCollection, MongoDbPool},
-    schemas,
-};
+use util_libs::db::{mongodb::MongoCollection, schemas};
 
 pub const WORKLOAD_SRV_OWNER_NAME: &str = "WORKLOAD_OWNER";
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
@@ -39,24 +36,42 @@ pub enum WorkloadState {
 
 #[derive(Debug, Clone)]
 pub struct WorkloadApi {
-    // >> TODO: Fill out with other endpoints when we use the DB
-    // pub workload_collection: MongoCollection<schemas::Workload>,
-    // pub host_collection: MongoCollection<schemas::Host>,
-    // pub user_collection: MongoCollection<schemas::User>,
+    pub workload_collection: MongoCollection<schemas::Workload>,
+    pub host_collection: MongoCollection<schemas::Host>,
+    pub user_collection: MongoCollection<schemas::User>,
 }
 
 impl WorkloadApi {
-    pub async fn new(/*client: &MongoDBClientA*/) -> Result<Self> {
-        // let workload_api: MongoCollection<schemas::Workload> =
-        //     MongoCollection::<schemas::Workload>::new(
-        //         client,
-        //         schemas::DATABASE_NAME,
-        //         schemas::HOST_COLLECTION_NAME,
-        //     )
-        //     .await?;
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        // Create a typed collection for Workload
+        let workload_api: MongoCollection<schemas::Workload> =
+            MongoCollection::<schemas::Workload>::new(
+                client,
+                schemas::DATABASE_NAME,
+                schemas::HOST_COLLECTION_NAME,
+            )
+            .await?;
+
+        // Create a typed collection for User
+        let user_api = MongoCollection::<schemas::User>::new(
+            &client,
+            schemas::DATABASE_NAME,
+            schemas::HOST_COLLECTION_NAME,
+        )
+        .await?;
+
+        // Create a typed collection for Host
+        let host_api = MongoCollection::<schemas::Host>::new(
+            &client,
+            schemas::DATABASE_NAME,
+            schemas::HOST_COLLECTION_NAME,
+        )
+        .await?;
 
         Ok(Self {
-            // workload_collection: workload_api,
+            workload_collection: workload_api,
+            host_collection: host_api,
+            user_collection: user_api,
         })
     }
 
@@ -94,7 +109,7 @@ impl WorkloadApi {
         log::warn!("INCOMING Message for 'WORKLOAD.start' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
-        let _workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
+        let _workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
 
         // TODO: Talk through with Stefan
         // 1. Connect to interface for Nix and instruct systemd to install workload...
@@ -109,7 +124,7 @@ impl WorkloadApi {
         log::warn!("INCOMING Message for 'WORKLOAD.remove' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
-        let workload_state: WorkloadState = serde_json::from_slice(&payload_buf)?;
+        let workload_state = serde_json::from_slice::<WorkloadState>(&payload_buf)?;
 
         // Send updated reponse:
         // NB: This will send the update to both the requester (if one exists)
@@ -119,7 +134,7 @@ impl WorkloadApi {
 
     pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
         let payload_buf = msg.payload.to_vec();
-        let _workload_id: String = serde_json::from_slice(&payload_buf)?;
+        let _workload_id = serde_json::from_slice::<String>(&payload_buf)?;
 
         // TODO: Talk through with Stefan
         // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
diff --git a/rust/util_libs/src/js_microservice.rs b/rust/util_libs/src/js_stream_service.rs
similarity index 94%
rename from rust/util_libs/src/js_microservice.rs
rename to rust/util_libs/src/js_stream_service.rs
index 9c87543..6669930 100644
--- a/rust/util_libs/src/js_microservice.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -1,10 +1,10 @@
-use super::nats_client::EndpointType;
+use super::nats_js_client::EndpointType;
 use anyhow::{anyhow, Result};
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
-use async_nats::jetstream::{self, Context};
-use async_nats::Client;
+use async_nats::jetstream::Context;
 use futures::StreamExt;
+use serde::Deserialize;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
@@ -38,9 +38,17 @@ pub struct ConsumerExt {
 #[allow(dead_code)]
 #[derive(Clone, Debug)]
 pub struct JsStreamServiceInfo<'a> {
-    name: &'a str,
-    version: &'a str,
-    service_subject: &'a str,
+    pub name: &'a str,
+    pub version: &'a str,
+    pub service_subject: &'a str,
+}
+
+#[derive(Deserialize, Default)]
+pub struct JsServiceParamsPartial {
+    pub name: String,
+    pub description: String,
+    pub version: String,
+    pub service_subject: String,
 }
 
 /// Microservice for Jetstream Streams
@@ -56,20 +64,6 @@ pub struct JsStreamService {
 }
 
 impl JsStreamService {
-    // Spin up a jetstream associated with the provided Nats Client
-    // NB: The context creation is separated out to allow already established js contexts to be passed into `new` instead of being re/created there.
-    pub fn get_context(client: Client) -> Context {
-        jetstream::new(client)
-    }
-
-    pub fn get_info(&self) -> JsStreamServiceInfo {
-        JsStreamServiceInfo {
-            name: self.name.as_ref(),
-            version: self.version.as_ref(),
-            service_subject: self.service_subject.as_ref(),
-        }
-    }
-
     /// Create a new MicroService instance
     // For when the consumer service also creates the stream
     pub async fn new(
@@ -124,6 +118,29 @@ impl JsStreamService {
         })
     }
 
+    pub async fn get_consumer_info(&self, consumer_name: &str) -> Result<Option<consumer::Info>> {
+        if let Some(consumer_ext) = self
+            .to_owned()
+            .local_consumers
+            .write()
+            .await
+            .get_mut(&consumer_name.to_string())
+        {
+            let info = consumer_ext.consumer.info().await?;
+            Ok(Some(info.to_owned()))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn get_service_info(&self) -> JsStreamServiceInfo {
+        JsStreamServiceInfo {
+            name: self.name.as_ref(),
+            version: self.version.as_ref(),
+            service_subject: self.service_subject.as_ref(),
+        }
+    }
+
     pub async fn add_local_consumer(
         &self,
         consumer_name: &str,
@@ -324,7 +341,7 @@ impl JsStreamService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use async_nats::ConnectOptions;
+    use async_nats::{jetstream, ConnectOptions};
     use std::sync::Arc;
 
     const NATS_SERVER_URL: &str = "nats://localhost:4222";
@@ -337,7 +354,7 @@ mod tests {
             .await
             .expect("Failed to connect to NATS");
 
-        JsStreamService::get_context(client)
+        jetstream::new(client)
     }
 
     pub async fn get_default_js_service(context: Context) -> JsStreamService {
diff --git a/rust/util_libs/src/lib.rs b/rust/util_libs/src/lib.rs
index 0dce1ed..861376d 100644
--- a/rust/util_libs/src/lib.rs
+++ b/rust/util_libs/src/lib.rs
@@ -1,5 +1,5 @@
 pub mod db;
-pub mod js_microservice;
-pub mod nats_client;
+pub mod js_stream_service;
+pub mod nats_js_client;
 pub mod nats_server;
 pub mod nats_types;
diff --git a/rust/util_libs/src/nats_client.rs b/rust/util_libs/src/nats_js_client.rs
similarity index 61%
rename from rust/util_libs/src/nats_client.rs
rename to rust/util_libs/src/nats_js_client.rs
index 84f3e32..5c3ca34 100644
--- a/rust/util_libs/src/nats_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -1,9 +1,9 @@
+use super::js_stream_service::{JsServiceParamsPartial, JsStreamService};
 use anyhow::{anyhow, Result};
 use async_nats::jetstream::context::PublishAckFuture;
 use async_nats::jetstream::{self, stream::Config};
-use async_nats::Message;
+use async_nats::{Message, ServerInfo};
 use async_trait::async_trait;
-use futures::StreamExt;
 use serde::Deserialize;
 use std::error::Error;
 use std::fmt;
@@ -13,9 +13,9 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type ClientOption = Box<dyn Fn(&mut DefaultClient)>;
-pub type EventListener = Box<dyn Fn(&mut DefaultClient)>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, Duration) + Send + Sync>>;
+pub type ClientOption = Box<dyn Fn(&mut DefaultJsClient)>;
+pub type EventListener = Box<dyn Fn(&mut DefaultJsClient)>;
+pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
 
 pub type EndpointHandler = Arc<dyn Fn(&Message) -> Result<Vec<u8>, anyhow::Error> + Send + Sync>;
 pub type AsyncEndpointHandler = Arc<
@@ -42,15 +42,16 @@ impl fmt::Display for ErrClientDisconnected {
 impl Error for ErrClientDisconnected {}
 
 #[async_trait]
-pub trait Client: Send + Sync {
+pub trait JsClient: Send + Sync {
     fn name(&self) -> &str;
     async fn monitor(&self) -> Result<(), async_nats::Error>;
     async fn close(&self) -> Result<(), async_nats::Error>;
+    async fn add_stream(&self, opts: &AddStreamOptions) -> Result<(), async_nats::Error>;
     async fn get_stream(
         &self,
         get_stream: &str,
     ) -> Result<jetstream::stream::Stream, async_nats::Error>;
-    async fn add_stream(&self, opts: &AddStreamOptions) -> Result<(), async_nats::Error>;
+    async fn request(&self, opts: &RequestOptions) -> Result<(), async_nats::Error>;
     async fn publish(&self, opts: &PublishOptions) -> Result<(), async_nats::Error>;
 }
 
@@ -59,6 +60,13 @@ pub struct AddStreamOptions {
     pub stream_name: String,
 }
 
+#[derive(Clone, Debug)]
+pub struct RequestOptions {
+    pub subject: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+}
+
 #[derive(Clone, Debug)]
 pub struct PublishOptions {
     pub subject: String,
@@ -66,33 +74,37 @@ pub struct PublishOptions {
     pub data: Vec<u8>,
 }
 
-impl std::fmt::Debug for DefaultClient {
+impl std::fmt::Debug for DefaultJsClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("DefaultClient")
+        f.debug_struct("DefaultJsClient")
             .field("url", &self.url)
             .field("name", &self.name)
             .field("client", &self.client)
             .field("js", &self.js)
+            .field("js_services", &self.js_services)
             .field("service_log_prefix", &self.service_log_prefix)
             .finish()
     }
 }
 
-pub struct DefaultClient {
+pub struct DefaultJsClient {
     url: String,
     name: String,
     on_msg_published_event: Option<EventHandler>,
     on_msg_failed_event: Option<EventHandler>,
-    pub client: async_nats::Client, // inner_client
-    js: jetstream::Context,
+    client: async_nats::Client, // inner_client
+    pub js: jetstream::Context,
+    pub js_services: Option<Vec<JsStreamService>>,
     service_log_prefix: String,
 }
 
 #[derive(Deserialize, Default)]
-pub struct NewDefaultClientParams {
+pub struct NewDefaultJsClientParams {
     pub nats_url: String,
     pub name: String,
     pub inbox_prefix: String,
+    #[serde(default)]
+    pub service_params: Vec<JsServiceParamsPartial>,
     #[serde(skip_deserializing)]
     pub opts: Vec<ClientOption>, // NB: These opts should not be required for client instantiation
     #[serde(default)]
@@ -103,49 +115,57 @@ pub struct NewDefaultClientParams {
     pub request_timeout: Option<Duration>, // Defaults to 5s
 }
 
-impl DefaultClient {
-    pub async fn new(p: NewDefaultClientParams) -> Result<Self, async_nats::Error> {
+impl DefaultJsClient {
+    pub async fn new(p: NewDefaultJsClientParams) -> Result<Self, async_nats::Error> {
+        let connect_options = async_nats::ConnectOptions::new()
+            // .require_tls(true)
+            .name(&p.name)
+            .ping_interval(p.ping_interval.unwrap_or(Duration::from_secs(120)))
+            .request_timeout(Some(p.request_timeout.unwrap_or(Duration::from_secs(10))))
+            .custom_inbox_prefix(&p.inbox_prefix);
+
         let client = match p.credentials_path {
             Some(cp) => {
                 let path = std::path::Path::new(&cp);
-                async_nats::ConnectOptions::new()
+                connect_options
                     .credentials_file(path)
                     .await?
-                    // .require_tls(true)
-                    .name(&p.name)
-                    .ping_interval(p.ping_interval.unwrap_or(Duration::from_secs(120)))
-                    .request_timeout(Some(p.request_timeout.unwrap_or(Duration::from_secs(10))))
-                    .custom_inbox_prefix(&p.inbox_prefix)
-                    .connect(&p.nats_url)
-                    .await?
-            }
-            None => {
-                async_nats::ConnectOptions::new()
-                    // .require_tls(true)
-                    .name(&p.name)
-                    .ping_interval(
-                        p.ping_interval
-                            .unwrap_or(std::time::Duration::from_secs(120)),
-                    )
-                    .request_timeout(Some(
-                        p.request_timeout
-                            .unwrap_or(std::time::Duration::from_secs(10)),
-                    ))
-                    .custom_inbox_prefix(&p.inbox_prefix)
                     .connect(&p.nats_url)
                     .await?
             }
+            None => connect_options.connect(&p.nats_url).await?,
+        };
+
+        let jetstream = jetstream::new(client.clone());
+        let mut services = vec![];
+        for params in p.service_params {
+            let service = JsStreamService::new(
+                jetstream.clone(),
+                &params.name,
+                &params.description,
+                &params.version,
+                &params.service_subject,
+            )
+            .await?;
+            services.push(service);
+        }
+
+        let js_services = if services.is_empty() {
+            None
+        } else {
+            Some(services)
         };
 
         let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
 
-        let mut default_client = DefaultClient {
+        let mut default_client = DefaultJsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
             on_msg_failed_event: None,
-            client: client.clone(),
-            js: jetstream::new(client),
+            client,
+            js: jetstream,
+            js_services,
             service_log_prefix: service_log_prefix.clone(),
         };
 
@@ -161,14 +181,34 @@ impl DefaultClient {
         Ok(default_client)
     }
 
+    pub fn get_server_info(&self) -> ServerInfo {
+        self.client.server_info()
+    }
+
+    pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
+        let mut current_services = self.js_services.unwrap_or_default();
+        current_services.extend(js_services);
+        self.js_services = Some(current_services);
+        self
+    }
+
+    pub async fn get_js_service(&self, js_service_name: String) -> Option<&JsStreamService> {
+        if let Some(services) = &self.js_services {
+            return services
+                .iter()
+                .find(|s| s.get_service_info().name == js_service_name);
+        }
+        None
+    }
+
     pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
         if let async_nats::connection::State::Disconnected = self.client.connection_state() {
             return Err(Box::new(ErrClientDisconnected));
         }
         let stream = &self.js.get_stream(stream_name).await?;
-        let info = stream.cached_info();
+        let info = stream.get_info().await?;
         log::debug!(
-            "{}JetStream (cached) info: stream:{}, info:{:?}",
+            "{}JetStream info: stream:{}, info:{:?}",
             self.service_log_prefix,
             stream_name,
             info
@@ -186,47 +226,6 @@ impl DefaultClient {
         Ok(())
     }
 
-    pub async fn subscribe(
-        &self,
-        subject: &str,
-        handler: EndpointType,
-    ) -> Result<(), async_nats::Error> {
-        let mut subscription = self.client.subscribe(subject.to_string()).await?;
-        let js_context = self.js.clone();
-        let service_log_prefix = self.service_log_prefix.clone();
-
-        tokio::spawn(async move {
-            while let Some(msg) = subscription.next().await {
-                // todo!: persist handler for reliability cases
-                log::info!("{}Received message: {:?}", service_log_prefix, msg);
-
-                let result = match handler.to_owned() {
-                    EndpointType::Sync(handler) => handler(&msg),
-                    EndpointType::Async(handler) => handler(Arc::new(msg.clone())).await,
-                };
-
-                let response_bytes: bytes::Bytes = match result {
-                    Ok(response) => response.into(),
-                    Err(err) => err.to_string().into(),
-                };
-
-                // (NB: Only return a response if a reply address exists...
-                // Otherwise, the underlying NATS system will receive a message it can't broker and will panic!)
-                if let Some(reply) = &msg.reply {
-                    if let Err(err) = js_context.publish(reply.to_owned(), response_bytes).await {
-                        log::error!(
-                            "{}Failed to send reply upon successful message consumption: subj='{}', err={:?}",
-                            service_log_prefix,
-                            reply,
-                            err
-                        );
-                    };
-                }
-            }
-        });
-        Ok(())
-    }
-
     pub async fn publish_with_retry(
         &self,
         opts: &PublishOptions,
@@ -251,7 +250,7 @@ impl DefaultClient {
 }
 
 #[async_trait]
-impl Client for DefaultClient {
+impl JsClient for DefaultJsClient {
     fn name(&self) -> &str {
         &self.name
     }
@@ -293,17 +292,21 @@ impl Client for DefaultClient {
         Ok(self.js.get_stream(stream_name).await?)
     }
 
+    async fn request(&self, opts: &RequestOptions) -> Result<(), async_nats::Error> {
+        Ok(())
+    }
+
     async fn publish(&self, opts: &PublishOptions) -> Result<(), async_nats::Error> {
+        let now = Instant::now();
         let result = self
             .js
             .publish(opts.subject.clone(), opts.data.clone().into())
             .await;
 
-        let now = Instant::now();
         let duration = now.elapsed();
         if let Err(err) = result {
             if let Some(ref on_failed) = self.on_msg_failed_event {
-                on_failed(&opts.subject, duration); // add msg_id
+                on_failed(&opts.subject, &self.name, duration); // add msg_id
             }
             return Err(Box::new(err));
         }
@@ -316,7 +319,7 @@ impl Client for DefaultClient {
             opts.data
         );
         if let Some(ref on_published) = self.on_msg_published_event {
-            on_published(&opts.subject, duration);
+            on_published(&opts.subject, &self.name, duration);
         }
         Ok(())
     }
@@ -341,8 +344,9 @@ where
     Err(anyhow!("Exceeded max retries"))
 }
 
+// Client Options:
 pub fn with_event_listeners(listeners: Vec<EventListener>) -> ClientOption {
-    Box::new(move |c: &mut DefaultClient| {
+    Box::new(move |c: &mut DefaultJsClient| {
         for listener in &listeners {
             listener(c);
         }
@@ -352,18 +356,18 @@ pub fn with_event_listeners(listeners: Vec<EventListener>) -> ClientOption {
 // Event Listener Options:
 pub fn on_msg_published_event<F>(f: F) -> EventListener
 where
-    F: Fn(&str, Duration) + Send + Sync + Clone + 'static,
+    F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
-    Box::new(move |c: &mut DefaultClient| {
+    Box::new(move |c: &mut DefaultJsClient| {
         c.on_msg_published_event = Some(Box::pin(f.clone()));
     })
 }
 
 pub fn on_msg_failed_event<F>(f: F) -> EventListener
 where
-    F: Fn(&str, Duration) + Send + Sync + Clone + 'static,
+    F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
-    Box::new(move |c: &mut DefaultClient| {
+    Box::new(move |c: &mut DefaultJsClient| {
         c.on_msg_failed_event = Some(Box::pin(f.clone()));
     })
 }
@@ -382,16 +386,37 @@ pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> Strin
     })
 }
 
+pub fn get_event_listeners() -> Vec<EventListener> {
+    // TODO: Use duration in handlers..
+    let published_msg_handler = move |msg: &str, client_name: &str, _duration: Duration| {
+        log::info!(
+            "Successfully published message for {}. Msg: {:?}",
+            client_name,
+            msg
+        );
+    };
+    let failure_handler = |err: &str, client_name: &str, _duration: Duration| {
+        log::info!("Failed to publish for {}. Err: {:?}", client_name, err);
+    };
+
+    let event_listeners = vec![
+        on_msg_published_event(published_msg_handler),
+        on_msg_failed_event(failure_handler),
+    ];
+
+    event_listeners
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::sync::Arc;
 
-    pub fn get_default_params() -> NewDefaultClientParams {
-        NewDefaultClientParams {
+    pub fn get_default_params() -> NewDefaultJsClientParams {
+        NewDefaultJsClientParams {
             nats_url: "localhost:4222".to_string(),
             name: "test_client".to_string(),
             inbox_prefix: "_UNIQUE_INBOX".to_string(),
+            service_params: vec![],
             credentials_path: None,
             ping_interval: Some(Duration::from_secs(10)),
             request_timeout: Some(Duration::from_secs(5)),
@@ -400,9 +425,9 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_nats_client_init() {
+    async fn test_nats_js_client_init() {
         let params = get_default_params();
-        let client = DefaultClient::new(params).await;
+        let client = DefaultJsClient::new(params).await;
         assert!(client.is_ok(), "Client initialization failed: {:?}", client);
 
         let client = client.unwrap();
@@ -410,9 +435,9 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_nats_client_add_stream() {
+    async fn test_nats_js_client_add_stream() {
         let params = get_default_params();
-        let client = DefaultClient::new(params).await.unwrap();
+        let client = DefaultJsClient::new(params).await.unwrap();
         let add_stream_options = AddStreamOptions {
             stream_name: "test_stream".to_string(),
         };
@@ -422,9 +447,9 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_nats_client_publish() {
+    async fn test_nats_js_client_publish() {
         let params = get_default_params();
-        let client = DefaultClient::new(params).await.unwrap();
+        let client = DefaultJsClient::new(params).await.unwrap();
         let publish_options = PublishOptions {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
@@ -436,63 +461,9 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_nats_client_subscribe_and_receive_message() {
-        let params = get_default_params();
-        let client = DefaultClient::new(params).await.unwrap();
-
-        // Set default message to false
-        let message_received = Arc::new(tokio::sync::Mutex::new(false));
-
-        async fn create_closure_async_block(
-            message: Arc<tokio::sync::Mutex<bool>>,
-        ) -> AsyncEndpointHandler {
-            Arc::new(move |_msg: Arc<Message>| {
-                let message = message.clone();
-                Box::pin(async move {
-                    tokio::time::sleep(Duration::from_millis(1)).await;
-                    let mut flag = message.lock().await;
-                    *flag = true;
-                    Ok(vec![])
-                })
-            })
-        }
-
-        // Update message from false to true via handler to test the handler
-        let handler =
-            EndpointType::Async(create_closure_async_block(message_received.clone()).await);
-
-        let subject = "test_subject";
-        let subscription_result = client.subscribe(subject, handler).await;
-        assert!(
-            subscription_result.is_ok(),
-            "Error: Subscription failed: {:?}",
-            subscription_result
-        );
-
-        // Publish a message to the subject
-        let publish_options = PublishOptions {
-            subject: subject.to_string(),
-            msg_id: "test_msg".to_string(),
-            data: b"Test Message".to_vec(),
-        };
-        let publication_result = client.publish(&publish_options).await;
-        assert!(
-            publication_result.is_ok(),
-            "Error: Publishing message failed: {:?}",
-            publication_result
-        );
-
-        tokio::time::sleep(Duration::from_secs(1)).await;
-
-        // Assert the message was received by the handler (ie: msg_flag should return true)
-        let msg_flag = message_received.lock().await;
-        assert!(*msg_flag);
-    }
-
-    #[tokio::test]
-    async fn test_nats_client_publish_with_retry() {
+    async fn test_nats_js_client_publish_with_retry() {
         let params = get_default_params();
-        let client = DefaultClient::new(params).await.unwrap();
+        let client = DefaultJsClient::new(params).await.unwrap();
 
         let publish_options = PublishOptions {
             subject: "test_subject".to_string(),

From 9b19e5a3e7cab2ecee9d201b5fc8803b794eec50 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Mon, 6 Jan 2025 23:18:41 -0600
Subject: [PATCH 05/91] restore db-url helper

---
 rust/util_libs/src/db/mongodb.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index 7e1ca17..dfc5ae4 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -139,6 +139,11 @@ where
     }
 }
 
+// Helpers:
+pub fn get_mongodb_url() -> String {
+    std::env::var("MONGO_URI").unwrap_or_else(|_| "mongodb://127.0.0.1:27017".to_string())
+}
+
 #[cfg(not(target_arch = "aarch64"))]
 #[cfg(test)]
 mod tests {

From cbaaac38fcd9daf22deaaec406c466cb5903d099 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Wed, 15 Jan 2025 15:06:07 -0600
Subject: [PATCH 06/91] update orchestrator workload to latest pattern

---
 rust/clients/host_agent/src/agent_cli.rs      |   2 +-
 rust/clients/host_agent/src/auth/mod.rs       |   1 +
 .../host_agent/src/{ => auth}/utils.rs        |   0
 .../src/{ => hostd}/gen_leaf_server.rs        |   0
 rust/clients/host_agent/src/hostd/mod.rs      |   2 +
 .../src/{ => hostd}/workload_manager.rs       |  12 +-
 rust/clients/host_agent/src/main.rs           |   8 +-
 .../orchestrator/src/workloads/controller.rs  |  95 ++++++++----
 .../orchestrator/src/workloads/endpoints.rs   |  40 -----
 .../clients/orchestrator/src/workloads/mod.rs |   1 -
 rust/services/workload/src/lib.rs             | 145 ++++++++----------
 rust/services/workload/src/types.rs           |   4 +-
 rust/util_libs/src/db/mongodb.rs              |   2 +-
 rust/util_libs/src/db/schemas.rs              |   5 +-
 14 files changed, 149 insertions(+), 168 deletions(-)
 rename rust/clients/host_agent/src/{ => auth}/utils.rs (100%)
 rename rust/clients/host_agent/src/{ => hostd}/gen_leaf_server.rs (100%)
 create mode 100644 rust/clients/host_agent/src/hostd/mod.rs
 rename rust/clients/host_agent/src/{ => hostd}/workload_manager.rs (94%)
 delete mode 100644 rust/clients/orchestrator/src/workloads/endpoints.rs

diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index e872ec1..21dda28 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -1,4 +1,4 @@
-/// MOdule containing all of the Clap Derive structs/definitions that make up the agent's
+/// Module containing all of the Clap Derive structs/definitions that make up the agent's
 /// command line. To start the agent daemon (usually from systemd), use `host_agent daemonize`.
 use clap::{Parser, Subcommand};
 
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
index 9c3b3b1..ef8adf5 100644
--- a/rust/clients/host_agent/src/auth/mod.rs
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -1,3 +1,4 @@
 pub mod agent_key;
 pub mod endpoints;
+pub mod utils;
 pub mod initializer;
diff --git a/rust/clients/host_agent/src/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
similarity index 100%
rename from rust/clients/host_agent/src/utils.rs
rename to rust/clients/host_agent/src/auth/utils.rs
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
similarity index 100%
rename from rust/clients/host_agent/src/gen_leaf_server.rs
rename to rust/clients/host_agent/src/hostd/gen_leaf_server.rs
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
new file mode 100644
index 0000000..6a971dc
--- /dev/null
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -0,0 +1,2 @@
+pub mod workload_manager;
+pub mod gen_leaf_server;
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
similarity index 94%
rename from rust/clients/host_agent/src/workload_manager.rs
rename to rust/clients/host_agent/src/hostd/workload_manager.rs
index 1eb6c8a..c44398f 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -12,17 +12,17 @@
 */
 
 use anyhow::{anyhow, Result};
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::{sync::Arc, time::Duration};
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use async_nats::Message;
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::{JsServiceParamsPartial, JsStreamService},
-    nats_js_client::{self, EndpointType, EventListener, JsClient},
+    js_stream_service::JsServiceParamsPartial,
+    nats_js_client::{self, EndpointType},
 };
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
-use async_nats::Message;
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
 const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
@@ -75,8 +75,8 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     let workload_api = WorkloadApi::new(&client).await?;
 
     // ==================== API ENDPOINTS ====================
-    // Register Workload Streams for Host Agent to consume
-    // NB: Subjects are published by orchestrator or nats-db-connector
+    // Register Workload Streams for Host Agent to consume and process
+    // NB: Subjects are published by orchestrator
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 44fccb6..2dd0c13 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -11,13 +11,11 @@ This client is responsible for subscribing the host agent to workload stream end
 */
 
 mod auth;
-mod utils;
-mod workload_manager;
+mod hostd;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
 pub mod agent_cli;
-pub mod gen_leaf_server;
 pub mod host_cmds;
 pub mod support_cmds;
 use thiserror::Error;
@@ -59,7 +57,7 @@ async fn daemonize() -> Result<(), async_nats::Error> {
     // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
     let host_creds_path = nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
     let host_pubkey = "host_id_placeholder>";
-    gen_leaf_server::run(&host_creds_path).await;
-    workload_manager::run(host_pubkey, &host_creds_path).await?;
+    hostd::gen_leaf_server::run(&host_creds_path).await;
+    hostd::workload_manager::run(host_pubkey, &host_creds_path).await?;
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/workloads/controller.rs b/rust/clients/orchestrator/src/workloads/controller.rs
index f427dc1..c2e20fb 100644
--- a/rust/clients/orchestrator/src/workloads/controller.rs
+++ b/rust/clients/orchestrator/src/workloads/controller.rs
@@ -6,14 +6,14 @@
 // This client is responsible for:
 */
 
-use super::endpoints;
-use anyhow::Result;
+use anyhow::{anyhow, Result};
+use std::{sync::Arc, time::Duration};
+use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use tokio::time::Duration;
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::JsStreamService,
-    nats_js_client::{self, EventListener, JsClient, NewJsClientParams},
+    js_stream_service::JsServiceParamsPartial,
+    nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
 };
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
@@ -28,27 +28,27 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let creds_path = nats_js_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
     let event_listeners = nats_js_client::get_event_listeners();
 
-    let workload_service =
+    // Setup JS Stream Service
+    let workload_stream_service_params = JsServiceParamsPartial {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+
+    let orchestrator_workload_client =
         JsClient::new(NewJsClientParams {
             nats_url,
             name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
             inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
-            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            service_params: vec![workload_stream_service_params],
             credentials_path: Some(creds_path),
-            ..Default::default()
+            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            ping_interval: Some(Duration::from_secs(10)),
+            request_timeout: Some(Duration::from_secs(5)),
         })
         .await?;
 
-    // Create a new Jetstream Microservice
-    let js_service = JsStreamService::new(
-        workload_service.js.clone(),
-        WORKLOAD_SRV_NAME,
-        WORKLOAD_SRV_DESC,
-        WORKLOAD_SRV_VERSION,
-        WORKLOAD_SRV_SUBJ,
-    )
-    .await?;
-
     // ==================== DB Setup ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
@@ -59,31 +59,62 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let workload_api = WorkloadApi::new(&client).await?;
 
     // ==================== API ENDPOINTS ====================
+    // Register Workload Streams for Orchestrator to consume and proceess
+    // NB: Subjects are published by external Developer, the Nats-DB-Connector, or the Host Agent
+    let workload_service = orchestrator_workload_client
+        .get_js_service(WORKLOAD_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate workload service. Unable to spin up Host Agent."
+        ))?;
 
-    // For ORCHESTRATOR to consume
-    // (subjects should be published by developer)
-    js_service
-        .add_local_consumer(
+    // Published by Developer
+    workload_service
+        .add_local_consumer::<workload::types::ApiResult>(
             "add_workload",
             "add",
-            nats_js_client::EndpointType::Async(endpoints::add_workload(workload_api).await),
+            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.add_workload(msg).await
+                }
+            })),
             None,
         )
         .await?;
 
-    js_service
-        .add_local_consumer(
-            "handle_changed_db_workload",
-            "handle_change",
-            nats_js_client::EndpointType::Async(endpoints::handle_db_change().await),
+    // Automatically published by the Nats-DB-Connector
+    workload_service
+        .add_local_consumer::<workload::types::ApiResult>(
+            "handle_db_insertion",
+            "insert",
+            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.handle_db_insertion(msg).await
+                }
+            })),
             None,
         )
         .await?;
+    
+    // Published by the Host Agent
+    workload_service
+    .add_local_consumer::<workload::types::ApiResult>(
+        "handle_status_update",
+        "read_status_update",
+        EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+            async move {
+                api.handle_status_update(msg).await
+            }
+        })),
+        None,
+    )
+    .await?;
+
 
-    log::trace!(
-        "{} Service is running. Waiting for requests...",
-        WORKLOAD_SRV_NAME
-    );
+    // Only exit program when explicitly requested
+    tokio::signal::ctrl_c().await?;
 
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    orchestrator_workload_client.close().await?;
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/workloads/endpoints.rs b/rust/clients/orchestrator/src/workloads/endpoints.rs
deleted file mode 100644
index 7ae1ba8..0000000
--- a/rust/clients/orchestrator/src/workloads/endpoints.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use anyhow::Result;
-use async_nats::Message;
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::Arc;
-use util_libs::nats_js_client;
-use workload::{self, WorkloadApi};
-
-/// TODO:
-pub async fn add_workload(workload_api: WorkloadApi) -> nats_js_client::AsyncEndpointHandler::<workload::types::ApiResult> {
-    Arc::new(
-            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-                log::warn!("INCOMING Message for 'WORKLOAD.add' : {:?}", msg);
-                let db_api = workload_api.clone();
-                Box::pin(async move {
-                    db_api.add_workload(msg).await
-                })
-            },
-        )
-}
-
-/// TODO:
-pub async fn handle_db_change() -> nats_js_client::AsyncEndpointHandler::<workload::types::ApiResult> {
-    Arc::new(
-        move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-            log::warn!("INCOMING Message for 'WORKLOAD.handle_change' : {:?}", msg);
-            Box::pin(async move {
-                // 1. Map over workload items in message and grab capacity requirements
-
-                // 2. Call mongodb to get host/node info and filter by capacity availability
-
-                // 3. Randomly choose host/node
-
-                // 4. Respond to endpoint request
-                let response = b"Successfully handled updated workload!".to_vec();
-                Ok(response)
-            })
-        },
-    )
-}
diff --git a/rust/clients/orchestrator/src/workloads/mod.rs b/rust/clients/orchestrator/src/workloads/mod.rs
index 32ba67b..cb9e0ac 100644
--- a/rust/clients/orchestrator/src/workloads/mod.rs
+++ b/rust/clients/orchestrator/src/workloads/mod.rs
@@ -1,2 +1 @@
 pub mod controller;
-pub mod endpoints;
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 2e50893..353d803 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -60,12 +60,11 @@ impl WorkloadApi {
             Box::pin(handler(api_clone, msg))
         })
     }
-    
+
     /*******************************  For Orchestrator   *********************************/
-    // For orchestrator
     pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.add'");
-        let (maybe_payload, status, maybe_response_subjects) = self.process_request(
+        Ok(self.process_request(
             msg,
             WorkloadState::Reported,
             |workload: schemas::Workload| async move {
@@ -75,9 +74,9 @@ impl WorkloadApi {
                     _id: Some(workload_id),
                     ..workload
                 };
-                Ok((
-                    Some(updated_workload),
+                Ok(types::ApiResult(
                     WorkloadStatus {
+                        id: updated_workload._id,
                         desired: WorkloadState::Reported,
                         actual: WorkloadState::Reported,
                     },
@@ -86,18 +85,12 @@ impl WorkloadApi {
             },
             WorkloadState::Error,
         )
-        .await;
-
-        if let Some(workload) = maybe_payload {
-            return Ok(types::ApiResult(workload._id, status, maybe_response_subjects))        ;
-        }
-        Ok(types::ApiResult(None, status, maybe_response_subjects))
+        .await)
     }
 
-    // For orchestrator
     pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-        let (maybe_payload, status, maybe_response_subjects) = self.process_request(
+        Ok(self.process_request(
             msg,
             WorkloadState::Running,
             |workload: schemas::Workload| async move {
@@ -105,9 +98,9 @@ impl WorkloadApi {
                 let updated_workload = to_document(&workload)?;
                 self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok((
-                    Some(workload),
+                Ok(types::ApiResult(
                     WorkloadStatus {
+                        id: workload._id,
                         desired: WorkloadState::Reported,
                         actual: WorkloadState::Reported,
                     },
@@ -116,18 +109,13 @@ impl WorkloadApi {
             },
             WorkloadState::Error,
         )
-        .await;
-        if let Some(workload) = maybe_payload {
-            return Ok(types::ApiResult(workload._id, status, maybe_response_subjects));
-        }
-        Ok(types::ApiResult(None, status, maybe_response_subjects))
+        .await)
 
     }
 
-    // For orchestrator
     pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
-        let (maybe_payload, status, maybe_response_subjects) = self.process_request(
+        Ok(self.process_request(
             msg,
             WorkloadState::Removed,
             |workload_id: schemas::MongoDbId| async move {
@@ -137,9 +125,9 @@ impl WorkloadApi {
                     "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
                     workload_id
                 );
-                Ok((
-                    Some(workload_id),
+                Ok(types::ApiResult(
                     WorkloadStatus {
+                        id: Some(workload_id),
                         desired: WorkloadState::Removed,
                         actual: WorkloadState::Removed,
                     },
@@ -148,19 +136,13 @@ impl WorkloadApi {
             },
             WorkloadState::Error,
         )
-        .await;
-        if let Some(workload_id) = maybe_payload {
-            return Ok(types::ApiResult(Some(workload_id), status, maybe_response_subjects));
-        }
-        Ok(types::ApiResult(None, status, maybe_response_subjects))
+        .await)
     }
 
-
-    // For orchestrator
-    // NB: This is the stream that is automatically published by the nats-db-connector
+    // NB: Automatically published by the nats-db-connector
     pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
-        let (maybe_payload, status, maybe_response_subjects) = self.process_request(
+        Ok(self.process_request(
             msg,
             WorkloadState::Assigned,
             |workload: schemas::Workload| async move {
@@ -177,8 +159,9 @@ impl WorkloadApi {
                 // todo: check for to ensure assigned host *still* has enough capacity for updated workload
                 if !workload.assigned_hosts.is_empty() {
                     log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok((Some(workload.clone()), 
+                    return Ok(types::ApiResult( 
                     WorkloadStatus {
+                        id: Some(workload_id),
                         desired: WorkloadState::Assigned,
                         actual: WorkloadState::Assigned,
                     },
@@ -234,9 +217,9 @@ impl WorkloadApi {
                     updated_host_result
                 );
      
-                Ok((
-                    Some(updated_workload.clone()),
+                Ok(types::ApiResult(
                     WorkloadStatus {
+                        id: Some(workload_id),
                         desired: WorkloadState::Assigned,
                         actual: WorkloadState::Assigned,
                     },
@@ -245,62 +228,68 @@ impl WorkloadApi {
         },
             WorkloadState::Error,
         )
-        .await;
-
-        // 6. Return status and host
-        if let Some(workload) = maybe_payload {
-            return Ok(types::ApiResult(workload._id, status, maybe_response_subjects));
-        }
-        Ok(types::ApiResult(None, status, maybe_response_subjects))
-    }    
+        .await)
+    }
 
     // Zeeshan to take a look:
-    // For orchestrator
-    // NB: This is the stream that is automatically published by the nats-db-connector
+    // NB: Automatically published by the nats-db-connector
     pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-
-        let success_status = WorkloadStatus {
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Running,
-        };
-
+        
         let payload_buf = msg.payload.to_vec();
         let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-
+        
         // TODO: ...handle the use case for the update entry change stream 
 
-        Ok(types::ApiResult(workload._id, success_status, None))
-    } 
+        let success_status = WorkloadStatus {
+            id: workload._id,
+            desired: WorkloadState::Running,
+            actual: WorkloadState::Running,
+        };
+        
+        Ok(types::ApiResult(success_status, None))
+    }
 
     // Zeeshan to take a look:
-    // For orchestrator
-    // NB: This is the stream that is automatically published by the nats-db-connector
+    // NB: Automatically published by the nats-db-connector
     pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.delete'");
+        
+        let payload_buf = msg.payload.to_vec();
+        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
+        log::trace!("New workload to assign. Workload={:#?}", workload);
+        
+        // TODO: ...handle the use case for the delete entry change stream
 
         let success_status = WorkloadStatus {
+            id: workload._id,
             desired: WorkloadState::Removed,
             actual: WorkloadState::Removed,
         };
+        
+        Ok(types::ApiResult(success_status, None))
+    }
+
+    // NB: Published by the Hosting Agent whenever the status of a workload changes
+    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+        log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
 
         let payload_buf = msg.payload.to_vec();
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("New workload to assign. Workload={:#?}", workload);
+        let workload_status: WorkloadStatus = serde_json::from_slice(&payload_buf)?;
+        log::trace!("Workload status to update. Status={:?}", workload_status);
 
-        // TODO: ...handle the use case for the delete entry change stream
+        // TODO: ...handle the use case for the workload status update
         
-        Ok(types::ApiResult(workload._id, success_status, None))
+        Ok(types::ApiResult(workload_status, None))
     }    
 
-     /*******************************   For Hosting Agent   *********************************/
-    // For hpos
+     /*******************************   For Host Agent   *********************************/
     pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
-        let _workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
+        let workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
 
         // TODO: Talk through with Stefan
         // 1. Connect to interface for Nix and instruct systemd to install workload...
@@ -308,18 +297,18 @@ impl WorkloadApi {
 
         // 2. Respond to endpoint request
         let status = WorkloadStatus {
+            id: workload._id,
             desired: WorkloadState::Running,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(None, status, None))
+        Ok(types::ApiResult(status, None))
     }
 
-    // For hpos
     pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
-        let _workload_id = serde_json::from_slice::<String>(&payload_buf)?;
+        let workload_id = serde_json::from_slice::<String>(&payload_buf)?;
 
         // TODO: Talk through with Stefan
         // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
@@ -327,13 +316,14 @@ impl WorkloadApi {
 
         // 2. Respond to endpoint request
         let status = WorkloadStatus {
+            id: Some(workload_id),
             desired: WorkloadState::Uninstalled,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(None, status, None))
+        Ok(types::ApiResult(status, None))
     }
 
-    // For hpos ? or elsewhere ?
+    // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
     pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!(
@@ -342,15 +332,12 @@ impl WorkloadApi {
         );
 
         let payload_buf = msg.payload.to_vec();
-        let workload_state = serde_json::from_slice::<WorkloadState>(&payload_buf)?;
+        let workload_status = serde_json::from_slice::<WorkloadStatus>(&payload_buf)?;
 
         // Send updated status:
         // NB: This will send the update to both the requester (if one exists)
         // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(None, WorkloadStatus {
-            desired: WorkloadState::Unknown("todo: pass-in/access desired state".to_string()),
-            actual: workload_state
-        }, None))
+        Ok(types::ApiResult(workload_status, None))
     }
 
     /*******************************  Helper Fns  *********************************/
@@ -373,10 +360,10 @@ impl WorkloadApi {
         desired_state: WorkloadState,
         cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> (Option<T>, WorkloadStatus, Option<Vec<String>>)
+    ) -> types::ApiResult
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<(Option<T>, WorkloadStatus, Option<Vec<String>>), anyhow::Error>> + Send,
+        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send,
     {
         // 1. Deserialize payload into the expected type
         let payload: T = match serde_json::from_slice(&msg.payload) {
@@ -385,10 +372,11 @@ impl WorkloadApi {
                 let err_msg = format!("Failed to deserialize payload for Workload Service Endpoint. Subject={} Error={:?}", msg.subject, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
+                    id: None,
                     desired: desired_state,
                     actual: error_state(err_msg),
                 };
-                return (None, status, None);
+                return types::ApiResult(status, None);
             }
         };
 
@@ -399,12 +387,13 @@ impl WorkloadApi {
                 let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
+                    id: None,
                     desired: desired_state,
                     actual: error_state(err_msg),
                 };
 
                 // 3. return response for stream
-                (Some(payload), status, None)
+                types::ApiResult(status, None)
             }
         }
     }
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 6b16ee9..cea7d1d 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -4,11 +4,11 @@ use serde::{Deserialize, Serialize};
 pub use String as WorkloadId;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub Option<WorkloadId>, pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct ApiResult (pub WorkloadStatus, pub Option<Vec<String>>);
 
 impl CreateTag for ApiResult {
     fn get_tags(&self) -> Option<Vec<String>> {
-        self.2.clone()
+        self.1.clone()
     }
 }
 
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index 8c35e6f..ecb579a 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -31,7 +31,7 @@ where
 }
 
 pub trait IntoIndexes {
-    fn into_indices(&self) -> Result<Vec<(Document, Option<IndexOptions>)>>;
+    fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>>;
 }
 
 #[derive(Debug, Clone)]
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 17f5bba..0a6ff82 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -131,7 +131,7 @@ pub struct Host {
 }
 
 impl IntoIndexes for Host {
-    fn into_indices(&self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
+    fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
 
         //  Add Device ID Index
@@ -163,6 +163,7 @@ pub enum WorkloadState {
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
+    pub id: Option<String>, 
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
@@ -218,7 +219,7 @@ impl Default for Workload {
 }
 
 impl IntoIndexes for Workload {
-    fn into_indices(&self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
+    fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
 
         //  Add Developer Index

From 6f18fd130f3075b307f0c8f8dc1c1daa32a1868f Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Wed, 15 Jan 2025 16:18:32 -0600
Subject: [PATCH 07/91] adjust auth to updated service pattern

---
 rust/clients/host_agent/src/auth/endpoints.rs |  34 -----
 .../host_agent/src/auth/initializer.rs        |  81 ++++++------
 rust/clients/host_agent/src/auth/mod.rs       |   3 +-
 rust/clients/host_agent/src/auth/utils.rs     |   2 +-
 .../orchestrator/src/auth/controller.rs       |  75 ++++++-----
 .../orchestrator/src/auth/endpoints.rs        |  19 ---
 rust/clients/orchestrator/src/auth/mod.rs     |   1 -
 .../orchestrator/src/workloads/controller.rs  |  10 +-
 rust/services/authentication/src/lib.rs       | 117 +++++++++++-------
 rust/services/authentication/src/types.rs     |  36 +++++-
 rust/services/workload/src/lib.rs             |   1 -
 rust/services/workload/src/types.rs           |   6 +-
 12 files changed, 203 insertions(+), 182 deletions(-)
 delete mode 100644 rust/clients/host_agent/src/auth/endpoints.rs
 delete mode 100644 rust/clients/orchestrator/src/auth/endpoints.rs

diff --git a/rust/clients/host_agent/src/auth/endpoints.rs b/rust/clients/host_agent/src/auth/endpoints.rs
deleted file mode 100644
index 1124ec6..0000000
--- a/rust/clients/host_agent/src/auth/endpoints.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use anyhow::Result;
-use async_nats::Message;
-use authentication::AuthApi;
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::Arc;
-use util_libs::nats_js_client;
-
-const USER_CREDENTIALS_PATH: &str = "./user_creds";
-
-pub async fn save_user_jwt(auth_api: &AuthApi) -> nats_js_client::AsyncEndpointHandler {
-    let api = auth_api.to_owned();
-    // let user_name_clone = user_name.clone();
-    Arc::new(
-            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-                let api_clone = api.clone();
-                Box::pin(async move {
-                    api_clone.save_user_jwt(msg, USER_CREDENTIALS_PATH).await
-                })
-            },
-        )
-}
-
-pub async fn save_hub_jwts(auth_api: &AuthApi) -> nats_js_client::AsyncEndpointHandler {
-    let api = auth_api.to_owned();
-    Arc::new(
-            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-                let api_clone = api.clone();
-                Box::pin(async move {
-                    api_clone.save_hub_jwts(msg).await
-                })
-            },
-        )
-}
diff --git a/rust/clients/host_agent/src/auth/initializer.rs b/rust/clients/host_agent/src/auth/initializer.rs
index 6b7a732..650d56d 100644
--- a/rust/clients/host_agent/src/auth/initializer.rs
+++ b/rust/clients/host_agent/src/auth/initializer.rs
@@ -16,23 +16,25 @@
 4. instantiate the leaf server via the leaf-server struct/service
 */
 
-use super::endpoints;
-use crate::utils;
+use super::utils;
 use anyhow::{anyhow, Result};
+use async_nats::Message;
 use authentication::{AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, JsStreamService},
     nats_js_client::{self, EndpointType, EventListener, JsClient},
 };
 
-pub const HOST_INIT_CLIENT_NAME: &str = "Host Initializer";
-pub const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_init_inbox";
+pub const HOST_INIT_CLIENT_NAME: &str = "Host Auth";
+pub const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_auth_inbox";
+
+const USER_CREDENTIALS_PATH: &str = "./user_creds";
 
 pub async fn run() -> Result<String, async_nats::Error> {
-    log::info!("Host Initializer Client: Connecting to server...");
+    log::info!("Host Auth Client: Connecting to server...");
 
     // ==================== NATS Setup ====================
     // Connect to Nats server
@@ -47,13 +49,13 @@ pub async fn run() -> Result<String, async_nats::Error> {
         service_subject: AUTH_SRV_SUBJ.to_string(),
     };
 
-    let initializer_client =
+    let host_auth_client =
         nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
             nats_url,
             name: HOST_INIT_CLIENT_NAME.to_string(),
             inbox_prefix: HOST_INIT_CLIENT_INBOX_PREFIX.to_string(),
-            credentials_path: None,
             service_params: vec![auth_stream_service_params],
+            credentials_path: None,
             opts: vec![nats_js_client::with_event_listeners(event_listeners)],
             ping_interval: Some(Duration::from_secs(10)),
             request_timeout: Some(Duration::from_secs(5)),
@@ -61,7 +63,6 @@ pub async fn run() -> Result<String, async_nats::Error> {
         .await?;
 
     // ==================== DB Setup ====================
-
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
@@ -73,39 +74,39 @@ pub async fn run() -> Result<String, async_nats::Error> {
     // ==================== Report Host to Orchestator ====================
 
     // Discover the server Node ID via INFO response
-    let server_node_id = initializer_client.get_server_info().server_id;
+    let server_node_id = host_auth_client.get_server_info().server_id;
     log::trace!(
-        "Host Initializer Client: Retrieved Node ID: {}",
+        "Host Auth Client: Retrieved Node ID: {}",
         server_node_id
     );
 
     // Publish a message with the Node ID as part of the subject
-    let publish_options = nats_js_client::PublishOptions {
+    let publish_options = nats_js_client::SendRequest {
         subject: format!("HPOS.init.{}", server_node_id),
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
-        data: b"Host Initializer Connected!".to_vec(),
+        data: b"Host Auth Connected!".to_vec(),
     };
 
-    match initializer_client
-        .publish_with_retry(&publish_options, 3)
+    match host_auth_client
+        .publish(&publish_options)
         .await
     {
         Ok(_r) => {
-            log::trace!("Host Initializer Client: Node ID published.");
+            log::trace!("Host Auth Client: Node ID published.");
         }
         Err(_e) => {}
     };
 
     // ==================== API ENDPOINTS ====================
-    // Register Workload Streams for Host Agent to consume
-    // (subjects should be published by orchestrator or nats-db-connector)
+    // Register Auth Streams for Orchestrator to consume and proceess
+    // NB: The subjects below are published by the Orchestrator
 
     // Call auth service and perform auth handshake
-    let auth_service = initializer_client
+    let auth_service = host_auth_client
         .get_js_service(AUTH_SRV_NAME.to_string())
         .await
         .ok_or(anyhow!(
-            "Failed to locate workload service. Unable to spin up Host Agent."
+            "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
         ))?;
 
     // i. register `save_hub_auth` consumer
@@ -121,22 +122,30 @@ pub async fn run() -> Result<String, async_nats::Error> {
     // to add_user_pubkey
     // then await the reply (which should include the user jwt)
 
-    // register save service for hub auth files (operator and sys)
+    // Register save service for hub auth files (operator and sys)
     auth_service
-        .add_local_consumer(
+        .add_local_consumer::<authentication::types::ApiResult>(
             "save_hub_auth",
             "save_hub_auth",
-            nats_js_client::EndpointType::Async(endpoints::save_hub_jwts(&auth_api).await),
+            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+                async move {
+                    api.save_hub_jwts(msg).await
+                }
+            })),
             None,
         )
         .await?;
-
-    // register save service for signed user jwt file
+        
+    // Register save service for signed user jwt file
     auth_service
-        .add_local_consumer(
-            "save_user_file",
+       .add_local_consumer::<authentication::types::ApiResult>(
+            "save_user_jwt",
             "end_hub_handshake",
-            EndpointType::Async(endpoints::save_user_jwt(&auth_api).await),
+            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+                async move {
+                    api.save_user_jwt(msg, USER_CREDENTIALS_PATH).await
+                }
+            })),
             None,
         )
         .await?;
@@ -151,27 +160,27 @@ pub async fn run() -> Result<String, async_nats::Error> {
     //         println!("Failed to get a response: {}", e);
     //     }
     // }
-    let req_hub_files_options = nats_js_client::PublishOptions {
+    let req_hub_files_options = nats_js_client::SendRequest {
         subject: format!("HPOS.init.{}", server_node_id),
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
-        data: b"Host Initializer Connected!".to_vec(),
+        data: b"Host Auth Connected!".to_vec(),
     };
-    initializer_client.publish(&req_hub_files_options);
+    host_auth_client.publish(&req_hub_files_options);
 
     // ...upon the reply to the above, do the following: publish user pubkey file
-    let send_user_pubkey_publish_options = nats_js_client::PublishOptions {
+    let send_user_pubkey_publish_options = nats_js_client::SendRequest {
         subject: format!("HPOS.init.{}", server_node_id),
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
-        data: b"Host Initializer Connected!".to_vec(),
+        data: b"Host Auth Connected!".to_vec(),
     };
-    // initializer_client.publish(send_user_pubkey_publish_options);
-    utils::chunk_file_and_publish(&initializer_client.js, "subject", "file_path");
+    // host_auth_client.publish(send_user_pubkey_publish_options);
+    utils::chunk_file_and_publish(&host_auth_client.js, "subject", "file_path");
 
     // 5. Generate user creds file
     let user_creds_path = utils::generate_creds_file();
 
     // 6. Close and drain internal buffer before exiting to make sure all messages are sent
-    initializer_client.close().await?;
+    host_auth_client.close().await?;
 
     Ok(user_creds_path)
 }
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
index ef8adf5..5f783ec 100644
--- a/rust/clients/host_agent/src/auth/mod.rs
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -1,4 +1,3 @@
-pub mod agent_key;
-pub mod endpoints;
+// pub mod agent_key;
 pub mod utils;
 pub mod initializer;
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index 2e5fb41..f082cac 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,4 +1,4 @@
-use super::auth::initializer::HOST_INIT_CLIENT_NAME;
+use super::initializer::HOST_INIT_CLIENT_NAME;
 use anyhow::Result;
 use async_nats::jetstream::Context;
 use std::process::Command;
diff --git a/rust/clients/orchestrator/src/auth/controller.rs b/rust/clients/orchestrator/src/auth/controller.rs
index 3037a09..efedcdc 100644
--- a/rust/clients/orchestrator/src/auth/controller.rs
+++ b/rust/clients/orchestrator/src/auth/controller.rs
@@ -1,12 +1,22 @@
-use super::endpoints;
+/*
+ This client is associated with the:
+- auth account
+- orchestrator user
+
+// This client is responsible for:
+*/
+
 use crate::utils;
-use anyhow::Result;
-use authentication::{AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+
+use anyhow::{anyhow, Result};
+use std::{sync::Arc, time::Duration};
+use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use authentication::{self, AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use std::process::Command;
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::JsStreamService,
+    js_stream_service::JsServiceParamsPartial,
     nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
 };
 
@@ -16,51 +26,57 @@ pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_orchestrator_auth_inbo
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== NATS Setup ====================
     let nats_url = nats_js_client::get_nats_url();
-    let creds_path = nats_js_client::get_nats_client_creds("HOLO", "ADMIN", "orchestrator");
     let event_listeners = nats_js_client::get_event_listeners();
 
+    // Setup JS Stream Service
+    let auth_stream_service_params = JsServiceParamsPartial {
+        name: AUTH_SRV_NAME.to_string(),
+        description: AUTH_SRV_DESC.to_string(),
+        version: AUTH_SRV_VERSION.to_string(),
+        service_subject: AUTH_SRV_SUBJ.to_string(),
+    };
+    
     let orchestrator_auth_client =
         JsClient::new(NewJsClientParams {
             nats_url,
             name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
             inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
+            service_params: vec![auth_stream_service_params],
+            credentials_path: None,
             opts: vec![nats_js_client::with_event_listeners(event_listeners)],
-            credentials_path: Some(creds_path),
-            ..Default::default()
+            ping_interval: Some(Duration::from_secs(10)),
+            request_timeout: Some(Duration::from_secs(5)),
         })
         .await?;
 
-    // Create a new Jetstream Microservice
-    let js_service = JsStreamService::new(
-        orchestrator_auth_client.js.clone(),
-        AUTH_SRV_NAME,
-        AUTH_SRV_DESC,
-        AUTH_SRV_VERSION,
-        AUTH_SRV_SUBJ,
-    )
-    .await?;
-
     // ==================== DB Setup ====================
-
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
     let client = MongoDBClient::with_options(client_options)?;
 
-    // Generate the Workload API with access to db
+    // Generate the Auth API with access to db
     let auth_api = AuthApi::new(&client).await?;
 
     // ==================== API ENDPOINTS ====================
-    // Register Workload Streams for Host Agent to consume
-    // (subjects should be published by orchestrator or nats-db-connector)
-
-    let auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", "host_id_placeholder"); // endpoint_subject
-
-    js_service
-        .add_local_consumer(
-            "add_user_pubkey", // called from orchestrator (no -auth service)
+    // Register Auth Streams for Orchestrator to consume and proceess
+    // NB: The subjects below are published by the Host Agent
+    let auth_service = orchestrator_auth_client
+        .get_js_service(AUTH_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
+        ))?;
+
+    auth_service
+        .add_local_consumer::<authentication::types::ApiResult>(
             "add_user_pubkey",
-            EndpointType::Async(endpoints::add_user_pubkey(&auth_api).await),
+            "add_user_pubkey",
+            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+                async move {
+                    api.add_user_pubkey(msg).await
+                }
+            })),
             None,
         )
         .await?;
@@ -70,8 +86,11 @@ pub async fn run() -> Result<(), async_nats::Error> {
         AUTH_SRV_NAME
     );
 
+
     let resolver_path = utils::get_resolver_path();
 
+    let auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", "host_id_placeholder"); // endpoint_subject
+ 
     // Generate resolver file and create resolver file
     Command::new("nsc")
         .arg("generate")
diff --git a/rust/clients/orchestrator/src/auth/endpoints.rs b/rust/clients/orchestrator/src/auth/endpoints.rs
deleted file mode 100644
index 6f6e755..0000000
--- a/rust/clients/orchestrator/src/auth/endpoints.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-use anyhow::Result;
-use async_nats::Message;
-use authentication::AuthApi;
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::Arc;
-use util_libs::nats_js_client::AsyncEndpointHandler;
-
-pub async fn add_user_pubkey(auth_api: &AuthApi) -> AsyncEndpointHandler {
-    let api = auth_api.to_owned();
-    Arc::new(
-            move |msg: Arc<Message>| -> Pin<Box<dyn Future<Output = Result<Vec<u8>, anyhow::Error>> + Send>> {
-                let api_clone = api.clone();
-                Box::pin(async move {
-                    api_clone.add_user_pubkey(msg).await
-                })
-            },
-        )
-}
diff --git a/rust/clients/orchestrator/src/auth/mod.rs b/rust/clients/orchestrator/src/auth/mod.rs
index 32ba67b..cb9e0ac 100644
--- a/rust/clients/orchestrator/src/auth/mod.rs
+++ b/rust/clients/orchestrator/src/auth/mod.rs
@@ -1,2 +1 @@
 pub mod controller;
-pub mod endpoints;
diff --git a/rust/clients/orchestrator/src/workloads/controller.rs b/rust/clients/orchestrator/src/workloads/controller.rs
index c2e20fb..e3c222d 100644
--- a/rust/clients/orchestrator/src/workloads/controller.rs
+++ b/rust/clients/orchestrator/src/workloads/controller.rs
@@ -10,14 +10,14 @@ use anyhow::{anyhow, Result};
 use std::{sync::Arc, time::Duration};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use workload::{
+    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::JsServiceParamsPartial,
     nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
 };
-use workload::{
-    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
-};
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
 const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_orchestrator_workload_inbox";
@@ -60,12 +60,12 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // ==================== API ENDPOINTS ====================
     // Register Workload Streams for Orchestrator to consume and proceess
-    // NB: Subjects are published by external Developer, the Nats-DB-Connector, or the Host Agent
+    // NB: These subjects below are published by external Developer, the Nats-DB-Connector, or the Host Agent
     let workload_service = orchestrator_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
         .ok_or(anyhow!(
-            "Failed to locate workload service. Unable to spin up Host Agent."
+            "Failed to locate Workload Service. Unable to spin up Orchestrator Workload Client."
         ))?;
 
     // Published by Developer
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 7370a75..172b457 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -17,14 +17,16 @@ Endpoints & Managed Subjects:
 
 */
 
-use anyhow::Result;
+pub mod types;
+use anyhow::{anyhow, Result};
 use async_nats::Message;
-use mongodb::Client as MongoDBClient;
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
 use std::process::Command;
 use std::sync::Arc;
-use util_libs::db::{mongodb::MongoCollection, schemas};
+use std::future::Future;
+use serde::{Deserialize, Serialize};
+use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}}, nats_js_client};
 
-pub const AUTH_SRV_OWNER_NAME: &str = "AUTH_OWNER";
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";
 pub const AUTH_SRV_VERSION: &str = "0.0.1";
@@ -40,42 +42,34 @@ pub struct AuthApi {
 
 impl AuthApi {
     pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        // Create a typed collection for User
-        let user_api: MongoCollection<schemas::User> = MongoCollection::<schemas::User>::new(
-            client,
-            schemas::DATABASE_NAME,
-            schemas::HOST_COLLECTION_NAME,
-        )
-        .await?;
-
-        // Create a typed collection for Hoster
-        let hoster_api = MongoCollection::<schemas::Hoster>::new(
-            &client,
-            schemas::DATABASE_NAME,
-            schemas::HOST_COLLECTION_NAME,
-        )
-        .await?;
-
-        // Create a typed collection for Host
-        let host_api = MongoCollection::<schemas::Host>::new(
-            &client,
-            schemas::DATABASE_NAME,
-            schemas::HOST_COLLECTION_NAME,
-        )
-        .await?;
-
         Ok(Self {
-            user_collection: user_api,
-            hoster_collection: hoster_api,
-            host_collection: host_api,
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME).await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
         })
     }
 
-    // For orchestrator
+
+    pub fn call<F, Fut>(
+        &self,
+        handler: F,
+    ) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    where
+        F: Fn(AuthApi, Arc<Message>) -> Fut + Send + Sync + 'static,
+        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+    {
+        let api = self.to_owned(); 
+        Arc::new(move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+            let api_clone = api.clone();
+            Box::pin(handler(api_clone, msg))
+        })
+    }
+    
+    /*******************************  For Orchestrator   *********************************/
     pub async fn receive_handshake_request(
         &self,
         msg: Arc<Message>,
-    ) -> Result<Vec<u8>, anyhow::Error> {
+    ) -> Result<types::ApiResult, anyhow::Error> {
         // 1. Verify expected data was received
         if msg.headers.is_none() {
             log::error!(
@@ -108,12 +102,18 @@ impl AuthApi {
         // let hub_operator_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
         // let hub_sys_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
 
-        let response = serde_json::to_vec(&"OK")?;
-        Ok(response)
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_id: "host_id_placeholder".to_string(),
+                status: types::AuthState::Requested
+            },
+            result: serde_json::to_vec(&"OK")?,
+            maybe_response_tags: None
+        })
     }
 
-    // For hpos
-    pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
+    /*******************************   For Host Agent   *********************************/
+     pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         // receive_and_write_file();
 
         // Respond to endpoint request
@@ -123,8 +123,8 @@ impl AuthApi {
         todo!();
     }
 
-    // For orchestrator
-    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
+    /*******************************  For Orchestrator   *********************************/
+    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
 
         // Add user with Keys and create jwt
@@ -144,24 +144,49 @@ impl AuthApi {
 
         // 2. Respond to endpoint request
         // let resposne = user_jwt_path;
-        let response = b"Hello, NATS!".to_vec();
-        Ok(response)
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_id: "host_id_placeholder".to_string(),
+                status: types::AuthState::ValidatedAgent
+            },
+            result: b"user_jwt_path_placeholder".to_vec(),
+            maybe_response_tags: None
+        })
     }
 
-    // For hpos
-    pub async fn save_user_jwt(
+    /*******************************   For Host Agent   *********************************/
+     pub async fn save_user_jwt(
         &self,
         msg: Arc<Message>,
         output_dir: &str,
-    ) -> Result<Vec<u8>, anyhow::Error> {
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
 
         // utils::receive_and_write_file(msg, output_dir, file_name).await?;
 
         // 2. Respond to endpoint request
-        let response = b"Hello, NATS!".to_vec();
-        Ok(response)
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_id: "host_id_placeholder".to_string(),
+                status: types::AuthState::Authenticated
+            },
+            result: b"Hello, NATS!".to_vec(),
+            maybe_response_tags: None
+        })
     }
+
+    /*******************************  Helper Fns  *********************************/
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+
 }
 
 // In orchestrator
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 266b02c..a62b3ea 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -1,13 +1,41 @@
+use util_libs::js_stream_service::{CreateTag, EndpointTraits};
 use serde::{Deserialize, Serialize};
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum AuthState {
+    Requested,
+    ValidatedAgent, //    AddedHostPubkey
+    SignedJWT,
+    Authenticated
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct AuthStatus {
+    pub host_id: String,
+    pub status: AuthState
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct AuthHeaders {
     signature: String,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
-pub struct AuthPayload {
-    email: String,
-    host_id: String,
-    pubkey: String,
+pub struct AuthRequestPayload {
+    pub email: String,
+    pub host_id: String,
+    pub pubkey: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ApiResult {
+    pub status: AuthStatus,
+    pub result: Vec<u8>,
+    pub maybe_response_tags: Option<Vec<String>>
+}
+impl EndpointTraits for ApiResult {}
+impl CreateTag for ApiResult {
+    fn get_tags(&self) -> Option<Vec<String>> {
+        self.maybe_response_tags.clone()
+    }
 }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 353d803..a1efb46 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -13,7 +13,6 @@ Endpoints & Managed Subjects:
 */
 
 pub mod types;
-
 use anyhow::{anyhow, Result};
 use async_nats::Message;
 use mongodb::{options::UpdateModifications, Client as MongoDBClient};
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index cea7d1d..3b0fdaf 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,15 +1,11 @@
 use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
 
-pub use String as WorkloadId;
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ApiResult (pub WorkloadStatus, pub Option<Vec<String>>);
-
+impl EndpointTraits for ApiResult {}
 impl CreateTag for ApiResult {
     fn get_tags(&self) -> Option<Vec<String>> {
         self.1.clone()
     }
 }
-
-impl EndpointTraits for ApiResult {}
\ No newline at end of file

From 9bc4b324fbe889443706358ceef252414de44603 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Wed, 15 Jan 2025 16:39:11 -0600
Subject: [PATCH 08/91] clean up

---
 .../host_agent/src/auth/initializer.rs        | 10 ++---
 rust/clients/host_agent/src/auth/utils.rs     | 40 -------------------
 .../orchestrator/src/auth/controller.rs       |  2 +-
 rust/clients/orchestrator/src/main.rs         |  4 +-
 rust/clients/orchestrator/src/utils.rs        |  9 ++---
 rust/services/authentication/src/lib.rs       | 15 ++++---
 6 files changed, 19 insertions(+), 61 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/initializer.rs b/rust/clients/host_agent/src/auth/initializer.rs
index 650d56d..55c50dd 100644
--- a/rust/clients/host_agent/src/auth/initializer.rs
+++ b/rust/clients/host_agent/src/auth/initializer.rs
@@ -24,8 +24,8 @@ use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::{sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::{JsServiceParamsPartial, JsStreamService},
-    nats_js_client::{self, EndpointType, EventListener, JsClient},
+    js_stream_service::JsServiceParamsPartial,
+    nats_js_client::{self, EndpointType},
 };
 
 pub const HOST_INIT_CLIENT_NAME: &str = "Host Auth";
@@ -165,16 +165,16 @@ pub async fn run() -> Result<String, async_nats::Error> {
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
         data: b"Host Auth Connected!".to_vec(),
     };
-    host_auth_client.publish(&req_hub_files_options);
+    let _ = host_auth_client.publish(&req_hub_files_options).await?;
 
     // ...upon the reply to the above, do the following: publish user pubkey file
-    let send_user_pubkey_publish_options = nats_js_client::SendRequest {
+    let _send_user_pubkey_publish_options = nats_js_client::SendRequest {
         subject: format!("HPOS.init.{}", server_node_id),
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
         data: b"Host Auth Connected!".to_vec(),
     };
     // host_auth_client.publish(send_user_pubkey_publish_options);
-    utils::chunk_file_and_publish(&host_auth_client.js, "subject", "file_path");
+    let _ = utils::chunk_file_and_publish(&host_auth_client.js, "subject", "file_path").await?;
 
     // 5. Generate user creds file
     let user_creds_path = utils::generate_creds_file();
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index f082cac..e69de29 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,40 +0,0 @@
-use super::initializer::HOST_INIT_CLIENT_NAME;
-use anyhow::Result;
-use async_nats::jetstream::Context;
-use std::process::Command;
-use std::time::Duration;
-use util_libs::{
-    js_stream_service::JsStreamService,
-    nats_js_client::{self, EventListener},
-};
-
-pub async fn chunk_file_and_publish(js: &Context, subject: &str, file_path: &str) -> Result<()> {
-    // let mut file = std::fs::File::open(file_path)?;
-    // let mut buffer = vec![0; CHUNK_SIZE];
-    // let mut chunk_id = 0;
-
-    // while let Ok(bytes_read) = file.read(mut buffer) {
-    //     if bytes_read == 0 {
-    //         break;
-    //     }
-    //     let chunk_data = &buffer[..bytes_read];
-    //     js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
-    //     chunk_id += 1;
-    // }
-
-    // // Send an EOF marker
-    // js.publish(subject.to_string(), "EOF".into()).await.unwrap();
-
-    Ok(())
-}
-
-pub fn generate_creds_file() -> String {
-    let user_creds_path = "/path/to/host/user.creds".to_string();
-    Command::new("nsc")
-        .arg(format!("... > {}", user_creds_path))
-        .output()
-        .expect("Failed to add user with provided keys")
-        .stdout;
-
-    "placeholder_user.creds".to_string()
-}
diff --git a/rust/clients/orchestrator/src/auth/controller.rs b/rust/clients/orchestrator/src/auth/controller.rs
index efedcdc..2d5db23 100644
--- a/rust/clients/orchestrator/src/auth/controller.rs
+++ b/rust/clients/orchestrator/src/auth/controller.rs
@@ -89,7 +89,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     let resolver_path = utils::get_resolver_path();
 
-    let auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", "host_id_placeholder"); // endpoint_subject
+    let _auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", "host_id_placeholder"); // endpoint_subject
  
     // Generate resolver file and create resolver file
     Command::new("nsc")
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index d541dc8..da6b49b 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -17,9 +17,9 @@ async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
 
-    auth::controller::run().await;
+    let _ = auth::controller::run().await?;
 
-    workloads::controller::run().await;
+    let _ = workloads::controller::run().await?;
 
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
index 7dfa582..997455e 100644
--- a/rust/clients/orchestrator/src/utils.rs
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -1,8 +1,7 @@
-use super::auth::controller::ORCHESTRATOR_AUTH_CLIENT_NAME;
+// use super::auth::controller::ORCHESTRATOR_AUTH_CLIENT_NAME;
 use anyhow::Result;
 use std::io::Read;
-use tokio::time::Duration;
-use util_libs::nats_js_client::{self, EventListener, JsClient, SendRequest};
+use util_libs::nats_js_client::{JsClient, SendRequest};
 
 const CHUNK_SIZE: usize = 1024; // 1 KB chunk size
 
@@ -35,7 +34,7 @@ pub async fn chunk_file_and_publish(
             msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
             data: chunk_data.into(),
         };
-        auth_client.publish(&send_user_jwt_publish_options).await;
+        let _ = auth_client.publish(&send_user_jwt_publish_options).await;
         chunk_id += 1;
     }
 
@@ -45,7 +44,7 @@ pub async fn chunk_file_and_publish(
         msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
         data: "EOF".into(),
     };
-    auth_client.publish(&send_user_jwt_publish_options);
+    let _ = auth_client.publish(&send_user_jwt_publish_options);
 
     Ok(())
 }
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 172b457..987c520 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -18,14 +18,14 @@ Endpoints & Managed Subjects:
 */
 
 pub mod types;
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use async_nats::Message;
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use mongodb::Client as MongoDBClient; // options::UpdateModifications, 
 use std::process::Command;
 use std::sync::Arc;
 use std::future::Future;
 use serde::{Deserialize, Serialize};
-use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}}, nats_js_client};
+use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection}, schemas}, nats_js_client};
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";
@@ -113,7 +113,7 @@ impl AuthApi {
     }
 
     /*******************************   For Host Agent   *********************************/
-     pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+     pub async fn save_hub_jwts(&self, _msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         // receive_and_write_file();
 
         // Respond to endpoint request
@@ -131,11 +131,10 @@ impl AuthApi {
         Command::new("nsc")
             .arg("...")
             .output()
-            .expect("Failed to add user with provided keys")
-            .stdout;
+            .expect("Failed to add user with provided keys");
 
         // Output jwt
-        let user_jwt_path = Command::new("nsc")
+        let _user_jwt_path = Command::new("nsc")
             .arg("...")
             // .arg(format!("> {}", output_dir))
             .output()
@@ -158,7 +157,7 @@ impl AuthApi {
      pub async fn save_user_jwt(
         &self,
         msg: Arc<Message>,
-        output_dir: &str,
+        _output_dir: &str,
     ) -> Result<types::ApiResult, anyhow::Error> {
         log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
 

From ebdf3bf17951e39d275fc88e78f5065ab79b9ba2 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Wed, 15 Jan 2025 16:40:54 -0600
Subject: [PATCH 09/91] restore util file

---
 rust/clients/host_agent/src/auth/utils.rs | 34 +++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index e69de29..c3d8529 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -0,0 +1,34 @@
+use anyhow::Result;
+use async_nats::jetstream::Context;
+use std::process::Command;
+
+pub async fn chunk_file_and_publish(_js: &Context, _subject: &str, _file_path: &str) -> Result<()> {
+    // let mut file = std::fs::File::open(file_path)?;
+    // let mut buffer = vec![0; CHUNK_SIZE];
+    // let mut chunk_id = 0;
+
+    // while let Ok(bytes_read) = file.read(mut buffer) {
+    //     if bytes_read == 0 {
+    //         break;
+    //     }
+    //     let chunk_data = &buffer[..bytes_read];
+    //     js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
+    //     chunk_id += 1;
+    // }
+
+    // // Send an EOF marker
+    // js.publish(subject.to_string(), "EOF".into()).await.unwrap();
+
+    Ok(())
+}
+
+pub fn generate_creds_file() -> String {
+    let user_creds_path = "/path/to/host/user.creds".to_string();
+    Command::new("nsc")
+        .arg(format!("... > {}", user_creds_path))
+        .output()
+        .expect("Failed to add user with provided keys")
+        .stdout;
+
+    "placeholder_user.creds".to_string()
+}

From 8137839a23458211b360799d35e8395cae54e3e7 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Wed, 15 Jan 2025 16:49:51 -0600
Subject: [PATCH 10/91] lint

---
 .../src/auth/{initializer.rs => init_agent.rs}           | 8 ++++----
 rust/clients/host_agent/src/auth/mod.rs                  | 2 +-
 rust/clients/host_agent/src/auth/utils.rs                | 3 +--
 rust/clients/host_agent/src/main.rs                      | 9 ++++-----
 rust/clients/orchestrator/src/auth/controller.rs         | 6 ++----
 rust/clients/orchestrator/src/main.rs                    | 7 ++-----
 rust/clients/orchestrator/src/utils.rs                   | 2 +-
 7 files changed, 15 insertions(+), 22 deletions(-)
 rename rust/clients/host_agent/src/auth/{initializer.rs => init_agent.rs} (96%)

diff --git a/rust/clients/host_agent/src/auth/initializer.rs b/rust/clients/host_agent/src/auth/init_agent.rs
similarity index 96%
rename from rust/clients/host_agent/src/auth/initializer.rs
rename to rust/clients/host_agent/src/auth/init_agent.rs
index 55c50dd..88b7e85 100644
--- a/rust/clients/host_agent/src/auth/initializer.rs
+++ b/rust/clients/host_agent/src/auth/init_agent.rs
@@ -33,7 +33,7 @@ pub const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_auth_inbox";
 
 const USER_CREDENTIALS_PATH: &str = "./user_creds";
 
-pub async fn run() -> Result<String, async_nats::Error> {
+pub async fn run() -> Result<(String, String), async_nats::Error> {
     log::info!("Host Auth Client: Connecting to server...");
 
     // ==================== NATS Setup ====================
@@ -165,7 +165,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
         data: b"Host Auth Connected!".to_vec(),
     };
-    let _ = host_auth_client.publish(&req_hub_files_options).await?;
+    host_auth_client.publish(&req_hub_files_options).await?;
 
     // ...upon the reply to the above, do the following: publish user pubkey file
     let _send_user_pubkey_publish_options = nats_js_client::SendRequest {
@@ -174,7 +174,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
         data: b"Host Auth Connected!".to_vec(),
     };
     // host_auth_client.publish(send_user_pubkey_publish_options);
-    let _ = utils::chunk_file_and_publish(&host_auth_client.js, "subject", "file_path").await?;
+    utils::chunk_file_and_publish(&host_auth_client.js, "subject", "file_path").await?;
 
     // 5. Generate user creds file
     let user_creds_path = utils::generate_creds_file();
@@ -182,5 +182,5 @@ pub async fn run() -> Result<String, async_nats::Error> {
     // 6. Close and drain internal buffer before exiting to make sure all messages are sent
     host_auth_client.close().await?;
 
-    Ok(user_creds_path)
+    Ok(("host_pubkey_placeholder".to_string(), user_creds_path))
 }
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
index 5f783ec..a155dc2 100644
--- a/rust/clients/host_agent/src/auth/mod.rs
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -1,3 +1,3 @@
 // pub mod agent_key;
 pub mod utils;
-pub mod initializer;
+pub mod init_agent;
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index c3d8529..d3ecea4 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -27,8 +27,7 @@ pub fn generate_creds_file() -> String {
     Command::new("nsc")
         .arg(format!("... > {}", user_creds_path))
         .output()
-        .expect("Failed to add user with provided keys")
-        .stdout;
+        .expect("Failed to add user with provided keys");
 
     "placeholder_user.creds".to_string()
 }
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 2dd0c13..efd4d30 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -19,7 +19,6 @@ pub mod agent_cli;
 pub mod host_cmds;
 pub mod support_cmds;
 use thiserror::Error;
-use util_libs::nats_js_client;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -54,10 +53,10 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize() -> Result<(), async_nats::Error> {
-    // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let host_creds_path = nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
-    let host_pubkey = "host_id_placeholder>";
+    // let (host_pubkey, host_creds_path) = auth::init_agent::run().await?;
+    let host_creds_path = util_libs::nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
+    let host_pubkey = "host_pubkey_placeholder>";
     hostd::gen_leaf_server::run(&host_creds_path).await;
-    hostd::workload_manager::run(host_pubkey, &host_creds_path).await?;
+    hostd::workload_manager::run(&host_pubkey, &host_creds_path).await?;
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/auth/controller.rs b/rust/clients/orchestrator/src/auth/controller.rs
index 2d5db23..fb81dca 100644
--- a/rust/clients/orchestrator/src/auth/controller.rs
+++ b/rust/clients/orchestrator/src/auth/controller.rs
@@ -100,15 +100,13 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .arg("--force")
         .arg(format!("--config-file {}", resolver_path))
         .output()
-        .expect("Failed to create resolver config file")
-        .stdout;
+        .expect("Failed to create resolver config file");
 
     // Push auth updates to hub server
     Command::new("nsc")
         .arg("push -A")
         .output()
-        .expect("Failed to create resolver config file")
-        .stdout;
+        .expect("Failed to create resolver config file");
 
     // publish user jwt file
     let server_node_id = "server_node_id_placeholder";
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index da6b49b..f625b41 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -16,10 +16,7 @@ use dotenv::dotenv;
 async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
-
-    let _ = auth::controller::run().await?;
-
-    let _ = workloads::controller::run().await?;
-
+    auth::controller::run().await?;
+    workloads::controller::run().await?;
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
index 997455e..e7ec3c1 100644
--- a/rust/clients/orchestrator/src/utils.rs
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -44,7 +44,7 @@ pub async fn chunk_file_and_publish(
         msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
         data: "EOF".into(),
     };
-    let _ = auth_client.publish(&send_user_jwt_publish_options);
+    let _ = auth_client.publish(&send_user_jwt_publish_options).await;
 
     Ok(())
 }

From 5e90d8fa10c9b3455cb6bd0114f90a8cedeca1fa Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Wed, 15 Jan 2025 17:24:42 -0600
Subject: [PATCH 11/91] clean-up dirs

---
 rust/clients/orchestrator/src/{auth/controller.rs => auth.rs} | 0
 rust/clients/orchestrator/src/auth/mod.rs                     | 1 -
 rust/clients/orchestrator/src/main.rs                         | 4 ++--
 .../src/{workloads/controller.rs => workloads.rs}             | 0
 rust/clients/orchestrator/src/workloads/mod.rs                | 1 -
 5 files changed, 2 insertions(+), 4 deletions(-)
 rename rust/clients/orchestrator/src/{auth/controller.rs => auth.rs} (100%)
 delete mode 100644 rust/clients/orchestrator/src/auth/mod.rs
 rename rust/clients/orchestrator/src/{workloads/controller.rs => workloads.rs} (100%)
 delete mode 100644 rust/clients/orchestrator/src/workloads/mod.rs

diff --git a/rust/clients/orchestrator/src/auth/controller.rs b/rust/clients/orchestrator/src/auth.rs
similarity index 100%
rename from rust/clients/orchestrator/src/auth/controller.rs
rename to rust/clients/orchestrator/src/auth.rs
diff --git a/rust/clients/orchestrator/src/auth/mod.rs b/rust/clients/orchestrator/src/auth/mod.rs
deleted file mode 100644
index cb9e0ac..0000000
--- a/rust/clients/orchestrator/src/auth/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod controller;
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index f625b41..adee43b 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -16,7 +16,7 @@ use dotenv::dotenv;
 async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
-    auth::controller::run().await?;
-    workloads::controller::run().await?;
+    auth::run().await?;
+    workloads::run().await?;
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/workloads/controller.rs b/rust/clients/orchestrator/src/workloads.rs
similarity index 100%
rename from rust/clients/orchestrator/src/workloads/controller.rs
rename to rust/clients/orchestrator/src/workloads.rs
diff --git a/rust/clients/orchestrator/src/workloads/mod.rs b/rust/clients/orchestrator/src/workloads/mod.rs
deleted file mode 100644
index cb9e0ac..0000000
--- a/rust/clients/orchestrator/src/workloads/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod controller;

From b065defa3dad09f75291fef094147ae9bab3b84f Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Thu, 16 Jan 2025 14:57:44 -0600
Subject: [PATCH 12/91] fix types

---
 rust/clients/orchestrator/src/main.rs | 13 +++++++++++--
 rust/util_libs/src/nats_js_client.rs  |  7 +++----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index adee43b..d00dde1 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -11,12 +11,21 @@ mod utils;
 mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
+use tokio::task::spawn;
 
 #[tokio::main]
 async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
-    auth::run().await?;
-    workloads::run().await?;
+    spawn(async move { 
+        if let Err(e) = auth::run().await {
+            log::error!("{}", e)
+        }
+    });
+    spawn(async move {
+        if let Err(e) = workloads::run().await {
+            log::error!("{}", e)
+        } 
+    });
     Ok(())
 }
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 597ef43..7eb43f1 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -11,8 +11,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type ClientOption = Box<dyn Fn(&mut JsClient)>;
-pub type EventListener = Box<dyn Fn(&mut JsClient)>;
+pub type EventListener = Box<dyn Fn(&mut JsClient) + Send + Sync>;
 pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
 pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
 pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
@@ -93,7 +92,7 @@ pub struct NewJsClientParams {
     #[serde(default)]
     pub service_params: Vec<JsServiceParamsPartial>,
     #[serde(skip_deserializing)]
-    pub opts: Vec<ClientOption>, // NB: These opts should not be required for client instantiation
+    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
     #[serde(default)]
     pub credentials_path: Option<String>,
     #[serde(default)]
@@ -250,7 +249,7 @@ impl JsClient {
 }
 
 // Client Options:
-pub fn with_event_listeners(listeners: Vec<EventListener>) -> ClientOption {
+pub fn with_event_listeners(listeners: Vec<EventListener>) -> EventListener {
     Box::new(move |c: &mut JsClient | {
         for listener in &listeners {
             listener(c);

From 58841506293e8bc1c82421a6043855a63ae5bdb0 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 17 Jan 2025 20:50:27 -0600
Subject: [PATCH 13/91] update names

---
 rust/services/workload/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index a1efb46..5b9c111 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -69,13 +69,13 @@ impl WorkloadApi {
             |workload: schemas::Workload| async move {
                 let workload_id = self.workload_collection.insert_one_into(workload.clone()).await?;
                 log::info!("Successfully added workload. MongodDB Workload ID={:?}", workload_id);
-                let updated_workload = schemas::Workload {
+                let new_workload = schemas::Workload {
                     _id: Some(workload_id),
                     ..workload
                 };
                 Ok(types::ApiResult(
                     WorkloadStatus {
-                        id: updated_workload._id,
+                        id: new_workload._id,
                         desired: WorkloadState::Reported,
                         actual: WorkloadState::Reported,
                     },
@@ -94,8 +94,8 @@ impl WorkloadApi {
             WorkloadState::Running,
             |workload: schemas::Workload| async move {
                 let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload = to_document(&workload)?;
-                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload)).await?;
+                let updated_workload_doc = to_document(&workload)?;
+                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
                 Ok(types::ApiResult(
                     WorkloadStatus {

From 0ecd15a5ee8a5011aee8ba884b78b11c71aa359f Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 00:20:57 -0600
Subject: [PATCH 14/91] auth service updates

---
 .env.example                                  |   3 +
 Cargo.lock                                    | 123 ++++++--
 rust/clients/host_agent/Cargo.toml            |   1 +
 rust/clients/host_agent/src/auth/agent_key.rs | 111 -------
 .../clients/host_agent/src/auth/init_agent.rs | 132 ++++----
 rust/clients/host_agent/src/auth/utils.rs     |  30 +-
 .../host_agent/src/hostd/workload_manager.rs  |  18 +-
 rust/clients/host_agent/src/main.rs           |  16 +-
 rust/clients/orchestrator/src/auth.rs         |  90 +++---
 rust/clients/orchestrator/src/utils.rs        |  86 +++--
 rust/clients/orchestrator/src/workloads.rs    |  30 +-
 rust/services/authentication/src/lib.rs       | 294 ++++++++++++------
 rust/services/authentication/src/types.rs     |  37 ++-
 rust/services/authentication/src/utils.rs     |  79 ++++-
 rust/services/workload/src/lib.rs             | 151 ++++-----
 rust/services/workload/src/types.rs           |   8 +-
 rust/util_libs/src/db/mongodb.rs              |  51 ++-
 rust/util_libs/src/db/schemas.rs              |  16 +-
 rust/util_libs/src/js_stream_service.rs       |  14 +-
 rust/util_libs/src/nats_js_client.rs          |  46 ++-
 20 files changed, 763 insertions(+), 573 deletions(-)
 delete mode 100644 rust/clients/host_agent/src/auth/agent_key.rs

diff --git a/.env.example b/.env.example
index bfe8157..634f1be 100644
--- a/.env.example
+++ b/.env.example
@@ -4,3 +4,6 @@ MONGO_URI = "mongodb://<host>:<port>"
 NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
 LEAF_SERVER_USER = "test-user"
 LEAF_SERVER_PW = "pw-123456789"
+
+HOST_CREDENTIALS_PATH: &str = "./host_user.creds"; 
+# USER_CREDENTIALS_PATH: &str = "./user_creds";
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index b88e140..f025399 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -146,7 +146,7 @@ dependencies = [
  "once_cell",
  "pin-project",
  "portable-atomic",
- "rand",
+ "rand 0.8.5",
  "regex",
  "ring",
  "rustls-native-certs 0.7.3",
@@ -222,6 +222,12 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -317,7 +323,7 @@ dependencies = [
  "indexmap 2.7.0",
  "js-sys",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -899,6 +905,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -907,7 +924,7 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
 [[package]]
@@ -971,7 +988,7 @@ dependencies = [
  "idna",
  "ipnet",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "thiserror 1.0.69",
  "tinyvec",
  "tokio",
@@ -992,7 +1009,7 @@ dependencies = [
  "lru-cache",
  "once_cell",
  "parking_lot",
- "rand",
+ "rand 0.8.5",
  "resolv-conf",
  "smallvec",
  "thiserror 1.0.69",
@@ -1027,9 +1044,10 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_json",
+ "textnonce",
  "thiserror 2.0.8",
  "tokio",
  "url",
@@ -1407,7 +1425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "windows-sys 0.52.0",
 ]
 
@@ -1437,7 +1455,7 @@ dependencies = [
  "once_cell",
  "pbkdf2",
  "percent-encoding",
- "rand",
+ "rand 0.8.5",
  "rustc_version_runtime",
  "rustls 0.21.12",
  "rustls-pemfile 1.0.4",
@@ -1492,9 +1510,9 @@ dependencies = [
  "data-encoding",
  "ed25519",
  "ed25519-dalek",
- "getrandom",
+ "getrandom 0.2.15",
  "log",
- "rand",
+ "rand 0.8.5",
  "signatory",
 ]
 
@@ -1504,7 +1522,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83"
 dependencies = [
- "rand",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -1559,7 +1577,7 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_json",
  "thiserror 2.0.8",
@@ -1767,6 +1785,19 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -1774,8 +1805,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -1785,7 +1826,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
 ]
 
 [[package]]
@@ -1794,7 +1844,16 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -1853,7 +1912,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "libc",
  "spin",
  "untrusted",
@@ -2222,7 +2281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31"
 dependencies = [
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "signature",
  "zeroize",
 ]
@@ -2234,7 +2293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -2388,6 +2447,16 @@ dependencies = [
  "touch",
 ]
 
+[[package]]
+name = "textnonce"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f8d70cd784ed1dc33106a18998d77758d281dc40dc3e6d050cf0f5286683"
+dependencies = [
+ "base64 0.12.3",
+ "rand 0.7.3",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -2526,7 +2595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
 dependencies = [
  "pin-project",
- "rand",
+ "rand 0.8.5",
  "tokio",
 ]
 
@@ -2576,7 +2645,7 @@ dependencies = [
  "futures-sink",
  "http",
  "httparse",
- "rand",
+ "rand 0.8.5",
  "ring",
  "rustls-native-certs 0.8.1",
  "rustls-pki-types",
@@ -2763,7 +2832,7 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
  "serde",
 ]
 
@@ -2782,6 +2851,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -3058,7 +3133,7 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "semver",
  "serde",
  "serde_json",
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index a465843..6813886 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -22,6 +22,7 @@ chrono = "0.4.0"
 bytes = "1.8.0"
 nkeys = "=0.4.4"
 rand = "0.8.5"
+textnonce = "1.0.0"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 authentication = { path = "../../services/authentication" }
diff --git a/rust/clients/host_agent/src/auth/agent_key.rs b/rust/clients/host_agent/src/auth/agent_key.rs
deleted file mode 100644
index 469cbb8..0000000
--- a/rust/clients/host_agent/src/auth/agent_key.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-// use std::sync::mpsc::{self, Receiver, Sender};
-// use std::thread;
-// use std::time::Duration;
-
-// fn obtain_authorization_token() -> String {
-//     // whatever you want, 3rd party token/username&password
-//     String::new()
-// }
-
-// fn is_token_authorized(token: &str) -> bool {
-//     // whatever logic to determine if the input authorizes the requester to obtain a user jwt
-//     token.is_empty()
-// }
-
-// // request struct to exchange data
-// struct UserRequest {
-//     user_jwt_response_chan: Sender<String>,
-//     user_public_key: String,
-//     auth_info: String,
-// }
-
-// fn start_user_provisioning_service(is_authorized_cb: fn(&str) -> bool) -> Receiver<UserRequest> {
-//     let (user_request_chan, user_request_receiver): (Sender<UserRequest>, Receiver<UserRequest>) =
-//         mpsc::channel();
-
-//     thread::spawn(move || {
-//         let account_signing_key = get_account_signing_key(); // Setup, obtain account signing key
-//         loop {
-//             if let Ok(req) = user_request_receiver.recv() {
-//                 // receive request
-//                 if !is_authorized_cb(&req.auth_info) {
-//                     println!("Request is not authorized to receive a JWT, timeout on purpose");
-//                 } else if let Some(user_jwt) =
-//                     generate_user_jwt(&req.user_public_key, &account_signing_key)
-//                 {
-//                     let _ = req.user_jwt_response_chan.send(user_jwt); // respond with jwt
-//                 }
-//             }
-//         }
-//     });
-
-//     user_request_chan
-// }
-
-// fn start_user_process(
-//     user_request_chan: Receiver<UserRequest>,
-//     obtain_authorization_cb: fn() -> String,
-// ) {
-//     let request_user = |user_request_chan: Receiver<UserRequest>, auth_info: String| {
-//         let (resp_chan, resp_receiver): (Sender<String>, Receiver<String>) = mpsc::channel();
-//         let (user_public_key, _, user_key_pair) = generate_user_key();
-
-//         // request jwt
-//         let _ = user_request_chan.send(UserRequest {
-//             user_jwt_response_chan: resp_chan,
-//             user_public_key,
-//             auth_info,
-//         });
-
-//         let user_jwt = resp_receiver.recv().unwrap(); // wait for response
-//                                                       // user_jwt and user_key_pair can be used in conjunction with this nats.Option
-//         let jwt_auth_option = nats::UserJWT::new(
-//             move || Ok(user_jwt.clone()),
-//             move |bytes| user_key_pair.sign(bytes),
-//         );
-
-//         // Alternatively you can create a creds file and use it as nats.Option
-//         jwt_auth_option
-//     };
-
-//     thread::spawn(move || {
-//         let jwt_auth_option = request_user(user_request_chan, obtain_authorization_cb());
-//         let nc = nats::connect("nats://localhost:4111", jwt_auth_option).unwrap();
-//         // simulate work one would want to do
-//         thread::sleep(Duration::from_secs(1));
-//     });
-// }
-
-// fn request_user_distributed() {
-//     let req_chan = start_user_provisioning_service(is_token_authorized);
-//     // start multiple user processes
-//     for _ in 0..4 {
-//         start_user_process(req_chan.clone(), obtain_authorization_token);
-//     }
-//     thread::sleep(Duration::from_secs(5));
-// }
-
-// // Placeholder functions for the missing implementations
-// fn get_account_signing_key() -> String {
-//     // Implementation here
-//     String::new()
-// }
-
-// fn generate_user_jwt(user_public_key: &str, account_signing_key: &str) -> Option<String> {
-//     // Implementation here
-//     Some(String::new())
-// }
-
-// fn generate_user_key() -> (String, String, UserKeyPair) {
-//     // Implementation here
-//     (String::new(), String::new(), UserKeyPair {})
-// }
-
-// struct UserKeyPair;
-
-// impl UserKeyPair {
-//     fn sign(&self, _bytes: &[u8]) -> Result<Vec<u8>, ()> {
-//         // Implementation here
-//         Ok(vec![])
-//     }
-// }
diff --git a/rust/clients/host_agent/src/auth/init_agent.rs b/rust/clients/host_agent/src/auth/init_agent.rs
index 88b7e85..ff2e468 100644
--- a/rust/clients/host_agent/src/auth/init_agent.rs
+++ b/rust/clients/host_agent/src/auth/init_agent.rs
@@ -16,27 +16,34 @@
 4. instantiate the leaf server via the leaf-server struct/service
 */
 
-use super::utils;
+use super::utils as local_utils;
 use anyhow::{anyhow, Result};
-use async_nats::Message;
-use authentication::{AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use nkeys::KeyPair;
+use std::str::FromStr;
+use async_nats::{HeaderMap, HeaderName, HeaderValue, Message};
+use authentication::{types, AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::{sync::Arc, time::Duration};
+use core::option::Option::{None, Some};
+use std::{collections::HashMap, sync::Arc, time::Duration};
+use textnonce::TextNonce;
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
+    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
     nats_js_client::{self, EndpointType},
 };
 
 pub const HOST_INIT_CLIENT_NAME: &str = "Host Auth";
 pub const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_auth_inbox";
 
-const USER_CREDENTIALS_PATH: &str = "./user_creds";
+pub fn create_callback_subject_to_orchestrator(sub_subject_name: String) -> ResponseSubjectsGenerator {
+    Arc::new(move |_: HashMap<String, String>| -> Vec<String> {
+        vec![format!("{}", sub_subject_name)]
+    })
+}
 
-pub async fn run() -> Result<(String, String), async_nats::Error> {
+pub async fn run() -> Result<String, async_nats::Error> {
     log::info!("Host Auth Client: Connecting to server...");
-
-    // ==================== NATS Setup ====================
+    // ==================== Setup NATS ============================================================
     // Connect to Nats server
     let nats_url = nats_js_client::get_nats_url();
     let event_listeners = nats_js_client::get_event_listeners();
@@ -62,7 +69,7 @@ pub async fn run() -> Result<(String, String), async_nats::Error> {
         })
         .await?;
 
-    // ==================== DB Setup ====================
+    // ==================== Setup DB ==============================================================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
@@ -71,8 +78,12 @@ pub async fn run() -> Result<(String, String), async_nats::Error> {
     // Generate the Auth API with access to db
     let auth_api = AuthApi::new(&client).await?;
 
-    // ==================== Report Host to Orchestator ====================
-
+    // ==================== Report Host to Orchestator ============================================
+    // Generate Host Pubkey && Fetch Hoster Pubkey (from config)..
+    // NB: This nkey keypair is a `ed25519_dalek::VerifyingKey` that is `BASE_32` encoded and returned as a String.
+    let host_user_keys = KeyPair::new_user();
+    let host_pubkey = host_user_keys.public_key();
+    
     // Discover the server Node ID via INFO response
     let server_node_id = host_auth_client.get_server_info().server_id;
     log::trace!(
@@ -81,14 +92,15 @@ pub async fn run() -> Result<(String, String), async_nats::Error> {
     );
 
     // Publish a message with the Node ID as part of the subject
-    let publish_options = nats_js_client::SendRequest {
+    let publish_options = nats_js_client::PublishInfo {
         subject: format!("HPOS.init.{}", server_node_id),
         msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
         data: b"Host Auth Connected!".to_vec(),
+        headers: None
     };
 
     match host_auth_client
-        .publish(&publish_options)
+        .publish(publish_options)
         .await
     {
         Ok(_r) => {
@@ -97,9 +109,13 @@ pub async fn run() -> Result<(String, String), async_nats::Error> {
         Err(_e) => {}
     };
 
-    // ==================== API ENDPOINTS ====================
+    // ==================== Register API ENDPOINTS ===============================================
     // Register Auth Streams for Orchestrator to consume and proceess
     // NB: The subjects below are published by the Orchestrator
+    
+    let auth_p1_subject = serde_json::to_string(&types::AuthServiceSubjects::HandleHandshakeP1)?;
+    let auth_p2_subject = serde_json::to_string(&types::AuthServiceSubjects::HandleHandshakeP2)?;
+    let auth_end_subject = serde_json::to_string(&types::AuthServiceSubjects::EndHandshake)?;
 
     // Call auth service and perform auth handshake
     let auth_service = host_auth_client
@@ -109,78 +125,70 @@ pub async fn run() -> Result<(String, String), async_nats::Error> {
             "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
         ))?;
 
-    // i. register `save_hub_auth` consumer
-    // ii. register `save_user_file` consumer
-    // iii. send req for `` /eg:`start_hub_handshake`
-    // iv. THEN (on get resp from start_handshake) `send_user_pubkey`
-
-    // 1. make req (with agent key & email & nonce in payload, & sig in headers)
-    // to receive_handhake_request
-    // then await the reply (which should include the hub jwts)
-
-    // 2. make req (with agent key as payload)
-    // to add_user_pubkey
-    // then await the reply (which should include the user jwt)
-
     // Register save service for hub auth files (operator and sys)
     auth_service
-        .add_local_consumer::<authentication::types::ApiResult>(
-            "save_hub_auth",
-            "save_hub_auth",
+        .add_consumer::<authentication::types::ApiResult>(
+            "save_hub_jwts", // consumer name
+            &format!("{}.{}", host_pubkey, auth_p1_subject), // consumer stream subj
             EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
                 async move {
                     api.save_hub_jwts(msg).await
                 }
             })),
-            None,
+            Some(create_callback_subject_to_orchestrator(auth_p2_subject)),
         )
         .await?;
         
     // Register save service for signed user jwt file
     auth_service
-       .add_local_consumer::<authentication::types::ApiResult>(
-            "save_user_jwt",
-            "end_hub_handshake",
+       .add_consumer::<authentication::types::ApiResult>(
+            "save_user_jwt", // consumer name
+            &format!("{}.{}", host_pubkey, auth_end_subject), // consumer stream subj
             EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
                 async move {
-                    api.save_user_jwt(msg, USER_CREDENTIALS_PATH).await
+                    api.save_user_jwt(msg, &local_utils::get_host_credentials_path()).await
                 }
             })),
             None,
         )
         .await?;
 
-    // Send the request (with payload) for the hub auth files and await a response
-    // match client.request(subject, payload.into()).await {
-    //     Ok(response) => {
-    //         let response_str = String::from_utf8_lossy(&response.payload);
-    //         println!("Received response: {}", response_str);
-    //     }
-    //     Err(e) => {
-    //         println!("Failed to get a response: {}", e);
-    //     }
-    // }
-    let req_hub_files_options = nats_js_client::SendRequest {
-        subject: format!("HPOS.init.{}", server_node_id),
-        msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
-        data: b"Host Auth Connected!".to_vec(),
+    // ==================== Publish Initial Auth Req =============================================
+    // Initialize auth handshake with Orchestrator
+    // by calling `AUTH.start_handshake` on the Auth Service
+    let payload = types::AuthRequestPayload {
+        host_pubkey: host_pubkey.clone(),
+        email: "config.test.email@holo.host".to_string(),
+        hoster_pubkey: "test_pubkey_from_config".to_string(),
+        nonce: TextNonce::new().to_string()
     };
-    host_auth_client.publish(&req_hub_files_options).await?;
 
-    // ...upon the reply to the above, do the following: publish user pubkey file
-    let _send_user_pubkey_publish_options = nats_js_client::SendRequest {
-        subject: format!("HPOS.init.{}", server_node_id),
-        msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
-        data: b"Host Auth Connected!".to_vec(),
+    let payload_bytes = serde_json::to_vec(&payload)?;
+    let signature: Vec<u8> = host_user_keys.sign(&payload_bytes)?;
+
+    let mut headers = HeaderMap::new();
+    headers.insert(HeaderName::from_static("X-Signature"), HeaderValue::from_str(&format!("{:?}",signature))?);
+
+    let publish_info = nats_js_client::PublishInfo {
+        subject: "AUTH.start_handshake".to_string(),
+        msg_id: format!("id={}", rand::random::<u8>()),
+        data: payload_bytes,
+        headers: Some(headers)
     };
-    // host_auth_client.publish(send_user_pubkey_publish_options);
-    utils::chunk_file_and_publish(&host_auth_client.js, "subject", "file_path").await?;
+    host_auth_client
+        .publish(publish_info)
+        .await?;
+
+    log::trace!(
+        "Init Host Agent Service is running. Waiting for requests..."
+    );
 
-    // 5. Generate user creds file
-    let user_creds_path = utils::generate_creds_file();
+    // ==================== Wait for Host Creds File & Safely Exit Auth Client ==================
+    // Register FILE WATCHER and WAIT FOR the Host Creds File to exist
+    // authentication::utils::get_file_path_buf(&host_creds_path).try_exists()?;
 
-    // 6. Close and drain internal buffer before exiting to make sure all messages are sent
+    // Close and drain internal buffer before exiting to make sure all messages are sent
     host_auth_client.close().await?;
 
-    Ok(("host_pubkey_placeholder".to_string(), user_creds_path))
+    Ok(host_pubkey)
 }
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index d3ecea4..6d76c1b 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,28 +1,10 @@
-use anyhow::Result;
-use async_nats::jetstream::Context;
 use std::process::Command;
 
-pub async fn chunk_file_and_publish(_js: &Context, _subject: &str, _file_path: &str) -> Result<()> {
-    // let mut file = std::fs::File::open(file_path)?;
-    // let mut buffer = vec![0; CHUNK_SIZE];
-    // let mut chunk_id = 0;
-
-    // while let Ok(bytes_read) = file.read(mut buffer) {
-    //     if bytes_read == 0 {
-    //         break;
-    //     }
-    //     let chunk_data = &buffer[..bytes_read];
-    //     js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
-    //     chunk_id += 1;
-    // }
-
-    // // Send an EOF marker
-    // js.publish(subject.to_string(), "EOF".into()).await.unwrap();
-
-    Ok(())
+pub fn _get_host_user_pubkey_path() -> String {
+    std::env::var("HOST_USER_PUBKEY").unwrap_or_else(|_| "./host_user.nk".to_string())
 }
 
-pub fn generate_creds_file() -> String {
+pub fn _generate_creds_file() -> String {
     let user_creds_path = "/path/to/host/user.creds".to_string();
     Command::new("nsc")
         .arg(format!("... > {}", user_creds_path))
@@ -31,3 +13,9 @@ pub fn generate_creds_file() -> String {
 
     "placeholder_user.creds".to_string()
 }
+
+pub fn get_host_credentials_path() -> String {
+    std::env::var("HOST_CREDENTIALS_PATH").unwrap_or_else(|_| {
+        util_libs::nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos")
+    })
+}
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index c44398f..5523ed6 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -85,9 +85,9 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         ))?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "start_workload",
-            "start",
+        .add_consumer::<workload::types::ApiResult>(
+            "start_workload", // consumer name
+            &format!("{}.start", host_pubkey), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.start_workload(msg).await
@@ -98,9 +98,9 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "send_workload_status",
-            "send_status",
+        .add_consumer::<workload::types::ApiResult>(
+            "send_workload_status", // consumer name
+            &format!("{}.send_status", host_pubkey), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.send_workload_status(msg).await
@@ -111,9 +111,9 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "uninstall_workload",
-            "uninstall",
+        .add_consumer::<workload::types::ApiResult>(
+            "uninstall_workload", // consumer name
+            &format!("{}.uninstall", host_pubkey), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.uninstall_workload(msg).await
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index efd4d30..ae60607 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -13,6 +13,7 @@ This client is responsible for subscribing the host agent to workload stream end
 mod auth;
 mod hostd;
 use anyhow::Result;
+use auth::utils as local_utils;
 use clap::Parser;
 use dotenv::dotenv;
 pub mod agent_cli;
@@ -53,9 +54,18 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize() -> Result<(), async_nats::Error> {
-    // let (host_pubkey, host_creds_path) = auth::init_agent::run().await?;
-    let host_creds_path = util_libs::nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
-    let host_pubkey = "host_pubkey_placeholder>";
+    let host_creds_path = local_utils::get_host_credentials_path();
+    let host_pubkey: String = match authentication::utils::get_file_path_buf(&host_creds_path).try_exists() {
+        Ok(_p) => {
+            // TODO: read creds file for pubkey OR call nsc and get pubkey (whichever is cleaner)
+            "host_pubkey_placeholder>".to_string()
+        },
+        Err(_) => {
+            log::debug!("About to run the Hosting Agent Initialization Service");
+            auth::init_agent::run().await?
+        }
+    };
+
     hostd::gen_leaf_server::run(&host_creds_path).await;
     hostd::workload_manager::run(&host_pubkey, &host_creds_path).await?;
     Ok(())
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index fb81dca..a9d82d9 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -4,25 +4,36 @@
 - orchestrator user
 
 // This client is responsible for:
+//// auth_endpoint_subject = "AUTH.{host_id}.file.transfer.JWT-User"
 */
 
-use crate::utils;
+use crate::utils as local_utils;
 
 use anyhow::{anyhow, Result};
-use std::{sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
+// use std::process::Command;
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use authentication::{self, AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
-use std::process::Command;
+use authentication::{self, types::AuthServiceSubjects, AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
+    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
     nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
 };
 
 pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Agent";
 pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_orchestrator_auth_inbox";
 
+pub fn create_callback_subject_to_host(tag_name: String, sub_subject_name: String) -> ResponseSubjectsGenerator {
+    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
+        if let Some(tag) = tag_map.get(&tag_name) {
+            return vec![format!("{}.{}", tag, &sub_subject_name)];
+        }
+        log::error!("Auth Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `AUTH.ERROR.INBOX`.", tag_name, sub_subject_name);
+        vec!["AUTH.ERROR.INBOX".to_string()]
+    })
+}
+
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== NATS Setup ====================
     let nats_url = nats_js_client::get_nats_url();
@@ -61,69 +72,50 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== API ENDPOINTS ====================
     // Register Auth Streams for Orchestrator to consume and proceess
     // NB: The subjects below are published by the Host Agent
+    let auth_start_subject = serde_json::to_string(&AuthServiceSubjects::StartHandshake)?;
+    let auth_p1_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP1)?;
+    let auth_p2_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP2)?;
+    let auth_end_subject = serde_json::to_string(&AuthServiceSubjects::EndHandshake)?;
+
     let auth_service = orchestrator_auth_client
         .get_js_service(AUTH_SRV_NAME.to_string())
         .await
         .ok_or(anyhow!(
             "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
-        ))?;
-
+        ))?;    
+    
     auth_service
-        .add_local_consumer::<authentication::types::ApiResult>(
-            "add_user_pubkey",
-            "add_user_pubkey",
+        .add_consumer::<authentication::types::ApiResult>(
+            "start_handshake", // consumer name
+            &auth_start_subject, // consumer stream subj
+            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+                async move {
+                    api.handle_handshake_request(msg, &local_utils::get_orchestrator_credentials_dir_path()).await
+                }
+            })),
+            Some(create_callback_subject_to_host("host_pubkey".to_string(), auth_p1_subject)),
+        )
+        .await?;
+        
+    auth_service
+        .add_consumer::<authentication::types::ApiResult>(
+            "add_user_pubkey", // consumer name
+            &auth_p2_subject, // consumer stream subj
             EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
                 async move {
                     api.add_user_pubkey(msg).await
                 }
             })),
-            None,
+            Some(create_callback_subject_to_host("host_pubkey".to_string(), auth_end_subject)),
         )
         .await?;
 
     log::trace!(
-        "{} Service is running. Waiting for requests...",
-        AUTH_SRV_NAME
+        "Orchestrator Auth Service is running. Waiting for requests..."
     );
 
-
-    let resolver_path = utils::get_resolver_path();
-
-    let _auth_endpoint_subject = format!("AUTH.{}.file.transfer.JWT-User", "host_id_placeholder"); // endpoint_subject
- 
-    // Generate resolver file and create resolver file
-    Command::new("nsc")
-        .arg("generate")
-        .arg("config")
-        .arg("--nats-resolver")
-        .arg("sys-account SYS")
-        .arg("--force")
-        .arg(format!("--config-file {}", resolver_path))
-        .output()
-        .expect("Failed to create resolver config file");
-
-    // Push auth updates to hub server
-    Command::new("nsc")
-        .arg("push -A")
-        .output()
-        .expect("Failed to create resolver config file");
-
-    // publish user jwt file
-    let server_node_id = "server_node_id_placeholder";
-    utils::chunk_file_and_publish(
-        &orchestrator_auth_client,
-        &format!("HPOS.init.{}", server_node_id),
-        "placeholder_user_id / pubkey",
-    )
-    .await?;
-
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
-    log::warn!("CTRL+C detected. Please press CTRL+C again within 5 seconds to confirm exit...");
-    tokio::select! {
-        _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => log::warn!("Resuming service."),
-        _ = tokio::signal::ctrl_c() => log::error!("Shutting down."),
-    }
 
     // Close client and drain internal buffer before exiting to make sure all messages are sent
     orchestrator_auth_client.close().await?;
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
index e7ec3c1..8262114 100644
--- a/rust/clients/orchestrator/src/utils.rs
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -1,50 +1,48 @@
-// use super::auth::controller::ORCHESTRATOR_AUTH_CLIENT_NAME;
-use anyhow::Result;
-use std::io::Read;
-use util_libs::nats_js_client::{JsClient, SendRequest};
+// use anyhow::Result;
+// use std::io::Read;
+// use util_libs::nats_js_client::{JsClient, PublishInfo};
 
-const CHUNK_SIZE: usize = 1024; // 1 KB chunk size
-
-pub fn get_hpos_users_pubkey_path() -> String {
+pub fn _get_resolver_path() -> String {
     std::env::var("RESOLVER_FILE_PATH").unwrap_or_else(|_| "./resolver.conf".to_string())
 }
 
-pub fn get_resolver_path() -> String {
-    std::env::var("RESOLVER_FILE_PATH").unwrap_or_else(|_| "./resolver.conf".to_string())
+pub fn get_orchestrator_credentials_dir_path() -> String {
+    std::env::var("ORCHESTRATOR_CREDENTIALS_DIR_PATH").unwrap_or_else(|e| panic!("Failed to locate 'ORCHESTRATOR_CREDENTIALS_DIR_PATH' env var.  Was it set?  Error={}", e))
 }
 
-pub async fn chunk_file_and_publish(
-    auth_client: &JsClient,
-    subject: &str,
-    host_id: &str,
-) -> Result<()> {
-    let file_path = format!("{}/{}.jwt", get_hpos_users_pubkey_path(), host_id);
-    let mut file = std::fs::File::open(file_path)?;
-    let mut buffer = vec![0; CHUNK_SIZE];
-    let mut chunk_id = 0;
-
-    while let Ok(bytes_read) = file.read(&mut buffer) {
-        if bytes_read == 0 {
-            break;
-        }
-        let chunk_data = &buffer[..bytes_read];
-
-        let send_user_jwt_publish_options = SendRequest {
-            subject: subject.to_string(),
-            msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
-            data: chunk_data.into(),
-        };
-        let _ = auth_client.publish(&send_user_jwt_publish_options).await;
-        chunk_id += 1;
-    }
-
-    // Send an EOF marker
-    let send_user_jwt_publish_options = SendRequest {
-        subject: subject.to_string(),
-        msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
-        data: "EOF".into(),
-    };
-    let _ = auth_client.publish(&send_user_jwt_publish_options).await;
-
-    Ok(())
-}
+// const CHUNK_SIZE: usize = 1024; // 1 KB chunk size
+// pub async fn chunk_file_and_publish(
+//     auth_client: &JsClient,
+//     subject: &str,
+//     host_id: &str,
+// ) -> Result<()> {
+//     let file_path = format!("{}/{}.jwt", get_host_user_pubkey_path(), host_id);
+//     let mut file = std::fs::File::open(file_path)?;
+//     let mut buffer = vec![0; CHUNK_SIZE];
+//     let mut chunk_id = 0;
+
+//     while let Ok(bytes_read) = file.read(&mut buffer) {
+//         if bytes_read == 0 {
+//             break;
+//         }
+//         let chunk_data = &buffer[..bytes_read];
+
+//         let send_user_jwt_publish_options = PublishInfo {
+//             subject: subject.to_string(),
+//             msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
+//             data: chunk_data.into(),
+//         };
+//         let _ = auth_client.publish(&send_user_jwt_publish_options).await;
+//         chunk_id += 1;
+//     }
+
+//     // Send an EOF marker
+//     let send_user_jwt_publish_options = PublishInfo {
+//         subject: subject.to_string(),
+//         msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
+//         data: "EOF".into(),
+//     };
+//     let _ = auth_client.publish(&send_user_jwt_publish_options).await;
+
+//     Ok(())
+// }
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index e3c222d..f3ef9f5 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -7,7 +7,7 @@
 */
 
 use anyhow::{anyhow, Result};
-use std::{sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use workload::{
@@ -15,13 +15,31 @@ use workload::{
 };
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
+    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
     nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
 const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_orchestrator_workload_inbox";
 
+pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_subject_name: String) -> ResponseSubjectsGenerator {
+    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
+        if is_prefix {
+            let matching_tags = tag_map.into_iter().fold(vec![], |mut acc, (k, v)| {
+                if k.starts_with(&tag_name) {
+                    acc.push(v)
+                }
+                acc
+            });
+            return matching_tags;
+        } else if let Some(tag) = tag_map.get(&tag_name) {
+            return vec![format!("{}.{}", tag, sub_subject_name)];
+        }
+        log::error!("WORKLOAD Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `WORKLOAD.ERROR.INBOX`.", tag_name, sub_subject_name);
+        vec!["WORKLOAD.ERROR.INBOX".to_string()]
+    })
+}
+
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== NATS Setup ====================
     let nats_url = nats_js_client::get_nats_url();
@@ -70,7 +88,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by Developer
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
+        .add_consumer::<workload::types::ApiResult>(
             "add_workload",
             "add",
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
@@ -84,7 +102,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Automatically published by the Nats-DB-Connector
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
+        .add_consumer::<workload::types::ApiResult>(
             "handle_db_insertion",
             "insert",
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
@@ -92,13 +110,13 @@ pub async fn run() -> Result<(), async_nats::Error> {
                     api.handle_db_insertion(msg).await
                 }
             })),
-            None,
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), "start".to_string())),
         )
         .await?;
     
     // Published by the Host Agent
     workload_service
-    .add_local_consumer::<workload::types::ApiResult>(
+    .add_consumer::<workload::types::ApiResult>(
         "handle_status_update",
         "read_status_update",
         EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 987c520..0079cc5 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -18,14 +18,36 @@ Endpoints & Managed Subjects:
 */
 
 pub mod types;
+pub mod utils;
+
 use anyhow::Result;
-use async_nats::Message;
-use mongodb::Client as MongoDBClient; // options::UpdateModifications, 
+use async_nats::{Message, HeaderValue};
+use async_nats::jetstream::ErrorCode;
+use nkeys::KeyPair;
+use types::AuthResult;
+use utils::handle_internal_err;
+use core::option::Option::None;
+use std::collections::HashMap;
 use std::process::Command;
 use std::sync::Arc;
 use std::future::Future;
 use serde::{Deserialize, Serialize};
-use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection}, schemas}, nats_js_client};
+use bson::{self, doc, to_document};
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use util_libs::{
+    nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{
+            self,
+            User,
+            Hoster,
+            Host,
+            Role,
+            RoleInfo,
+        }
+    },
+};
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";
@@ -33,11 +55,12 @@ pub const AUTH_SRV_VERSION: &str = "0.0.1";
 pub const AUTH_SRV_DESC: &str =
     "This service handles the Authentication flow the HPOS and the Orchestrator.";
 
-#[derive(Debug, Clone)]
+#[derive(Debug,
+    Clone)]
 pub struct AuthApi {
-    pub user_collection: MongoCollection<schemas::User>,
-    pub hoster_collection: MongoCollection<schemas::Hoster>,
-    pub host_collection: MongoCollection<schemas::Host>,
+    pub user_collection: MongoCollection<User>,
+    pub hoster_collection: MongoCollection<Hoster>,
+    pub host_collection: MongoCollection<Host>,
 }
 
 impl AuthApi {
@@ -49,106 +72,203 @@ impl AuthApi {
         })
     }
 
-
     pub fn call<F, Fut>(
         &self,
         handler: F,
-    ) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    ) -> AsyncEndpointHandler<types::ApiResult>
     where
         F: Fn(AuthApi, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
     {
         let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
             let api_clone = api.clone();
             Box::pin(handler(api_clone, msg))
         })
     }
     
     /*******************************  For Orchestrator   *********************************/
-    pub async fn receive_handshake_request(
+    // nb: returns to the `save_hub_files` subject
+    pub async fn handle_handshake_request(
         &self,
         msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        // 1. Verify expected data was received
-        if msg.headers.is_none() {
-            log::error!(
-                "Error: Missing headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'"
-            );
-            // anyhow!(ErrorCode::BAD_REQUEST)
-        }
+        creds_dir_path: &str,
+    ) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
 
-        // let signature = msg_clone.headers.unwrap().get("Signature").unwrap_or(&HeaderValue::new());
-
-        // match  serde_json::from_str::<types::AuthHeaders>(signature.as_str()) {
-        //     Ok(r) => {}
-        //     Err(e) => {
-        //         log::error!("Error: Failed to deserialize headers. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
-        //         // anyhow!(ErrorCode::BAD_REQUEST)
-        //     }
-        // }
+        // 1. Verify expected data was received
+        let signature: &[u8] = match &msg.headers {
+            Some(h) => {
+                HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
+                    log::error!(
+                        "Error: Missing x-signature header. Subject='AUTH.authorize'"
+                    );
+                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
+                })?)
+            },
+            None => {
+                log::error!(
+                    "Error: Missing message headers. Subject='AUTH.authorize'"
+                );
+                return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+            }
+        };
+
+        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, nonce: _ } = Self::convert_to_type::<types::AuthRequestPayload>(msg.clone())?;
+
+        // 2. Validate signature
+        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
+            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
+            return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+        };
+
+        // 3. Authenticate the Hosting Agent (via email and host id info?)
+        match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_pubkey.clone() }).await? {
+            Some(u) => {
+                // If hoster exists with pubkey, verify email
+                if u.email != email {
+                    log::error!("Error: Failed to validate user email. Subject='{}'.", msg.subject);
+                    return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+                }
+
+                // ...then find the host collection that contains the provided host pubkey
+                match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
+                    Some(host_collection) => {
+                        // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
+                        if host_collection.assigned_hoster != hoster_pubkey {
+                            let host_query: bson::Document = doc! { "_id":  host_collection._id.clone() };
+                            let updated_host_doc = to_document(& Host{
+                                assigned_hoster: hoster_pubkey,
+                                ..host_collection
+                            }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                            self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
+                        }
+                    },
+                    None => {
+                        let err_msg = format!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
+                        return Err(handle_internal_err(&err_msg));
+                    }
+                }
+
+                // Find the mongo_id ref for the hoster associated with this user
+                let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
+                    let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
+                    handle_internal_err(&err_msg)
+                })?;
+                
+                // Finally, find the hoster collection
+                match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
+                    Some(hoster_collection) => {
+                        // ...and pair the hoster with host (if the host is not already assiged to the hoster)
+                        let mut updated_assigned_hosts = hoster_collection.assigned_hosts;
+                        if !updated_assigned_hosts.contains(&host_pubkey) {
+                        let hoster_query: bson::Document = doc! { "_id":  hoster_collection._id.clone() };
+                        updated_assigned_hosts.push(host_pubkey.clone());
+                        let updated_hoster_doc = to_document(& Hoster {
+                            assigned_hosts: updated_assigned_hosts,
+                            ..hoster_collection
+                        }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                        self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
+                        }
+                    },
+                    None => {
+                        let err_msg = format!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
+                        return Err(handle_internal_err(&err_msg));
+                    }
+                }
+            },
+            None => {
+                let err_msg = format!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
+                return Err(handle_internal_err(&err_msg));
+            }
+        };
 
-        // match serde_json::from_slice::<types::AuthPayload>(msg.payload.as_ref()) {
-        //     Ok(r) => {}
-        //     Err(e) => {
-        //         log::error!("Error: Failed to deserialize payload. Consumer=authorize_ext_client, Subject='/AUTH/authorize'")
-        //         // anyhow!(ErrorCode::BAD_REQUEST)
-        //     }
-        // }
+        // 4. Read operator and sys account jwts and prepare them to be sent as a payload in the publication callback
+        let operator_path = utils::get_file_path_buf(&format!("{}/operator.creds", creds_dir_path));
+        let hub_operator_creds: Vec<u8> = std::fs::read(operator_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        // 2. Authenticate the HPOS client(?via email and host id info?)
+        let sys_path = utils::get_file_path_buf(&format!("{}/sys.creds", creds_dir_path));
+        let hub_sys_creds: Vec<u8> = std::fs::read(sys_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        // 3. Publish operator and sys account jwts for orchestrator
-        // let hub_operator_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
-        // let hub_sys_account = chunk_and_publish().await; // returns to the `save_hub_files` subject
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
 
         Ok(types::ApiResult {
             status: types::AuthStatus { 
-                host_id: "host_id_placeholder".to_string(),
+                host_pubkey: host_pubkey.clone(),
                 status: types::AuthState::Requested
             },
-            result: serde_json::to_vec(&"OK")?,
-            maybe_response_tags: None
+            result: AuthResult {
+                data: types::AuthResultType::Multiple(vec![hub_operator_creds, hub_sys_creds])
+            },
+            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
         })
     }
 
     /*******************************   For Host Agent   *********************************/
-     pub async fn save_hub_jwts(&self, _msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+     pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
+
         // receive_and_write_file();
 
         // Respond to endpoint request
         // let response = b"Hello, NATS!".to_vec();
+
+        // let resolver_path = utils::get_resolver_path();
+
+        // // Generate resolver file and create resolver file
+        // Command::new("nsc")
+        //     .arg("generate")
+        //     .arg("config")
+        //     .arg("--nats-resolver")
+        //     .arg("sys-account SYS")
+        //     .arg("--force")
+        //     .arg(format!("--config-file {}", resolver_path))
+        //     .output()
+        //     .expect("Failed to create resolver config file");
+
+        // // Push auth updates to hub server
+        // Command::new("nsc")
+        //     .arg("push -A")
+        //     .output()
+        //     .expect("Failed to create resolver config file");
+
         // Ok(response)
 
         todo!();
     }
 
     /*******************************  For Orchestrator   *********************************/
-    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.handle_handshake_p2' : {:?}", msg);
+
+        // 1. Verify expected payload was received
+        let host_pubkey = Self::convert_to_type::<String>(msg.clone())?;
 
-        // Add user with Keys and create jwt
+        // 2. Add User keys to Orchestrator nsc resolver
         Command::new("nsc")
             .arg("...")
             .output()
             .expect("Failed to add user with provided keys");
+        
+        // 3. Create and sign User JWT
+        let account_signing_key = utils::get_account_signing_key();
+        utils::generate_user_jwt(&host_pubkey, &account_signing_key);
 
-        // Output jwt
-        let _user_jwt_path = Command::new("nsc")
-            .arg("...")
-            // .arg(format!("> {}", output_dir))
-            .output()
-            .expect("Failed to output user jwt to file")
-            .stdout;
+        // 4. Prepare User JWT to be sent as a payload in the publication callback
+        let sys_path = utils::get_file_path_buf("user_jwt_path");
+        let user_jwt: Vec<u8> = std::fs::read(sys_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        // 2. Respond to endpoint request
-        // let resposne = user_jwt_path;
+        // 5. Respond to endpoint request
         Ok(types::ApiResult {
             status: types::AuthStatus { 
-                host_id: "host_id_placeholder".to_string(),
+                host_pubkey,
                 status: types::AuthState::ValidatedAgent
             },
-            result: b"user_jwt_path_placeholder".to_vec(),
+            result: AuthResult {
+                data: types::AuthResultType::Single(user_jwt)
+            },
             maybe_response_tags: None
         })
     }
@@ -158,18 +278,24 @@ impl AuthApi {
         &self,
         msg: Arc<Message>,
         _output_dir: &str,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
+    ) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.end_handshake' : {:?}", msg);
 
+        // Generate user jwt file
         // utils::receive_and_write_file(msg, output_dir, file_name).await?;
+        
+        // Generate user creds file
+        // let _user_creds_path = utils::generate_creds_file();
 
         // 2. Respond to endpoint request
         Ok(types::ApiResult {
             status: types::AuthStatus { 
-                host_id: "host_id_placeholder".to_string(),
+                host_pubkey: "host_id_placeholder".to_string(),
                 status: types::AuthState::Authenticated
             },
-            result: b"Hello, NATS!".to_vec(),
+            result: AuthResult {
+                data: types::AuthResultType::Single(b"Hello, NATS!".to_vec())
+            },
             maybe_response_tags: None
         })
     }
@@ -186,24 +312,22 @@ impl AuthApi {
         Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
     }
 
-}
-
-// In orchestrator
-// pub async fn send_hub_jwts(
-//     &self,
-//     msg: Arc<Message>,
-// ) -> Result<Vec<u8>, anyhow::Error> {
-//     log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
-
-//     utils::chunk_file_and_publish(msg, output_dir, file_name).await?;
+    fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
+    where
+        T: for<'de> Deserialize<'de> + Send + Sync,
+    {
+        let payload_buf = msg.payload.to_vec();
+        serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
+            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject, e);
+            log::error!("{}", err_msg);
+            ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
+        })
+    }
 
-//     // 2. Respond to endpoint request
-//     let response = b"Hello, NATS!".to_vec();
-//     Ok(response)
-// }
+}
 
 // In hpos
-// pub async fn send_user_pubkey(&self, msg: Arc<Message>) -> Result<Vec<u8>, anyhow::Error> {
+// pub async fn send_user_pubkey(&self, msg: Arc<Message>) -> Result<Vec<u8>, ServiceError> {
 //     // 1. validate nk key...
 //     // let auth_endpoint_subject =
 //     // format!("AUTH.{}.file.transfer.JWT-operator", "host_id_placeholder"); // endpoint_subject
@@ -212,7 +336,7 @@ impl AuthApi {
 
 //     // 3. create signed jwt
 
-//     // 4. `Ack last request and publish the new jwt to for hpos
+//     // 4. `Ack last msg and publish the new jwt to for hpos
 
 //     // 5. Respond to endpoint request
 //     // let response = b"Hello, NATS!".to_vec();
@@ -220,17 +344,3 @@ impl AuthApi {
 
 //     todo!()
 // }
-
-// In orchestrator
-// pub async fn send_user_file(
-//     &self,
-//     msg: Arc<Message>,
-// ) -> Result<Vec<u8>, anyhow::Error> {
-//     log::warn!("INCOMING Message for 'AUTH.add' : {:?}", msg);
-
-//     utils::chunk_file_and_publish(msg, output_dir, file_name).await?;
-
-//     // 2. Respond to endpoint request
-//     let response = b"Hello, NATS!".to_vec();
-//     Ok(response)
-// }
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index a62b3ea..9478c8c 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -1,6 +1,17 @@
+use std::collections::HashMap;
+
 use util_libs::js_stream_service::{CreateTag, EndpointTraits};
 use serde::{Deserialize, Serialize};
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum AuthServiceSubjects {
+    StartHandshake,
+    HandleHandshakeP1,
+    HandleHandshakeP2,
+    EndHandshake,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthState {
     Requested,
@@ -11,7 +22,7 @@ pub enum AuthState {
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthStatus {
-    pub host_id: String,
+    pub host_pubkey: String,
     pub status: AuthState
 }
 
@@ -22,20 +33,32 @@ pub struct AuthHeaders {
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct AuthRequestPayload {
+    pub hoster_pubkey: String,
     pub email: String,
-    pub host_id: String,
-    pub pubkey: String,
+    pub host_pubkey: String,
+    pub nonce: String
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum AuthResultType {
+    Single(Vec<u8>),
+    Multiple(Vec<Vec<u8>>)
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct AuthResult {
+    pub data: AuthResultType,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct ApiResult {
     pub status: AuthStatus,
-    pub result: Vec<u8>,
-    pub maybe_response_tags: Option<Vec<String>>
+    pub result: AuthResult,
+    pub maybe_response_tags: Option<HashMap<String, String>>
 }
 impl EndpointTraits for ApiResult {}
 impl CreateTag for ApiResult {
-    fn get_tags(&self) -> Option<Vec<String>> {
-        self.maybe_response_tags.clone()
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.maybe_response_tags.clone().unwrap_or_default()
     }
 }
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index 8f9c3a3..532a841 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -1,8 +1,22 @@
 use anyhow::Result;
 use async_nats::jetstream::Context;
-use async_nats::{jetstream::consumer::PullConsumer, Message};
+use async_nats::Message;
+use util_libs::nats_js_client::ServiceError;
 use std::sync::Arc;
-use tokio::{fs::File, io::AsyncWriteExt};
+use std::io::Write;
+use std::path::PathBuf;
+
+pub fn handle_internal_err(err_msg: &str) -> ServiceError {
+    log::error!("{}", err_msg);
+    ServiceError::Internal(err_msg.to_string())
+}
+
+pub fn get_file_path_buf(
+    file_name: &str,
+) -> PathBuf {
+    let root_path = std::env::current_dir().expect("Failed to locate root directory.");
+    root_path.join(file_name)
+}
 
 pub async fn receive_and_write_file(
     msg: Arc<Message>,
@@ -10,21 +24,58 @@ pub async fn receive_and_write_file(
     file_name: &str,
 ) -> Result<()> {
     let output_path = format!("{}/{}", output_dir, file_name);
-    let mut file = tokio::fs::OpenOptions::new()
+    let mut file = std::fs::OpenOptions::new()
         .create(true)
         .append(true)
-        .open(&output_path)
-        .await?;
+        .open(&output_path)?;
 
-    let payload_buf = msg.payload.to_vec();
-    let payload = serde_json::from_slice::<String>(&payload_buf)?;
-    if payload.to_string().contains("EOF") {
-        log::info!("File transfer complete.");
-        return Ok(());
-    }
-
-    file.write_all(&msg.payload).await?;
-    file.flush().await?;
+    file.write_all(&msg.payload)?;
+    file.flush()?;
+    Ok(())
+}
 
+pub async fn publish_chunks(js: &Context, subject: &str, file_name: &str, data: Vec<u8>) -> Result<()> {
+    // let data: Vec<u8> = std::fs::read(file_path)?;
+    js.publish(format!("{}.{} ", subject, file_name), data.into()).await?;
     Ok(())
 }
+
+// Placeholder functions for the missing implementations
+pub fn get_account_signing_key() -> String {
+    // Implementation here
+    String::new()
+}
+
+pub fn generate_user_jwt(_user_public_key: &str, _account_signing_key: &str) -> Option<String> {
+    // Implementation here
+
+    // // Output jwt with nsc
+    // let user_jwt_path = Command::new("nsc")
+    //     .arg("...")
+    //     // .arg(format!("> {}", output_dir))
+    //     .output()
+    //     .expect("Failed to output user jwt to file")
+    //     .stdout;
+
+        Some(String::new())
+}
+
+// pub async fn chunk_file_and_publish(_js: &Context, _subject: &str, _file_path: &str) -> Result<()> {
+    // let mut file = std::fs::File::open(file_path)?;
+    // let mut buffer = vec![0; CHUNK_SIZE];
+    // let mut chunk_id = 0;
+
+    // while let Ok(bytes_read) = file.read(mut buffer) {
+    //     if bytes_read == 0 {
+    //         break;
+    //     }
+    //     let chunk_data = &buffer[..bytes_read];
+    //     js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
+    //     chunk_id += 1;
+    // }
+
+    // // Send an EOF marker
+    // js.publish(subject.to_string(), "EOF".into()).await.unwrap();
+
+//     Ok(())
+// }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 5b9c111..96a592f 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -13,15 +13,23 @@ Endpoints & Managed Subjects:
 */
 
 pub mod types;
-use anyhow::{anyhow, Result};
+use anyhow::Result;
+use core::option::Option::None;
+use async_nats::jetstream::ErrorCode;
+use std::{collections::HashMap, fmt::Debug, sync::Arc};
 use async_nats::Message;
 use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use std::{fmt::Debug, sync::Arc};
-use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}}, nats_js_client};
 use rand::seq::SliceRandom;
 use std::future::Future;
 use serde::{Deserialize, Serialize};
 use bson::{self, doc, to_document};
+use util_libs::{
+    nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}
+    }
+};
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
@@ -48,22 +56,22 @@ impl WorkloadApi {
     pub fn call<F, Fut>(
         &self,
         handler: F,
-    ) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    ) -> AsyncEndpointHandler<types::ApiResult>
     where
         F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
     {
         let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
             let api_clone = api.clone();
             Box::pin(handler(api_clone, msg))
         })
     }
 
     /*******************************  For Orchestrator   *********************************/
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self.process_request(
+        self.process_request(
             msg,
             WorkloadState::Reported,
             |workload: schemas::Workload| async move {
@@ -84,17 +92,17 @@ impl WorkloadApi {
             },
             WorkloadState::Error,
         )
-        .await)
+        .await
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self.process_request(
+        self.process_request(
             msg,
             WorkloadState::Running,
             |workload: schemas::Workload| async move {
                 let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload_doc = to_document(&workload)?;
+                let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
                 Ok(types::ApiResult(
@@ -108,13 +116,13 @@ impl WorkloadApi {
             },
             WorkloadState::Error,
         )
-        .await)
+        .await
 
     }
 
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
-        Ok(self.process_request(
+        self.process_request(
             msg,
             WorkloadState::Removed,
             |workload_id: schemas::MongoDbId| async move {
@@ -135,13 +143,13 @@ impl WorkloadApi {
             },
             WorkloadState::Error,
         )
-        .await)
+        .await
     }
 
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
-        Ok(self.process_request(
+        self.process_request(
             msg,
             WorkloadState::Assigned,
             |workload: schemas::Workload| async move {
@@ -150,7 +158,7 @@ impl WorkloadApi {
                 // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
                 let workload_id = if let Some(id) = workload.clone()._id { id } else {
                     let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
-                    return Err(anyhow!(err_msg));
+                    return Err(ServiceError::Internal(err_msg));
                 };
 
                 // 1. Perform sanity check to ensure workload is not already assigned to a host
@@ -158,13 +166,18 @@ impl WorkloadApi {
                 // todo: check for to ensure assigned host *still* has enough capacity for updated workload
                 if !workload.assigned_hosts.is_empty() {
                     log::warn!("Attempted to assign host for new workload, but host already exists.");
+                    let mut tag_map: HashMap<String, String> = HashMap::new();
+                    for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
+                        tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
+                    }
                     return Ok(types::ApiResult( 
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
-                    },
-                    Some(workload.assigned_hosts)));
+                        WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Assigned,
+                            actual: WorkloadState::Assigned,
+                        },
+                        Some(tag_map)
+                    ));
                 }
 
                 // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
@@ -182,7 +195,7 @@ impl WorkloadApi {
                     None => {
                         // todo: Try to get another host up to 5 times, if fails thereafter, return error
                         let err_msg = format!("Failed to locate an eligible host to support the required workload capacity. Workload={:?}", workload);
-                        return Err(anyhow!(err_msg));
+                        return Err(ServiceError::Internal(err_msg));
                     }
                 };
 
@@ -197,7 +210,7 @@ impl WorkloadApi {
                     assigned_hosts: vec![host_id],
                     ..workload.clone()
                 };
-                let updated_workload_doc = to_document(updated_workload)?;
+                let updated_workload_doc = to_document(updated_workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::trace!(
                     "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
@@ -209,34 +222,36 @@ impl WorkloadApi {
                 let updated_host_doc =  to_document(&Host {
                     assigned_workloads: vec![workload_id.clone()],
                     ..host.to_owned()
-                })?;
+                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 let updated_host_result = self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;
                 log::trace!(
                     "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
                     updated_host_result
                 );
-     
+                let mut tag_map: HashMap<String, String> = HashMap::new();
+                for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
+                }
                 Ok(types::ApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Assigned,
                         actual: WorkloadState::Assigned,
                     },
-                    Some(updated_workload.assigned_hosts.to_owned())
+                    Some(tag_map)
                 ))
         },
             WorkloadState::Error,
         )
-        .await)
+        .await
     }
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
         
-        let payload_buf = msg.payload.to_vec();
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
         
         // TODO: ...handle the use case for the update entry change stream 
@@ -252,13 +267,12 @@ impl WorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.delete'");
         
-        let payload_buf = msg.payload.to_vec();
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("New workload to assign. Workload={:#?}", workload);
-        
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+        log::trace!("New workload to assign. Workload={:#?}", workload);   
+
         // TODO: ...handle the use case for the delete entry change stream
 
         let success_status = WorkloadStatus {
@@ -271,24 +285,21 @@ impl WorkloadApi {
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
 
-        let payload_buf = msg.payload.to_vec();
-        let workload_status: WorkloadStatus = serde_json::from_slice(&payload_buf)?;
+        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
         log::trace!("Workload status to update. Status={:?}", workload_status);
 
-        // TODO: ...handle the use case for the workload status update
+        // TODO: ...handle the use case for the workload status update within the orchestrator
         
         Ok(types::ApiResult(workload_status, None))
     }    
 
      /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
         // TODO: Talk through with Stefan
         // 1. Connect to interface for Nix and instruct systemd to install workload...
@@ -303,11 +314,9 @@ impl WorkloadApi {
         Ok(types::ApiResult(status, None))
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_id = serde_json::from_slice::<String>(&payload_buf)?;
+        let workload_id = Self::convert_to_type::<String>(msg)?;
 
         // TODO: Talk through with Stefan
         // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
@@ -324,14 +333,13 @@ impl WorkloadApi {
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!(
             "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
             msg
         );
 
-        let payload_buf = msg.payload.to_vec();
-        let workload_status = serde_json::from_slice::<WorkloadStatus>(&payload_buf)?;
+        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
 
         // Send updated status:
         // NB: This will send the update to both the requester (if one exists)
@@ -351,6 +359,19 @@ impl WorkloadApi {
         Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
     }
 
+    fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
+    where
+        T: for<'de> Deserialize<'de> + Send + Sync,
+    {
+        let payload_buf = msg.payload.to_vec();
+        serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
+            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject, e);
+            log::error!("{}", err_msg);
+            ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
+        })
+        
+    }
+
     // Helper function to streamline the processing of incoming workload messages
     // NB: Currently used to process requests for MongoDB ops and the subsequent db change streams these db edits create (via the mongodb<>nats connector)
     async fn process_request<T, Fut>(
@@ -359,28 +380,16 @@ impl WorkloadApi {
         desired_state: WorkloadState,
         cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> types::ApiResult
+    ) -> Result<types::ApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send,
+        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = match serde_json::from_slice(&msg.payload) {
-            Ok(r) => r,
-            Err(e) => {
-                let err_msg = format!("Failed to deserialize payload for Workload Service Endpoint. Subject={} Error={:?}", msg.subject, e);
-                log::error!("{}", err_msg);
-                let status = WorkloadStatus {
-                    id: None,
-                    desired: desired_state,
-                    actual: error_state(err_msg),
-                };
-                return types::ApiResult(status, None);
-            }
-        };
+        let payload: T = Self::convert_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
-        match cb_fn(payload.clone()).await {
+        Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
                 let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
@@ -394,6 +403,6 @@ impl WorkloadApi {
                 // 3. return response for stream
                 types::ApiResult(status, None)
             }
-        }
+        })
     }
 }
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 3b0fdaf..3774ad6 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,11 +1,13 @@
+use std::collections::HashMap;
+
 use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct ApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
 impl EndpointTraits for ApiResult {}
 impl CreateTag for ApiResult {
-    fn get_tags(&self) -> Option<Vec<String>> {
-        self.1.clone()
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.1.clone().unwrap_or_default()
     }
 }
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index ecb579a..28956a2 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use async_trait::async_trait;
 use bson::{self, doc, Document};
 use futures::stream::TryStreamExt;
@@ -7,27 +7,22 @@ use mongodb::results::{DeleteResult, UpdateResult};
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
-
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Internal Error: {0}")]
-    Internal(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-}
+use crate::nats_js_client::ServiceError;
 
 #[async_trait]
 pub trait MongoDbAPI<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync,
 {
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>>;
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
-    async fn insert_one_into(&self, item: T) -> Result<String>;
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult>;
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
-    async fn delete_all_from(&self) -> Result<DeleteResult>;
+    type Error;
+
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error>;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error>;
+    async fn insert_one_into(&self, item: T) -> Result<String, Self::Error>;
+    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>, Self::Error>;
+    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult, Self::Error>;
+    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult, Self::Error>;
+    async fn delete_all_from(&self) -> Result<DeleteResult, Self::Error>;
 }
 
 pub trait IntoIndexes {
@@ -90,7 +85,9 @@ impl<T> MongoDbAPI<T> for MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes + Debug,
 {
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>> {
+    type Error = ServiceError;
+
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error> {
         log::info!("get_one_from filter {:?}", filter);
 
         let item = self
@@ -103,13 +100,13 @@ where
         Ok(item)
     }
 
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>> {
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error> {
         let cursor = self.collection.find(filter).await?;
         let results: Vec<T> = cursor.try_collect().await.map_err(ServiceError::Database)?;
         Ok(results)
     }
 
-    async fn insert_one_into(&self, item: T) -> Result<String> {
+    async fn insert_one_into(&self, item: T) -> Result<String, Self::Error> {
         let result = self
             .collection
             .insert_one(item)
@@ -119,7 +116,7 @@ where
         Ok(result.inserted_id.to_string())
     }
 
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>> {
+    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>, Self::Error> {
         let result = self
             .collection
             .insert_many(items)
@@ -134,25 +131,25 @@ where
         Ok(ids)
     }
 
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult> {
+    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult, Self::Error> {
         self.collection
             .update_one(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult> {
+    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult, Self::Error> {
         self.collection
             .delete_one(query)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn delete_all_from(&self) -> Result<DeleteResult> {
+    async fn delete_all_from(&self) -> Result<DeleteResult, Self::Error> {
         self.collection
             .delete_many(doc! {})
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 }
 
@@ -268,7 +265,7 @@ mod tests {
         fn get_mock_host() -> schemas::Host {
             schemas::Host {
                 _id: Some(oid::ObjectId::new().to_string()),
-                device_id: "Vf3IceiD".to_string(),
+                pubkey: "placeholder_pubkey".to_string(),
                 ip_address: "127.0.0.1".to_string(),
                 remaining_capacity: Capacity {
                     memory: 16,
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 0a6ff82..6275c3c 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -28,10 +28,10 @@ pub use String as SemVer;
 pub use String as MongoDbId;
 
 // ==================== User Schema ====================
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
 pub enum Role {
     Developer(DeveloperJWT), // jwt string
-    Host(HosterPubKey),      // host pubkey
+    Hoster(HosterPubKey),      // host pubkey
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -120,14 +120,14 @@ pub struct Capacity {
 pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<MongoDbId>,
-    pub device_id: String, // *INDEXED*, Auto-generated Nats server ID
+    pub pubkey: String, // *INDEXED* // the HPOS/Device pubkey // nb: Unlike the hoster and developer pubkeys, this pubkey is not considered peronal info as it is not directly connected to a "natural person".
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
     pub avg_network_speed: i64,
     pub avg_latency: i64,
     pub assigned_workloads: Vec<String>, // MongoDB ID refs to `workload._id`
-    pub assigned_hoster: HosterPubKey,   // *INDEXED*, Hoster pubkey
+    pub assigned_hoster: HosterPubKey,   // *INDEXED*
 }
 
 impl IntoIndexes for Host {
@@ -135,13 +135,13 @@ impl IntoIndexes for Host {
         let mut indices = vec![];
 
         //  Add Device ID Index
-        let device_id_index_doc = doc! { "device_id": 1 };
-        let device_id_index_opts = Some(
+        let pubkey_index_doc = doc! { "pubkey": 1 };
+        let pubkey_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("device_id_index".to_string()))
+                .name(Some("pubkey_index".to_string()))
                 .build(),
         );
-        indices.push((device_id_index_doc, device_id_index_opts));
+        indices.push((pubkey_index_doc, pubkey_index_opts));
 
         Ok(indices)
     }
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index fce50ea..154d63f 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -14,10 +14,10 @@ use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
+pub type ResponseSubjectsGenerator = Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
 
 pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> Option<Vec<String>>;
+    fn get_tags(&self) -> HashMap<String, String>;
 }
 
 pub trait EndpointTraits:  Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static {}
@@ -197,7 +197,7 @@ impl JsStreamService {
         })
     }
 
-    pub async fn add_local_consumer<T>(
+    pub async fn add_consumer<T>(
         &self,
         consumer_name: &str,
         endpoint_subject: &str,
@@ -345,7 +345,7 @@ impl JsStreamService {
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 },
-                Err(err) => (err.to_string().into(), None),
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
@@ -496,7 +496,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_js_service_add_local_consumer() {
+    async fn test_js_service_add_consumer() {
         let context = setup_jetstream().await;
         let service = get_default_js_service(context).await;
 
@@ -506,7 +506,7 @@ mod tests {
         let response_subject = Some("response.subject".to_string());
 
         let consumer = service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
@@ -531,7 +531,7 @@ mod tests {
         let response_subject = None;
 
         service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 7eb43f1..099b6e1 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -1,8 +1,9 @@
 use super::js_stream_service::{JsServiceParamsPartial, JsStreamService, CreateTag};
 use anyhow::Result;
-use async_nats::jetstream;
+use async_nats::{jetstream, HeaderMap};
 use async_nats::{Message, ServerInfo};
 use serde::{Deserialize, Serialize};
+use core::option::Option::None;
 use std::future::Future;
 use std::error::Error;
 use std::fmt;
@@ -11,12 +12,24 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
+
 pub type EventListener = Box<dyn Fn(&mut JsClient) + Send + Sync>;
 pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
 pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
         + Send
         + Sync,
 >;
@@ -45,10 +58,11 @@ where
 }
 
 #[derive(Clone, Debug)]
-pub struct SendRequest {
+pub struct PublishInfo {
     pub subject: String,
     pub msg_id: String,
     pub data: Vec<u8>,
+    pub headers: Option<HeaderMap> 
 }
 
 #[derive(Debug)]
@@ -198,17 +212,19 @@ impl JsClient {
         );
         Ok(())
     }
-    
-    pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
-        Ok(())
-    }
 
-    pub async fn publish(&self, payload: &SendRequest) -> Result<(), async_nats::Error> {
+    pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
         let now = Instant::now();
-        let result = self
-            .js
-            .publish(payload.subject.clone(), payload.data.clone().into())
-            .await;
+        let result = match payload.headers {
+            Some(h) => self
+                .js
+                .publish_with_headers(payload.subject.clone(), h, payload.data.clone().into())
+                .await,
+            None => self
+                .js
+                .publish(payload.subject.clone(), payload.data.clone().into())
+                .await
+        };
 
         let duration = now.elapsed();
         if let Err(err) = result {
@@ -343,7 +359,7 @@ mod tests {
     async fn test_nats_js_client_publish() {
         let params = get_default_params();
         let client = JsClient::new(params).await.unwrap();
-        let payload = SendRequest {
+        let payload = PublishInfo {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
             data: b"Hello, NATS!".to_vec(),

From 74c7f0611afa5b69eeb798bd9344d422e09750e0 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 01:01:38 -0600
Subject: [PATCH 15/91] clean inline comments

---
 .../host_agent/src/hostd/workload_manager.rs  |   7 +-
 rust/clients/orchestrator/src/auth.rs         |   7 +-
 rust/clients/orchestrator/src/workloads.rs    |   8 +-
 rust/services/authentication/src/lib.rs       | 103 ++++++++----------
 4 files changed, 59 insertions(+), 66 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 5523ed6..3d6c479 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -33,7 +33,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     log::info!("host_creds_path : {}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
-    // ==================== NATS Setup ====================
+    // ==================== Setup NATS ====================
     // Connect to Nats server
     let nats_url = nats_js_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
@@ -65,7 +65,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         })
         .await?;
 
-    // ==================== DB Setup ====================
+    // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
@@ -74,7 +74,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     // Generate the Workload API with access to db
     let workload_api = WorkloadApi::new(&client).await?;
 
-    // ==================== API ENDPOINTS ====================
+    // ==================== Register API Endpoints ====================
     // Register Workload Streams for Host Agent to consume and process
     // NB: Subjects are published by orchestrator
     let workload_service = host_workload_client
@@ -123,6 +123,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         )
         .await?;
 
+    // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index a9d82d9..a7a8d09 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -35,7 +35,7 @@ pub fn create_callback_subject_to_host(tag_name: String, sub_subject_name: Strin
 }
 
 pub async fn run() -> Result<(), async_nats::Error> {
-    // ==================== NATS Setup ====================
+    // ==================== Setup NATS ====================
     let nats_url = nats_js_client::get_nats_url();
     let event_listeners = nats_js_client::get_event_listeners();
 
@@ -60,7 +60,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         })
         .await?;
 
-    // ==================== DB Setup ====================
+    // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
@@ -69,7 +69,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Generate the Auth API with access to db
     let auth_api = AuthApi::new(&client).await?;
 
-    // ==================== API ENDPOINTS ====================
+    // ==================== Register API Endpoints ====================
     // Register Auth Streams for Orchestrator to consume and proceess
     // NB: The subjects below are published by the Host Agent
     let auth_start_subject = serde_json::to_string(&AuthServiceSubjects::StartHandshake)?;
@@ -114,6 +114,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         "Orchestrator Auth Service is running. Waiting for requests..."
     );
 
+    // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index f3ef9f5..2d54df2 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -41,7 +41,7 @@ pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_su
 }
 
 pub async fn run() -> Result<(), async_nats::Error> {
-    // ==================== NATS Setup ====================
+    // ==================== Setup NATS ====================
     let nats_url = nats_js_client::get_nats_url();
     let creds_path = nats_js_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
     let event_listeners = nats_js_client::get_event_listeners();
@@ -67,7 +67,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         })
         .await?;
 
-    // ==================== DB Setup ====================
+    // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
@@ -76,7 +76,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Generate the Workload API with access to db
     let workload_api = WorkloadApi::new(&client).await?;
 
-    // ==================== API ENDPOINTS ====================
+    // ==================== Register API Endpoints ====================
     // Register Workload Streams for Orchestrator to consume and proceess
     // NB: These subjects below are published by external Developer, the Nats-DB-Connector, or the Host Agent
     let workload_service = orchestrator_workload_client
@@ -128,7 +128,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     )
     .await?;
 
-
+    // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 0079cc5..df44149 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -206,40 +206,6 @@ impl AuthApi {
         })
     }
 
-    /*******************************   For Host Agent   *********************************/
-     pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
-
-        // receive_and_write_file();
-
-        // Respond to endpoint request
-        // let response = b"Hello, NATS!".to_vec();
-
-        // let resolver_path = utils::get_resolver_path();
-
-        // // Generate resolver file and create resolver file
-        // Command::new("nsc")
-        //     .arg("generate")
-        //     .arg("config")
-        //     .arg("--nats-resolver")
-        //     .arg("sys-account SYS")
-        //     .arg("--force")
-        //     .arg(format!("--config-file {}", resolver_path))
-        //     .output()
-        //     .expect("Failed to create resolver config file");
-
-        // // Push auth updates to hub server
-        // Command::new("nsc")
-        //     .arg("push -A")
-        //     .output()
-        //     .expect("Failed to create resolver config file");
-
-        // Ok(response)
-
-        todo!();
-    }
-
-    /*******************************  For Orchestrator   *********************************/
     pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.handle_handshake_p2' : {:?}", msg);
 
@@ -257,8 +223,8 @@ impl AuthApi {
         utils::generate_user_jwt(&host_pubkey, &account_signing_key);
 
         // 4. Prepare User JWT to be sent as a payload in the publication callback
-        let sys_path = utils::get_file_path_buf("user_jwt_path");
-        let user_jwt: Vec<u8> = std::fs::read(sys_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let user_jwt_path = utils::get_file_path_buf("user_jwt_path");
+        let user_jwt: Vec<u8> = std::fs::read(user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         // 5. Respond to endpoint request
         Ok(types::ApiResult {
@@ -274,7 +240,51 @@ impl AuthApi {
     }
 
     /*******************************   For Host Agent   *********************************/
-     pub async fn save_user_jwt(
+    pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
+
+        // utils::receive_and_write_file();
+
+        // // Generate resolver file and create resolver file
+        // let resolver_path = utils::get_resolver_path();
+        // Command::new("nsc")
+        //     .arg("generate")
+        //     .arg("config")
+        //     .arg("--nats-resolver")
+        //     .arg("sys-account SYS")
+        //     .arg("--force")
+        //     .arg(format!("--config-file {}", resolver_path))
+        //     .output()
+        //     .expect("Failed to create resolver config file");
+
+        // // Push auth updates to hub server
+        // Command::new("nsc")
+        //     .arg("push -A")
+        //     .output()
+        //     .expect("Failed to create resolver config file");
+
+        // Prepare to send over user pubkey(to trigger the user jwt gen on hub)
+        let user_nkey_path = utils::get_file_path_buf("user_jwt_path");
+        let user_nkey: Vec<u8> = std::fs::read(user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let host_pubkey = serde_json::to_string(&user_nkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
+        
+        // Respond to endpoint request
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_pubkey: host_pubkey.clone(),
+                status: types::AuthState::Requested
+            },
+            result: AuthResult {
+                data: types::AuthResultType::Single(user_nkey)
+            },
+            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
+        })
+    }
+
+    pub async fn save_user_jwt(
         &self,
         msg: Arc<Message>,
         _output_dir: &str,
@@ -325,22 +335,3 @@ impl AuthApi {
     }
 
 }
-
-// In hpos
-// pub async fn send_user_pubkey(&self, msg: Arc<Message>) -> Result<Vec<u8>, ServiceError> {
-//     // 1. validate nk key...
-//     // let auth_endpoint_subject =
-//     // format!("AUTH.{}.file.transfer.JWT-operator", "host_id_placeholder"); // endpoint_subject
-
-//     // 2. Update the hub nsc with user pubkey
-
-//     // 3. create signed jwt
-
-//     // 4. `Ack last msg and publish the new jwt to for hpos
-
-//     // 5. Respond to endpoint request
-//     // let response = b"Hello, NATS!".to_vec();
-//     // Ok(response)
-
-//     todo!()
-// }

From 3f9b4387b4698265af1e2cab81193081da4aa81c Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 01:45:26 -0600
Subject: [PATCH 16/91] standarize service name ref

---
 .../host_agent/src/hostd/workload_manager.rs  | 37 +++++++---
 rust/clients/orchestrator/src/workloads.rs    | 73 ++++++++++++++++---
 rust/services/workload/src/lib.rs             | 48 ++++++------
 rust/services/workload/src/types.rs           | 15 ++++
 rust/util_libs/src/db/schemas.rs              |  1 +
 5 files changed, 128 insertions(+), 46 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 3d6c479..f8eea45 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -22,6 +22,7 @@ use util_libs::{
 };
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+    types::{WorkloadServiceSubjects, ApiResult}
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
@@ -77,6 +78,11 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     // ==================== Register API Endpoints ====================
     // Register Workload Streams for Host Agent to consume and process
     // NB: Subjects are published by orchestrator
+    let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
+    let workload_send_status_subject = serde_json::to_string(&WorkloadServiceSubjects::SendStatus)?;
+    let workload_uninstall_subject = serde_json::to_string(&WorkloadServiceSubjects::Uninstall)?;
+    let workload_handle_update_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleUpdate)?;
+
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -85,12 +91,12 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         ))?;
 
     workload_service
-        .add_consumer::<workload::types::ApiResult>(
+        .add_consumer::<ApiResult>(
             "start_workload", // consumer name
-            &format!("{}.start", host_pubkey), // consumer stream subj
+            &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.start_workload(msg).await
+                    api.start_workload_on_host(msg).await
                 }
             })),
             None,
@@ -98,12 +104,12 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_consumer::<workload::types::ApiResult>(
+        .add_consumer::<ApiResult>(
             "send_workload_status", // consumer name
-            &format!("{}.send_status", host_pubkey), // consumer stream subj
+            &format!("{}.{}", host_pubkey, workload_handle_update_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.send_workload_status(msg).await
+                    api.update_workload_on_host(msg).await
                 }
             })),
             None,
@@ -111,12 +117,25 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_consumer::<workload::types::ApiResult>(
+        .add_consumer::<ApiResult>(
             "uninstall_workload", // consumer name
-            &format!("{}.uninstall", host_pubkey), // consumer stream subj
+            &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.uninstall_workload_from_host(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+
+        workload_service
+        .add_consumer::<ApiResult>(
+            "send_workload_status", // consumer name
+            &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.uninstall_workload(msg).await
+                    api.send_workload_status_from_host(msg).await
                 }
             })),
             None,
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 2d54df2..8bf0749 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -11,7 +11,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use workload::{
-    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, ApiResult}
 };
 use util_libs::{
     db::mongodb::get_mongodb_url,
@@ -79,6 +79,15 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== Register API Endpoints ====================
     // Register Workload Streams for Orchestrator to consume and proceess
     // NB: These subjects below are published by external Developer, the Nats-DB-Connector, or the Host Agent
+    let workload_add_subject = serde_json::to_string(&WorkloadServiceSubjects::Add)?;
+    let workload_update_subject = serde_json::to_string(&WorkloadServiceSubjects::Update)?;
+    let workload_remove_subject = serde_json::to_string(&WorkloadServiceSubjects::Remove)?;
+    let workload_db_insert_subject = serde_json::to_string(&WorkloadServiceSubjects::Insert)?;
+    let workload_db_modification_subject = serde_json::to_string(&WorkloadServiceSubjects::Modify)?;
+    let workload_handle_status_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
+    let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
+    let workload_handle_update_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleUpdate)?;
+
     let workload_service = orchestrator_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -88,9 +97,9 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by Developer
     workload_service
-        .add_consumer::<workload::types::ApiResult>(
-            "add_workload",
-            "add",
+        .add_consumer::<ApiResult>(
+            "add_workload", // consumer name
+             &workload_add_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.add_workload(msg).await
@@ -100,25 +109,65 @@ pub async fn run() -> Result<(), async_nats::Error> {
         )
         .await?;
 
+        workload_service
+        .add_consumer::<ApiResult>(
+            "update_workload", // consumer name
+             &workload_update_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.update_workload(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+
+    
+    workload_service
+        .add_consumer::<ApiResult>(
+            "remove_workload", // consumer name
+             &workload_remove_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.remove_workload(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+    
     // Automatically published by the Nats-DB-Connector
     workload_service
-        .add_consumer::<workload::types::ApiResult>(
-            "handle_db_insertion",
-            "insert",
+        .add_consumer::<ApiResult>(
+            "handle_db_insertion", // consumer name
+             &workload_db_insert_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.handle_db_insertion(msg).await
                 }
             })),
-            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), "start".to_string())),
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_start_subject)),
         )
         .await?;
-    
+
+    workload_service
+        .add_consumer::<ApiResult>(
+            "handle_db_modification", // consumer name
+                &workload_db_modification_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.handle_db_modification(msg).await
+                }
+            })),
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_handle_update_subject)),
+        )
+        .await?;
+
     // Published by the Host Agent
     workload_service
-    .add_consumer::<workload::types::ApiResult>(
-        "handle_status_update",
-        "read_status_update",
+    .add_consumer::<ApiResult>(
+        "handle_status_update", // consumer name
+        &workload_handle_status_subject, // consumer stream subj
         EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
             async move {
                 api.handle_status_update(msg).await
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 96a592f..78f17d8 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -248,8 +248,8 @@ impl WorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
+    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.modify'");
         
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
@@ -265,28 +265,9 @@ impl WorkloadApi {
         Ok(types::ApiResult(success_status, None))
     }
 
-    // Zeeshan to take a look:
-    // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.delete'");
-        
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
-        log::trace!("New workload to assign. Workload={:#?}", workload);   
-
-        // TODO: ...handle the use case for the delete entry change stream
-
-        let success_status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Removed,
-            actual: WorkloadState::Removed,
-        };
-        
-        Ok(types::ApiResult(success_status, None))
-    }
-
     // NB: Published by the Hosting Agent whenever the status of a workload changes
     pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
+        log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
         let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
         log::trace!("Workload status to update. Status={:?}", workload_status);
@@ -297,7 +278,7 @@ impl WorkloadApi {
     }    
 
      /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn start_workload_on_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
@@ -314,7 +295,24 @@ impl WorkloadApi {
         Ok(types::ApiResult(status, None))
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn update_workload_on_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.handle_update' : {:?}", msg);
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+
+        // TODO: Talk through with Stefan
+        // 1. Connect to interface for Nix and instruct systemd to install workload...
+        // eg: nix_install_with(workload)
+
+        // 2. Respond to endpoint request
+        let status = WorkloadStatus {
+            id: workload._id,
+            desired: WorkloadState::Updating,
+            actual: WorkloadState::Unknown("..".to_string()),
+        };
+        Ok(types::ApiResult(status, None))
+    }
+
+    pub async fn uninstall_workload_from_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
         let workload_id = Self::convert_to_type::<String>(msg)?;
 
@@ -333,7 +331,7 @@ impl WorkloadApi {
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn send_workload_status_from_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!(
             "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
             msg
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 3774ad6..d24e3dc 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -3,6 +3,21 @@ use std::collections::HashMap;
 use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum WorkloadServiceSubjects {
+    Add,
+    Update,
+    Remove,
+    Insert, // db change stream trigger
+    Modify, // db change stream trigger
+    HandleStatusUpdate,
+    SendStatus,
+    Start,
+    Uninstall,
+    HandleUpdate
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
 impl EndpointTraits for ApiResult {}
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 6275c3c..963daff 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -155,6 +155,7 @@ pub enum WorkloadState {
     Pending,
     Installed,
     Running,
+    Updating,
     Removed,
     Uninstalled,
     Error(String), // String = error message

From 287cd7e64741e2ebc4c90c6f61a378748b60924a Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 13 Jan 2025 16:49:10 +0100
Subject: [PATCH 17/91] temporary(flake): switch to blueprint fork

---
 flake.lock                                  | 11 ++++---
 flake.nix                                   |  2 +-
 nix/checks/holo-agent-integration-nixos.nix | 34 +++++++++++++++++++++
 3 files changed, 41 insertions(+), 6 deletions(-)
 create mode 100644 nix/checks/holo-agent-integration-nixos.nix

diff --git a/flake.lock b/flake.lock
index 2e2f35e..92d3a83 100644
--- a/flake.lock
+++ b/flake.lock
@@ -8,15 +8,16 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1733562445,
-        "narHash": "sha256-gLmqbX40Qos+EeBvmlzvntWB3NrdiDaFxhr3VAmhrf4=",
-        "owner": "numtide",
+        "lastModified": 1737018163,
+        "narHash": "sha256-rIkW13S9BLcdPYRVwGJfPUj1PctAfFOsJp03de2XxzE=",
+        "owner": "steveej-forks",
         "repo": "blueprint",
-        "rev": "97ef7fba3a6eec13e12a108ce4b3473602eb424f",
+        "rev": "9c3872fec6ff50525178dcb0857f2b234299ecff",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "steveej-forks",
+        "ref": "fix-checks-import",
         "repo": "blueprint",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 9e3a1df..f7b1916 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,7 +4,7 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.11";
     nixpkgs-unstable.url = "github:NixOS/nixpkgs?ref=nixos-unstable";
-    blueprint.url = "github:numtide/blueprint";
+    blueprint.url = "github:steveej-forks/blueprint/fix-checks-import";
     blueprint.inputs.nixpkgs.follows = "nixpkgs";
     treefmt-nix.url = "github:numtide/treefmt-nix";
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
new file mode 100644
index 0000000..2d03ccf
--- /dev/null
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -0,0 +1,34 @@
+{
+  pkgs,
+  flake,
+  ...
+}:
+
+pkgs.testers.runNixOSTest (_: {
+  name = "holo-agent-nixostest-basic";
+
+  nodes.machine =
+    _:
+
+    {
+      imports = [
+        flake.nixosModules.holo-agent
+      ];
+
+      holo.agent = {
+        enable = true;
+        rust = {
+          log = "trace";
+          backtrace = "trace";
+        };
+      };
+    };
+
+  # takes args which are currently removed by deadnix:
+  # { nodes, ... }
+  testScript = _: ''
+    machine.start()
+    # machine.wait_for_unit("holo-agent.service")
+    machine.wait_for_unit("default.target")
+  '';
+})

From f79b43110493a870473672d6b65bb5095be1f595 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Tue, 14 Jan 2025 21:32:42 +0100
Subject: [PATCH 18/91] feat(nix/lib): wrap runNixOSTest with defaults

this is required when VM tests use nixos modules that live in a
blueprint repository like this.
---
 nix/lib/default.nix | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/nix/lib/default.nix b/nix/lib/default.nix
index 3c1aa40..85b81a4 100644
--- a/nix/lib/default.nix
+++ b/nix/lib/default.nix
@@ -10,4 +10,28 @@
       );
     in
     craneLib.overrideToolchain toolchain;
+
+  runNixOSTest' =
+    { pkgs, system }:
+
+    /*
+      looks like this:
+      args: { fn body that has access to args}
+      or this:
+      { static attrs }
+    */
+    callerArg:
+
+    let
+      callerFn = pkgs.lib.toFunction callerArg;
+    in
+
+    pkgs.testers.runNixOSTest (
+      args:
+      (pkgs.lib.recursiveUpdate {
+        defaults._module.args = {
+          inherit flake inputs system;
+        };
+      } (callerFn args))
+    );
 }

From 829b064d7e59adaf73ae032f5eab35cf27a1154c Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 13 Jan 2025 12:34:44 +0100
Subject: [PATCH 19/91] feat(nix/packages/rust-workspace): expose rust binaries

previously it would only expose the target directory as an archive.
---
 nix/packages/rust-workspace.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/packages/rust-workspace.nix b/nix/packages/rust-workspace.nix
index 5203e81..8d7d029 100644
--- a/nix/packages/rust-workspace.nix
+++ b/nix/packages/rust-workspace.nix
@@ -35,7 +35,7 @@ let
   # cache misses when building individual top-level-crates
   cargoArtifacts = craneLib.buildDepsOnly commonArgs;
 in
-craneLib.cargoBuild (
+craneLib.buildPackage (
   commonArgs
   // {
     inherit cargoArtifacts;

From f609b569c5d7d316231699ef5e3e113120497708 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 13 Jan 2025 16:51:12 +0100
Subject: [PATCH 20/91] WIP: feat(holo-agent): add nixos module with
 integration test

---
 flake.lock                                  | 71 +++++++++++++++++---
 flake.nix                                   |  3 +
 nix/checks/holo-agent-integration-nixos.nix | 67 ++++++++++++-------
 nix/modules/nixos/holo-agent.nix            | 74 ++++++++++++++++++---
 4 files changed, 175 insertions(+), 40 deletions(-)

diff --git a/flake.lock b/flake.lock
index 92d3a83..5824692 100644
--- a/flake.lock
+++ b/flake.lock
@@ -57,13 +57,37 @@
         "type": "github"
       }
     },
+    "extra-container": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1734542275,
+        "narHash": "sha256-wnRkafo4YrIuvJeRsOmfStxIzi7ty2I0OtGMO9chwJc=",
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "rev": "fa723fb67201c1b4610fd3d608681da362f800eb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "type": "github"
+      }
+    },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1653893745,
-        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
         "type": "github"
       },
       "original": {
@@ -132,9 +156,24 @@
         "type": "github"
       }
     },
+    "flake-utils_6": {
+      "locked": {
+        "lastModified": 1653893745,
+        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "nixago": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "flake-utils": "flake-utils_2",
         "nixago-exts": "nixago-exts",
         "nixpkgs": [
           "nixpkgs"
@@ -156,7 +195,7 @@
     },
     "nixago-exts": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_3",
         "nixago": "nixago_2",
         "nixpkgs": [
           "nixago",
@@ -179,7 +218,7 @@
     },
     "nixago-exts_2": {
       "inputs": {
-        "flake-utils": "flake-utils_4",
+        "flake-utils": "flake-utils_5",
         "nixago": "nixago_3",
         "nixpkgs": [
           "nixago",
@@ -204,7 +243,7 @@
     },
     "nixago_2": {
       "inputs": {
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_4",
         "nixago-exts": "nixago-exts_2",
         "nixpkgs": [
           "nixago",
@@ -229,7 +268,7 @@
     },
     "nixago_3": {
       "inputs": {
-        "flake-utils": "flake-utils_5",
+        "flake-utils": "flake-utils_6",
         "nixpkgs": [
           "nixago",
           "nixago-exts",
@@ -289,6 +328,7 @@
         "blueprint": "blueprint",
         "crane": "crane",
         "disko": "disko",
+        "extra-container": "extra-container",
         "nixago": "nixago",
         "nixpkgs": "nixpkgs",
         "nixpkgs-unstable": "nixpkgs-unstable",
@@ -352,6 +392,21 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "treefmt-nix": {
       "inputs": {
         "nixpkgs": [
diff --git a/flake.nix b/flake.nix
index f7b1916..5ebd332 100644
--- a/flake.nix
+++ b/flake.nix
@@ -21,6 +21,9 @@
       url = "github:oxalica/rust-overlay";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    extra-container.url = "github:erikarvstedt/extra-container";
+    extra-container.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs =
diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index 2d03ccf..278e1e9 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -1,34 +1,53 @@
 {
-  pkgs,
   flake,
+  pkgs,
   ...
 }:
 
-pkgs.testers.runNixOSTest (_: {
-  name = "holo-agent-nixostest-basic";
+pkgs.testers.runNixOSTest (
+  { nodes, lib, ... }:
+  {
+    name = "host-agent-integration-nixos";
+    meta.platforms = lib.lists.intersectLists lib.platforms.linux lib.platforms.x86_64;
+
+    # TODO: add a NATS server which is used to test the leaf connection
+    # nodes.hub = { };
+
+    nodes.agent =
+      { config, ... }:
+      {
+        imports = [
+          flake.nixosModules.holo-nats-server
+          flake.nixosModules.holo-agent
+        ];
 
-  nodes.machine =
-    _:
+        holo.nats-server.enable = true;
 
-    {
-      imports = [
-        flake.nixosModules.holo-agent
-      ];
+        holo.agent = {
+          enable = true;
+          autoStart = false;
+          rust = {
+            log = "trace";
+            backtrace = "trace";
+          };
 
-      holo.agent = {
-        enable = true;
-        rust = {
-          log = "trace";
-          backtrace = "trace";
+          nats = {
+            url = "127.0.0.1:${builtins.toString config.services.nats.port}";
+            hubServerUrl = "127.0.0.1:${builtins.toString config.services.nats.settings.leafnodes.port}";
+          };
         };
       };
-    };
-
-  # takes args which are currently removed by deadnix:
-  # { nodes, ... }
-  testScript = _: ''
-    machine.start()
-    # machine.wait_for_unit("holo-agent.service")
-    machine.wait_for_unit("default.target")
-  '';
-})
+
+    # takes args which are currently removed by deadnix:
+    # { nodes, ... }
+    testScript = _: ''
+      agent.start()
+
+      agent.wait_for_unit("nats.service")
+
+      # TODO: fix after/require settings of the holo-agent service to make autoStart work
+      agent.succeed("systemctl start holo-agent")
+      agent.wait_for_unit("holo-agent")
+    '';
+  }
+)
diff --git a/nix/modules/nixos/holo-agent.nix b/nix/modules/nixos/holo-agent.nix
index d4fc43b..d901abf 100644
--- a/nix/modules/nixos/holo-agent.nix
+++ b/nix/modules/nixos/holo-agent.nix
@@ -1,29 +1,87 @@
 # Module to configure a machine as a holo-agent.
-
 {
-  inputs,
   lib,
   config,
+  pkgs,
+  system,
+  flake,
   ...
 }:
 
 let
-  cfg = config.holo.nats-server;
+  cfg = config.holo.agent;
 in
 {
   imports = [
-    inputs.extra-container.nixosModules.default
+    # TODO: this causes an infinite recursion. we do need this to run workloads.
+    # flake.inputs.extra-container.nixosModules.default
   ];
 
-  options.holo.agent = with lib; {
-    enable = mkOption {
+  options.holo.agent = {
+    enable = lib.mkOption {
       description = "enable holo-agent";
       default = true;
     };
+
+    autoStart = lib.mkOption {
+      default = true;
+    };
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = flake.packages.${system}.rust-workspace;
+    };
+
+    rust = {
+      log = lib.mkOption {
+        type = lib.types.str;
+        default = "debug";
+      };
+
+      backtrace = lib.mkOption {
+        type = lib.types.str;
+        default = "1";
+      };
+    };
+
+    nats = {
+      useOsNats = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+      };
+
+      url = lib.mkOption {
+        type = lib.types.str;
+      };
+      hubServerUrl = lib.mkOption {
+        type = lib.types.str;
+      };
+    };
   };
 
   config = lib.mkIf cfg.enable {
-    # TODO: add holo-agent systemd service
-    # TODO: add nats client here or is it started by the agent?
+    systemd.services.holo-agent = {
+      enable = true;
+
+      requires = lib.lists.optional cfg.nats.useOsNats "nats.service";
+      after = lib.lists.optional cfg.nats.useOsNats "nats.service";
+
+      requiredBy = lib.optional cfg.autoStart "multi-user.target";
+
+      environment = {
+        RUST_LOG = cfg.rust.log;
+        RUST_BACKTRACE = cfg.rust.backtrace;
+        NATS_URL = cfg.nats.url;
+        NATS_HUB_SERVER_URL = cfg.nats.hubServerUrl;
+      };
+
+      script = builtins.toString (
+        pkgs.writeShellScript "holo-agent" ''
+          ${lib.getExe' cfg.package "host_agent"} daemonize
+        ''
+      );
+    };
+
+    # TODO: add nats server here or is it started by the agent?
   };
 }

From 94509066c9d4dea8e0ba0576dee059b9a5339b04 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Wed, 15 Jan 2025 22:47:14 +0100
Subject: [PATCH 21/91] FIXME: this commit needs splitting up

iterate on holo-agent-integration-nixos with code changes all over the place.

test can be run with:

nix build -vL .\#checks.x86_64-linux.holo-agent-integration-nixos
---
 nix/checks/holo-agent-integration-nixos.nix   |  56 ++++++---
 nix/lib/default.nix                           |  30 +----
 nix/modules/nixos/holo-agent.nix              |  87 --------------
 nix/modules/nixos/holo-host-agent.nix         | 111 ++++++++++++++++++
 nix/modules/nixos/holo-nats-server.nix        |  19 ++-
 nix/packages/rust-workspace.nix               |   2 -
 .../clients/host_agent/src/gen_leaf_server.rs |  12 +-
 rust/util_libs/src/nats_js_client.rs          |  53 +++++----
 rust/util_libs/src/nats_server.rs             |  18 +--
 9 files changed, 221 insertions(+), 167 deletions(-)
 delete mode 100644 nix/modules/nixos/holo-agent.nix
 create mode 100644 nix/modules/nixos/holo-host-agent.nix

diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index 278e1e9..6e4cd5a 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -10,20 +10,29 @@ pkgs.testers.runNixOSTest (
     name = "host-agent-integration-nixos";
     meta.platforms = lib.lists.intersectLists lib.platforms.linux lib.platforms.x86_64;
 
-    # TODO: add a NATS server which is used to test the leaf connection
-    # nodes.hub = { };
+    nodes.hub =
+      {
+        ...
 
-    nodes.agent =
-      { config, ... }:
+      }:
       {
         imports = [
           flake.nixosModules.holo-nats-server
-          flake.nixosModules.holo-agent
+          # flake.nixosModules.holo-orchestrator
         ];
 
+        # holo.orchestrator.enable = true;
         holo.nats-server.enable = true;
+      };
+
+    nodes.host =
+      { ... }:
+      {
+        imports = [
+          flake.nixosModules.holo-host-agent
+        ];
 
-        holo.agent = {
+        holo.host-agent = {
           enable = true;
           autoStart = false;
           rust = {
@@ -32,22 +41,39 @@ pkgs.testers.runNixOSTest (
           };
 
           nats = {
-            url = "127.0.0.1:${builtins.toString config.services.nats.port}";
-            hubServerUrl = "127.0.0.1:${builtins.toString config.services.nats.settings.leafnodes.port}";
+            # url = "agent:${builtins.toString config.services.nats.port}";
+            hubServerUrl = "hub:${builtins.toString nodes.hub.holo.nats-server.leafnodePort}";
           };
         };
       };
 
     # takes args which are currently removed by deadnix:
     # { nodes, ... }
-    testScript = _: ''
-      agent.start()
+    testScript =
+      _:
+      let
+        natsCli = lib.getExe pkgs.natscli;
+        hubTestScript = pkgs.writeShellScript "cmd" ''
+          ${natsCli} pub -s 'nats://127.0.0.1:${builtins.toString nodes.hub.holo.nats-server.port}' --count=10 WORKLOAD.start '{"message":"hello"}'
+        '';
+
+        hostTestScript = pkgs.writeShellScript "cmd" ''
+          ${natsCli} sub -s 'nats://127.0.0.1:${builtins.toString nodes.host.holo.host-agent.nats.listenPort}' --count=10 'WORKLOAD.>'
+        '';
+      in
+      ''
+        hub.start()
+        hub.wait_for_unit("nats.service")
+        hub.succeed("${hubTestScript}")
+
+        host.start()
+        # agent.wait_for_unit("nats.service")
 
-      agent.wait_for_unit("nats.service")
+        # TODO: fix after/require settings of the host-agent service to make autoStart work
+        host.succeed("systemctl start holo-host-agent")
+        host.wait_for_unit("holo-host-agent")
 
-      # TODO: fix after/require settings of the holo-agent service to make autoStart work
-      agent.succeed("systemctl start holo-agent")
-      agent.wait_for_unit("holo-agent")
-    '';
+        host.succeed("${hostTestScript}")
+      '';
   }
 )
diff --git a/nix/lib/default.nix b/nix/lib/default.nix
index 85b81a4..f2cd411 100644
--- a/nix/lib/default.nix
+++ b/nix/lib/default.nix
@@ -1,4 +1,8 @@
-{ inputs, flake, ... }:
+{
+  inputs,
+  flake,
+  ...
+}:
 
 {
   mkCraneLib =
@@ -10,28 +14,4 @@
       );
     in
     craneLib.overrideToolchain toolchain;
-
-  runNixOSTest' =
-    { pkgs, system }:
-
-    /*
-      looks like this:
-      args: { fn body that has access to args}
-      or this:
-      { static attrs }
-    */
-    callerArg:
-
-    let
-      callerFn = pkgs.lib.toFunction callerArg;
-    in
-
-    pkgs.testers.runNixOSTest (
-      args:
-      (pkgs.lib.recursiveUpdate {
-        defaults._module.args = {
-          inherit flake inputs system;
-        };
-      } (callerFn args))
-    );
 }
diff --git a/nix/modules/nixos/holo-agent.nix b/nix/modules/nixos/holo-agent.nix
deleted file mode 100644
index d901abf..0000000
--- a/nix/modules/nixos/holo-agent.nix
+++ /dev/null
@@ -1,87 +0,0 @@
-# Module to configure a machine as a holo-agent.
-{
-  lib,
-  config,
-  pkgs,
-  system,
-  flake,
-  ...
-}:
-
-let
-  cfg = config.holo.agent;
-in
-{
-  imports = [
-    # TODO: this causes an infinite recursion. we do need this to run workloads.
-    # flake.inputs.extra-container.nixosModules.default
-  ];
-
-  options.holo.agent = {
-    enable = lib.mkOption {
-      description = "enable holo-agent";
-      default = true;
-    };
-
-    autoStart = lib.mkOption {
-      default = true;
-    };
-
-    package = lib.mkOption {
-      type = lib.types.package;
-      default = flake.packages.${system}.rust-workspace;
-    };
-
-    rust = {
-      log = lib.mkOption {
-        type = lib.types.str;
-        default = "debug";
-      };
-
-      backtrace = lib.mkOption {
-        type = lib.types.str;
-        default = "1";
-      };
-    };
-
-    nats = {
-      useOsNats = lib.mkOption {
-        type = lib.types.bool;
-        default = true;
-      };
-
-      url = lib.mkOption {
-        type = lib.types.str;
-      };
-      hubServerUrl = lib.mkOption {
-        type = lib.types.str;
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    systemd.services.holo-agent = {
-      enable = true;
-
-      requires = lib.lists.optional cfg.nats.useOsNats "nats.service";
-      after = lib.lists.optional cfg.nats.useOsNats "nats.service";
-
-      requiredBy = lib.optional cfg.autoStart "multi-user.target";
-
-      environment = {
-        RUST_LOG = cfg.rust.log;
-        RUST_BACKTRACE = cfg.rust.backtrace;
-        NATS_URL = cfg.nats.url;
-        NATS_HUB_SERVER_URL = cfg.nats.hubServerUrl;
-      };
-
-      script = builtins.toString (
-        pkgs.writeShellScript "holo-agent" ''
-          ${lib.getExe' cfg.package "host_agent"} daemonize
-        ''
-      );
-    };
-
-    # TODO: add nats server here or is it started by the agent?
-  };
-}
diff --git a/nix/modules/nixos/holo-host-agent.nix b/nix/modules/nixos/holo-host-agent.nix
new file mode 100644
index 0000000..0f0079e
--- /dev/null
+++ b/nix/modules/nixos/holo-host-agent.nix
@@ -0,0 +1,111 @@
+# Module to configure a machine as a holo-host-agent.
+
+# blueprint specific first level argument that's referred to as "publisherArgs"
+{
+  inputs,
+  ...
+}:
+
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.holo.host-agent;
+in
+{
+  imports = [
+    inputs.extra-container.nixosModules.default
+  ];
+
+  options.holo.host-agent = {
+    enable = lib.mkOption {
+      description = "enable holo-host-agent";
+      default = true;
+    };
+
+    autoStart = lib.mkOption {
+      default = true;
+    };
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = inputs.self.packages.${pkgs.stdenv.system}.rust-workspace;
+    };
+
+    rust = {
+      log = lib.mkOption {
+        type = lib.types.str;
+        default = "debug";
+      };
+
+      backtrace = lib.mkOption {
+        type = lib.types.str;
+        default = "1";
+      };
+    };
+
+    nats = {
+      useOsNats = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+      };
+
+      listenHost = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+      };
+
+      listenPort = lib.mkOption {
+        type = lib.types.int;
+        default = 4222;
+      };
+
+      url = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = "${cfg.nats.listenHost}:${builtins.toString cfg.nats.listenPort}";
+      };
+
+      hubServerUrl = lib.mkOption {
+        type = lib.types.str;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.holo-host-agent = {
+      enable = true;
+
+      requires = lib.lists.optional cfg.nats.useOsNats "nats.service";
+      after = lib.lists.optional cfg.nats.useOsNats "nats.service";
+
+      requiredBy = lib.optional cfg.autoStart "multi-user.target";
+
+      environment =
+        {
+          RUST_LOG = cfg.rust.log;
+          RUST_BACKTRACE = cfg.rust.backtrace;
+          NATS_HUB_SERVER_URL = cfg.nats.hubServerUrl;
+          NATS_LISTEN_PORT = builtins.toString cfg.nats.listenPort;
+        }
+        // lib.attrsets.optionalAttrs (cfg.nats.url != null) {
+          NATS_URL = cfg.nats.url;
+        };
+
+      path = [
+        pkgs.nats-server
+      ];
+
+      script = builtins.toString (
+        pkgs.writeShellScript "holo-host-agent" ''
+          ${lib.getExe' cfg.package "host_agent"} daemonize
+        ''
+      );
+    };
+
+    # TODO: add nats server here or is it started by the host-agent?
+  };
+}
diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index e39e1be..a18b268 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -5,11 +5,23 @@ in
 {
   imports = [ ];
 
-  options.holo.nats-server = with lib; {
-    enable = mkOption {
+  options.holo.nats-server = {
+    enable = lib.mkOption {
       description = "enable holo NATS server";
       default = true;
     };
+
+    port = lib.mkOption {
+      description = "enable holo NATS server";
+      type = lib.types.int;
+      default = 4222;
+    };
+
+    leafnodePort = lib.mkOption {
+      description = "enable holo NATS server";
+      type = lib.types.int;
+      default = 7422;
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -24,7 +36,8 @@ in
       jetstream = true;
 
       settings = {
-        leafnodes.port = 7422;
+        inherit (cfg) port;
+        leafnodes.port = cfg.leafnodePort;
       };
     };
   };
diff --git a/nix/packages/rust-workspace.nix b/nix/packages/rust-workspace.nix
index 8d7d029..ae42a82 100644
--- a/nix/packages/rust-workspace.nix
+++ b/nix/packages/rust-workspace.nix
@@ -97,8 +97,6 @@ craneLib.buildPackage (
           partitionType = "count";
         }
       );
-
     };
-
   }
 )
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/gen_leaf_server.rs
index 718267b..4e6063c 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/gen_leaf_server.rs
@@ -1,12 +1,14 @@
-use util_libs::nats_server::{self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions};
-
-const LEAF_SERVE_NAME: &str = "test_leaf_server";
-const LEAF_SERVER_CONFIG_PATH: &str = "test_leaf_server.conf";
+use util_libs::nats_server::{
+    self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
+    LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
+};
 
 pub async fn run(user_creds_path: &str) {
     let leaf_server_remote_conn_url = nats_server::get_hub_server_url();
     let leaf_client_conn_domain = "127.0.0.1";
-    let leaf_client_conn_port = 4111;
+    let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
+        .map(|var| var.parse().expect("can't parse into number"))
+        .unwrap_or_else(|_| LEAF_SERVER_DEFAULT_LISTEN_PORT);
 
     let nsc_path =
         std::env::var("NSC_PATH").unwrap_or_else(|_| ".local/share/nats/nsc".to_string());
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 7eb43f1..4671eef 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -1,12 +1,13 @@
-use super::js_stream_service::{JsServiceParamsPartial, JsStreamService, CreateTag};
+use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
+use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
+
 use anyhow::Result;
-use async_nats::jetstream;
-use async_nats::{Message, ServerInfo};
+use async_nats::{jetstream, Message, ServerInfo};
 use serde::{Deserialize, Serialize};
-use std::future::Future;
 use std::error::Error;
 use std::fmt;
 use std::fmt::Debug;
+use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -22,7 +23,7 @@ pub type AsyncEndpointHandler<T> = Arc<
 >;
 
 #[derive(Clone)]
-pub enum EndpointType<T> 
+pub enum EndpointType<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
 {
@@ -136,7 +137,11 @@ impl JsClient {
             services.push(service);
         }
 
-        let js_services = if services.is_empty() { None } else { Some(services) };
+        let js_services = if services.is_empty() {
+            None
+        } else {
+            Some(services)
+        };
 
         let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
 
@@ -198,7 +203,7 @@ impl JsClient {
         );
         Ok(())
     }
-    
+
     pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
         Ok(())
     }
@@ -250,7 +255,7 @@ impl JsClient {
 
 // Client Options:
 pub fn with_event_listeners(listeners: Vec<EventListener>) -> EventListener {
-    Box::new(move |c: &mut JsClient | {
+    Box::new(move |c: &mut JsClient| {
         for listener in &listeners {
             listener(c);
         }
@@ -260,7 +265,7 @@ pub fn with_event_listeners(listeners: Vec<EventListener>) -> EventListener {
 // Event Listener Options:
 pub fn on_msg_published_event<F>(f: F) -> EventListener
 where
-        F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
+    F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Box::new(move |c: &mut JsClient| {
         c.on_msg_published_event = Some(Box::pin(f.clone()));
@@ -269,7 +274,7 @@ where
 
 pub fn on_msg_failed_event<F>(f: F) -> EventListener
 where
-        F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
+    F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Box::new(move |c: &mut JsClient| {
         c.on_msg_failed_event = Some(Box::pin(f.clone()));
@@ -277,6 +282,21 @@ where
 }
 
 // Helpers:
+// TODO: there's overlap with the NATS_LISTEN_PORT. refactor this to e.g. read NATS_LISTEN_HOST and NATS_LISTEN_PORT
+pub fn get_nats_url() -> String {
+    std::env::var("NATS_URL")
+        .unwrap_or_else(|_| format!("127.0.0.1:{}", LEAF_SERVER_DEFAULT_LISTEN_PORT))
+}
+
+pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> String {
+    std::env::var("HOST_CREDS_FILE_PATH").unwrap_or_else(|_| {
+        format!(
+            "/.local/share/nats/nsc/keys/creds/{}/{}/{}.creds",
+            operator, account, user
+        )
+    })
+}
+
 pub fn get_event_listeners() -> Vec<EventListener> {
     // TODO: Use duration in handlers..
     let published_msg_handler = move |msg: &str, client_name: &str, _duration: Duration| {
@@ -298,19 +318,6 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     event_listeners
 }
 
-pub fn get_nats_url() -> String {
-    std::env::var("NATS_URL").unwrap_or_else(|_| "127.0.0.1:4111".to_string())
-}
-
-pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> String {
-    std::env::var("HOST_CREDS_FILE_PATH").unwrap_or_else(|_| {
-        format!(
-            "/.local/share/nats/nsc/keys/creds/{}/{}/{}.creds",
-            operator, account, user
-        )
-    })
-}
-
 #[cfg(feature = "tests_integration_nats")]
 #[cfg(test)]
 mod tests {
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index 47a88f8..afe84b1 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -10,6 +10,10 @@ use std::process::{Child, Command, Stdio};
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
+pub const LEAF_SERVE_NAME: &str = "test_leaf_server";
+pub const LEAF_SERVER_CONFIG_PATH: &str = "test_leaf_server.conf";
+pub const LEAF_SERVER_DEFAULT_LISTEN_PORT: u16 = 4111;
+
 #[derive(Debug, Clone)]
 pub struct JetStreamConfig {
     pub store_dir: String,
@@ -124,11 +128,11 @@ jetstream {{
 
 leafnodes {{
     remotes = [
-        {}
+        {leafnodes_config}
     ]
 }}
 
-{}
+{logging_config}
 "#,
             self.name,
             self.host,
@@ -136,8 +140,6 @@ leafnodes {{
             self.jetstream_config.store_dir,
             self.jetstream_config.max_memory_store,
             self.jetstream_config.max_file_store,
-            leafnodes_config,
-            logging_config
         )?;
 
         // Run the server with the generated config
@@ -425,7 +427,7 @@ mod tests {
             .await
             .expect("Failed to publish jetstream message.");
 
-        // Force shut down the Hub Server (note: leaf server run on port 4111)
+        // Force shut down the Hub Server (note: leaf server run on port LEAF_SERVER_DEFAULT_LISTEN_PORT)
         let test_stream_consumer_name = "test_stream_consumer".to_string();
         let consumer = stream
             .get_or_create_consumer(
@@ -470,6 +472,7 @@ mod tests {
             log::error!("Failed to shut down Leaf Server.  Err:{:#?}", err);
 
             // Force the port to close
+            // TODO(techdebt): use the command child handle to terminate the process.
             Command::new("kill")
                 .arg("-9")
                 .arg(format!("`lsof -t -i:{}`", leaf_client_conn_port))
@@ -480,10 +483,11 @@ mod tests {
         }
         log::info!("Leaf Server has shut down successfully");
 
-        // Force shut down the Hub Server (note: leaf server run on port 4111)
+        // Force shut down the Hub Server (note: leaf server run on port LEAF_SERVER_DEFAULT_LISTEN_PORT)
+        // TODO(techdebt): use the command child handle to terminate the process.
         Command::new("kill")
             .arg("-9")
-            .arg("`lsof -t -i:4111`")
+            .arg(format!("`lsof -t -i:{LEAF_SERVER_DEFAULT_LISTEN_PORT}`"))
             .spawn()
             .expect("Error spawning kill command")
             .wait()

From aa3d0f754b369efd428cbea937507078ddc36352 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Fri, 17 Jan 2025 21:46:32 +0100
Subject: [PATCH 22/91] fix(nix/modules/host-agent): wait for network
 connectivity

---
 nix/checks/holo-agent-integration-nixos.nix |  5 -----
 nix/modules/nixos/holo-host-agent.nix       | 10 ++--------
 2 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index 6e4cd5a..ea14310 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -34,7 +34,6 @@ pkgs.testers.runNixOSTest (
 
         holo.host-agent = {
           enable = true;
-          autoStart = false;
           rust = {
             log = "trace";
             backtrace = "trace";
@@ -67,10 +66,6 @@ pkgs.testers.runNixOSTest (
         hub.succeed("${hubTestScript}")
 
         host.start()
-        # agent.wait_for_unit("nats.service")
-
-        # TODO: fix after/require settings of the host-agent service to make autoStart work
-        host.succeed("systemctl start holo-host-agent")
         host.wait_for_unit("holo-host-agent")
 
         host.succeed("${hostTestScript}")
diff --git a/nix/modules/nixos/holo-host-agent.nix b/nix/modules/nixos/holo-host-agent.nix
index 0f0079e..19ed3ef 100644
--- a/nix/modules/nixos/holo-host-agent.nix
+++ b/nix/modules/nixos/holo-host-agent.nix
@@ -49,11 +49,6 @@ in
     };
 
     nats = {
-      useOsNats = lib.mkOption {
-        type = lib.types.bool;
-        default = false;
-      };
-
       listenHost = lib.mkOption {
         type = lib.types.str;
         default = "127.0.0.1";
@@ -65,7 +60,7 @@ in
       };
 
       url = lib.mkOption {
-        type = lib.types.nullOr lib.types.str;
+        type = lib.types.str;
         default = "${cfg.nats.listenHost}:${builtins.toString cfg.nats.listenPort}";
       };
 
@@ -79,8 +74,7 @@ in
     systemd.services.holo-host-agent = {
       enable = true;
 
-      requires = lib.lists.optional cfg.nats.useOsNats "nats.service";
-      after = lib.lists.optional cfg.nats.useOsNats "nats.service";
+      wants = [ "network-online.target" ];
 
       requiredBy = lib.optional cfg.autoStart "multi-user.target";
 

From fd8cef247805e72771baf452b255faf5d1c3e0be Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Fri, 17 Jan 2025 23:29:37 +0100
Subject: [PATCH 23/91] holo-host-agent: use wantedBy and increase logging

---
 nix/modules/nixos/holo-host-agent.nix | 3 +--
 rust/util_libs/src/nats_js_client.rs  | 7 +++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/nix/modules/nixos/holo-host-agent.nix b/nix/modules/nixos/holo-host-agent.nix
index 19ed3ef..52fc4b8 100644
--- a/nix/modules/nixos/holo-host-agent.nix
+++ b/nix/modules/nixos/holo-host-agent.nix
@@ -75,8 +75,7 @@ in
       enable = true;
 
       wants = [ "network-online.target" ];
-
-      requiredBy = lib.optional cfg.autoStart "multi-user.target";
+      wantedBy = lib.lists.optional cfg.autoStart "multi-user.target";
 
       environment =
         {
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 4671eef..399fcd0 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -284,8 +284,11 @@ where
 // Helpers:
 // TODO: there's overlap with the NATS_LISTEN_PORT. refactor this to e.g. read NATS_LISTEN_HOST and NATS_LISTEN_PORT
 pub fn get_nats_url() -> String {
-    std::env::var("NATS_URL")
-        .unwrap_or_else(|_| format!("127.0.0.1:{}", LEAF_SERVER_DEFAULT_LISTEN_PORT))
+    std::env::var("NATS_URL").unwrap_or_else(|_| {
+        let default = format!("127.0.0.1:{}", LEAF_SERVER_DEFAULT_LISTEN_PORT);
+        log::debug!("using default for NATS_URL: {default}");
+        default
+    })
 }
 
 pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> String {

From 7da629e07dfa7d19a33da7a7bc8addc61d6bf5a0 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 20 Jan 2025 17:13:03 +0100
Subject: [PATCH 24/91] feat(host_agent): add leafnode creds CLI arg and
 improve handling consistency

this also takes out the hardoded path for the credentials path which has
been panicing in the integration tests.
---
 rust/clients/host_agent/src/agent_cli.rs      | 12 +++-
 .../clients/host_agent/src/gen_leaf_server.rs |  6 +-
 rust/clients/host_agent/src/main.rs           | 15 ++---
 .../host_agent/src/workload_manager.rs        | 65 +++++++++----------
 rust/util_libs/src/nats_server.rs             |  7 +-
 5 files changed, 57 insertions(+), 48 deletions(-)

diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index e872ec1..a5bfade 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -1,6 +1,8 @@
+use std::path::PathBuf;
+
 /// MOdule containing all of the Clap Derive structs/definitions that make up the agent's
 /// command line. To start the agent daemon (usually from systemd), use `host_agent daemonize`.
-use clap::{Parser, Subcommand};
+use clap::{Args, Parser, Subcommand};
 
 #[derive(Parser)]
 #[command(
@@ -17,7 +19,7 @@ pub struct Root {
 #[derive(Subcommand, Clone)]
 pub enum CommandScopes {
     /// Start the Holo Hosting Agent Daemon.
-    Daemonize,
+    Daemonize(DaemonzeArgs),
     /// Commmands for managing this host.
     Host {
         #[command(subcommand)]
@@ -30,6 +32,12 @@ pub enum CommandScopes {
     },
 }
 
+#[derive(Args, Clone, Debug, Default)]
+pub struct DaemonzeArgs {
+    #[arg(help = "path to NATS credentials used for the LeafNode client connection")]
+    pub(crate) nats_leafnode_client_creds_path: Option<PathBuf>,
+}
+
 /// A set of commands for being able to manage the local host. We may (later) want to gate some
 /// of these behind a global `--advanced` option to deter hosters from certain commands, but in the
 /// meantime, everything is safe to leave open.
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/gen_leaf_server.rs
index 4e6063c..f5367e2 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/gen_leaf_server.rs
@@ -1,9 +1,11 @@
+use std::path::PathBuf;
+
 use util_libs::nats_server::{
     self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
     LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
 };
 
-pub async fn run(user_creds_path: &str) {
+pub async fn run(user_creds_path: &Option<PathBuf>) {
     let leaf_server_remote_conn_url = nats_server::get_hub_server_url();
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
@@ -29,7 +31,7 @@ pub async fn run(user_creds_path: &str) {
     let leaf_node_remotes = vec![LeafNodeRemote {
         // sys account user (automated)
         url: leaf_server_remote_conn_url.to_string(),
-        credentials_path: Some(user_creds_path.to_string()),
+        credentials_path: user_creds_path.clone(),
     }];
 
     // Create a new Leaf Server instance
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index e68fb58..241e7a0 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -11,6 +11,7 @@ This client is responsible for subscribing the host agent to workload stream end
 */
 
 mod workload_manager;
+use agent_cli::DaemonzeArgs;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
@@ -19,7 +20,6 @@ pub mod gen_leaf_server;
 pub mod host_cmds;
 pub mod support_cmds;
 use thiserror::Error;
-use util_libs::nats_js_client;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -36,9 +36,9 @@ async fn main() -> Result<(), AgentCliError> {
 
     let cli = agent_cli::Root::parse();
     match &cli.scope {
-        Some(agent_cli::CommandScopes::Daemonize) => {
+        Some(agent_cli::CommandScopes::Daemonize(daemonize_args)) => {
             log::info!("Spawning host agent.");
-            daemonize().await?;
+            daemonize(daemonize_args).await?;
         }
         Some(agent_cli::CommandScopes::Host { command }) => host_cmds::host_command(command)?,
         Some(agent_cli::CommandScopes::Support { command }) => {
@@ -46,18 +46,17 @@ async fn main() -> Result<(), AgentCliError> {
         }
         None => {
             log::warn!("No arguments given. Spawning host agent.");
-            daemonize().await?;
+            daemonize(&Default::default()).await?;
         }
     }
 
     Ok(())
 }
 
-async fn daemonize() -> Result<(), async_nats::Error> {
+async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let host_creds_path = nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
     let host_pubkey = "host_id_placeholder>";
-    gen_leaf_server::run(&host_creds_path).await;
-    workload_manager::run(host_pubkey, &host_creds_path).await?;
+    gen_leaf_server::run(&args.nats_leafnode_client_creds_path).await;
+    workload_manager::run(host_pubkey, &args.nats_leafnode_client_creds_path).await?;
     Ok(())
 }
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
index 2d61850..c30334f 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/workload_manager.rs
@@ -12,25 +12,28 @@
 */
 
 use anyhow::{anyhow, Result};
+use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::{sync::Arc, time::Duration};
+use std::{path::PathBuf, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType,  },
+    nats_js_client::{self, EndpointType},
 };
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
-use async_nats::Message;
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
 const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
 
 // TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
-pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_nats::Error> {
+pub async fn run(
+    host_pubkey: &str,
+    host_creds_path: &Option<PathBuf>,
+) -> Result<(), async_nats::Error> {
     log::info!("HPOS Agent Client: Connecting to server...");
-    log::info!("host_creds_path : {}", host_creds_path);
+    log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
     // ==================== NATS Setup ====================
@@ -49,21 +52,19 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     };
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client =
-        nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-            nats_url,
-            name: HOST_AGENT_CLIENT_NAME.to_string(),
-            inbox_prefix: format!(
-                "{}_{}",
-                HOST_AGENT_INBOX_PREFIX, host_pubkey
-            ),
-            service_params: vec![workload_stream_service_params],
-            credentials_path: Some(host_creds_path.to_string()),
-            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
-            ping_interval: Some(Duration::from_secs(10)),
-            request_timeout: Some(Duration::from_secs(5)),
-        })
-        .await?;
+    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+        nats_url,
+        name: HOST_AGENT_CLIENT_NAME.to_string(),
+        inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
+        service_params: vec![workload_stream_service_params],
+        credentials_path: host_creds_path
+            .as_ref()
+            .map(|path| path.to_string_lossy().to_string()),
+        opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(5)),
+    })
+    .await?;
 
     // ==================== DB Setup ====================
     // Create a new MongoDB Client and connect it to the cluster
@@ -88,11 +89,9 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .add_local_consumer::<workload::types::ApiResult>(
             "start_workload",
             "start",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
-                async move {
-                    api.start_workload(msg).await
-                }
-            })),
+            EndpointType::Async(workload_api.call(
+                |api: WorkloadApi, msg: Arc<Message>| async move { api.start_workload(msg).await },
+            )),
             None,
         )
         .await?;
@@ -101,11 +100,11 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .add_local_consumer::<workload::types::ApiResult>(
             "send_workload_status",
             "send_status",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
-                async move {
+            EndpointType::Async(
+                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
                     api.send_workload_status(msg).await
-                }
-            })),
+                }),
+            ),
             None,
         )
         .await?;
@@ -114,11 +113,11 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .add_local_consumer::<workload::types::ApiResult>(
             "uninstall_workload",
             "uninstall",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
-                async move {
+            EndpointType::Async(
+                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
-                }
-            })),
+                }),
+            ),
             None,
         )
         .await?;
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index afe84b1..28aaac5 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -6,6 +6,7 @@ use anyhow::Context;
 use std::fmt::Debug;
 use std::fs::File;
 use std::io::Write;
+use std::path::PathBuf;
 use std::process::{Child, Command, Stdio};
 use std::sync::Arc;
 use tokio::sync::Mutex;
@@ -31,7 +32,7 @@ pub struct LoggingOptions {
 #[derive(Debug, Clone)]
 pub struct LeafNodeRemote {
     pub url: String,
-    pub credentials_path: Option<String>,
+    pub credentials_path: Option<PathBuf>,
 }
 
 #[derive(Debug, Clone)]
@@ -87,7 +88,7 @@ impl LeafServer {
             .leaf_node_remotes
             .iter()
             .map(|remote| {
-                if remote.credentials_path.is_some() {
+                if let Some(credentials_path) = &remote.credentials_path {
                     format!(
                         r#"
         {{
@@ -96,7 +97,7 @@ impl LeafServer {
         }}
                     "#,
                         remote.url,
-                        remote.credentials_path.as_ref().unwrap() // Unwrap is safe here as the check for `Some()` wraps this condition
+                        credentials_path.as_path().as_os_str().to_string_lossy(), // Unwrap is safe here as the check for `Some()` wraps this condition
                     )
                 } else {
                     format!(

From 06c91c6a0ea9c4b031d01912ce94c29682095c24 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 20 Jan 2025 21:57:19 +0100
Subject: [PATCH 25/91] fix(host-agent): continously try to connect to spawned
 NATS leaf server

when running the host-agent on system startup there seems to be a race
condition that prevents the agent from connecting to the spawned NATS
instance.

the root cause for this _might_ be a race condition between the network
stack availability and spawning Nats, however that's a guess. it might
also just take a 100-200ms for Nats to start servicing the TCP port.

either way, the boot log in the integration test looks like this with
the fix applied. the loop fails once and then succeeds after waiting
100ms:

```
[    6.690765] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  util_libs::nats_server] NATS Leaf Server is running at 127.0.0.1:4222
[    6.692975] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] HPOS Agent Client: Connecting to server...
[    6.695163] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] host_creds_path : None
[    6.696391] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] host_pubkey : host_id_placeholder>
[    6.698881] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] nats_url : 127.0.0.1:4222
[    6.720665] systemd-logind[707]: New seat seat0.
[    6.723219] holo-host-agent-start[695]: [2025-01-20T20:53:16Z WARN  host_agent::workload_manager] connecting to NATS via 127.0.0.1:4222: IO error: Connection refused (os error 111), retrying in 100ms
[    6.726726] systemd-logind[707]: Watching system buttons on /dev/input/event2 (Power Button)
[    6.727999] systemd-logind[707]: Watching system buttons on /dev/input/event3 (QEMU Virtio Keyboard)
[    6.731311] systemd-logind[707]: Watching system buttons on /dev/input/event0 (AT Translated Set 2 keyboard)
[    6.734306] systemd[1]: Started User Login Management.
[    6.762172] systemd[1]: Started Name Service Cache Daemon (nsncd).
[    6.764253] nsncd[750]: Jan 20 20:53:16.581 INFO started, config: Config { ignored_request_types: {}, worker_count: 8, handoff_timeout: 3s }, path: "/var/run/nscd/socket"
[    6.767328] systemd[1]: Reached target Host and Network Name Lookups.
[    6.768655] systemd[1]: Reached target User and Group Name Lookups.
[    6.771096] systemd[1]: Finished resolvconf update.
[    6.771760] systemd[1]: Reached target Preparation for Network.
[    6.776104] systemd[1]: Starting DHCP Client...
[    6.779801] systemd[1]: Starting Address configuration of eth1...
[    6.862637] network-addresses-eth1-start[775]: adding address 192.168.1.1/24... done
[    6.872977] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  util_libs::nats_js_client] NATS-CLIENT-LOG::Host Agent::Connected to NATS server at 127.0.0.1:4222
[    6.880800] network-addresses-eth1-start[775]: adding address 2001:db8:1::1/64... done
[    6.903973] systemd[1]: Finished Address configuration of eth1.
```
---
 rust/clients/host_agent/src/agent_cli.rs      |  6 +++
 rust/clients/host_agent/src/main.rs           |  7 ++-
 .../host_agent/src/workload_manager.rs        | 46 +++++++++++++------
 rust/util_libs/src/js_stream_service.rs       |  2 +-
 rust/util_libs/src/nats_js_client.rs          | 14 +++---
 5 files changed, 53 insertions(+), 22 deletions(-)

diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index a5bfade..620dd8c 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -36,6 +36,12 @@ pub enum CommandScopes {
 pub struct DaemonzeArgs {
     #[arg(help = "path to NATS credentials used for the LeafNode client connection")]
     pub(crate) nats_leafnode_client_creds_path: Option<PathBuf>,
+
+    #[arg(
+        help = "try to connect to the (internally spawned) Nats instance for the given duration in seconds before giving up",
+        default_value = "10"
+    )]
+    pub(crate) nats_connect_timeout_secs: u64,
 }
 
 /// A set of commands for being able to manage the local host. We may (later) want to gate some
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 241e7a0..3cb1b2b 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -57,6 +57,11 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
     let host_pubkey = "host_id_placeholder>";
     gen_leaf_server::run(&args.nats_leafnode_client_creds_path).await;
-    workload_manager::run(host_pubkey, &args.nats_leafnode_client_creds_path).await?;
+    workload_manager::run(
+        host_pubkey,
+        &args.nats_leafnode_client_creds_path,
+        args.nats_connect_timeout_secs,
+    )
+    .await?;
     Ok(())
 }
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
index c30334f..fdb8978 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/workload_manager.rs
@@ -31,6 +31,7 @@ const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
 pub async fn run(
     host_pubkey: &str,
     host_creds_path: &Option<PathBuf>,
+    nats_connect_timeout_secs: u64,
 ) -> Result<(), async_nats::Error> {
     log::info!("HPOS Agent Client: Connecting to server...");
     log::info!("host_creds_path : {:?}", host_creds_path);
@@ -52,19 +53,38 @@ pub async fn run(
     };
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-        nats_url,
-        name: HOST_AGENT_CLIENT_NAME.to_string(),
-        inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
-        service_params: vec![workload_stream_service_params],
-        credentials_path: host_creds_path
-            .as_ref()
-            .map(|path| path.to_string_lossy().to_string()),
-        opts: vec![nats_js_client::with_event_listeners(event_listeners)],
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(5)),
-    })
-    .await?;
+    // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
+    // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
+    let host_workload_client = tokio::select! {
+        client = async {loop {
+                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                    nats_url: nats_url.clone(),
+                    name: HOST_AGENT_CLIENT_NAME.to_string(),
+                    inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
+                    service_params: vec![workload_stream_service_params.clone()],
+                    credentials_path: host_creds_path
+                        .as_ref()
+                        .map(|path| path.to_string_lossy().to_string()),
+                    opts: vec![nats_js_client::with_event_listeners(event_listeners.clone())],
+                    ping_interval: Some(Duration::from_secs(10)),
+                    request_timeout: Some(Duration::from_secs(29)),
+                })
+                .await
+                .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
+
+                match host_workload_client {
+                    Ok(client) => break client,
+                    Err(e) => {
+                        let duration =tokio::time::Duration::from_millis(100);
+                        log::warn!("{}, retrying in {duration:?}", e);
+                        tokio::time::sleep(duration).await;
+                    }
+                }
+            }} => client,
+        _ = tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs)) => {
+            return Err(format!("timed out waiting for NATS on {nats_url}").into());
+        }
+    };
 
     // ==================== DB Setup ====================
     // Create a new MongoDB Client and connect it to the cluster
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index fce50ea..337a762 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -92,7 +92,7 @@ struct LogInfo {
     endpoint_subject: String,
 }
 
-#[derive(Deserialize, Default)]
+#[derive(Clone, Deserialize, Default)]
 pub struct JsServiceParamsPartial {
     pub name: String,
     pub description: String,
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 399fcd0..cbd1fed 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -12,7 +12,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type EventListener = Box<dyn Fn(&mut JsClient) + Send + Sync>;
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
 pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
 pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
 pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
@@ -255,11 +255,11 @@ impl JsClient {
 
 // Client Options:
 pub fn with_event_listeners(listeners: Vec<EventListener>) -> EventListener {
-    Box::new(move |c: &mut JsClient| {
+    Arc::new(Box::new(move |c: &mut JsClient| {
         for listener in &listeners {
             listener(c);
         }
-    })
+    }))
 }
 
 // Event Listener Options:
@@ -267,18 +267,18 @@ pub fn on_msg_published_event<F>(f: F) -> EventListener
 where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
-    Box::new(move |c: &mut JsClient| {
+    Arc::new(Box::new(move |c: &mut JsClient| {
         c.on_msg_published_event = Some(Box::pin(f.clone()));
-    })
+    }))
 }
 
 pub fn on_msg_failed_event<F>(f: F) -> EventListener
 where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
-    Box::new(move |c: &mut JsClient| {
+    Arc::new(Box::new(move |c: &mut JsClient| {
         c.on_msg_failed_event = Some(Box::pin(f.clone()));
-    })
+    }))
 }
 
 // Helpers:

From 394ad5babf803e9cee8050d6fd64f50a62a99ebe Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 15:01:24 -0600
Subject: [PATCH 26/91] separate out orchetrator client into own feature pr

---
 .env.example                                  |   3 +
 Cargo.lock                                    | 147 +++++--
 Cargo.toml                                    |   1 +
 rust/clients/host_agent/Cargo.toml            |   1 +
 rust/clients/host_agent/src/agent_cli.rs      |   2 +-
 .../src/{ => hostd}/gen_leaf_server.rs        |   0
 rust/clients/host_agent/src/hostd/mod.rs      |   2 +
 .../src/{ => hostd}/workload_manager.rs       |  80 ++--
 rust/clients/host_agent/src/main.rs           |  16 +-
 rust/clients/orchestrator/Cargo.toml          |  25 ++
 rust/clients/orchestrator/src/main.rs         |  29 ++
 rust/clients/orchestrator/src/workloads.rs    | 187 +++++++++
 rust/services/workload/Cargo.toml             |   1 +
 rust/services/workload/src/host_api.rs        |  93 +++++
 rust/services/workload/src/lib.rs             | 369 ++----------------
 .../services/workload/src/orchestrator_api.rs | 265 +++++++++++++
 rust/services/workload/src/types.rs           |  27 +-
 rust/util_libs/src/db/mongodb.rs              |  51 ++-
 rust/util_libs/src/db/schemas.rs              |  17 +-
 rust/util_libs/src/js_stream_service.rs       |  14 +-
 rust/util_libs/src/nats_js_client.rs          |  46 ++-
 21 files changed, 914 insertions(+), 462 deletions(-)
 rename rust/clients/host_agent/src/{ => hostd}/gen_leaf_server.rs (100%)
 create mode 100644 rust/clients/host_agent/src/hostd/mod.rs
 rename rust/clients/host_agent/src/{ => hostd}/workload_manager.rs (54%)
 create mode 100644 rust/clients/orchestrator/Cargo.toml
 create mode 100644 rust/clients/orchestrator/src/main.rs
 create mode 100644 rust/clients/orchestrator/src/workloads.rs
 create mode 100644 rust/services/workload/src/host_api.rs
 create mode 100644 rust/services/workload/src/orchestrator_api.rs

diff --git a/.env.example b/.env.example
index bfe8157..634f1be 100644
--- a/.env.example
+++ b/.env.example
@@ -4,3 +4,6 @@ MONGO_URI = "mongodb://<host>:<port>"
 NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
 LEAF_SERVER_USER = "test-user"
 LEAF_SERVER_PW = "pw-123456789"
+
+HOST_CREDENTIALS_PATH: &str = "./host_user.creds"; 
+# USER_CREDENTIALS_PATH: &str = "./user_creds";
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 7b76bc2..578fdea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -146,7 +146,7 @@ dependencies = [
  "once_cell",
  "pin-project",
  "portable-atomic",
- "rand",
+ "rand 0.8.5",
  "regex",
  "ring",
  "rustls-native-certs 0.7.3",
@@ -199,6 +199,12 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -294,7 +300,7 @@ dependencies = [
  "indexmap 2.7.0",
  "js-sys",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -876,6 +882,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -884,7 +901,7 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
 [[package]]
@@ -948,7 +965,7 @@ dependencies = [
  "idna",
  "ipnet",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "thiserror 1.0.69",
  "tinyvec",
  "tokio",
@@ -969,7 +986,7 @@ dependencies = [
  "lru-cache",
  "once_cell",
  "parking_lot",
- "rand",
+ "rand 0.8.5",
  "resolv-conf",
  "smallvec",
  "thiserror 1.0.69",
@@ -1003,9 +1020,10 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_json",
+ "textnonce",
  "thiserror 2.0.8",
  "tokio",
  "url",
@@ -1383,7 +1401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "windows-sys 0.52.0",
 ]
 
@@ -1413,7 +1431,7 @@ dependencies = [
  "once_cell",
  "pbkdf2",
  "percent-encoding",
- "rand",
+ "rand 0.8.5",
  "rustc_version_runtime",
  "rustls 0.21.12",
  "rustls-pemfile 1.0.4",
@@ -1468,9 +1486,9 @@ dependencies = [
  "data-encoding",
  "ed25519",
  "ed25519-dalek",
- "getrandom",
+ "getrandom 0.2.15",
  "log",
- "rand",
+ "rand 0.8.5",
  "signatory",
 ]
 
@@ -1480,7 +1498,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83"
 dependencies = [
- "rand",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -1519,6 +1537,31 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "orchestrator"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "async-nats",
+ "bson",
+ "bytes",
+ "chrono",
+ "dotenv",
+ "env_logger",
+ "futures",
+ "log",
+ "mongodb",
+ "nkeys",
+ "rand 0.8.5",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.8",
+ "tokio",
+ "url",
+ "util_libs",
+ "workload",
+]
+
 [[package]]
 name = "owo-colors"
 version = "3.5.0"
@@ -1717,6 +1760,19 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -1724,8 +1780,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -1735,7 +1801,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
 ]
 
 [[package]]
@@ -1744,7 +1819,16 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -1803,7 +1887,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "libc",
  "spin",
  "untrusted",
@@ -2172,7 +2256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31"
 dependencies = [
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "signature",
  "zeroize",
 ]
@@ -2184,7 +2268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -2338,6 +2422,16 @@ dependencies = [
  "touch",
 ]
 
+[[package]]
+name = "textnonce"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f8d70cd784ed1dc33106a18998d77758d281dc40dc3e6d050cf0f5286683"
+dependencies = [
+ "base64 0.12.3",
+ "rand 0.7.3",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -2476,7 +2570,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
 dependencies = [
  "pin-project",
- "rand",
+ "rand 0.8.5",
  "tokio",
 ]
 
@@ -2526,7 +2620,7 @@ dependencies = [
  "futures-sink",
  "http",
  "httparse",
- "rand",
+ "rand 0.8.5",
  "ring",
  "rustls-native-certs 0.8.1",
  "rustls-pki-types",
@@ -2713,7 +2807,7 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
  "serde",
 ]
 
@@ -2732,6 +2826,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -2999,6 +3099,7 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
+ "async-trait",
  "bson",
  "bytes",
  "chrono",
@@ -3008,7 +3109,7 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "semver",
  "serde",
  "serde_json",
diff --git a/Cargo.toml b/Cargo.toml
index 780343a..4af3e41 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,6 +6,7 @@ metadata.crane.name = "holo-host-workspace"
 members = [
     "rust/hpos-hal",
     "rust/clients/host_agent",
+    "rust/clients/orchestrator",
     "rust/services/workload",
     "rust/util_libs",
 ]
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 4e8e3b2..352e655 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -22,6 +22,7 @@ chrono = "0.4.0"
 bytes = "1.8.0"
 nkeys = "=0.4.4"
 rand = "0.8.5"
+textnonce = "1.0.0"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 hpos-hal = { path = "../../hpos-hal" }
diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index e872ec1..21dda28 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -1,4 +1,4 @@
-/// MOdule containing all of the Clap Derive structs/definitions that make up the agent's
+/// Module containing all of the Clap Derive structs/definitions that make up the agent's
 /// command line. To start the agent daemon (usually from systemd), use `host_agent daemonize`.
 use clap::{Parser, Subcommand};
 
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
similarity index 100%
rename from rust/clients/host_agent/src/gen_leaf_server.rs
rename to rust/clients/host_agent/src/hostd/gen_leaf_server.rs
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
new file mode 100644
index 0000000..6a971dc
--- /dev/null
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -0,0 +1,2 @@
+pub mod workload_manager;
+pub mod gen_leaf_server;
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
similarity index 54%
rename from rust/clients/host_agent/src/workload_manager.rs
rename to rust/clients/host_agent/src/hostd/workload_manager.rs
index 2d61850..488d65f 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -12,17 +12,16 @@
 */
 
 use anyhow::{anyhow, Result};
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::{sync::Arc, time::Duration};
+use async_nats::Message;
 use util_libs::{
-    db::mongodb::get_mongodb_url,
     js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType,  },
+    nats_js_client::{self, EndpointType},
 };
 use workload::{
-    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+    WorkloadServiceApi, host_api::HostWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+    types::{WorkloadServiceSubjects, ApiResult}
 };
-use async_nats::Message;
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
 const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
@@ -33,7 +32,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     log::info!("host_creds_path : {}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
-    // ==================== NATS Setup ====================
+    // ==================== Setup NATS ====================
     // Connect to Nats server
     let nats_url = nats_js_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
@@ -63,20 +62,19 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
             ping_interval: Some(Duration::from_secs(10)),
             request_timeout: Some(Duration::from_secs(5)),
         })
-        .await?;
+        .await?;    
+
+    // ==================== Setup API & Register Endpoints ====================
+    // Instantiate the Workload API
+    let workload_api = HostWorkloadApi::default();
+    
+    // Register Workload Streams for Host Agent to consume and process
+    // NB: Subjects are published by orchestrator
+    let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
+    let workload_send_status_subject = serde_json::to_string(&WorkloadServiceSubjects::SendStatus)?;
+    let workload_uninstall_subject = serde_json::to_string(&WorkloadServiceSubjects::Uninstall)?;
+    let workload_handle_update_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleUpdate)?;
 
-    // ==================== DB Setup ====================
-    // Create a new MongoDB Client and connect it to the cluster
-    let mongo_uri = get_mongodb_url();
-    let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-
-    // Generate the Workload API with access to db
-    let workload_api = WorkloadApi::new(&client).await?;
-
-    // ==================== API ENDPOINTS ====================
-    // Register Workload Streams for Host Agent to consume
-    // NB: Subjects are published by orchestrator or nats-db-connector
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -85,12 +83,12 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         ))?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "start_workload",
-            "start",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+        .add_consumer::<ApiResult>(
+            "start_workload", // consumer name
+            &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.start_workload(msg).await
+                    api.start_workload_on_host(msg).await
                 }
             })),
             None,
@@ -98,12 +96,12 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "send_workload_status",
-            "send_status",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+        .add_consumer::<ApiResult>(
+            "send_workload_status", // consumer name
+            &format!("{}.{}", host_pubkey, workload_handle_update_subject), // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.send_workload_status(msg).await
+                    api.update_workload_on_host(msg).await
                 }
             })),
             None,
@@ -111,18 +109,32 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "uninstall_workload",
-            "uninstall",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
+        .add_consumer::<ApiResult>(
+            "uninstall_workload", // consumer name
+            &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.uninstall_workload_from_host(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+
+        workload_service
+        .add_consumer::<ApiResult>(
+            "send_workload_status", // consumer name
+            &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.uninstall_workload(msg).await
+                    api.send_workload_status_from_host(msg).await
                 }
             })),
             None,
         )
         .await?;
 
+    // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index e68fb58..84e6adf 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -10,16 +10,14 @@ This client is responsible for subscribing the host agent to workload stream end
   - sending workload status upon request
 */
 
-mod workload_manager;
+mod hostd;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
 pub mod agent_cli;
-pub mod gen_leaf_server;
 pub mod host_cmds;
 pub mod support_cmds;
 use thiserror::Error;
-use util_libs::nats_js_client;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -54,10 +52,12 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize() -> Result<(), async_nats::Error> {
-    // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let host_creds_path = nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
-    let host_pubkey = "host_id_placeholder>";
-    gen_leaf_server::run(&host_creds_path).await;
-    workload_manager::run(host_pubkey, &host_creds_path).await?;
+    let host_creds_path = std::env::var("HOST_CREDENTIALS_PATH").unwrap_or_else(|_| {
+        util_libs::nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos")
+    });
+    // let host_pubkey = auth::init_agent::run().await?;
+    let host_pubkey = "host_pubkey_placeholder>".to_string();
+    hostd::gen_leaf_server::run(&host_creds_path).await;
+    hostd::workload_manager::run(&host_pubkey, &host_creds_path).await?;
     Ok(())
 }
diff --git a/rust/clients/orchestrator/Cargo.toml b/rust/clients/orchestrator/Cargo.toml
new file mode 100644
index 0000000..ae867cb
--- /dev/null
+++ b/rust/clients/orchestrator/Cargo.toml
@@ -0,0 +1,25 @@
+[package]
+name = "orchestrator"
+version = "0.0.1"
+edition = "2021"
+
+[dependencies]
+async-nats = { workspace = true }
+anyhow = { workspace = true }
+tokio = { workspace = true }
+futures = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+log = { workspace = true }
+dotenv = { workspace = true }
+thiserror = "2.0"
+url = { version = "2", features = ["serde"] }
+bson = { version = "2.6.1", features = ["chrono-0_4"] }
+env_logger = { workspace = true }
+mongodb = "3.1"
+chrono = "0.4.0"
+bytes = "1.8.0"
+nkeys = "=0.4.4"
+rand = "0.8.5"
+util_libs = { path = "../../util_libs" }
+workload = { path = "../../services/workload" }
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
new file mode 100644
index 0000000..ebb30eb
--- /dev/null
+++ b/rust/clients/orchestrator/src/main.rs
@@ -0,0 +1,29 @@
+/*
+ This client is associated with the:
+- WORKLOAD account
+- orchestrator user
+// This client is responsible for:
+    - handling requests to add workloads
+    - handling requests to update workloads
+    - handling requests to remove workloads
+    - handling workload status updates
+    - interfacing with mongodb DB
+*/
+
+mod workloads;
+use anyhow::Result;
+use dotenv::dotenv;
+
+#[tokio::main]
+async fn main() -> Result<(), async_nats::Error> {
+    dotenv().ok();
+    env_logger::init();
+    // Run auth service
+    // TODO: invoke auth service (once ready)
+
+    // Run workload service
+    if let Err(e) = workloads::run().await {
+        log::error!("{}", e)
+    }
+    Ok(())
+}
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
new file mode 100644
index 0000000..d8b9425
--- /dev/null
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -0,0 +1,187 @@
+/*
+ This client is associated with the:
+- WORKLOAD account
+- orchestrator user
+
+// This client is responsible for:
+*/
+
+use anyhow::{anyhow, Result};
+use std::{collections::HashMap, sync::Arc, time::Duration};
+use async_nats::Message;
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use workload::{
+    WorkloadServiceApi, orchestrator_api::OrchestratorWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, ApiResult}
+};
+use util_libs::{
+    db::mongodb::get_mongodb_url,
+    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
+    nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
+};
+
+const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_orchestrator_workload_inbox";
+
+pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_subject_name: String) -> ResponseSubjectsGenerator {
+    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
+        if is_prefix {
+            let matching_tags = tag_map.into_iter().fold(vec![], |mut acc, (k, v)| {
+                if k.starts_with(&tag_name) {
+                    acc.push(v)
+                }
+                acc
+            });
+            return matching_tags;
+        } else if let Some(tag) = tag_map.get(&tag_name) {
+            return vec![format!("{}.{}", tag, sub_subject_name)];
+        }
+        log::error!("WORKLOAD Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `WORKLOAD.ERROR.INBOX`.", tag_name, sub_subject_name);
+        vec!["WORKLOAD.ERROR.INBOX".to_string()]
+    })
+}
+
+pub async fn run() -> Result<(), async_nats::Error> {
+    // ==================== Setup NATS ====================
+    let nats_url = nats_js_client::get_nats_url();
+    let creds_path = nats_js_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
+    let event_listeners = nats_js_client::get_event_listeners();
+
+    // Setup JS Stream Service
+    let workload_stream_service_params = JsServiceParamsPartial {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+
+    let orchestrator_workload_client =
+        JsClient::new(NewJsClientParams {
+            nats_url,
+            name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
+            inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
+            service_params: vec![workload_stream_service_params],
+            credentials_path: Some(creds_path),
+            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            ping_interval: Some(Duration::from_secs(10)),
+            request_timeout: Some(Duration::from_secs(5)),
+        })
+        .await?;
+
+    // ==================== Setup DB ====================
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
+    
+    // ==================== Setup API & Register Endpoints ====================
+    // Instantiate the Workload API (requires access to db client)
+    let workload_api = OrchestratorWorkloadApi::new(&client).await?;
+
+    // Register Workload Streams for Orchestrator to consume and proceess
+    // NB: These subjects below are published by external Developer, the Nats-DB-Connector, or the Host Agent
+    let workload_add_subject = serde_json::to_string(&WorkloadServiceSubjects::Add)?;
+    let workload_update_subject = serde_json::to_string(&WorkloadServiceSubjects::Update)?;
+    let workload_remove_subject = serde_json::to_string(&WorkloadServiceSubjects::Remove)?;
+    let workload_db_insert_subject = serde_json::to_string(&WorkloadServiceSubjects::Insert)?;
+    let workload_db_modification_subject = serde_json::to_string(&WorkloadServiceSubjects::Modify)?;
+    let workload_handle_status_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
+    let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
+    let workload_handle_update_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleUpdate)?;
+
+    let workload_service = orchestrator_workload_client
+        .get_js_service(WORKLOAD_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate Workload Service. Unable to spin up Orchestrator Workload Client."
+        ))?;
+
+    // Published by Developer
+    workload_service
+        .add_consumer::<ApiResult>(
+            "add_workload", // consumer name
+             &workload_add_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.add_workload(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+
+        workload_service
+        .add_consumer::<ApiResult>(
+            "update_workload", // consumer name
+             &workload_update_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.update_workload(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+
+    
+    workload_service
+        .add_consumer::<ApiResult>(
+            "remove_workload", // consumer name
+             &workload_remove_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.remove_workload(msg).await
+                }
+            })),
+            None,
+        )
+        .await?;
+    
+    // Automatically published by the Nats-DB-Connector
+    workload_service
+        .add_consumer::<ApiResult>(
+            "handle_db_insertion", // consumer name
+             &workload_db_insert_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.handle_db_insertion(msg).await
+                }
+            })),
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_start_subject)),
+        )
+        .await?;
+
+    workload_service
+        .add_consumer::<ApiResult>(
+            "handle_db_modification", // consumer name
+                &workload_db_modification_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
+                async move {
+                    api.handle_db_modification(msg).await
+                }
+            })),
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_handle_update_subject)),
+        )
+        .await?;
+
+    // Published by the Host Agent
+    workload_service
+    .add_consumer::<ApiResult>(
+        "handle_status_update", // consumer name
+        &workload_handle_status_subject, // consumer stream subj
+        EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
+            async move {
+                api.handle_status_update(msg).await
+            }
+        })),
+        None,
+    )
+    .await?;
+
+    // ==================== Close and Clean Client ====================
+    // Only exit program when explicitly requested
+    tokio::signal::ctrl_c().await?;
+
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    orchestrator_workload_client.close().await?;
+    Ok(())
+}
diff --git a/rust/services/workload/Cargo.toml b/rust/services/workload/Cargo.toml
index abc7708..fb929d3 100644
--- a/rust/services/workload/Cargo.toml
+++ b/rust/services/workload/Cargo.toml
@@ -14,6 +14,7 @@ env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
 thiserror = { workspace = true }
+async-trait = "0.1.83"
 semver = "1.0.24"
 rand = "0.8.5"
 mongodb = "3.1"
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
new file mode 100644
index 0000000..ad0cc87
--- /dev/null
+++ b/rust/services/workload/src/host_api.rs
@@ -0,0 +1,93 @@
+/*
+Endpoints & Managed Subjects:
+- `add_workload`: handles the "WORKLOAD.add" subject
+- `remove_workload`: handles the "WORKLOAD.remove" subject
+- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
+- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
+- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
+- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+*/
+
+use super::{types, WorkloadServiceApi};
+use anyhow::Result;
+use core::option::Option::None;
+use std::{fmt::Debug, sync::Arc};
+use async_nats::Message;
+use util_libs::{
+    nats_js_client::ServiceError,
+    db::schemas::{self, WorkloadState, WorkloadStatus}
+};
+
+#[derive(Debug, Clone, Default)]
+pub struct HostWorkloadApi {}
+
+impl WorkloadServiceApi for HostWorkloadApi {}
+
+impl HostWorkloadApi {
+    pub async fn start_workload_on_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+
+        // TODO: Talk through with Stefan
+        // 1. Connect to interface for Nix and instruct systemd to install workload...
+        // eg: nix_install_with(workload)
+
+        // 2. Respond to endpoint request
+        let status = WorkloadStatus {
+            id: workload._id,
+            desired: WorkloadState::Running,
+            actual: WorkloadState::Unknown("..".to_string()),
+        };
+        Ok(types::ApiResult(status, None))
+    }
+
+    pub async fn update_workload_on_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.handle_update' : {:?}", msg);
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+
+        // TODO: Talk through with Stefan
+        // 1. Connect to interface for Nix and instruct systemd to install workload...
+        // eg: nix_install_with(workload)
+
+        // 2. Respond to endpoint request
+        let status = WorkloadStatus {
+            id: workload._id,
+            desired: WorkloadState::Updating,
+            actual: WorkloadState::Unknown("..".to_string()),
+        };
+        Ok(types::ApiResult(status, None))
+    }
+
+    pub async fn uninstall_workload_from_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
+        let workload_id = Self::convert_to_type::<String>(msg)?;
+
+        // TODO: Talk through with Stefan
+        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
+        // nix_uninstall_with(workload_id)
+
+        // 2. Respond to endpoint request
+        let status = WorkloadStatus {
+            id: Some(workload_id),
+            desired: WorkloadState::Uninstalled,
+            actual: WorkloadState::Unknown("..".to_string()),
+        };
+        Ok(types::ApiResult(status, None))
+    }
+
+    // For host agent ? or elsewhere ?
+    // TODO: Talk through with Stefan
+    pub async fn send_workload_status_from_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!(
+            "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
+            msg
+        );
+
+        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
+
+        // Send updated status:
+        // NB: This will send the update to both the requester (if one exists)
+        // and will broadcast the update to for any `response_subject` address registred for the endpoint
+        Ok(types::ApiResult(workload_status, None))
+    }
+}
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 353d803..3bd65e9 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -12,17 +12,22 @@ Endpoints & Managed Subjects:
 - TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
 */
 
+pub mod orchestrator_api;
+pub mod host_api;
 pub mod types;
 
-use anyhow::{anyhow, Result};
-use async_nats::Message;
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use anyhow::Result;
+use core::option::Option::None;
+use async_nats::jetstream::ErrorCode;
+use async_trait::async_trait;
 use std::{fmt::Debug, sync::Arc};
-use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}}, nats_js_client};
-use rand::seq::SliceRandom;
+use async_nats::Message;
 use std::future::Future;
-use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
+use serde::Deserialize;
+use util_libs::{
+    nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
+    db::schemas::{WorkloadState, WorkloadStatus}
+};
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
@@ -30,326 +35,38 @@ pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
 pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
 
 
-#[derive(Debug, Clone)]
-pub struct WorkloadApi {
-    pub workload_collection: MongoCollection<schemas::Workload>,
-    pub host_collection: MongoCollection<schemas::Host>,
-    pub user_collection: MongoCollection<schemas::User>,
-}
-
-impl WorkloadApi {
-    pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME).await?,
-            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
-            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-        })
-    }
-
-    pub fn call<F, Fut>(
+#[async_trait]
+pub trait WorkloadServiceApi
+where
+    Self: std::fmt::Debug + Clone + 'static,
+{
+    fn call<F, Fut>(
         &self,
         handler: F,
-    ) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    ) -> AsyncEndpointHandler<types::ApiResult>
     where
-        F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+        F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
+        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
+        Self: Send + Sync
     {
         let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
             let api_clone = api.clone();
             Box::pin(handler(api_clone, msg))
         })
     }
 
-    /*******************************  For Orchestrator   *********************************/
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Reported,
-            |workload: schemas::Workload| async move {
-                let workload_id = self.workload_collection.insert_one_into(workload.clone()).await?;
-                log::info!("Successfully added workload. MongodDB Workload ID={:?}", workload_id);
-                let updated_workload = schemas::Workload {
-                    _id: Some(workload_id),
-                    ..workload
-                };
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: updated_workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Running,
-            |workload: schemas::Workload| async move {
-                let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload = to_document(&workload)?;
-                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload)).await?;
-                log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-
-    }
-
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.remove'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Removed,
-            |workload_id: schemas::MongoDbId| async move {
-                let workload_query = doc! { "_id":  workload_id.clone() };
-                self.workload_collection.delete_one_from(workload_query).await?;
-                log::info!(
-                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
-                    workload_id
-                );
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Removed,
-                        actual: WorkloadState::Removed,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.insert'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Assigned,
-            |workload: schemas::Workload| async move {
-                log::debug!("New workload to assign. Workload={:#?}", workload);
-
-                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
-                let workload_id = if let Some(id) = workload.clone()._id { id } else {
-                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
-                    return Err(anyhow!(err_msg));
-                };
-
-                // 1. Perform sanity check to ensure workload is not already assigned to a host
-                // ...and if so, exit fn
-                // todo: check for to ensure assigned host *still* has enough capacity for updated workload
-                if !workload.assigned_hosts.is_empty() {
-                    log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok(types::ApiResult( 
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
-                    },
-                    Some(workload.assigned_hosts)));
-                }
-
-                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
-                let host_filter = doc! {
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },      
-                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
-                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk }
-                };
-                let eligible_hosts = self.host_collection.get_many_from(host_filter).await? ;
-                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_hosts);
-
-                // 3. Randomly choose host/node
-                let host = match eligible_hosts.choose(&mut rand::thread_rng()) {
-                    Some(h) => h,
-                    None => {
-                        // todo: Try to get another host up to 5 times, if fails thereafter, return error
-                        let err_msg = format!("Failed to locate an eligible host to support the required workload capacity. Workload={:?}", workload);
-                        return Err(anyhow!(err_msg));
-                    }
-                };
-
-                // Note: The `_id` is an option because it is only generated upon the intial insertion of a record in
-                // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` feild.
-                // Using `unwrap` is therefore safe.
-                let host_id = host._id.to_owned().unwrap();
-                
-                // 4. Update the Workload Collection with the assigned Host ID
-                let workload_query = doc! { "_id":  workload_id.clone() };
-                let updated_workload = &Workload {
-                    assigned_hosts: vec![host_id],
-                    ..workload.clone()
-                };
-                let updated_workload_doc = to_document(updated_workload)?;
-                let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
-                    updated_workload_result
-                );
-                
-                // 5. Update the Host Collection with the assigned Workload ID
-                let host_query = doc! { "_id":  host.clone()._id };
-                let updated_host_doc =  to_document(&Host {
-                    assigned_workloads: vec![workload_id.clone()],
-                    ..host.to_owned()
-                })?;
-                let updated_host_result = self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
-                    updated_host_result
-                );
-     
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
-                    },
-                    Some(updated_workload.assigned_hosts.to_owned())
-                ))
-        },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // Zeeshan to take a look:
-    // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-        
-        let payload_buf = msg.payload.to_vec();
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("New workload to assign. Workload={:#?}", workload);
-        
-        // TODO: ...handle the use case for the update entry change stream 
-
-        let success_status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Running,
-        };
-        
-        Ok(types::ApiResult(success_status, None))
-    }
-
-    // Zeeshan to take a look:
-    // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.delete'");
-        
-        let payload_buf = msg.payload.to_vec();
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("New workload to assign. Workload={:#?}", workload);
-        
-        // TODO: ...handle the use case for the delete entry change stream
-
-        let success_status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Removed,
-            actual: WorkloadState::Removed,
-        };
-        
-        Ok(types::ApiResult(success_status, None))
-    }
-
-    // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status: WorkloadStatus = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload status to update. Status={:?}", workload_status);
-
-        // TODO: ...handle the use case for the workload status update
-        
-        Ok(types::ApiResult(workload_status, None))
-    }    
-
-     /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_id = serde_json::from_slice::<String>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
-        // nix_uninstall_with(workload_id)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: Some(workload_id),
-            desired: WorkloadState::Uninstalled,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    // For host agent ? or elsewhere ?
-    // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!(
-            "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
-            msg
-        );
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status = serde_json::from_slice::<WorkloadStatus>(&payload_buf)?;
-
-        // Send updated status:
-        // NB: This will send the update to both the requester (if one exists)
-        // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************  Helper Fns  *********************************/
-    // Helper function to initialize mongodb collections
-    async fn init_collection<T>(
-        client: &MongoDBClient,
-        collection_name: &str,
-    ) -> Result<MongoCollection<T>>
+    fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
-        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+        T: for<'de> Deserialize<'de> + Send + Sync,
     {
-        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+        let payload_buf = msg.payload.to_vec();
+        serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
+            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject, e);
+            log::error!("{}", err_msg);
+            ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
+        })
+        
     }
 
     // Helper function to streamline the processing of incoming workload messages
@@ -360,28 +77,16 @@ impl WorkloadApi {
         desired_state: WorkloadState,
         cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> types::ApiResult
+    ) -> Result<types::ApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send,
+        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = match serde_json::from_slice(&msg.payload) {
-            Ok(r) => r,
-            Err(e) => {
-                let err_msg = format!("Failed to deserialize payload for Workload Service Endpoint. Subject={} Error={:?}", msg.subject, e);
-                log::error!("{}", err_msg);
-                let status = WorkloadStatus {
-                    id: None,
-                    desired: desired_state,
-                    actual: error_state(err_msg),
-                };
-                return types::ApiResult(status, None);
-            }
-        };
+        let payload: T = Self::convert_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
-        match cb_fn(payload.clone()).await {
+        Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
                 let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
@@ -395,6 +100,6 @@ impl WorkloadApi {
                 // 3. return response for stream
                 types::ApiResult(status, None)
             }
-        }
+        })
     }
 }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
new file mode 100644
index 0000000..cd8a4e3
--- /dev/null
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -0,0 +1,265 @@
+/*
+Endpoints & Managed Subjects:
+- `add_workload`: handles the "WORKLOAD.add" subject
+- `remove_workload`: handles the "WORKLOAD.remove" subject
+- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
+- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
+- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
+- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+*/
+
+use super::{types, WorkloadServiceApi};
+use anyhow::Result;
+use core::option::Option::None;
+use std::{collections::HashMap, fmt::Debug, sync::Arc};
+use async_nats::Message;
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use rand::seq::SliceRandom;
+use serde::{Deserialize, Serialize};
+use bson::{self, doc, to_document};
+use util_libs::{
+    nats_js_client::ServiceError,
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}
+    }
+};
+
+#[derive(Debug, Clone)]
+pub struct OrchestratorWorkloadApi {
+    pub workload_collection: MongoCollection<schemas::Workload>,
+    pub host_collection: MongoCollection<schemas::Host>,
+    pub user_collection: MongoCollection<schemas::User>,
+}
+
+impl WorkloadServiceApi for OrchestratorWorkloadApi {}
+
+impl OrchestratorWorkloadApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        Ok(Self {
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME).await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+        })
+    }
+
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.add'");
+        self.process_request(
+            msg,
+            WorkloadState::Reported,
+            |workload: schemas::Workload| async move {
+                let workload_id = self.workload_collection.insert_one_into(workload.clone()).await?;
+                log::info!("Successfully added workload. MongodDB Workload ID={:?}", workload_id);
+                let new_workload = schemas::Workload {
+                    _id: Some(workload_id),
+                    ..workload
+                };
+                Ok(types::ApiResult(
+                    WorkloadStatus {
+                        id: new_workload._id,
+                        desired: WorkloadState::Reported,
+                        actual: WorkloadState::Reported,
+                    },
+                    None
+                ))
+            },
+            WorkloadState::Error,
+        )
+        .await
+    }
+
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.update'");
+        self.process_request(
+            msg,
+            WorkloadState::Running,
+            |workload: schemas::Workload| async move {
+                let workload_query = doc! { "_id":  workload._id.clone() };
+                let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
+                log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
+                Ok(types::ApiResult(
+                    WorkloadStatus {
+                        id: workload._id,
+                        desired: WorkloadState::Reported,
+                        actual: WorkloadState::Reported,
+                    },
+                    None
+                ))
+            },
+            WorkloadState::Error,
+        )
+        .await
+
+    }
+
+    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.remove'");
+        self.process_request(
+            msg,
+            WorkloadState::Removed,
+            |workload_id: schemas::MongoDbId| async move {
+                let workload_query = doc! { "_id":  workload_id.clone() };
+                self.workload_collection.delete_one_from(workload_query).await?;
+                log::info!(
+                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+                Ok(types::ApiResult(
+                    WorkloadStatus {
+                        id: Some(workload_id),
+                        desired: WorkloadState::Removed,
+                        actual: WorkloadState::Removed,
+                    },
+                    None
+                ))
+            },
+            WorkloadState::Error,
+        )
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.insert'");
+        self.process_request(
+            msg,
+            WorkloadState::Assigned,
+            |workload: schemas::Workload| async move {
+                log::debug!("New workload to assign. Workload={:#?}", workload);
+
+                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
+                let workload_id = if let Some(id) = workload.clone()._id { id } else {
+                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
+                    return Err(ServiceError::Internal(err_msg));
+                };
+
+                // 1. Perform sanity check to ensure workload is not already assigned to a host
+                // ...and if so, exit fn
+                // todo: check for to ensure assigned host *still* has enough capacity for updated workload
+                if !workload.assigned_hosts.is_empty() {
+                    log::warn!("Attempted to assign host for new workload, but host already exists.");
+                    let mut tag_map: HashMap<String, String> = HashMap::new();
+                    for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
+                        tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
+                    }
+                    return Ok(types::ApiResult( 
+                        WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Assigned,
+                            actual: WorkloadState::Assigned,
+                        },
+                        Some(tag_map)
+                    ));
+                }
+
+                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
+                let host_filter = doc! {
+                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },      
+                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
+                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk }
+                };
+                let eligible_hosts = self.host_collection.get_many_from(host_filter).await? ;
+                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_hosts);
+
+                // 3. Randomly choose host/node
+                let host = match eligible_hosts.choose(&mut rand::thread_rng()) {
+                    Some(h) => h,
+                    None => {
+                        // todo: Try to get another host up to 5 times, if fails thereafter, return error
+                        let err_msg = format!("Failed to locate an eligible host to support the required workload capacity. Workload={:?}", workload);
+                        return Err(ServiceError::Internal(err_msg));
+                    }
+                };
+
+                // Note: The `_id` is an option because it is only generated upon the intial insertion of a record in
+                // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` feild.
+                // Using `unwrap` is therefore safe.
+                let host_id = host._id.to_owned().unwrap();
+                
+                // 4. Update the Workload Collection with the assigned Host ID
+                let workload_query = doc! { "_id":  workload_id.clone() };
+                let updated_workload = &Workload {
+                    assigned_hosts: vec![host_id],
+                    ..workload.clone()
+                };
+                let updated_workload_doc = to_document(updated_workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
+                log::trace!(
+                    "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
+                    updated_workload_result
+                );
+                
+                // 5. Update the Host Collection with the assigned Workload ID
+                let host_query = doc! { "_id":  host.clone()._id };
+                let updated_host_doc =  to_document(&Host {
+                    assigned_workloads: vec![workload_id.clone()],
+                    ..host.to_owned()
+                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                let updated_host_result = self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;
+                log::trace!(
+                    "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
+                    updated_host_result
+                );
+                let mut tag_map: HashMap<String, String> = HashMap::new();
+                for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
+                }
+                Ok(types::ApiResult(
+                    WorkloadStatus {
+                        id: Some(workload_id),
+                        desired: WorkloadState::Assigned,
+                        actual: WorkloadState::Assigned,
+                    },
+                    Some(tag_map)
+                ))
+        },
+            WorkloadState::Error,
+        )
+        .await
+    }
+
+    // Zeeshan to take a look:
+    // NB: Automatically published by the nats-db-connector
+    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.modify'");
+        
+        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+        log::trace!("New workload to assign. Workload={:#?}", workload);
+        
+        // TODO: ...handle the use case for the update entry change stream 
+
+        let success_status = WorkloadStatus {
+            id: workload._id,
+            desired: WorkloadState::Running,
+            actual: WorkloadState::Running,
+        };
+        
+        Ok(types::ApiResult(success_status, None))
+    }
+
+    // NB: Published by the Hosting Agent whenever the status of a workload changes
+    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
+
+        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
+        log::trace!("Workload status to update. Status={:?}", workload_status);
+
+        // TODO: ...handle the use case for the workload status update within the orchestrator
+        
+        Ok(types::ApiResult(workload_status, None))
+    }
+
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+
+}
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index cea7d1d..d24e3dc 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,15 +1,28 @@
+use std::collections::HashMap;
+
 use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
 
-pub use String as WorkloadId;
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum WorkloadServiceSubjects {
+    Add,
+    Update,
+    Remove,
+    Insert, // db change stream trigger
+    Modify, // db change stream trigger
+    HandleStatusUpdate,
+    SendStatus,
+    Start,
+    Uninstall,
+    HandleUpdate
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub WorkloadStatus, pub Option<Vec<String>>);
-
+pub struct ApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
+impl EndpointTraits for ApiResult {}
 impl CreateTag for ApiResult {
-    fn get_tags(&self) -> Option<Vec<String>> {
-        self.1.clone()
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.1.clone().unwrap_or_default()
     }
 }
-
-impl EndpointTraits for ApiResult {}
\ No newline at end of file
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index ecb579a..28956a2 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use async_trait::async_trait;
 use bson::{self, doc, Document};
 use futures::stream::TryStreamExt;
@@ -7,27 +7,22 @@ use mongodb::results::{DeleteResult, UpdateResult};
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
-
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Internal Error: {0}")]
-    Internal(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-}
+use crate::nats_js_client::ServiceError;
 
 #[async_trait]
 pub trait MongoDbAPI<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync,
 {
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>>;
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
-    async fn insert_one_into(&self, item: T) -> Result<String>;
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult>;
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
-    async fn delete_all_from(&self) -> Result<DeleteResult>;
+    type Error;
+
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error>;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error>;
+    async fn insert_one_into(&self, item: T) -> Result<String, Self::Error>;
+    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>, Self::Error>;
+    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult, Self::Error>;
+    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult, Self::Error>;
+    async fn delete_all_from(&self) -> Result<DeleteResult, Self::Error>;
 }
 
 pub trait IntoIndexes {
@@ -90,7 +85,9 @@ impl<T> MongoDbAPI<T> for MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes + Debug,
 {
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>> {
+    type Error = ServiceError;
+
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error> {
         log::info!("get_one_from filter {:?}", filter);
 
         let item = self
@@ -103,13 +100,13 @@ where
         Ok(item)
     }
 
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>> {
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error> {
         let cursor = self.collection.find(filter).await?;
         let results: Vec<T> = cursor.try_collect().await.map_err(ServiceError::Database)?;
         Ok(results)
     }
 
-    async fn insert_one_into(&self, item: T) -> Result<String> {
+    async fn insert_one_into(&self, item: T) -> Result<String, Self::Error> {
         let result = self
             .collection
             .insert_one(item)
@@ -119,7 +116,7 @@ where
         Ok(result.inserted_id.to_string())
     }
 
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>> {
+    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>, Self::Error> {
         let result = self
             .collection
             .insert_many(items)
@@ -134,25 +131,25 @@ where
         Ok(ids)
     }
 
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult> {
+    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult, Self::Error> {
         self.collection
             .update_one(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult> {
+    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult, Self::Error> {
         self.collection
             .delete_one(query)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn delete_all_from(&self) -> Result<DeleteResult> {
+    async fn delete_all_from(&self) -> Result<DeleteResult, Self::Error> {
         self.collection
             .delete_many(doc! {})
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 }
 
@@ -268,7 +265,7 @@ mod tests {
         fn get_mock_host() -> schemas::Host {
             schemas::Host {
                 _id: Some(oid::ObjectId::new().to_string()),
-                device_id: "Vf3IceiD".to_string(),
+                pubkey: "placeholder_pubkey".to_string(),
                 ip_address: "127.0.0.1".to_string(),
                 remaining_capacity: Capacity {
                     memory: 16,
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 0a6ff82..963daff 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -28,10 +28,10 @@ pub use String as SemVer;
 pub use String as MongoDbId;
 
 // ==================== User Schema ====================
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
 pub enum Role {
     Developer(DeveloperJWT), // jwt string
-    Host(HosterPubKey),      // host pubkey
+    Hoster(HosterPubKey),      // host pubkey
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -120,14 +120,14 @@ pub struct Capacity {
 pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<MongoDbId>,
-    pub device_id: String, // *INDEXED*, Auto-generated Nats server ID
+    pub pubkey: String, // *INDEXED* // the HPOS/Device pubkey // nb: Unlike the hoster and developer pubkeys, this pubkey is not considered peronal info as it is not directly connected to a "natural person".
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
     pub avg_network_speed: i64,
     pub avg_latency: i64,
     pub assigned_workloads: Vec<String>, // MongoDB ID refs to `workload._id`
-    pub assigned_hoster: HosterPubKey,   // *INDEXED*, Hoster pubkey
+    pub assigned_hoster: HosterPubKey,   // *INDEXED*
 }
 
 impl IntoIndexes for Host {
@@ -135,13 +135,13 @@ impl IntoIndexes for Host {
         let mut indices = vec![];
 
         //  Add Device ID Index
-        let device_id_index_doc = doc! { "device_id": 1 };
-        let device_id_index_opts = Some(
+        let pubkey_index_doc = doc! { "pubkey": 1 };
+        let pubkey_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("device_id_index".to_string()))
+                .name(Some("pubkey_index".to_string()))
                 .build(),
         );
-        indices.push((device_id_index_doc, device_id_index_opts));
+        indices.push((pubkey_index_doc, pubkey_index_opts));
 
         Ok(indices)
     }
@@ -155,6 +155,7 @@ pub enum WorkloadState {
     Pending,
     Installed,
     Running,
+    Updating,
     Removed,
     Uninstalled,
     Error(String), // String = error message
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index fce50ea..154d63f 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -14,10 +14,10 @@ use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
+pub type ResponseSubjectsGenerator = Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
 
 pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> Option<Vec<String>>;
+    fn get_tags(&self) -> HashMap<String, String>;
 }
 
 pub trait EndpointTraits:  Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static {}
@@ -197,7 +197,7 @@ impl JsStreamService {
         })
     }
 
-    pub async fn add_local_consumer<T>(
+    pub async fn add_consumer<T>(
         &self,
         consumer_name: &str,
         endpoint_subject: &str,
@@ -345,7 +345,7 @@ impl JsStreamService {
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 },
-                Err(err) => (err.to_string().into(), None),
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
@@ -496,7 +496,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_js_service_add_local_consumer() {
+    async fn test_js_service_add_consumer() {
         let context = setup_jetstream().await;
         let service = get_default_js_service(context).await;
 
@@ -506,7 +506,7 @@ mod tests {
         let response_subject = Some("response.subject".to_string());
 
         let consumer = service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
@@ -531,7 +531,7 @@ mod tests {
         let response_subject = None;
 
         service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 7eb43f1..099b6e1 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -1,8 +1,9 @@
 use super::js_stream_service::{JsServiceParamsPartial, JsStreamService, CreateTag};
 use anyhow::Result;
-use async_nats::jetstream;
+use async_nats::{jetstream, HeaderMap};
 use async_nats::{Message, ServerInfo};
 use serde::{Deserialize, Serialize};
+use core::option::Option::None;
 use std::future::Future;
 use std::error::Error;
 use std::fmt;
@@ -11,12 +12,24 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
+
 pub type EventListener = Box<dyn Fn(&mut JsClient) + Send + Sync>;
 pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
 pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
         + Send
         + Sync,
 >;
@@ -45,10 +58,11 @@ where
 }
 
 #[derive(Clone, Debug)]
-pub struct SendRequest {
+pub struct PublishInfo {
     pub subject: String,
     pub msg_id: String,
     pub data: Vec<u8>,
+    pub headers: Option<HeaderMap> 
 }
 
 #[derive(Debug)]
@@ -198,17 +212,19 @@ impl JsClient {
         );
         Ok(())
     }
-    
-    pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
-        Ok(())
-    }
 
-    pub async fn publish(&self, payload: &SendRequest) -> Result<(), async_nats::Error> {
+    pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
         let now = Instant::now();
-        let result = self
-            .js
-            .publish(payload.subject.clone(), payload.data.clone().into())
-            .await;
+        let result = match payload.headers {
+            Some(h) => self
+                .js
+                .publish_with_headers(payload.subject.clone(), h, payload.data.clone().into())
+                .await,
+            None => self
+                .js
+                .publish(payload.subject.clone(), payload.data.clone().into())
+                .await
+        };
 
         let duration = now.elapsed();
         if let Err(err) = result {
@@ -343,7 +359,7 @@ mod tests {
     async fn test_nats_js_client_publish() {
         let params = get_default_params();
         let client = JsClient::new(params).await.unwrap();
-        let payload = SendRequest {
+        let payload = PublishInfo {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
             data: b"Hello, NATS!".to_vec(),

From d1f21bc0c4bef541c5ccb86c86232dad578b8c87 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 20 Jan 2025 22:02:03 +0100
Subject: [PATCH 27/91] chore: nix fmt

---
 rust/services/workload/src/lib.rs       | 211 +++++++++++++++---------
 rust/services/workload/src/types.rs     |   9 +-
 rust/util_libs/src/db/mongodb.rs        |  14 +-
 rust/util_libs/src/db/schemas.rs        |  19 +--
 rust/util_libs/src/js_stream_service.rs |  70 ++++----
 rust/util_libs/src/nats_types.rs        |   2 +-
 6 files changed, 192 insertions(+), 133 deletions(-)

diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 353d803..3bcc279 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -16,20 +16,25 @@ pub mod types;
 
 use anyhow::{anyhow, Result};
 use async_nats::Message;
+use bson::{self, doc, to_document};
 use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use std::{fmt::Debug, sync::Arc};
-use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}}, nats_js_client};
 use rand::seq::SliceRandom;
-use std::future::Future;
 use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
+use std::future::Future;
+use std::{fmt::Debug, sync::Arc};
+use util_libs::{
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
+    },
+    nats_js_client,
+};
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
 pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
 
-
 #[derive(Debug, Clone)]
 pub struct WorkloadApi {
     pub workload_collection: MongoCollection<schemas::Workload>,
@@ -40,80 +45,101 @@ pub struct WorkloadApi {
 impl WorkloadApi {
     pub async fn new(client: &MongoDBClient) -> Result<Self> {
         Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME).await?,
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
+                .await?,
             host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
             user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
         })
     }
 
-    pub fn call<F, Fut>(
-        &self,
-        handler: F,
-    ) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    pub fn call<F, Fut>(&self, handler: F) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
     where
         F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
     {
-        let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
-            let api_clone = api.clone();
-            Box::pin(handler(api_clone, msg))
-        })
+        let api = self.to_owned();
+        Arc::new(
+            move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+                let api_clone = api.clone();
+                Box::pin(handler(api_clone, msg))
+            },
+        )
     }
 
     /*******************************  For Orchestrator   *********************************/
     pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Reported,
-            |workload: schemas::Workload| async move {
-                let workload_id = self.workload_collection.insert_one_into(workload.clone()).await?;
-                log::info!("Successfully added workload. MongodDB Workload ID={:?}", workload_id);
-                let updated_workload = schemas::Workload {
-                    _id: Some(workload_id),
-                    ..workload
-                };
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: updated_workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
+        Ok(self
+            .process_request(
+                msg,
+                WorkloadState::Reported,
+                |workload: schemas::Workload| async move {
+                    let workload_id = self
+                        .workload_collection
+                        .insert_one_into(workload.clone())
+                        .await?;
+                    log::info!(
+                        "Successfully added workload. MongodDB Workload ID={:?}",
+                        workload_id
+                    );
+                    let updated_workload = schemas::Workload {
+                        _id: Some(workload_id),
+                        ..workload
+                    };
+                    Ok(types::ApiResult(
+                        WorkloadStatus {
+                            id: updated_workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        None,
+                    ))
+                },
+                WorkloadState::Error,
+            )
+            .await)
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Running,
-            |workload: schemas::Workload| async move {
-                let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload = to_document(&workload)?;
-                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload)).await?;
-                log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-
+        Ok(self
+            .process_request(
+                msg,
+                WorkloadState::Running,
+                |workload: schemas::Workload| async move {
+                    let workload_query = doc! { "_id":  workload._id.clone() };
+                    let updated_workload = to_document(&workload)?;
+                    self.workload_collection
+                        .update_one_within(
+                            workload_query,
+                            UpdateModifications::Document(updated_workload),
+                        )
+                        .await?;
+                    log::info!(
+                        "Successfully updated workload. MongodDB Workload ID={:?}",
+                        workload._id
+                    );
+                    Ok(types::ApiResult(
+                        WorkloadStatus {
+                            id: workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        None,
+                    ))
+                },
+                WorkloadState::Error,
+            )
+            .await)
     }
 
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn remove_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
         Ok(self.process_request(
             msg,
@@ -140,7 +166,10 @@ impl WorkloadApi {
     }
 
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_insertion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
         Ok(self.process_request(
             msg,
@@ -159,7 +188,7 @@ impl WorkloadApi {
                 // todo: check for to ensure assigned host *still* has enough capacity for updated workload
                 if !workload.assigned_hosts.is_empty() {
                     log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok(types::ApiResult( 
+                    return Ok(types::ApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Assigned,
@@ -170,7 +199,7 @@ impl WorkloadApi {
 
                 // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
                 let host_filter = doc! {
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },      
+                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
                     "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
                     "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk }
                 };
@@ -191,7 +220,7 @@ impl WorkloadApi {
                 // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` feild.
                 // Using `unwrap` is therefore safe.
                 let host_id = host._id.to_owned().unwrap();
-                
+
                 // 4. Update the Workload Collection with the assigned Host ID
                 let workload_query = doc! { "_id":  workload_id.clone() };
                 let updated_workload = &Workload {
@@ -204,7 +233,7 @@ impl WorkloadApi {
                     "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
                     updated_workload_result
                 );
-                
+
                 // 5. Update the Host Collection with the assigned Workload ID
                 let host_query = doc! { "_id":  host.clone()._id };
                 let updated_host_doc =  to_document(&Host {
@@ -216,7 +245,7 @@ impl WorkloadApi {
                     "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
                     updated_host_result
                 );
-     
+
                 Ok(types::ApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
@@ -233,33 +262,39 @@ impl WorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-        
+
         let payload_buf = msg.payload.to_vec();
         let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-        
-        // TODO: ...handle the use case for the update entry change stream 
+
+        // TODO: ...handle the use case for the update entry change stream
 
         let success_status = WorkloadStatus {
             id: workload._id,
             desired: WorkloadState::Running,
             actual: WorkloadState::Running,
         };
-        
+
         Ok(types::ApiResult(success_status, None))
     }
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_deletion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.delete'");
-        
+
         let payload_buf = msg.payload.to_vec();
         let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-        
+
         // TODO: ...handle the use case for the delete entry change stream
 
         let success_status = WorkloadStatus {
@@ -267,12 +302,15 @@ impl WorkloadApi {
             desired: WorkloadState::Removed,
             actual: WorkloadState::Removed,
         };
-        
+
         Ok(types::ApiResult(success_status, None))
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
 
         let payload_buf = msg.payload.to_vec();
@@ -280,12 +318,15 @@ impl WorkloadApi {
         log::trace!("Workload status to update. Status={:?}", workload_status);
 
         // TODO: ...handle the use case for the workload status update
-        
+
         Ok(types::ApiResult(workload_status, None))
-    }    
+    }
 
-     /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    /*******************************   For Host Agent   *********************************/
+    pub async fn start_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
@@ -304,7 +345,10 @@ impl WorkloadApi {
         Ok(types::ApiResult(status, None))
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
@@ -325,7 +369,10 @@ impl WorkloadApi {
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!(
             "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
             msg
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index cea7d1d..912ffb6 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,10 +1,13 @@
-use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
+use util_libs::{
+    db::schemas::WorkloadStatus,
+    js_stream_service::{CreateTag, EndpointTraits},
+};
 
 pub use String as WorkloadId;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct ApiResult(pub WorkloadStatus, pub Option<Vec<String>>);
 
 impl CreateTag for ApiResult {
     fn get_tags(&self) -> Option<Vec<String>> {
@@ -12,4 +15,4 @@ impl CreateTag for ApiResult {
     }
 }
 
-impl EndpointTraits for ApiResult {}
\ No newline at end of file
+impl EndpointTraits for ApiResult {}
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index ecb579a..7709901 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -25,7 +25,11 @@ where
     async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
     async fn insert_one_into(&self, item: T) -> Result<String>;
     async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult>;
+    async fn update_one_within(
+        &self,
+        query: Document,
+        updated_doc: UpdateModifications,
+    ) -> Result<UpdateResult>;
     async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
     async fn delete_all_from(&self) -> Result<DeleteResult>;
 }
@@ -134,7 +138,11 @@ where
         Ok(ids)
     }
 
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult> {
+    async fn update_one_within(
+        &self,
+        query: Document,
+        updated_doc: UpdateModifications,
+    ) -> Result<UpdateResult> {
         self.collection
             .update_one(query, updated_doc)
             .await
@@ -273,7 +281,7 @@ mod tests {
                 remaining_capacity: Capacity {
                     memory: 16,
                     disk: 200,
-                    cores: 16
+                    cores: 16,
                 },
                 avg_uptime: 95,
                 avg_network_speed: 500,
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 0a6ff82..6771d26 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -111,8 +111,8 @@ impl IntoIndexes for Hoster {
 // ==================== Host Schema ====================
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct Capacity {
-    pub memory: i64,  // GiB
-    pub disk: i64,  // ssd; GiB 
+    pub memory: i64, // GiB
+    pub disk: i64,   // ssd; GiB
     pub cores: i64,
 }
 
@@ -157,22 +157,21 @@ pub enum WorkloadState {
     Running,
     Removed,
     Uninstalled,
-    Error(String), // String = error message
+    Error(String),   // String = error message
     Unknown(String), // String = context message
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
-    pub id: Option<String>, 
+    pub id: Option<String>,
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SystemSpecs {
-    pub capacity: Capacity
-    // network_speed: i64
-    // uptime: i64
+    pub capacity: Capacity, // network_speed: i64
+                            // uptime: i64
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -185,7 +184,7 @@ pub struct Workload {
     pub min_hosts: u16,
     pub system_specs: SystemSpecs,
     pub assigned_hosts: Vec<String>, // Host Device IDs (eg: assigned nats server id)
-    // pub status: WorkloadStatus,
+                                     // pub status: WorkloadStatus,
 }
 
 impl Default for Workload {
@@ -210,8 +209,8 @@ impl Default for Workload {
                 capacity: Capacity {
                     memory: 64,
                     disk: 400,
-                    cores: 20
-                }
+                    cores: 20,
+                },
             },
             assigned_hosts: Vec::new(),
         }
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 337a762..0860c3b 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -3,10 +3,10 @@ use super::nats_js_client::EndpointType;
 use anyhow::{anyhow, Result};
 use std::any::Any;
 // use async_nats::jetstream::message::Message;
-use async_trait::async_trait;
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
+use async_trait::async_trait;
 use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -20,7 +20,10 @@ pub trait CreateTag: Send + Sync {
     fn get_tags(&self) -> Option<Vec<String>>;
 }
 
-pub trait EndpointTraits:  Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static {}
+pub trait EndpointTraits:
+    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
+{
+}
 
 #[async_trait]
 pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
@@ -46,7 +49,7 @@ where
 }
 
 #[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T> 
+pub struct ConsumerExt<T>
 where
     T: EndpointTraits,
 {
@@ -54,11 +57,11 @@ where
     consumer: PullConsumer,
     handler: EndpointType<T>,
     #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>
+    response_subject_fn: Option<ResponseSubjectsGenerator>,
 }
 
 #[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T> 
+impl<T> ConsumerExtTrait for ConsumerExt<T>
 where
     T: EndpointTraits,
 {
@@ -123,9 +126,9 @@ impl JsStreamService {
         description: &str,
         version: &str,
         service_subject: &str,
-    ) -> Result<Self, async_nats::Error> 
+    ) -> Result<Self, async_nats::Error>
     where
-        Self: 'static
+        Self: 'static,
     {
         let stream = context
             .get_or_create_stream(&stream::Config {
@@ -158,7 +161,10 @@ impl JsStreamService {
         }
     }
 
-    pub async fn get_consumer_stream_info(&self, consumer_name: &str) -> Result<Option<consumer::Info>> {
+    pub async fn get_consumer_stream_info(
+        &self,
+        consumer_name: &str,
+    ) -> Result<Option<consumer::Info>> {
         if let Some(consumer_ext) = self
             .to_owned()
             .local_consumers
@@ -191,9 +197,9 @@ impl JsStreamService {
 
         Ok(ConsumerExt {
             name: consumer_ext.get_name().to_string(),
-            consumer:consumer_ext.get_consumer(),
+            consumer: consumer_ext.get_consumer(),
             handler,
-            response_subject_fn: consumer_ext.get_response()
+            response_subject_fn: consumer_ext.get_response(),
         })
     }
 
@@ -203,7 +209,7 @@ impl JsStreamService {
         endpoint_subject: &str,
         endpoint_type: EndpointType<T>,
         response_subject_fn: Option<ResponseSubjectsGenerator>,
-    ) -> Result<ConsumerExt<T>, async_nats::Error>  
+    ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
@@ -251,19 +257,20 @@ impl JsStreamService {
     pub async fn spawn_consumer_handler<T>(
         &self,
         consumer_name: &str,
-    ) -> Result<(), async_nats::Error> 
+    ) -> Result<(), async_nats::Error>
     where
         T: EndpointTraits,
     {
         if let Some(consumer_ext) = self
-        .to_owned()
-        .local_consumers
-        .write()
-        .await
-        .get_mut(&consumer_name.to_string())
+            .to_owned()
+            .local_consumers
+            .write()
+            .await
+            .get_mut(&consumer_name.to_string())
         {
             let consumer_details = consumer_ext.to_owned();
-            let endpoint_handler: EndpointType<T> = EndpointType::try_from(consumer_details.get_endpoint())?;
+            let endpoint_handler: EndpointType<T> =
+                EndpointType::try_from(consumer_details.get_endpoint())?;
             let maybe_response_generator = consumer_ext.get_response();
             let mut consumer = consumer_details.get_consumer();
             let messages = consumer
@@ -271,22 +278,17 @@ impl JsStreamService {
                 .heartbeat(std::time::Duration::from_secs(10))
                 .messages()
                 .await?;
-            
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
                 endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer
-                .info()
-                .await?
-                .config
-                .filter_subject
-                .clone()
+                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
             };
-            
+
             let service_context = self.js_context.clone();
-            
+
             tokio::spawn(async move {
                 Self::process_messages(
                     log_info,
@@ -331,20 +333,18 @@ impl JsStreamService {
 
             let result = match endpoint_handler {
                 EndpointType::Sync(ref handler) => handler(&js_msg.message),
-                EndpointType::Async(ref handler) => {
-                    handler(Arc::new(js_msg.clone().message)).await
-                }
+                EndpointType::Async(ref handler) => handler(Arc::new(js_msg.clone().message)).await,
             };
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
                     let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
                         Ok(r) => r.into(),
-                        Err(e) => e.to_string().into()
+                        Err(e) => e.to_string().into(),
                     };
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
-                },
+                }
                 Err(err) => (err.to_string().into(), None),
             };
 
@@ -355,7 +355,10 @@ impl JsStreamService {
                     .read()
                     .await
                     .publish(
-                        format!("{}.{}.{}", reply, log_info.service_subject, log_info.endpoint_subject),
+                        format!(
+                            "{}.{}.{}",
+                            reply, log_info.service_subject, log_info.endpoint_subject
+                        ),
                         response_bytes.clone(),
                     )
                     .await
@@ -418,7 +421,6 @@ impl JsStreamService {
             }
         }
     }
-
 }
 
 #[cfg(feature = "tests_integration_nats")]
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
index 9342371..ece916b 100644
--- a/rust/util_libs/src/nats_types.rs
+++ b/rust/util_libs/src/nats_types.rs
@@ -2,7 +2,7 @@
 NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
 IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
 TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */ 
+-------- */
 
 use serde::{Deserialize, Serialize};
 

From a03d0de55dbf7e39c5eac2e7eccbbcedddb0e66d Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 15:03:48 -0600
Subject: [PATCH 28/91] update naming

---
 rust/services/authentication/src/lib.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index df44149..6ae178d 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -134,13 +134,13 @@ impl AuthApi {
 
                 // ...then find the host collection that contains the provided host pubkey
                 match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
-                    Some(host_collection) => {
+                    Some(h) => {
                         // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
-                        if host_collection.assigned_hoster != hoster_pubkey {
-                            let host_query: bson::Document = doc! { "_id":  host_collection._id.clone() };
+                        if h.assigned_hoster != hoster_pubkey {
+                            let host_query: bson::Document = doc! { "_id":  h._id.clone() };
                             let updated_host_doc = to_document(& Host{
                                 assigned_hoster: hoster_pubkey,
-                                ..host_collection
+                                ..h
                             }).map_err(|e| ServiceError::Internal(e.to_string()))?;
                             self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
                         }
@@ -159,15 +159,15 @@ impl AuthApi {
                 
                 // Finally, find the hoster collection
                 match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
-                    Some(hoster_collection) => {
+                    Some(hr) => {
                         // ...and pair the hoster with host (if the host is not already assiged to the hoster)
-                        let mut updated_assigned_hosts = hoster_collection.assigned_hosts;
+                        let mut updated_assigned_hosts = hr.assigned_hosts;
                         if !updated_assigned_hosts.contains(&host_pubkey) {
-                        let hoster_query: bson::Document = doc! { "_id":  hoster_collection._id.clone() };
+                        let hoster_query: bson::Document = doc! { "_id":  hr._id.clone() };
                         updated_assigned_hosts.push(host_pubkey.clone());
                         let updated_hoster_doc = to_document(& Hoster {
                             assigned_hosts: updated_assigned_hosts,
-                            ..hoster_collection
+                            ..hr
                         }).map_err(|e| ServiceError::Internal(e.to_string()))?;
                         self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
                         }

From 7a012e78f0d4ebcc7ba4c58621e6d65a28d8fd80 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 16:07:01 -0600
Subject: [PATCH 29/91] clean up

---
 .env.example                                  |   3 +-
 Cargo.lock                                    | 123 ++++--------------
 rust/clients/host_agent/Cargo.toml            |   1 -
 .../host_agent/src/hostd/workload_manager.rs  |  23 ++--
 rust/clients/orchestrator/src/main.rs         |  12 --
 rust/clients/orchestrator/src/workloads.rs    |  10 +-
 rust/services/workload/src/host_api.rs        |  24 ++--
 rust/services/workload/src/lib.rs             |   7 -
 .../services/workload/src/orchestrator_api.rs |   8 +-
 rust/services/workload/src/types.rs           |   2 +-
 10 files changed, 59 insertions(+), 154 deletions(-)

diff --git a/.env.example b/.env.example
index 634f1be..aa999e7 100644
--- a/.env.example
+++ b/.env.example
@@ -5,5 +5,4 @@ NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
 LEAF_SERVER_USER = "test-user"
 LEAF_SERVER_PW = "pw-123456789"
 
-HOST_CREDENTIALS_PATH: &str = "./host_user.creds"; 
-# USER_CREDENTIALS_PATH: &str = "./user_creds";
\ No newline at end of file
+HOST_CREDENTIALS_PATH: &str = "./host_user.creds";
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 578fdea..38a360f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
- "getrandom 0.2.15",
+ "getrandom",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -146,7 +146,7 @@ dependencies = [
  "once_cell",
  "pin-project",
  "portable-atomic",
- "rand 0.8.5",
+ "rand",
  "regex",
  "ring",
  "rustls-native-certs 0.7.3",
@@ -199,12 +199,6 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "base64"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
-
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -300,7 +294,7 @@ dependencies = [
  "indexmap 2.7.0",
  "js-sys",
  "once_cell",
- "rand 0.8.5",
+ "rand",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -882,17 +876,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "getrandom"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.9.0+wasi-snapshot-preview1",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -901,7 +884,7 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
 ]
 
 [[package]]
@@ -965,7 +948,7 @@ dependencies = [
  "idna",
  "ipnet",
  "once_cell",
- "rand 0.8.5",
+ "rand",
  "thiserror 1.0.69",
  "tinyvec",
  "tokio",
@@ -986,7 +969,7 @@ dependencies = [
  "lru-cache",
  "once_cell",
  "parking_lot",
- "rand 0.8.5",
+ "rand",
  "resolv-conf",
  "smallvec",
  "thiserror 1.0.69",
@@ -1020,10 +1003,9 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand 0.8.5",
+ "rand",
  "serde",
  "serde_json",
- "textnonce",
  "thiserror 2.0.8",
  "tokio",
  "url",
@@ -1401,7 +1383,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
  "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
  "windows-sys 0.52.0",
 ]
 
@@ -1431,7 +1413,7 @@ dependencies = [
  "once_cell",
  "pbkdf2",
  "percent-encoding",
- "rand 0.8.5",
+ "rand",
  "rustc_version_runtime",
  "rustls 0.21.12",
  "rustls-pemfile 1.0.4",
@@ -1486,9 +1468,9 @@ dependencies = [
  "data-encoding",
  "ed25519",
  "ed25519-dalek",
- "getrandom 0.2.15",
+ "getrandom",
  "log",
- "rand 0.8.5",
+ "rand",
  "signatory",
 ]
 
@@ -1498,7 +1480,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83"
 dependencies = [
- "rand 0.8.5",
+ "rand",
 ]
 
 [[package]]
@@ -1552,7 +1534,7 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand 0.8.5",
+ "rand",
  "serde",
  "serde_json",
  "thiserror 2.0.8",
@@ -1760,19 +1742,6 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
-[[package]]
-name = "rand"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
-dependencies = [
- "getrandom 0.1.16",
- "libc",
- "rand_chacha 0.2.2",
- "rand_core 0.5.1",
- "rand_hc",
-]
-
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -1780,18 +1749,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha 0.3.1",
- "rand_core 0.6.4",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.5.1",
+ "rand_chacha",
+ "rand_core",
 ]
 
 [[package]]
@@ -1801,16 +1760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core 0.6.4",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
-dependencies = [
- "getrandom 0.1.16",
+ "rand_core",
 ]
 
 [[package]]
@@ -1819,16 +1769,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.15",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
-dependencies = [
- "rand_core 0.5.1",
+ "getrandom",
 ]
 
 [[package]]
@@ -1887,7 +1828,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom 0.2.15",
+ "getrandom",
  "libc",
  "spin",
  "untrusted",
@@ -2256,7 +2197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31"
 dependencies = [
  "pkcs8",
- "rand_core 0.6.4",
+ "rand_core",
  "signature",
  "zeroize",
 ]
@@ -2268,7 +2209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core 0.6.4",
+ "rand_core",
 ]
 
 [[package]]
@@ -2422,16 +2363,6 @@ dependencies = [
  "touch",
 ]
 
-[[package]]
-name = "textnonce"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7743f8d70cd784ed1dc33106a18998d77758d281dc40dc3e6d050cf0f5286683"
-dependencies = [
- "base64 0.12.3",
- "rand 0.7.3",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -2570,7 +2501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
 dependencies = [
  "pin-project",
- "rand 0.8.5",
+ "rand",
  "tokio",
 ]
 
@@ -2620,7 +2551,7 @@ dependencies = [
  "futures-sink",
  "http",
  "httparse",
- "rand 0.8.5",
+ "rand",
  "ring",
  "rustls-native-certs 0.8.1",
  "rustls-pki-types",
@@ -2807,7 +2738,7 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
 dependencies = [
- "getrandom 0.2.15",
+ "getrandom",
  "serde",
 ]
 
@@ -2826,12 +2757,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -3109,7 +3034,7 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand 0.8.5",
+ "rand",
  "semver",
  "serde",
  "serde_json",
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 352e655..4e8e3b2 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -22,7 +22,6 @@ chrono = "0.4.0"
 bytes = "1.8.0"
 nkeys = "=0.4.4"
 rand = "0.8.5"
-textnonce = "1.0.0"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 hpos-hal = { path = "../../hpos-hal" }
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 488d65f..f1ff531 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -3,12 +3,11 @@
 - WORKLOAD account
 - hpos user
 
-// This client is responsible for:
-  - subscribing to workload streams
-    - installing new workloads
-    - removing workloads
+// This client is responsible for subscribing to workload streams that handle:
+    - installing new workloads onto the hosting device
+    - removing workloads from the hosting device
     - sending workload status upon request
-    - sending active periodic workload reports
+    - sending out active periodic workload reports
 */
 
 use anyhow::{anyhow, Result};
@@ -73,7 +72,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
     let workload_send_status_subject = serde_json::to_string(&WorkloadServiceSubjects::SendStatus)?;
     let workload_uninstall_subject = serde_json::to_string(&WorkloadServiceSubjects::Uninstall)?;
-    let workload_handle_update_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleUpdate)?;
+    let workload_update_installed_subject = serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
 
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
@@ -88,7 +87,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
             &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.start_workload_on_host(msg).await
+                    api.start_workload(msg).await
                 }
             })),
             None,
@@ -97,11 +96,11 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
 
     workload_service
         .add_consumer::<ApiResult>(
-            "send_workload_status", // consumer name
-            &format!("{}.{}", host_pubkey, workload_handle_update_subject), // consumer stream subj
+            "update_installed_workload", // consumer name
+            &format!("{}.{}", host_pubkey, workload_update_installed_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.update_workload_on_host(msg).await
+                    api.update_workload(msg).await
                 }
             })),
             None,
@@ -114,7 +113,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
             &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.uninstall_workload_from_host(msg).await
+                    api.uninstall_workload(msg).await
                 }
             })),
             None,
@@ -127,7 +126,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
             &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
                 async move {
-                    api.send_workload_status_from_host(msg).await
+                    api.send_workload_status(msg).await
                 }
             })),
             None,
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index ebb30eb..8e5aa3e 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -1,15 +1,3 @@
-/*
- This client is associated with the:
-- WORKLOAD account
-- orchestrator user
-// This client is responsible for:
-    - handling requests to add workloads
-    - handling requests to update workloads
-    - handling requests to remove workloads
-    - handling workload status updates
-    - interfacing with mongodb DB
-*/
-
 mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index d8b9425..a4be330 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -2,8 +2,12 @@
  This client is associated with the:
 - WORKLOAD account
 - orchestrator user
-
 // This client is responsible for:
+    - handling requests to add workloads
+    - handling requests to update workloads
+    - handling requests to remove workloads
+    - handling workload status updates
+    - interfacing with mongodb DB
 */
 
 use anyhow::{anyhow, Result};
@@ -86,7 +90,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let workload_db_modification_subject = serde_json::to_string(&WorkloadServiceSubjects::Modify)?;
     let workload_handle_status_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
     let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
-    let workload_handle_update_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleUpdate)?;
+    let workload_update_installed_subject = serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
 
     let workload_service = orchestrator_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
@@ -159,7 +163,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
                     api.handle_db_modification(msg).await
                 }
             })),
-            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_handle_update_subject)),
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_update_installed_subject)),
         )
         .await?;
 
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index ad0cc87..381489c 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -1,11 +1,9 @@
 /*
-Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
-- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
-- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
-- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+Endpoint Subjects:
+- `start_workload`: handles the "WORKLOAD.<host_pukey>.start." subject
+- `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
+- `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
+- `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
 */
 
 use super::{types, WorkloadServiceApi};
@@ -24,7 +22,7 @@ pub struct HostWorkloadApi {}
 impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
-    pub async fn start_workload_on_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
@@ -41,8 +39,8 @@ impl HostWorkloadApi {
         Ok(types::ApiResult(status, None))
     }
 
-    pub async fn update_workload_on_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.handle_update' : {:?}", msg);
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.update_installed' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
         // TODO: Talk through with Stefan
@@ -58,7 +56,7 @@ impl HostWorkloadApi {
         Ok(types::ApiResult(status, None))
     }
 
-    pub async fn uninstall_workload_from_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
         let workload_id = Self::convert_to_type::<String>(msg)?;
 
@@ -77,9 +75,9 @@ impl HostWorkloadApi {
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status_from_host(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
         log::debug!(
-            "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
+            "Incoming message for 'WORKLOAD.send_status' : {:?}",
             msg
         );
 
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 3bd65e9..f99e6c5 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -3,13 +3,6 @@ Service Name: WORKLOAD
 Subject: "WORKLOAD.>"
 Provisioning Account: WORKLOAD
 Users: orchestrator & hpos
-Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
-- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
-- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
-- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
 */
 
 pub mod orchestrator_api;
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index cd8a4e3..72e3b4e 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -1,11 +1,11 @@
 /*
 Endpoints & Managed Subjects:
 - `add_workload`: handles the "WORKLOAD.add" subject
+- `update_workload`: handles the "WORKLOAD.update" subject
 - `remove_workload`: handles the "WORKLOAD.remove" subject
-- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
-- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
-- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
-- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+- `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
+- `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
+- `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
 */
 
 use super::{types, WorkloadServiceApi};
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index d24e3dc..7ee6395 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -15,7 +15,7 @@ pub enum WorkloadServiceSubjects {
     SendStatus,
     Start,
     Uninstall,
-    HandleUpdate
+    UpdateInstalled
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

From d8a5c976906b7eb274fed57f051c5d9407f71555 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 16:54:37 -0600
Subject: [PATCH 30/91] clean auth api pattern

---
 Cargo.lock                                    |   1 +
 .../clients/host_agent/src/auth/init_agent.rs |  34 +-
 rust/clients/orchestrator/src/auth.rs         |  14 +-
 rust/services/authentication/Cargo.toml       |   3 +-
 rust/services/authentication/src/host_api.rs  | 104 ++++++
 rust/services/authentication/src/lib.rs       | 299 +-----------------
 .../authentication/src/orchestrator_api.rs    | 228 +++++++++++++
 7 files changed, 369 insertions(+), 314 deletions(-)
 create mode 100644 rust/services/authentication/src/host_api.rs
 create mode 100644 rust/services/authentication/src/orchestrator_api.rs

diff --git a/Cargo.lock b/Cargo.lock
index 525ced6..6b8dc90 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -185,6 +185,7 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
+ "async-trait",
  "bson",
  "bytes",
  "chrono",
diff --git a/rust/clients/host_agent/src/auth/init_agent.rs b/rust/clients/host_agent/src/auth/init_agent.rs
index ff2e468..7bb6f57 100644
--- a/rust/clients/host_agent/src/auth/init_agent.rs
+++ b/rust/clients/host_agent/src/auth/init_agent.rs
@@ -21,13 +21,11 @@ use anyhow::{anyhow, Result};
 use nkeys::KeyPair;
 use std::str::FromStr;
 use async_nats::{HeaderMap, HeaderName, HeaderValue, Message};
-use authentication::{types, AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use authentication::{types, AuthServiceApi, host_api::HostAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use core::option::Option::{None, Some};
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use textnonce::TextNonce;
 use util_libs::{
-    db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
     nats_js_client::{self, EndpointType},
 };
@@ -68,16 +66,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
             request_timeout: Some(Duration::from_secs(5)),
         })
         .await?;
-
-    // ==================== Setup DB ==============================================================
-    // Create a new MongoDB Client and connect it to the cluster
-    let mongo_uri = get_mongodb_url();
-    let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-
-    // Generate the Auth API with access to db
-    let auth_api = AuthApi::new(&client).await?;
-
+    
     // ==================== Report Host to Orchestator ============================================
     // Generate Host Pubkey && Fetch Hoster Pubkey (from config)..
     // NB: This nkey keypair is a `ed25519_dalek::VerifyingKey` that is `BASE_32` encoded and returned as a String.
@@ -90,7 +79,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
         "Host Auth Client: Retrieved Node ID: {}",
         server_node_id
     );
-
+    
     // Publish a message with the Node ID as part of the subject
     let publish_options = nats_js_client::PublishInfo {
         subject: format!("HPOS.init.{}", server_node_id),
@@ -98,18 +87,21 @@ pub async fn run() -> Result<String, async_nats::Error> {
         data: b"Host Auth Connected!".to_vec(),
         headers: None
     };
-
+    
     match host_auth_client
-        .publish(publish_options)
-        .await
+    .publish(publish_options)
+    .await
     {
         Ok(_r) => {
             log::trace!("Host Auth Client: Node ID published.");
         }
         Err(_e) => {}
     };
-
-    // ==================== Register API ENDPOINTS ===============================================
+    
+    // ==================== Setup API & Register Endpoints ===============================================
+    // Generate the Auth API with access to db
+    let auth_api = HostAuthApi::default();
+    
     // Register Auth Streams for Orchestrator to consume and proceess
     // NB: The subjects below are published by the Orchestrator
     
@@ -130,7 +122,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
         .add_consumer::<authentication::types::ApiResult>(
             "save_hub_jwts", // consumer name
             &format!("{}.{}", host_pubkey, auth_p1_subject), // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+            EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
                 async move {
                     api.save_hub_jwts(msg).await
                 }
@@ -144,7 +136,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
        .add_consumer::<authentication::types::ApiResult>(
             "save_user_jwt", // consumer name
             &format!("{}.{}", host_pubkey, auth_end_subject), // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+            EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
                 async move {
                     api.save_user_jwt(msg, &local_utils::get_host_credentials_path()).await
                 }
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index a7a8d09..8cfebc7 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -14,7 +14,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 // use std::process::Command;
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use authentication::{self, types::AuthServiceSubjects, AuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use authentication::{self, types::AuthServiceSubjects, AuthServiceApi, orchestrator_api::OrchestratorAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
@@ -65,11 +65,11 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
     let client = MongoDBClient::with_options(client_options)?;
-
+    
+    // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
-    let auth_api = AuthApi::new(&client).await?;
-
-    // ==================== Register API Endpoints ====================
+    let auth_api = OrchestratorAuthApi::new(&client).await?;
+    
     // Register Auth Streams for Orchestrator to consume and proceess
     // NB: The subjects below are published by the Host Agent
     let auth_start_subject = serde_json::to_string(&AuthServiceSubjects::StartHandshake)?;
@@ -88,7 +88,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .add_consumer::<authentication::types::ApiResult>(
             "start_handshake", // consumer name
             &auth_start_subject, // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+            EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
                 async move {
                     api.handle_handshake_request(msg, &local_utils::get_orchestrator_credentials_dir_path()).await
                 }
@@ -101,7 +101,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .add_consumer::<authentication::types::ApiResult>(
             "add_user_pubkey", // consumer name
             &auth_p2_subject, // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: AuthApi, msg: Arc<Message>| {
+            EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
                 async move {
                     api.add_user_pubkey(msg).await
                 }
diff --git a/rust/services/authentication/Cargo.toml b/rust/services/authentication/Cargo.toml
index 8b0ed90..fba30e9 100644
--- a/rust/services/authentication/Cargo.toml
+++ b/rust/services/authentication/Cargo.toml
@@ -13,7 +13,8 @@ tokio = { workspace = true }
 env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
-thiserror = "2.0"
+thiserror = { workspace = true }
+async-trait = "0.1.83"
 mongodb = "3.1"
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
 url = { version = "2", features = ["serde"] }
diff --git a/rust/services/authentication/src/host_api.rs b/rust/services/authentication/src/host_api.rs
new file mode 100644
index 0000000..97bf541
--- /dev/null
+++ b/rust/services/authentication/src/host_api.rs
@@ -0,0 +1,104 @@
+/*
+Service Name: AUTH
+Subject: "AUTH.>"
+Provisioning Account: AUTH Account
+Importing Account: Auth/NoAuth Account
+
+This service should be run on the ORCHESTRATOR side and called from the HPOS side.
+The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
+NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
+This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
+
+Endpoints & Managed Subjects:
+    - start_hub_handshake
+    - end_hub_handshake
+    - save_hub_auth
+    - save_user_auth
+
+*/
+
+use super::{AuthServiceApi, types, utils};
+use anyhow::Result;
+use async_nats::Message;
+use types::AuthResult;
+use core::option::Option::None;
+use std::collections::HashMap;
+use std::sync::Arc;
+use util_libs::nats_js_client::ServiceError;
+
+#[derive(Debug, Clone, Default)]
+pub struct HostAuthApi {}
+
+impl AuthServiceApi for HostAuthApi {}
+
+impl HostAuthApi {
+   pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
+
+        // utils::receive_and_write_file();
+
+        // // Generate resolver file and create resolver file
+        // let resolver_path = utils::get_resolver_path();
+        // Command::new("nsc")
+        //     .arg("generate")
+        //     .arg("config")
+        //     .arg("--nats-resolver")
+        //     .arg("sys-account SYS")
+        //     .arg("--force")
+        //     .arg(format!("--config-file {}", resolver_path))
+        //     .output()
+        //     .expect("Failed to create resolver config file");
+
+        // // Push auth updates to hub server
+        // Command::new("nsc")
+        //     .arg("push -A")
+        //     .output()
+        //     .expect("Failed to create resolver config file");
+
+        // Prepare to send over user pubkey(to trigger the user jwt gen on hub)
+        let user_nkey_path = utils::get_file_path_buf("user_jwt_path");
+        let user_nkey: Vec<u8> = std::fs::read(user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let host_pubkey = serde_json::to_string(&user_nkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
+        
+        // Respond to endpoint request
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_pubkey: host_pubkey.clone(),
+                status: types::AuthState::Requested
+            },
+            result: AuthResult {
+                data: types::AuthResultType::Single(user_nkey)
+            },
+            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
+        })
+    }
+
+    pub async fn save_user_jwt(
+        &self,
+        msg: Arc<Message>,
+        _output_dir: &str,
+    ) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.end_handshake' : {:?}", msg);
+
+        // Generate user jwt file
+        // utils::receive_and_write_file(msg, output_dir, file_name).await?;
+        
+        // Generate user creds file
+        // let _user_creds_path = utils::generate_creds_file();
+
+        // 2. Respond to endpoint request
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_pubkey: "host_id_placeholder".to_string(),
+                status: types::AuthState::Authenticated
+            },
+            result: AuthResult {
+                data: types::AuthResultType::Single(b"Hello, NATS!".to_vec())
+            },
+            maybe_response_tags: None
+        })
+    }
+}
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 6ae178d..c9f2889 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -8,46 +8,21 @@ This service should be run on the ORCHESTRATOR side and called from the HPOS sid
 The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
 NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
 This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
-
-Endpoints & Managed Subjects:
-    - start_hub_handshake
-    - end_hub_handshake
-    - save_hub_auth
-    - save_user_auth
-
 */
 
+pub mod orchestrator_api;
+pub mod host_api;
 pub mod types;
 pub mod utils;
 
 use anyhow::Result;
-use async_nats::{Message, HeaderValue};
+use async_nats::Message;
 use async_nats::jetstream::ErrorCode;
-use nkeys::KeyPair;
-use types::AuthResult;
-use utils::handle_internal_err;
-use core::option::Option::None;
-use std::collections::HashMap;
-use std::process::Command;
+use async_trait::async_trait;
 use std::sync::Arc;
 use std::future::Future;
-use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use util_libs::{
-    nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
-    db::{
-        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{
-            self,
-            User,
-            Hoster,
-            Host,
-            Role,
-            RoleInfo,
-        }
-    },
-};
+use serde::Deserialize;
+use util_libs::nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse};
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";
@@ -55,30 +30,19 @@ pub const AUTH_SRV_VERSION: &str = "0.0.1";
 pub const AUTH_SRV_DESC: &str =
     "This service handles the Authentication flow the HPOS and the Orchestrator.";
 
-#[derive(Debug,
-    Clone)]
-pub struct AuthApi {
-    pub user_collection: MongoCollection<User>,
-    pub hoster_collection: MongoCollection<Hoster>,
-    pub host_collection: MongoCollection<Host>,
-}
-
-impl AuthApi {
-    pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        Ok(Self {
-            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME).await?,
-            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
-        })
-    }
-
-    pub fn call<F, Fut>(
+#[async_trait]
+pub trait AuthServiceApi
+where
+    Self: std::fmt::Debug + Clone + 'static,
+{
+    fn call<F, Fut>(
         &self,
         handler: F,
     ) -> AsyncEndpointHandler<types::ApiResult>
     where
-        F: Fn(AuthApi, Arc<Message>) -> Fut + Send + Sync + 'static,
+        F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
+        Self: Send + Sync
     {
         let api = self.to_owned(); 
         Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
@@ -86,241 +50,6 @@ impl AuthApi {
             Box::pin(handler(api_clone, msg))
         })
     }
-    
-    /*******************************  For Orchestrator   *********************************/
-    // nb: returns to the `save_hub_files` subject
-    pub async fn handle_handshake_request(
-        &self,
-        msg: Arc<Message>,
-        creds_dir_path: &str,
-    ) -> Result<types::ApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
-
-        // 1. Verify expected data was received
-        let signature: &[u8] = match &msg.headers {
-            Some(h) => {
-                HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
-                    log::error!(
-                        "Error: Missing x-signature header. Subject='AUTH.authorize'"
-                    );
-                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
-                })?)
-            },
-            None => {
-                log::error!(
-                    "Error: Missing message headers. Subject='AUTH.authorize'"
-                );
-                return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-            }
-        };
-
-        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, nonce: _ } = Self::convert_to_type::<types::AuthRequestPayload>(msg.clone())?;
-
-        // 2. Validate signature
-        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
-            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
-            return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-        };
-
-        // 3. Authenticate the Hosting Agent (via email and host id info?)
-        match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_pubkey.clone() }).await? {
-            Some(u) => {
-                // If hoster exists with pubkey, verify email
-                if u.email != email {
-                    log::error!("Error: Failed to validate user email. Subject='{}'.", msg.subject);
-                    return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-                }
-
-                // ...then find the host collection that contains the provided host pubkey
-                match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
-                    Some(h) => {
-                        // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
-                        if h.assigned_hoster != hoster_pubkey {
-                            let host_query: bson::Document = doc! { "_id":  h._id.clone() };
-                            let updated_host_doc = to_document(& Host{
-                                assigned_hoster: hoster_pubkey,
-                                ..h
-                            }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                            self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
-                        }
-                    },
-                    None => {
-                        let err_msg = format!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
-                        return Err(handle_internal_err(&err_msg));
-                    }
-                }
-
-                // Find the mongo_id ref for the hoster associated with this user
-                let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
-                    let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
-                    handle_internal_err(&err_msg)
-                })?;
-                
-                // Finally, find the hoster collection
-                match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
-                    Some(hr) => {
-                        // ...and pair the hoster with host (if the host is not already assiged to the hoster)
-                        let mut updated_assigned_hosts = hr.assigned_hosts;
-                        if !updated_assigned_hosts.contains(&host_pubkey) {
-                        let hoster_query: bson::Document = doc! { "_id":  hr._id.clone() };
-                        updated_assigned_hosts.push(host_pubkey.clone());
-                        let updated_hoster_doc = to_document(& Hoster {
-                            assigned_hosts: updated_assigned_hosts,
-                            ..hr
-                        }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                        self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
-                        }
-                    },
-                    None => {
-                        let err_msg = format!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
-                        return Err(handle_internal_err(&err_msg));
-                    }
-                }
-            },
-            None => {
-                let err_msg = format!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
-                return Err(handle_internal_err(&err_msg));
-            }
-        };
-
-        // 4. Read operator and sys account jwts and prepare them to be sent as a payload in the publication callback
-        let operator_path = utils::get_file_path_buf(&format!("{}/operator.creds", creds_dir_path));
-        let hub_operator_creds: Vec<u8> = std::fs::read(operator_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let sys_path = utils::get_file_path_buf(&format!("{}/sys.creds", creds_dir_path));
-        let hub_sys_creds: Vec<u8> = std::fs::read(sys_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let mut tag_map: HashMap<String, String> = HashMap::new();
-        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
-
-        Ok(types::ApiResult {
-            status: types::AuthStatus { 
-                host_pubkey: host_pubkey.clone(),
-                status: types::AuthState::Requested
-            },
-            result: AuthResult {
-                data: types::AuthResultType::Multiple(vec![hub_operator_creds, hub_sys_creds])
-            },
-            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
-        })
-    }
-
-    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.handle_handshake_p2' : {:?}", msg);
-
-        // 1. Verify expected payload was received
-        let host_pubkey = Self::convert_to_type::<String>(msg.clone())?;
-
-        // 2. Add User keys to Orchestrator nsc resolver
-        Command::new("nsc")
-            .arg("...")
-            .output()
-            .expect("Failed to add user with provided keys");
-        
-        // 3. Create and sign User JWT
-        let account_signing_key = utils::get_account_signing_key();
-        utils::generate_user_jwt(&host_pubkey, &account_signing_key);
-
-        // 4. Prepare User JWT to be sent as a payload in the publication callback
-        let user_jwt_path = utils::get_file_path_buf("user_jwt_path");
-        let user_jwt: Vec<u8> = std::fs::read(user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        // 5. Respond to endpoint request
-        Ok(types::ApiResult {
-            status: types::AuthStatus { 
-                host_pubkey,
-                status: types::AuthState::ValidatedAgent
-            },
-            result: AuthResult {
-                data: types::AuthResultType::Single(user_jwt)
-            },
-            maybe_response_tags: None
-        })
-    }
-
-    /*******************************   For Host Agent   *********************************/
-    pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
-
-        // utils::receive_and_write_file();
-
-        // // Generate resolver file and create resolver file
-        // let resolver_path = utils::get_resolver_path();
-        // Command::new("nsc")
-        //     .arg("generate")
-        //     .arg("config")
-        //     .arg("--nats-resolver")
-        //     .arg("sys-account SYS")
-        //     .arg("--force")
-        //     .arg(format!("--config-file {}", resolver_path))
-        //     .output()
-        //     .expect("Failed to create resolver config file");
-
-        // // Push auth updates to hub server
-        // Command::new("nsc")
-        //     .arg("push -A")
-        //     .output()
-        //     .expect("Failed to create resolver config file");
-
-        // Prepare to send over user pubkey(to trigger the user jwt gen on hub)
-        let user_nkey_path = utils::get_file_path_buf("user_jwt_path");
-        let user_nkey: Vec<u8> = std::fs::read(user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        let host_pubkey = serde_json::to_string(&user_nkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let mut tag_map: HashMap<String, String> = HashMap::new();
-        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
-        
-        // Respond to endpoint request
-        Ok(types::ApiResult {
-            status: types::AuthStatus { 
-                host_pubkey: host_pubkey.clone(),
-                status: types::AuthState::Requested
-            },
-            result: AuthResult {
-                data: types::AuthResultType::Single(user_nkey)
-            },
-            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
-        })
-    }
-
-    pub async fn save_user_jwt(
-        &self,
-        msg: Arc<Message>,
-        _output_dir: &str,
-    ) -> Result<types::ApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.end_handshake' : {:?}", msg);
-
-        // Generate user jwt file
-        // utils::receive_and_write_file(msg, output_dir, file_name).await?;
-        
-        // Generate user creds file
-        // let _user_creds_path = utils::generate_creds_file();
-
-        // 2. Respond to endpoint request
-        Ok(types::ApiResult {
-            status: types::AuthStatus { 
-                host_pubkey: "host_id_placeholder".to_string(),
-                status: types::AuthState::Authenticated
-            },
-            result: AuthResult {
-                data: types::AuthResultType::Single(b"Hello, NATS!".to_vec())
-            },
-            maybe_response_tags: None
-        })
-    }
-
-    /*******************************  Helper Fns  *********************************/
-    // Helper function to initialize mongodb collections
-    async fn init_collection<T>(
-        client: &MongoDBClient,
-        collection_name: &str,
-    ) -> Result<MongoCollection<T>>
-    where
-        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
-    {
-        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
-    }
 
     fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
diff --git a/rust/services/authentication/src/orchestrator_api.rs b/rust/services/authentication/src/orchestrator_api.rs
new file mode 100644
index 0000000..74ed258
--- /dev/null
+++ b/rust/services/authentication/src/orchestrator_api.rs
@@ -0,0 +1,228 @@
+/*
+Service Name: AUTH
+Subject: "AUTH.>"
+Provisioning Account: AUTH Account
+Importing Account: Auth/NoAuth Account
+
+This service should be run on the ORCHESTRATOR side and called from the HPOS side.
+The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
+NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
+This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
+
+Endpoints & Managed Subjects:
+    - start_hub_handshake
+    - end_hub_handshake
+    - save_hub_auth
+    - save_user_auth
+
+*/
+use super::{AuthServiceApi, types, utils};
+use anyhow::Result;
+use async_nats::{Message, HeaderValue};
+use async_nats::jetstream::ErrorCode;
+use nkeys::KeyPair;
+use types::AuthResult;
+use utils::handle_internal_err;
+use core::option::Option::None;
+use std::collections::HashMap;
+use std::process::Command;
+use std::sync::Arc;
+use serde::{Deserialize, Serialize};
+use bson::{self, doc, to_document};
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use util_libs::{
+    nats_js_client::ServiceError,
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{
+            self,
+            User,
+            Hoster,
+            Host,
+            Role,
+            RoleInfo,
+        }
+    },
+};
+
+#[derive(Debug, Clone)]
+pub struct OrchestratorAuthApi {
+    pub user_collection: MongoCollection<User>,
+    pub hoster_collection: MongoCollection<Hoster>,
+    pub host_collection: MongoCollection<Host>,
+}
+
+impl AuthServiceApi for OrchestratorAuthApi {}
+
+impl OrchestratorAuthApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        Ok(Self {
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME).await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
+        })
+    }
+
+    /*******************************  For Orchestrator   *********************************/
+    // nb: returns to the `save_hub_files` subject
+    pub async fn handle_handshake_request(
+        &self,
+        msg: Arc<Message>,
+        creds_dir_path: &str,
+    ) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
+
+        // 1. Verify expected data was received
+        let signature: &[u8] = match &msg.headers {
+            Some(h) => {
+                HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
+                    log::error!(
+                        "Error: Missing x-signature header. Subject='AUTH.authorize'"
+                    );
+                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
+                })?)
+            },
+            None => {
+                log::error!(
+                    "Error: Missing message headers. Subject='AUTH.authorize'"
+                );
+                return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+            }
+        };
+
+        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, nonce: _ } = Self::convert_to_type::<types::AuthRequestPayload>(msg.clone())?;
+
+        // 2. Validate signature
+        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
+            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
+            return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+        };
+
+        // 3. Authenticate the Hosting Agent (via email and host id info?)
+        match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_pubkey.clone() }).await? {
+            Some(u) => {
+                // If hoster exists with pubkey, verify email
+                if u.email != email {
+                    log::error!("Error: Failed to validate user email. Subject='{}'.", msg.subject);
+                    return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+                }
+
+                // ...then find the host collection that contains the provided host pubkey
+                match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
+                    Some(h) => {
+                        // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
+                        if h.assigned_hoster != hoster_pubkey {
+                            let host_query: bson::Document = doc! { "_id":  h._id.clone() };
+                            let updated_host_doc = to_document(& Host{
+                                assigned_hoster: hoster_pubkey,
+                                ..h
+                            }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                            self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
+                        }
+                    },
+                    None => {
+                        let err_msg = format!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
+                        return Err(handle_internal_err(&err_msg));
+                    }
+                }
+
+                // Find the mongo_id ref for the hoster associated with this user
+                let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
+                    let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
+                    handle_internal_err(&err_msg)
+                })?;
+                
+                // Finally, find the hoster collection
+                match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
+                    Some(hr) => {
+                        // ...and pair the hoster with host (if the host is not already assiged to the hoster)
+                        let mut updated_assigned_hosts = hr.assigned_hosts;
+                        if !updated_assigned_hosts.contains(&host_pubkey) {
+                        let hoster_query: bson::Document = doc! { "_id":  hr._id.clone() };
+                        updated_assigned_hosts.push(host_pubkey.clone());
+                        let updated_hoster_doc = to_document(& Hoster {
+                            assigned_hosts: updated_assigned_hosts,
+                            ..hr
+                        }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                        self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
+                        }
+                    },
+                    None => {
+                        let err_msg = format!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
+                        return Err(handle_internal_err(&err_msg));
+                    }
+                }
+            },
+            None => {
+                let err_msg = format!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
+                return Err(handle_internal_err(&err_msg));
+            }
+        };
+
+        // 4. Read operator and sys account jwts and prepare them to be sent as a payload in the publication callback
+        let operator_path = utils::get_file_path_buf(&format!("{}/operator.creds", creds_dir_path));
+        let hub_operator_creds: Vec<u8> = std::fs::read(operator_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let sys_path = utils::get_file_path_buf(&format!("{}/sys.creds", creds_dir_path));
+        let hub_sys_creds: Vec<u8> = std::fs::read(sys_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
+
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_pubkey: host_pubkey.clone(),
+                status: types::AuthState::Requested
+            },
+            result: AuthResult {
+                data: types::AuthResultType::Multiple(vec![hub_operator_creds, hub_sys_creds])
+            },
+            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
+        })
+    }
+
+    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.handle_handshake_p2' : {:?}", msg);
+
+        // 1. Verify expected payload was received
+        let host_pubkey = Self::convert_to_type::<String>(msg.clone())?;
+
+        // 2. Add User keys to Orchestrator nsc resolver
+        Command::new("nsc")
+            .arg("...")
+            .output()
+            .expect("Failed to add user with provided keys");
+        
+        // 3. Create and sign User JWT
+        let account_signing_key = utils::get_account_signing_key();
+        utils::generate_user_jwt(&host_pubkey, &account_signing_key);
+
+        // 4. Prepare User JWT to be sent as a payload in the publication callback
+        let user_jwt_path = utils::get_file_path_buf("user_jwt_path");
+        let user_jwt: Vec<u8> = std::fs::read(user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        // 5. Respond to endpoint request
+        Ok(types::ApiResult {
+            status: types::AuthStatus { 
+                host_pubkey,
+                status: types::AuthState::ValidatedAgent
+            },
+            result: AuthResult {
+                data: types::AuthResultType::Single(user_jwt)
+            },
+            maybe_response_tags: None
+        })
+    }
+
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+}

From f63286f73d650786654f4fcb3dd4b4ae70b44fb7 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 19:54:26 -0600
Subject: [PATCH 31/91] tidy descriptions and typing

---
 .../clients/host_agent/src/auth/init_agent.rs | 44 +++++++++----------
 .../host_agent/src/hostd/workload_manager.rs  | 18 ++++----
 rust/clients/orchestrator/src/auth.rs         | 27 ++++++++----
 rust/clients/orchestrator/src/workloads.rs    | 36 ++++++++-------
 rust/services/authentication/src/host_api.rs  | 27 +++---------
 rust/services/authentication/src/lib.rs       | 15 +++----
 .../authentication/src/orchestrator_api.rs    | 28 ++++--------
 rust/services/authentication/src/types.rs     |  6 +--
 rust/services/workload/src/host_api.rs        | 28 ++++++------
 rust/services/workload/src/lib.rs             | 13 +++---
 .../services/workload/src/orchestrator_api.rs | 40 ++++++++---------
 rust/services/workload/src/types.rs           |  6 +--
 12 files changed, 138 insertions(+), 150 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init_agent.rs b/rust/clients/host_agent/src/auth/init_agent.rs
index 7bb6f57..e8107e9 100644
--- a/rust/clients/host_agent/src/auth/init_agent.rs
+++ b/rust/clients/host_agent/src/auth/init_agent.rs
@@ -1,19 +1,19 @@
 /*
- This client is associated with the:
-- ADMIN account
-- noauth user
-
-...once this the host and hoster are validated, this client should close and the hpos manager should spin up.
-
-// This client is responsible for:
-1. generating new key / re-using the user key from provided file
-2. calling the auth service to:
-  - verify host/hoster via `auth/start_hub_handshake` call
-  - get hub operator jwt and hub sys account jwt via `auth/start_hub_handshake`
-  - send "nkey" version of pubkey as file to hub via via `auth/end_hub_handshake`
-  - get user jwt from hub via `auth/save_`
-3. create user creds file with file path
-4. instantiate the leaf server via the leaf-server struct/service
+This client is associated with the:
+    - AUTH account
+    - noauth user
+
+Nb: Once the host and hoster are validated, and the host creds file is created,
+...this client should close and the hostd workload manager should spin up.
+
+This client is responsible for:
+    - generating new key for host / and accessing hoster key from provided config file
+    - registering with the host auth service to:
+        - get hub operator jwt and hub sys account jwt
+        - send "nkey" version of host pubkey as file to hub
+        - get user jwt from hub and create user creds file with provided file path
+    - publishing to `auth.start` to initilize the auth handshake and validate the host/hoster
+    - returning the host pubkey and closing client cleanly
 */
 
 use super::utils as local_utils;
@@ -21,7 +21,7 @@ use anyhow::{anyhow, Result};
 use nkeys::KeyPair;
 use std::str::FromStr;
 use async_nats::{HeaderMap, HeaderName, HeaderValue, Message};
-use authentication::{types, AuthServiceApi, host_api::HostAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use authentication::{types::{AuthServiceSubjects, AuthRequestPayload, AuthApiResult}, AuthServiceApi, host_api::HostAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use core::option::Option::{None, Some};
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use textnonce::TextNonce;
@@ -105,9 +105,9 @@ pub async fn run() -> Result<String, async_nats::Error> {
     // Register Auth Streams for Orchestrator to consume and proceess
     // NB: The subjects below are published by the Orchestrator
     
-    let auth_p1_subject = serde_json::to_string(&types::AuthServiceSubjects::HandleHandshakeP1)?;
-    let auth_p2_subject = serde_json::to_string(&types::AuthServiceSubjects::HandleHandshakeP2)?;
-    let auth_end_subject = serde_json::to_string(&types::AuthServiceSubjects::EndHandshake)?;
+    let auth_p1_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP1)?;
+    let auth_p2_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP2)?;
+    let auth_end_subject = serde_json::to_string(&AuthServiceSubjects::EndHandshake)?;
 
     // Call auth service and perform auth handshake
     let auth_service = host_auth_client
@@ -119,7 +119,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
 
     // Register save service for hub auth files (operator and sys)
     auth_service
-        .add_consumer::<authentication::types::ApiResult>(
+        .add_consumer::<AuthApiResult>(
             "save_hub_jwts", // consumer name
             &format!("{}.{}", host_pubkey, auth_p1_subject), // consumer stream subj
             EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
@@ -133,7 +133,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
         
     // Register save service for signed user jwt file
     auth_service
-       .add_consumer::<authentication::types::ApiResult>(
+       .add_consumer::<AuthApiResult>(
             "save_user_jwt", // consumer name
             &format!("{}.{}", host_pubkey, auth_end_subject), // consumer stream subj
             EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
@@ -148,7 +148,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
     // ==================== Publish Initial Auth Req =============================================
     // Initialize auth handshake with Orchestrator
     // by calling `AUTH.start_handshake` on the Auth Service
-    let payload = types::AuthRequestPayload {
+    let payload = AuthRequestPayload {
         host_pubkey: host_pubkey.clone(),
         email: "config.test.email@holo.host".to_string(),
         hoster_pubkey: "test_pubkey_from_config".to_string(),
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index f1ff531..bbbeb02 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -1,9 +1,9 @@
 /*
- This client is associated with the:
-- WORKLOAD account
-- hpos user
+This client is associated with the:
+    - WORKLOAD account
+    - hpos user
 
-// This client is responsible for subscribing to workload streams that handle:
+This client is responsible for subscribing to workload streams that handle:
     - installing new workloads onto the hosting device
     - removing workloads from the hosting device
     - sending workload status upon request
@@ -19,7 +19,7 @@ use util_libs::{
 };
 use workload::{
     WorkloadServiceApi, host_api::HostWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
-    types::{WorkloadServiceSubjects, ApiResult}
+    types::{WorkloadServiceSubjects, WorkloadApiResult}
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
@@ -82,7 +82,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         ))?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "start_workload", // consumer name
             &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
@@ -95,7 +95,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "update_installed_workload", // consumer name
             &format!("{}.{}", host_pubkey, workload_update_installed_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
@@ -108,7 +108,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "uninstall_workload", // consumer name
             &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
@@ -121,7 +121,7 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .await?;
 
         workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "send_workload_status", // consumer name
             &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
             EndpointType::Async(workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| {
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 8cfebc7..f54604a 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -1,10 +1,21 @@
 /*
- This client is associated with the:
-- auth account
-- orchestrator user
+This client is associated with the:
+    - AUTH account
+    - orchestrator user
 
-// This client is responsible for:
-//// auth_endpoint_subject = "AUTH.{host_id}.file.transfer.JWT-User"
+This client is responsible for:
+    - initalizing connection and handling interface with db
+    - registering with the host auth service to:
+        - handling inital auth requests
+            - validating user signature
+            - validating hoster pubkey
+            - validating hoster email
+            - bidirectionally pairing hoster and host
+            - sending hub jwt files back to user (once validated) 
+        - handling request to add user pubkey and generate signed user jwt 
+            - interfacing with hub nsc resolver and hub credential files
+            - sending user jwt file back to user
+    - keeping service running until explicitly cancelled out
 */
 
 use crate::utils as local_utils;
@@ -14,7 +25,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 // use std::process::Command;
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use authentication::{self, types::AuthServiceSubjects, AuthServiceApi, orchestrator_api::OrchestratorAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use authentication::{self, types::{AuthServiceSubjects, AuthApiResult}, AuthServiceApi, orchestrator_api::OrchestratorAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
@@ -85,7 +96,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         ))?;    
     
     auth_service
-        .add_consumer::<authentication::types::ApiResult>(
+        .add_consumer::<AuthApiResult>(
             "start_handshake", // consumer name
             &auth_start_subject, // consumer stream subj
             EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
@@ -98,7 +109,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .await?;
         
     auth_service
-        .add_consumer::<authentication::types::ApiResult>(
+        .add_consumer::<AuthApiResult>(
             "add_user_pubkey", // consumer name
             &auth_p2_subject, // consumer stream subj
             EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index a4be330..c5bd37e 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -1,13 +1,17 @@
 /*
- This client is associated with the:
-- WORKLOAD account
-- orchestrator user
-// This client is responsible for:
-    - handling requests to add workloads
-    - handling requests to update workloads
-    - handling requests to remove workloads
-    - handling workload status updates
-    - interfacing with mongodb DB
+This client is associated with the:
+    - WORKLOAD account
+    - orchestrator user
+
+This client is responsible for:
+    - initalizing connection and handling interface with db
+    - registering with the host worklload service to:
+        - handling requests to add workloads
+        - handling requests to update workloads
+        - handling requests to remove workloads
+        - handling workload status updates
+        - interfacing with mongodb DB
+    - keeping service running until explicitly cancelled out
 */
 
 use anyhow::{anyhow, Result};
@@ -15,7 +19,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use workload::{
-    WorkloadServiceApi, orchestrator_api::OrchestratorWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, ApiResult}
+    WorkloadServiceApi, orchestrator_api::OrchestratorWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, WorkloadApiResult}
 };
 use util_libs::{
     db::mongodb::get_mongodb_url,
@@ -101,7 +105,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by Developer
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "add_workload", // consumer name
              &workload_add_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -114,7 +118,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .await?;
 
         workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "update_workload", // consumer name
              &workload_update_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -128,7 +132,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "remove_workload", // consumer name
              &workload_remove_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -142,7 +146,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     
     // Automatically published by the Nats-DB-Connector
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "handle_db_insertion", // consumer name
              &workload_db_insert_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -155,7 +159,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "handle_db_modification", // consumer name
                 &workload_db_modification_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -169,7 +173,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by the Host Agent
     workload_service
-    .add_consumer::<ApiResult>(
+    .add_consumer::<WorkloadApiResult>(
         "handle_status_update", // consumer name
         &workload_handle_status_subject, // consumer stream subj
         EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
diff --git a/rust/services/authentication/src/host_api.rs b/rust/services/authentication/src/host_api.rs
index 97bf541..5f959a4 100644
--- a/rust/services/authentication/src/host_api.rs
+++ b/rust/services/authentication/src/host_api.rs
@@ -1,26 +1,13 @@
 /*
-Service Name: AUTH
-Subject: "AUTH.>"
-Provisioning Account: AUTH Account
-Importing Account: Auth/NoAuth Account
-
-This service should be run on the ORCHESTRATOR side and called from the HPOS side.
-The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
-NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
-This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
-
 Endpoints & Managed Subjects:
-    - start_hub_handshake
-    - end_hub_handshake
-    - save_hub_auth
-    - save_user_auth
-
+    - save_hub_jwts: AUTH.<host_pubkey>.handle_handshake_p1
+    - save_user_jwt: AUTH.<host_pubkey>.end_hub_handshake
 */
 
 use super::{AuthServiceApi, types, utils};
 use anyhow::Result;
 use async_nats::Message;
-use types::AuthResult;
+use types::{AuthApiResult, AuthResult};
 use core::option::Option::None;
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -32,7 +19,7 @@ pub struct HostAuthApi {}
 impl AuthServiceApi for HostAuthApi {}
 
 impl HostAuthApi {
-   pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<AuthApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
 
         // utils::receive_and_write_file();
@@ -64,7 +51,7 @@ impl HostAuthApi {
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
         
         // Respond to endpoint request
-        Ok(types::ApiResult {
+        Ok(AuthApiResult {
             status: types::AuthStatus { 
                 host_pubkey: host_pubkey.clone(),
                 status: types::AuthState::Requested
@@ -80,7 +67,7 @@ impl HostAuthApi {
         &self,
         msg: Arc<Message>,
         _output_dir: &str,
-    ) -> Result<types::ApiResult, ServiceError> {
+    ) -> Result<AuthApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.end_handshake' : {:?}", msg);
 
         // Generate user jwt file
@@ -90,7 +77,7 @@ impl HostAuthApi {
         // let _user_creds_path = utils::generate_creds_file();
 
         // 2. Respond to endpoint request
-        Ok(types::ApiResult {
+        Ok(AuthApiResult {
             status: types::AuthStatus { 
                 host_pubkey: "host_id_placeholder".to_string(),
                 status: types::AuthState::Authenticated
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index c9f2889..bfc8447 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -1,13 +1,9 @@
 /*
 Service Name: AUTH
 Subject: "AUTH.>"
-Provisioning Account: AUTH Account
-Importing Account: Auth/NoAuth Account
+Provisioning Account: AUTH Account (ie: This service is exclusively permissioned to the AUTH account.)
+Users: orchestrator & noauth
 
-This service should be run on the ORCHESTRATOR side and called from the HPOS side.
-The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
-NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
-This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
 */
 
 pub mod orchestrator_api;
@@ -22,6 +18,7 @@ use async_trait::async_trait;
 use std::sync::Arc;
 use std::future::Future;
 use serde::Deserialize;
+use types::AuthApiResult;
 use util_libs::nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse};
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
@@ -38,14 +35,14 @@ where
     fn call<F, Fut>(
         &self,
         handler: F,
-    ) -> AsyncEndpointHandler<types::ApiResult>
+    ) -> AsyncEndpointHandler<AuthApiResult>
     where
         F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
+        Fut: Future<Output = Result<AuthApiResult, ServiceError>> + Send + 'static,
         Self: Send + Sync
     {
         let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
+        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<AuthApiResult> {
             let api_clone = api.clone();
             Box::pin(handler(api_clone, msg))
         })
diff --git a/rust/services/authentication/src/orchestrator_api.rs b/rust/services/authentication/src/orchestrator_api.rs
index 74ed258..15e3a2b 100644
--- a/rust/services/authentication/src/orchestrator_api.rs
+++ b/rust/services/authentication/src/orchestrator_api.rs
@@ -1,27 +1,15 @@
 /*
-Service Name: AUTH
-Subject: "AUTH.>"
-Provisioning Account: AUTH Account
-Importing Account: Auth/NoAuth Account
-
-This service should be run on the ORCHESTRATOR side and called from the HPOS side.
-The NoAuth/Auth Server will import this service on the hub side and read local jwt files once the agent is validated.
-NB: subject pattern = "<SERVICE_NAME>.<Subject>.<DirectObject>.<Verb>.<Details>"
-This service handles the the "AUTH.<host_id>.file.transfer.JWT-<hoster_pubkey>.<chunk_id>" subject
-
 Endpoints & Managed Subjects:
-    - start_hub_handshake
-    - end_hub_handshake
-    - save_hub_auth
-    - save_user_auth
-
+    - handle_handshake_request: AUTH.start_handshake
+    - add_user_pubkey: AUTH.handle_handshake_p2
 */
+
 use super::{AuthServiceApi, types, utils};
 use anyhow::Result;
 use async_nats::{Message, HeaderValue};
 use async_nats::jetstream::ErrorCode;
 use nkeys::KeyPair;
-use types::AuthResult;
+use types::{AuthApiResult, AuthResult};
 use utils::handle_internal_err;
 use core::option::Option::None;
 use std::collections::HashMap;
@@ -69,7 +57,7 @@ impl OrchestratorAuthApi {
         &self,
         msg: Arc<Message>,
         creds_dir_path: &str,
-    ) -> Result<types::ApiResult, ServiceError> {
+    ) -> Result<AuthApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
 
         // 1. Verify expected data was received
@@ -170,7 +158,7 @@ impl OrchestratorAuthApi {
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
 
-        Ok(types::ApiResult {
+        Ok(AuthApiResult {
             status: types::AuthStatus { 
                 host_pubkey: host_pubkey.clone(),
                 status: types::AuthState::Requested
@@ -182,7 +170,7 @@ impl OrchestratorAuthApi {
         })
     }
 
-    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<AuthApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.handle_handshake_p2' : {:?}", msg);
 
         // 1. Verify expected payload was received
@@ -203,7 +191,7 @@ impl OrchestratorAuthApi {
         let user_jwt: Vec<u8> = std::fs::read(user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         // 5. Respond to endpoint request
-        Ok(types::ApiResult {
+        Ok(AuthApiResult {
             status: types::AuthStatus { 
                 host_pubkey,
                 status: types::AuthState::ValidatedAgent
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 9478c8c..64bd8d7 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -51,13 +51,13 @@ pub struct AuthResult {
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct ApiResult {
+pub struct AuthApiResult {
     pub status: AuthStatus,
     pub result: AuthResult,
     pub maybe_response_tags: Option<HashMap<String, String>>
 }
-impl EndpointTraits for ApiResult {}
-impl CreateTag for ApiResult {
+impl EndpointTraits for AuthApiResult {}
+impl CreateTag for AuthApiResult {
     fn get_tags(&self) -> HashMap<String, String> {
         self.maybe_response_tags.clone().unwrap_or_default()
     }
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index 381489c..56b8d84 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -1,12 +1,12 @@
 /*
-Endpoint Subjects:
-- `start_workload`: handles the "WORKLOAD.<host_pukey>.start." subject
-- `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
-- `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
-- `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
+Endpoints & Managed Subjects:
+    - `start_workload`: handles the "WORKLOAD.<host_pukey>.start." subject
+    - `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
+    - `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
+    - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
 */
 
-use super::{types, WorkloadServiceApi};
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
 use std::{fmt::Debug, sync::Arc};
@@ -22,7 +22,7 @@ pub struct HostWorkloadApi {}
 impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
@@ -36,10 +36,10 @@ impl HostWorkloadApi {
             desired: WorkloadState::Running,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(status, None))
+        Ok(WorkloadApiResult(status, None))
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update_installed' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
@@ -53,10 +53,10 @@ impl HostWorkloadApi {
             desired: WorkloadState::Updating,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(status, None))
+        Ok(WorkloadApiResult(status, None))
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
         let workload_id = Self::convert_to_type::<String>(msg)?;
 
@@ -70,12 +70,12 @@ impl HostWorkloadApi {
             desired: WorkloadState::Uninstalled,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(status, None))
+        Ok(WorkloadApiResult(status, None))
     }
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!(
             "Incoming message for 'WORKLOAD.send_status' : {:?}",
             msg
@@ -86,6 +86,6 @@ impl HostWorkloadApi {
         // Send updated status:
         // NB: This will send the update to both the requester (if one exists)
         // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(workload_status, None))
+        Ok(WorkloadApiResult(workload_status, None))
     }
 }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index f99e6c5..61d66b0 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -17,6 +17,7 @@ use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
 use std::future::Future;
 use serde::Deserialize;
+use types::WorkloadApiResult;
 use util_libs::{
     nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
     db::schemas::{WorkloadState, WorkloadStatus}
@@ -36,14 +37,14 @@ where
     fn call<F, Fut>(
         &self,
         handler: F,
-    ) -> AsyncEndpointHandler<types::ApiResult>
+    ) -> AsyncEndpointHandler<WorkloadApiResult>
     where
         F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
         Self: Send + Sync
     {
         let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
+        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
             let api_clone = api.clone();
             Box::pin(handler(api_clone, msg))
         })
@@ -70,10 +71,10 @@ where
         desired_state: WorkloadState,
         cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> Result<types::ApiResult, ServiceError>
+    ) -> Result<WorkloadApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
         let payload: T = Self::convert_to_type::<T>(msg.clone())?;
@@ -91,7 +92,7 @@ where
                 };
 
                 // 3. return response for stream
-                types::ApiResult(status, None)
+                WorkloadApiResult(status, None)
             }
         })
     }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index 72e3b4e..918f6e2 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -1,14 +1,14 @@
 /*
 Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `update_workload`: handles the "WORKLOAD.update" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
-- `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
-- `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
+    - `add_workload`: handles the "WORKLOAD.add" subject
+    - `update_workload`: handles the "WORKLOAD.update" subject
+    - `remove_workload`: handles the "WORKLOAD.remove" subject
+    - `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
+    - `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
+    - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
 */
 
-use super::{types, WorkloadServiceApi};
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
 use std::{collections::HashMap, fmt::Debug, sync::Arc};
@@ -43,7 +43,7 @@ impl OrchestratorWorkloadApi {
         })
     }
 
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.add'");
         self.process_request(
             msg,
@@ -55,7 +55,7 @@ impl OrchestratorWorkloadApi {
                     _id: Some(workload_id),
                     ..workload
                 };
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: new_workload._id,
                         desired: WorkloadState::Reported,
@@ -69,7 +69,7 @@ impl OrchestratorWorkloadApi {
         .await
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
         self.process_request(
             msg,
@@ -79,7 +79,7 @@ impl OrchestratorWorkloadApi {
                 let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: workload._id,
                         desired: WorkloadState::Reported,
@@ -94,7 +94,7 @@ impl OrchestratorWorkloadApi {
 
     }
 
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
         self.process_request(
             msg,
@@ -106,7 +106,7 @@ impl OrchestratorWorkloadApi {
                     "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
                     workload_id
                 );
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Removed,
@@ -121,7 +121,7 @@ impl OrchestratorWorkloadApi {
     }
 
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
         self.process_request(
             msg,
@@ -144,7 +144,7 @@ impl OrchestratorWorkloadApi {
                     for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
                         tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                     }
-                    return Ok(types::ApiResult( 
+                    return Ok(WorkloadApiResult( 
                         WorkloadStatus {
                             id: Some(workload_id),
                             desired: WorkloadState::Assigned,
@@ -206,7 +206,7 @@ impl OrchestratorWorkloadApi {
                 for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
                     tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                 }
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Assigned,
@@ -222,7 +222,7 @@ impl OrchestratorWorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.modify'");
         
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
@@ -236,11 +236,11 @@ impl OrchestratorWorkloadApi {
             actual: WorkloadState::Running,
         };
         
-        Ok(types::ApiResult(success_status, None))
+        Ok(WorkloadApiResult(success_status, None))
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
         let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
@@ -248,7 +248,7 @@ impl OrchestratorWorkloadApi {
 
         // TODO: ...handle the use case for the workload status update within the orchestrator
         
-        Ok(types::ApiResult(workload_status, None))
+        Ok(WorkloadApiResult(workload_status, None))
     }
 
     // Helper function to initialize mongodb collections
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 7ee6395..1fce9eb 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -19,9 +19,9 @@ pub enum WorkloadServiceSubjects {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
-impl EndpointTraits for ApiResult {}
-impl CreateTag for ApiResult {
+pub struct WorkloadApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
+impl EndpointTraits for WorkloadApiResult {}
+impl CreateTag for WorkloadApiResult {
     fn get_tags(&self) -> HashMap<String, String> {
         self.1.clone().unwrap_or_default()
     }

From 258f2ff0ba045e079e9e689f35fcdf7d23a68dc2 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 20:36:42 -0600
Subject: [PATCH 32/91] clean merge

---
 rust/clients/host_agent/src/hostd/workload_manager.rs | 1 -
 rust/services/workload/src/types.rs                   | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 6bc3030..09d2b20 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -12,7 +12,6 @@
 
 use anyhow::{anyhow, Result};
 use async_nats::Message;
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use util_libs::{
     js_stream_service::JsServiceParamsPartial,
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 92948ae..1a1124f 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,10 +1,6 @@
 use std::collections::HashMap;
 use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
-use util_libs::{
-    db::schemas::WorkloadStatus,
-    js_stream_service::{CreateTag, EndpointTraits},
-};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "snake_case")]

From 07f0c126c21bbf4f013ef69363973ce813730ea9 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 21:27:23 -0600
Subject: [PATCH 33/91] update hpos naming

---
 rust/clients/host_agent/src/hostd/workload_manager.rs | 4 ++--
 rust/clients/host_agent/src/main.rs                   | 2 +-
 rust/services/workload/src/lib.rs                     | 4 ++--
 rust/util_libs/src/db/schemas.rs                      | 2 +-
 rust/util_libs/src/nats_server.rs                     | 8 ++++----
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 09d2b20..c687de3 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -1,7 +1,7 @@
 /*
  This client is associated with the:
 - WORKLOAD account
-- hpos user
+- host user
 
 // This client is responsible for subscribing to workload streams that handle:
     - installing new workloads onto the hosting device
@@ -31,7 +31,7 @@ pub async fn run(
     host_creds_path: &Option<PathBuf>,
     nats_connect_timeout_secs: u64,
 ) -> Result<(), async_nats::Error> {
-    log::info!("HPOS Agent Client: Connecting to server...");
+    log::info!("Host Agent Client: Connecting to server...");
     log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index de603fe..3030a3a 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -1,7 +1,7 @@
 /*
 This client is associated with the:
   - WORKLOAD account
-  - hpos user
+  - host user
 
 This client is responsible for subscribing the host agent to workload stream endpoints:
   - installing new workloads
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index f99e6c5..7a77c2b 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -2,7 +2,7 @@
 Service Name: WORKLOAD
 Subject: "WORKLOAD.>"
 Provisioning Account: WORKLOAD
-Users: orchestrator & hpos
+Users: orchestrator & host
 */
 
 pub mod orchestrator_api;
@@ -25,7 +25,7 @@ use util_libs::{
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
-pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
+pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
 
 
 #[async_trait]
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 3675abb..1239448 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -120,7 +120,7 @@ pub struct Capacity {
 pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<MongoDbId>,
-    pub pubkey: String, // *INDEXED* // the HPOS/Device pubkey // nb: Unlike the hoster and developer pubkeys, this pubkey is not considered peronal info as it is not directly connected to a "natural person".
+    pub pubkey: String, // *INDEXED* // = the host pubkey // nb: Unlike the hoster and developer pubkeys, this pubkey is not considered peronal info as it is not directly connected to a "natural person".
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index 28aaac5..601cd75 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -208,8 +208,8 @@ mod tests {
     const TMP_JS_DIR: &str = "./tmp";
     const TEST_AUTH_DIR: &str = "./tmp/test-auth";
     const OPERATOR_NAME: &str = "test-operator";
-    const USER_ACCOUNT_NAME: &str = "hpos-account";
-    const USER_NAME: &str = "hpos-user";
+    const USER_ACCOUNT_NAME: &str = "host-account";
+    const USER_NAME: &str = "host-user";
     const NEW_LEAF_CONFIG_PATH: &str = "./test_configs/leaf_server.conf";
     // NB: if changed, the resolver file path must also be changed in the `hub-server.conf` iteself as well.
     const RESOLVER_FILE_PATH: &str = "./test_configs/resolver.conf";
@@ -243,7 +243,7 @@ mod tests {
             .output()
             .expect("Failed to create edit operator");
 
-        // Create hpos account (with js enabled)
+        // Create host account (with js enabled)
         Command::new("nsc")
             .args(["add", "account", USER_ACCOUNT_NAME])
             .output()
@@ -261,7 +261,7 @@ mod tests {
             .output()
             .expect("Failed to create edit account");
 
-        // Create user for hpos account
+        // Create user for host account
         Command::new("nsc")
             .args(["add", "user", USER_NAME])
             .args(["--account", USER_ACCOUNT_NAME])

From 78b69384efdafeca95c73b347c9f6b8b95b28221 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 21:35:51 -0600
Subject: [PATCH 34/91] clean up naming

---
 rust/clients/orchestrator/src/workloads.rs    | 17 ++++----
 rust/services/workload/src/host_api.rs        | 28 ++++++-------
 rust/services/workload/src/lib.rs             | 13 +++---
 .../services/workload/src/orchestrator_api.rs | 40 +++++++++----------
 rust/services/workload/src/types.rs           |  6 +--
 5 files changed, 53 insertions(+), 51 deletions(-)

diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index a4be330..551c1e7 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -2,7 +2,8 @@
  This client is associated with the:
 - WORKLOAD account
 - orchestrator user
-// This client is responsible for:
+
+This client is responsible for:
     - handling requests to add workloads
     - handling requests to update workloads
     - handling requests to remove workloads
@@ -15,7 +16,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use workload::{
-    WorkloadServiceApi, orchestrator_api::OrchestratorWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, ApiResult}
+    WorkloadServiceApi, orchestrator_api::OrchestratorWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, WorkloadApiResult}
 };
 use util_libs::{
     db::mongodb::get_mongodb_url,
@@ -101,7 +102,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by Developer
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "add_workload", // consumer name
              &workload_add_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -114,7 +115,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .await?;
 
         workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "update_workload", // consumer name
              &workload_update_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -128,7 +129,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "remove_workload", // consumer name
              &workload_remove_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -142,7 +143,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     
     // Automatically published by the Nats-DB-Connector
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "handle_db_insertion", // consumer name
              &workload_db_insert_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -155,7 +156,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "handle_db_modification", // consumer name
                 &workload_db_modification_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
@@ -169,7 +170,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by the Host Agent
     workload_service
-    .add_consumer::<ApiResult>(
+    .add_consumer::<WorkloadApiResult>(
         "handle_status_update", // consumer name
         &workload_handle_status_subject, // consumer stream subj
         EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index 381489c..56b8d84 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -1,12 +1,12 @@
 /*
-Endpoint Subjects:
-- `start_workload`: handles the "WORKLOAD.<host_pukey>.start." subject
-- `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
-- `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
-- `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
+Endpoints & Managed Subjects:
+    - `start_workload`: handles the "WORKLOAD.<host_pukey>.start." subject
+    - `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
+    - `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
+    - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
 */
 
-use super::{types, WorkloadServiceApi};
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
 use std::{fmt::Debug, sync::Arc};
@@ -22,7 +22,7 @@ pub struct HostWorkloadApi {}
 impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
@@ -36,10 +36,10 @@ impl HostWorkloadApi {
             desired: WorkloadState::Running,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(status, None))
+        Ok(WorkloadApiResult(status, None))
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update_installed' : {:?}", msg);
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
 
@@ -53,10 +53,10 @@ impl HostWorkloadApi {
             desired: WorkloadState::Updating,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(status, None))
+        Ok(WorkloadApiResult(status, None))
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
         let workload_id = Self::convert_to_type::<String>(msg)?;
 
@@ -70,12 +70,12 @@ impl HostWorkloadApi {
             desired: WorkloadState::Uninstalled,
             actual: WorkloadState::Unknown("..".to_string()),
         };
-        Ok(types::ApiResult(status, None))
+        Ok(WorkloadApiResult(status, None))
     }
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!(
             "Incoming message for 'WORKLOAD.send_status' : {:?}",
             msg
@@ -86,6 +86,6 @@ impl HostWorkloadApi {
         // Send updated status:
         // NB: This will send the update to both the requester (if one exists)
         // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(workload_status, None))
+        Ok(WorkloadApiResult(workload_status, None))
     }
 }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 7a77c2b..3508061 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -17,6 +17,7 @@ use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
 use std::future::Future;
 use serde::Deserialize;
+use types::WorkloadApiResult;
 use util_libs::{
     nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
     db::schemas::{WorkloadState, WorkloadStatus}
@@ -36,14 +37,14 @@ where
     fn call<F, Fut>(
         &self,
         handler: F,
-    ) -> AsyncEndpointHandler<types::ApiResult>
+    ) -> AsyncEndpointHandler<WorkloadApiResult>
     where
         F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send + 'static,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
         Self: Send + Sync
     {
         let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
+        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
             let api_clone = api.clone();
             Box::pin(handler(api_clone, msg))
         })
@@ -70,10 +71,10 @@ where
         desired_state: WorkloadState,
         cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> Result<types::ApiResult, ServiceError>
+    ) -> Result<WorkloadApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, ServiceError>> + Send,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
         let payload: T = Self::convert_to_type::<T>(msg.clone())?;
@@ -91,7 +92,7 @@ where
                 };
 
                 // 3. return response for stream
-                types::ApiResult(status, None)
+                WorkloadApiResult(status, None)
             }
         })
     }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index 72e3b4e..918f6e2 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -1,14 +1,14 @@
 /*
 Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `update_workload`: handles the "WORKLOAD.update" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
-- `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
-- `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
+    - `add_workload`: handles the "WORKLOAD.add" subject
+    - `update_workload`: handles the "WORKLOAD.update" subject
+    - `remove_workload`: handles the "WORKLOAD.remove" subject
+    - `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
+    - `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
+    - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
 */
 
-use super::{types, WorkloadServiceApi};
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
 use std::{collections::HashMap, fmt::Debug, sync::Arc};
@@ -43,7 +43,7 @@ impl OrchestratorWorkloadApi {
         })
     }
 
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.add'");
         self.process_request(
             msg,
@@ -55,7 +55,7 @@ impl OrchestratorWorkloadApi {
                     _id: Some(workload_id),
                     ..workload
                 };
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: new_workload._id,
                         desired: WorkloadState::Reported,
@@ -69,7 +69,7 @@ impl OrchestratorWorkloadApi {
         .await
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
         self.process_request(
             msg,
@@ -79,7 +79,7 @@ impl OrchestratorWorkloadApi {
                 let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: workload._id,
                         desired: WorkloadState::Reported,
@@ -94,7 +94,7 @@ impl OrchestratorWorkloadApi {
 
     }
 
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
         self.process_request(
             msg,
@@ -106,7 +106,7 @@ impl OrchestratorWorkloadApi {
                     "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
                     workload_id
                 );
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Removed,
@@ -121,7 +121,7 @@ impl OrchestratorWorkloadApi {
     }
 
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
         self.process_request(
             msg,
@@ -144,7 +144,7 @@ impl OrchestratorWorkloadApi {
                     for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
                         tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                     }
-                    return Ok(types::ApiResult( 
+                    return Ok(WorkloadApiResult( 
                         WorkloadStatus {
                             id: Some(workload_id),
                             desired: WorkloadState::Assigned,
@@ -206,7 +206,7 @@ impl OrchestratorWorkloadApi {
                 for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
                     tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                 }
-                Ok(types::ApiResult(
+                Ok(WorkloadApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Assigned,
@@ -222,7 +222,7 @@ impl OrchestratorWorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.modify'");
         
         let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
@@ -236,11 +236,11 @@ impl OrchestratorWorkloadApi {
             actual: WorkloadState::Running,
         };
         
-        Ok(types::ApiResult(success_status, None))
+        Ok(WorkloadApiResult(success_status, None))
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, ServiceError> {
+    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
         let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
@@ -248,7 +248,7 @@ impl OrchestratorWorkloadApi {
 
         // TODO: ...handle the use case for the workload status update within the orchestrator
         
-        Ok(types::ApiResult(workload_status, None))
+        Ok(WorkloadApiResult(workload_status, None))
     }
 
     // Helper function to initialize mongodb collections
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 1a1124f..4229577 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -18,9 +18,9 @@ pub enum WorkloadServiceSubjects {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
-impl EndpointTraits for ApiResult {}
-impl CreateTag for ApiResult {
+pub struct WorkloadApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
+impl EndpointTraits for WorkloadApiResult {}
+impl CreateTag for WorkloadApiResult {
     fn get_tags(&self) -> HashMap<String, String> {
         self.1.clone().unwrap_or_default()
     }

From 5da647a4900b182af45ed4abc14ad8c45d84ae52 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 21:37:47 -0600
Subject: [PATCH 35/91] improve workload desc

---
 rust/clients/orchestrator/src/workloads.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 551c1e7..c5bd37e 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -1,14 +1,17 @@
 /*
- This client is associated with the:
-- WORKLOAD account
-- orchestrator user
+This client is associated with the:
+    - WORKLOAD account
+    - orchestrator user
 
 This client is responsible for:
-    - handling requests to add workloads
-    - handling requests to update workloads
-    - handling requests to remove workloads
-    - handling workload status updates
-    - interfacing with mongodb DB
+    - initalizing connection and handling interface with db
+    - registering with the host worklload service to:
+        - handling requests to add workloads
+        - handling requests to update workloads
+        - handling requests to remove workloads
+        - handling workload status updates
+        - interfacing with mongodb DB
+    - keeping service running until explicitly cancelled out
 */
 
 use anyhow::{anyhow, Result};

From 2bc73c45fbe27d9578e516fd88d9269235644259 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 20 Jan 2025 21:40:44 -0600
Subject: [PATCH 36/91] restore `WorkloadApiResult`

---
 .../host_agent/src/hostd/workload_manager.rs     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index c687de3..7117d4e 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -1,9 +1,9 @@
 /*
  This client is associated with the:
-- WORKLOAD account
-- host user
+    - WORKLOAD account
+    - host user
 
-// This client is responsible for subscribing to workload streams that handle:
+This client is responsible for subscribing to workload streams that handle:
     - installing new workloads onto the hosting device
     - removing workloads from the hosting device
     - sending workload status upon request
@@ -19,7 +19,7 @@ use util_libs::{
 };
 use workload::{
     WorkloadServiceApi, host_api::HostWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
-    types::{WorkloadServiceSubjects, ApiResult}
+    types::{WorkloadServiceSubjects, WorkloadApiResult}
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
@@ -103,7 +103,7 @@ pub async fn run(
         ))?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "start_workload", // consumer name
             &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
             EndpointType::Async(
@@ -116,7 +116,7 @@ pub async fn run(
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "update_installed_workload", // consumer name
             &format!("{}.{}", host_pubkey, workload_update_installed_subject), // consumer stream subj
             EndpointType::Async(
@@ -129,7 +129,7 @@ pub async fn run(
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "uninstall_workload", // consumer name
             &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
             EndpointType::Async(
@@ -142,7 +142,7 @@ pub async fn run(
         .await?;
 
     workload_service
-        .add_consumer::<ApiResult>(
+        .add_consumer::<WorkloadApiResult>(
             "send_workload_status", // consumer name
             &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
             EndpointType::Async(

From 514960d9c553151c7a8d1f53b5cb5eb21de4c1e8 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 21 Jan 2025 14:37:04 -0600
Subject: [PATCH 37/91] remove host env var

---
 .env.example | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.env.example b/.env.example
index aa999e7..38d15ca 100644
--- a/.env.example
+++ b/.env.example
@@ -3,6 +3,4 @@ HOST_CREDS_FILE_PATH = "ops/admin.creds"
 MONGO_URI = "mongodb://<host>:<port>"
 NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
-
-HOST_CREDENTIALS_PATH: &str = "./host_user.creds";
\ No newline at end of file
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file

From 3a88a94c5079fe1fc4cb3745f763a3de2e5416fd Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 21 Jan 2025 14:37:39 -0600
Subject: [PATCH 38/91] clean up env vars

---
 .env.example | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.env.example b/.env.example
index 38d15ca..e4722cb 100644
--- a/.env.example
+++ b/.env.example
@@ -3,4 +3,5 @@ HOST_CREDS_FILE_PATH = "ops/admin.creds"
 MONGO_URI = "mongodb://<host>:<port>"
 NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
+LEAF_SERVER_PW = "pw-123456789"
+HOST_CREDENTIALS_PATH: &str = "./host_user.creds";

From 9f43a01853bf4b8a997abbe8e4451c07e409c24d Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Wed, 22 Jan 2025 01:30:08 -0600
Subject: [PATCH 39/91] auth updates

---
 .../clients/host_agent/src/auth/init_agent.rs |   3 +-
 rust/clients/orchestrator/src/auth.rs         |   2 +-
 rust/services/authentication/src/host_api.rs  | 165 +++++++++++++-----
 rust/services/authentication/src/lib.rs       |  16 +-
 .../authentication/src/orchestrator_api.rs    | 103 ++++++++---
 rust/services/authentication/src/types.rs     |  20 ++-
 rust/services/authentication/src/utils.rs     |  10 +-
 rust/services/workload/src/host_api.rs        | 159 ++++++++++++-----
 rust/services/workload/src/lib.rs             |  18 +-
 .../services/workload/src/orchestrator_api.rs | 116 +++++++-----
 rust/services/workload/src/types.rs           |  24 ++-
 rust/util_libs/src/js_stream_service.rs       |  15 +-
 12 files changed, 463 insertions(+), 188 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init_agent.rs b/rust/clients/host_agent/src/auth/init_agent.rs
index 1d16ab3..a7c8a50 100644
--- a/rust/clients/host_agent/src/auth/init_agent.rs
+++ b/rust/clients/host_agent/src/auth/init_agent.rs
@@ -71,6 +71,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
     // NB: This nkey keypair is a `ed25519_dalek::VerifyingKey` that is `BASE_32` encoded and returned as a String.
     let host_user_keys = KeyPair::new_user();
     let host_pubkey = host_user_keys.public_key();
+    // QUESTION: Where is this nkey file saved?
     
     // Discover the server Node ID via INFO response
     let server_node_id = host_auth_client.get_server_info().server_id;
@@ -123,7 +124,7 @@ pub async fn run() -> Result<String, async_nats::Error> {
             &format!("{}.{}", host_pubkey, auth_p1_subject), // consumer stream subj
             EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
                 async move {
-                    api.save_hub_jwts(msg).await
+                    api.save_hub_jwts(msg, &get_nats_client_creds("HOLO", "HPOS", "host")).await
                 }
             })),
             Some(create_callback_subject_to_orchestrator(auth_p2_subject)),
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index f54604a..3d7a92a 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -114,7 +114,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
             &auth_p2_subject, // consumer stream subj
             EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
                 async move {
-                    api.add_user_pubkey(msg).await
+                    api.add_user_nkey(msg, &local_utils::get_orchestrator_credentials_dir_path()).await
                 }
             })),
             Some(create_callback_subject_to_host("host_pubkey".to_string(), auth_end_subject)),
diff --git a/rust/services/authentication/src/host_api.rs b/rust/services/authentication/src/host_api.rs
index 5f959a4..592f5a7 100644
--- a/rust/services/authentication/src/host_api.rs
+++ b/rust/services/authentication/src/host_api.rs
@@ -5,11 +5,13 @@ Endpoints & Managed Subjects:
 */
 
 use super::{AuthServiceApi, types, utils};
+use utils::handle_internal_err;
 use anyhow::Result;
 use async_nats::Message;
 use types::{AuthApiResult, AuthResult};
 use core::option::Option::None;
 use std::collections::HashMap;
+use std::process::Command;
 use std::sync::Arc;
 use util_libs::nats_js_client::ServiceError;
 
@@ -19,45 +21,99 @@ pub struct HostAuthApi {}
 impl AuthServiceApi for HostAuthApi {}
 
 impl HostAuthApi {
-    pub async fn save_hub_jwts(&self, msg: Arc<Message>) -> Result<AuthApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.handle_handshake_p1' : {:?}", msg);
+    pub async fn save_hub_jwts(
+        &self,
+        msg: Arc<Message>,
+        output_dir: &str
+    ) -> Result<AuthApiResult, ServiceError> {
+        let msg_subject = &msg.subject.clone().into_string(); // AUTH.<host_pubkey>.handle_handshake_p1
+        log::trace!("Incoming message for '{}'", msg_subject);
 
-        // utils::receive_and_write_file();
+        // 1. Verify expected payload was received
+        let message_payload = Self::convert_msg_to_type::<AuthResult>(msg.clone())?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
 
-        // // Generate resolver file and create resolver file
-        // let resolver_path = utils::get_resolver_path();
-        // Command::new("nsc")
-        //     .arg("generate")
-        //     .arg("config")
-        //     .arg("--nats-resolver")
-        //     .arg("sys-account SYS")
-        //     .arg("--force")
-        //     .arg(format!("--config-file {}", resolver_path))
-        //     .output()
-        //     .expect("Failed to create resolver config file");
+        let operator_jwt_bytes = message_payload.data.inner.get("operator_jwt").ok_or_else(|| {
+            let err_msg = format!("Error: Failed to find operator jwt in message payload. Subject='{}'.", msg_subject);
+            handle_internal_err(&err_msg)
+        })?;
+
+        let sys_account_jwt_bytes = message_payload.data.inner.get("sys_account_jwt").ok_or_else(|| {
+            let err_msg = format!("Error: Failed to find sys jwt in message payload. Subject='{}'.", msg_subject);
+            handle_internal_err(&err_msg)
+        })?;
+        
+        let workload_account_jwt_bytes = message_payload.data.inner.get("workload_account_jwt").ok_or_else(|| {
+            let err_msg = format!("Error: Failed to find sys jwt in message payload. Subject='{}'.", msg_subject);
+            handle_internal_err(&err_msg)
+        })?;
+
+        // 2. Save operator_jwt, sys_account_jwt, and workload_account_jwt local to hosting agent
+        let operator_jwt_file = utils::receive_and_write_file(operator_jwt_bytes.to_owned(), output_dir, "operator.jwt").await.map_err(|e| {
+            let err_msg = format!("Failed to save operator jwt. Subject='{}' Error={}.", msg_subject, e);
+            handle_internal_err(&err_msg)
+        })?;
+
+        let sys_jwt_file = utils::receive_and_write_file(sys_account_jwt_bytes.to_owned(), output_dir, "account_sys.jwt").await.map_err(|e| {
+            let err_msg = format!("Failed to save sys jwt. Subject='{}' Error={}.", msg_subject, e);
+            handle_internal_err(&err_msg)
+        })?;
+
+        let workload_jwt_file = utils::receive_and_write_file(workload_account_jwt_bytes.to_owned(), output_dir, "account_sys.jwt").await.map_err(|e| {
+            let err_msg = format!("Failed to save sys jwt. Subject='{}' Error={}.", msg_subject, e);
+            handle_internal_err(&err_msg)
+        })?;
+
+        Command::new("nsc")
+            .arg(format!("add operator -u {} --force", operator_jwt_file))
+            .output()
+            .expect("Failed to add operator with provided operator jwt file");
+
+        Command::new("nsc")
+            .arg(format!("add import account --file {}", sys_jwt_file))
+            .output()
+            .expect("Failed to add sys with provided sys jwt file");
+
+        Command::new("nsc")
+            .arg(format!("add import account --file {}", workload_jwt_file))
+            .output()
+            .expect("Failed to add workload account with provided workload jwt file");
 
-        // // Push auth updates to hub server
         // Command::new("nsc")
-        //     .arg("push -A")
+        //     .arg(format!("generate nkey -o --store > operator_sk.nk"))
         //     .output()
-        //     .expect("Failed to create resolver config file");
+        //     .expect("Failed to add new operator signing key on hosting agent");
+
+        let host_sys_user_file_name = format!("{}/user_sys_host_{}.nk", output_dir, message_payload.status.host_pubkey);
+        Command::new("nsc")
+            .arg(format!("generate nkey -u --store > {}", host_sys_user_file_name))
+            .output()
+            .expect("Failed to add new sys user key on hosting agent");
 
-        // Prepare to send over user pubkey(to trigger the user jwt gen on hub)
-        let user_nkey_path = utils::get_file_path_buf("user_jwt_path");
-        let user_nkey: Vec<u8> = std::fs::read(user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        let host_pubkey = serde_json::to_string(&user_nkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        // 3. Prepare to send over user pubkey(to trigger the user jwt gen on hub)
+        let sys_user_nkey_path = utils::get_file_path_buf(&host_sys_user_file_name);
+        let sys_user_nkey: Vec<u8> = std::fs::read(sys_user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
+        let host_user_file_name = format!("{}/user_host_{}.nk", output_dir, message_payload.status.host_pubkey);
+        let host_user_nkey_path = utils::get_file_path_buf(&host_user_file_name);
+        let host_user_nkey: Vec<u8> = std::fs::read(host_user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+    
+        // let host_pubkey = serde_json::to_string(&user_nkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
         let mut tag_map: HashMap<String, String> = HashMap::new();
-        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
+        tag_map.insert("host_pubkey".to_string(), message_payload.status.host_pubkey.clone());
+
+        let mut result_hash_map: HashMap<String, Vec<u8>> = HashMap::new();
+        result_hash_map.insert("sys_user_nkey".to_string(), sys_user_nkey);
+        result_hash_map.insert("host_user_nkey".to_string(), host_user_nkey);
         
-        // Respond to endpoint request
+        // 4. Respond to endpoint request
         Ok(AuthApiResult {
-            status: types::AuthStatus { 
-                host_pubkey: host_pubkey.clone(),
-                status: types::AuthState::Requested
-            },
             result: AuthResult {
-                data: types::AuthResultType::Single(user_nkey)
+                status: types::AuthStatus { 
+                    host_pubkey: message_payload.status.host_pubkey,
+                    status: types::AuthState::Requested
+                },
+                data: types::AuthResultData { inner: result_hash_map }
             },
             maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
         })
@@ -66,24 +122,51 @@ impl HostAuthApi {
     pub async fn save_user_jwt(
         &self,
         msg: Arc<Message>,
-        _output_dir: &str,
+        output_dir: &str,
     ) -> Result<AuthApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.<host_pubkey>.end_handshake' : {:?}", msg);
+        let msg_subject = &msg.subject.clone().into_string(); // AUTH.<host_pubkey>.end_handshake
+        log::trace!("Incoming message for '{}'", msg_subject);
 
-        // Generate user jwt file
-        // utils::receive_and_write_file(msg, output_dir, file_name).await?;
-        
-        // Generate user creds file
-        // let _user_creds_path = utils::generate_creds_file();
+        // 1. Verify expected payload was received
+        let message_payload = Self::convert_msg_to_type::<AuthResult>(msg.clone())?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let host_sys_user_jwt_bytes = message_payload.data.inner.get("host_sys_user_jwt").ok_or_else(|| {
+            let err_msg = format!("Error: . Subject='{}'.", msg_subject);
+            handle_internal_err(&err_msg)
+        })?;
 
-        // 2. Respond to endpoint request
+        let host_user_jwt_bytes = message_payload.data.inner.get("host_user_jwt").ok_or_else(|| {
+            let err_msg = format!("Error: Failed to find sys jwt in message payload. Subject='{}'.", msg_subject);
+            handle_internal_err(&err_msg)
+        })?;
+
+        // 2. Save user_jwt and sys_jwt local to hosting agent
+        utils::receive_and_write_file(host_sys_user_jwt_bytes.to_owned(), output_dir, "operator.jwt").await.map_err(|e| {
+            let err_msg = format!("Failed to save operator jwt. Subject='{}' Error={}.", msg_subject, e);
+            handle_internal_err(&err_msg)
+        })?;
+
+        utils::receive_and_write_file(host_user_jwt_bytes.to_owned(), output_dir, "account_sys.jwt").await.map_err(|e| {
+            let err_msg = format!("Failed to save sys jwt. Subject='{}' Error={}.", msg_subject, e);
+            handle_internal_err(&err_msg)
+        })?;
+
+        let host_user_log = Command::new("nsc") 
+            .arg(format!("describe user -a WORKLOAD -n user_host_{} --json", message_payload.status.host_pubkey))
+            .output()
+            .expect("Failed to add user with provided keys");    
+
+        log::debug!("HOST USER JWT: {:?}", host_user_log);
+
+        // 3. Respond to endpoint request
         Ok(AuthApiResult {
-            status: types::AuthStatus { 
-                host_pubkey: "host_id_placeholder".to_string(),
-                status: types::AuthState::Authenticated
-            },
             result: AuthResult {
-                data: types::AuthResultType::Single(b"Hello, NATS!".to_vec())
+                status: types::AuthStatus { 
+                    host_pubkey: message_payload.status.host_pubkey,
+                    status: types::AuthState::Authenticated
+                },
+                data: types::AuthResultData { inner: HashMap::new() }
             },
             maybe_response_tags: None
         })
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 944eb3b..88a81a6 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -48,13 +48,25 @@ where
         })
     }
 
-    fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
+    fn convert_to_type<T>(data: Vec<u8>, msg_subject: &str) -> Result<T, ServiceError>
+    where
+        T: for<'de> Deserialize<'de> + Send + Sync,
+    {
+        serde_json::from_slice::<T>(&data).map_err(|e| {
+            let err_msg = format!("Error: Failed to deserialize payload data. Subject='{}' Err={}", msg_subject, e);
+            log::error!("{}", err_msg);
+            ServiceError::Internal(err_msg.to_string())
+        })
+        
+    }
+
+    fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Send + Sync,
     {
         let payload_buf = msg.payload.to_vec();
         serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject, e);
+            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject.clone().into_string(), e);
             log::error!("{}", err_msg);
             ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
         })
diff --git a/rust/services/authentication/src/orchestrator_api.rs b/rust/services/authentication/src/orchestrator_api.rs
index 15e3a2b..bfcdda0 100644
--- a/rust/services/authentication/src/orchestrator_api.rs
+++ b/rust/services/authentication/src/orchestrator_api.rs
@@ -78,7 +78,7 @@ impl OrchestratorAuthApi {
             }
         };
 
-        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, nonce: _ } = Self::convert_to_type::<types::AuthRequestPayload>(msg.clone())?;
+        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, nonce: _ } = Self::convert_msg_to_type::<types::AuthRequestPayload>(msg.clone())?;
 
         // 2. Validate signature
         let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
@@ -150,54 +150,105 @@ impl OrchestratorAuthApi {
 
         // 4. Read operator and sys account jwts and prepare them to be sent as a payload in the publication callback
         let operator_path = utils::get_file_path_buf(&format!("{}/operator.creds", creds_dir_path));
-        let hub_operator_creds: Vec<u8> = std::fs::read(operator_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let hub_operator_jwt: Vec<u8> = std::fs::read(operator_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        let sys_path = utils::get_file_path_buf(&format!("{}/sys.creds", creds_dir_path));
-        let hub_sys_creds: Vec<u8> = std::fs::read(sys_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let sys_account_path = utils::get_file_path_buf(&format!("{}/account_sys.creds", creds_dir_path));
+        let hub_sys_account_jwt: Vec<u8> = std::fs::read(sys_account_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let workload_account_path = utils::get_file_path_buf(&format!("{}/account_workload.creds", creds_dir_path));
+        let hub_workload_account_jwt: Vec<u8> = std::fs::read(workload_account_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
 
+        let mut result_hash_map: HashMap<String, Vec<u8>> = HashMap::new();
+        result_hash_map.insert("operator_jwt".to_string(), hub_operator_jwt);
+        result_hash_map.insert("sys_account_jwt".to_string(), hub_sys_account_jwt);
+        result_hash_map.insert("workload_account_jwt".to_string(), hub_workload_account_jwt);
+
         Ok(AuthApiResult {
-            status: types::AuthStatus { 
-                host_pubkey: host_pubkey.clone(),
-                status: types::AuthState::Requested
-            },
             result: AuthResult {
-                data: types::AuthResultType::Multiple(vec![hub_operator_creds, hub_sys_creds])
+                status: types::AuthStatus { 
+                    host_pubkey: host_pubkey.clone(),
+                    status: types::AuthState::Requested
+                },
+                data: types::AuthResultData { inner: result_hash_map }
             },
             maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
         })
     }
 
-    pub async fn add_user_pubkey(&self, msg: Arc<Message>) -> Result<AuthApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.handle_handshake_p2' : {:?}", msg);
+    pub async fn add_user_nkey(&self,
+        msg: Arc<Message>,
+        creds_dir_path: &str,
+    ) -> Result<AuthApiResult, ServiceError> {
+        let msg_subject = &msg.subject.clone().into_string(); // AUTH.handle_handshake_p2
+        log::trace!("Incoming message for '{}'", msg_subject);
 
         // 1. Verify expected payload was received
-        let host_pubkey = Self::convert_to_type::<String>(msg.clone())?;
+        let message_payload = Self::convert_msg_to_type::<AuthResult>(msg.clone())?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
 
-        // 2. Add User keys to Orchestrator nsc resolver
+        let host_user_nkey_bytes = message_payload.data.inner.get("host_user_nkey").ok_or_else(|| {
+            let err_msg = format!("Error: . Subject='{}'.", msg_subject);
+            handle_internal_err(&err_msg)
+        })?;
+        let host_user_nkey = Self::convert_to_type::<String>(host_user_nkey_bytes.to_owned(), msg_subject)?;
+
+        let host_pubkey = &message_payload.status.host_pubkey;
+
+        // 2. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
         Command::new("nsc")
-            .arg("...")
+            .arg(format!("add user -a SYS -n user_sys_host_{} -k {}", host_pubkey, host_user_nkey))
             .output()
-            .expect("Failed to add user with provided keys");
-        
-        // 3. Create and sign User JWT
-        let account_signing_key = utils::get_account_signing_key();
-        utils::generate_user_jwt(&host_pubkey, &account_signing_key);
+            .expect("Failed to add host sys user with provided keys");
+
+        Command::new("nsc")
+            .arg(format!("add user -a WORKLOAD -n user_host_{} -k {}", host_pubkey, host_user_nkey))
+            .output()
+            .expect("Failed to add host user with provided keys");
+
+        // ..and push auth updates to hub server
+        Command::new("nsc")
+            .arg("push -A")
+            .output()
+            .expect("Failed to update resolver config file");    
+
+        // 3. Create User JWT files (automatically signed with respective account key)
+        let host_sys_user_file_name = format!("{}/user_sys_host_{}.jwt", creds_dir_path, host_pubkey);
+        Command::new("nsc") 
+            .arg(format!("describe user -a SYS -n user_sys_host_{} --raw --output-file {}", host_pubkey, host_sys_user_file_name))
+            .output()
+            .expect("Failed to generate host sys user jwt file");
+
+        let host_user_file_name = format!("{}/user_host_{}.jwt", creds_dir_path, host_pubkey);
+        Command::new("nsc") 
+            .arg(format!("describe user -a WORKLOAD -n user_host_{} --raw --output-file {}", host_pubkey, host_user_file_name))
+            .output()
+            .expect("Failed to generate host user jwt file");        
+    
+        // let account_signing_key = utils::get_account_signing_key();
+        // utils::generate_user_jwt(&user_nkey, &account_signing_key);
 
         // 4. Prepare User JWT to be sent as a payload in the publication callback
-        let user_jwt_path = utils::get_file_path_buf("user_jwt_path");
-        let user_jwt: Vec<u8> = std::fs::read(user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let host_sys_user_jwt_path = utils::get_file_path_buf(&host_sys_user_file_name);
+        let host_sys_user_jwt: Vec<u8> = std::fs::read(host_sys_user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let host_user_jwt_path = utils::get_file_path_buf(&host_user_file_name);
+        let host_user_jwt: Vec<u8> = std::fs::read(host_user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        
+        let mut result_hash_map: HashMap<String, Vec<u8>> = HashMap::new();
+        result_hash_map.insert("host_sys_user_jwt".to_string(), host_sys_user_jwt);
+        result_hash_map.insert("host_user_jwt".to_string(), host_user_jwt);
 
         // 5. Respond to endpoint request
         Ok(AuthApiResult {
-            status: types::AuthStatus { 
-                host_pubkey,
-                status: types::AuthState::ValidatedAgent
-            },
             result: AuthResult {
-                data: types::AuthResultType::Single(user_jwt)
+                status: types::AuthStatus { 
+                    host_pubkey: message_payload.status.host_pubkey,
+                    status: types::AuthState::ValidatedAgent
+                },
+                data: types::AuthResultData { inner: result_hash_map }
             },
             maybe_response_tags: None
         })
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 64bd8d7..e2a89dd 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use util_libs::js_stream_service::{CreateTag, EndpointTraits};
+use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -40,19 +40,18 @@ pub struct AuthRequestPayload {
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
-pub enum AuthResultType {
-    Single(Vec<u8>),
-    Multiple(Vec<Vec<u8>>)
+pub struct AuthResultData {
+    pub inner: HashMap<String,Vec<u8>>
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthResult {
-    pub data: AuthResultType,
+    pub status: AuthStatus,
+    pub data: AuthResultData,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthApiResult {
-    pub status: AuthStatus,
     pub result: AuthResult,
     pub maybe_response_tags: Option<HashMap<String, String>>
 }
@@ -62,3 +61,12 @@ impl CreateTag for AuthApiResult {
         self.maybe_response_tags.clone().unwrap_or_default()
     }
 }
+impl CreateResponse for AuthApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        let r = self.result.clone();
+        match serde_json::to_vec(&r) {
+            Ok(r) => r.into(),
+            Err(e) => e.to_string().into(),
+        }
+    }
+}
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index 532a841..79dbc5c 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -1,8 +1,6 @@
 use anyhow::Result;
 use async_nats::jetstream::Context;
-use async_nats::Message;
 use util_libs::nats_js_client::ServiceError;
-use std::sync::Arc;
 use std::io::Write;
 use std::path::PathBuf;
 
@@ -19,19 +17,19 @@ pub fn get_file_path_buf(
 }
 
 pub async fn receive_and_write_file(
-    msg: Arc<Message>,
+    data: Vec<u8>,
     output_dir: &str,
     file_name: &str,
-) -> Result<()> {
+) -> Result<String> {
     let output_path = format!("{}/{}", output_dir, file_name);
     let mut file = std::fs::OpenOptions::new()
         .create(true)
         .append(true)
         .open(&output_path)?;
 
-    file.write_all(&msg.payload)?;
+    file.write_all(&data)?;
     file.flush()?;
-    Ok(())
+    Ok(output_path)
 }
 
 pub async fn publish_chunks(js: &Context, subject: &str, file_name: &str, data: Vec<u8>) -> Result<()> {
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index 56b8d84..a24fdc4 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -6,6 +6,8 @@ Endpoints & Managed Subjects:
     - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
 */
 
+use crate::types::WorkloadResult;
+
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
@@ -13,7 +15,7 @@ use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
 use util_libs::{
     nats_js_client::ServiceError,
-    db::schemas::{self, WorkloadState, WorkloadStatus}
+    db::schemas::{WorkloadState, WorkloadStatus}
 };
 
 #[derive(Debug, Clone, Default)]
@@ -23,69 +25,134 @@ impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
     pub async fn start_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Unknown("..".to_string()),
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Running,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
         };
-        Ok(WorkloadApiResult(status, None))
+
+        Ok(WorkloadApiResult  {
+            result: WorkloadResult {
+                status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.update_installed' : {:?}", msg);
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Updating,
-            actual: WorkloadState::Unknown("..".to_string()),
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
         };
-        Ok(WorkloadApiResult(status, None))
+            
+        Ok(WorkloadApiResult  {
+            result: WorkloadResult {
+                status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-        let workload_id = Self::convert_to_type::<String>(msg)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
-        // nix_uninstall_with(workload_id)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: Some(workload_id),
-            desired: WorkloadState::Uninstalled,
-            actual: WorkloadState::Unknown("..".to_string()),
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
+            // nix_uninstall_with(workload_id)
+    
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Error(err_msg),
+            }
         };
-        Ok(WorkloadApiResult(status, None))
+
+        Ok(WorkloadApiResult  {
+            result: WorkloadResult {
+                status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
     pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!(
-            "Incoming message for 'WORKLOAD.send_status' : {:?}",
-            msg
-        );
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
 
-        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
 
         // Send updated status:
         // NB: This will send the update to both the requester (if one exists)
         // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(WorkloadApiResult(workload_status, None))
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 3508061..46aeeab 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -17,7 +17,7 @@ use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
 use std::future::Future;
 use serde::Deserialize;
-use types::WorkloadApiResult;
+use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
     nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
     db::schemas::{WorkloadState, WorkloadStatus}
@@ -50,13 +50,13 @@ where
         })
     }
 
-    fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
+    fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Send + Sync,
     {
         let payload_buf = msg.payload.to_vec();
         serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject, e);
+            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject.clone().into_string(), e);
             log::error!("{}", err_msg);
             ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
         })
@@ -77,13 +77,13 @@ where
         Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = Self::convert_to_type::<T>(msg.clone())?;
+        let payload: T = Self::convert_msg_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
         Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
-                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
+                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject.clone().into_string(), payload, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
                     id: None,
@@ -92,7 +92,13 @@ where
                 };
 
                 // 3. return response for stream
-                WorkloadApiResult(status, None)
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None
+                    },
+                    maybe_response_tags: None
+                }
             }
         })
     }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index 918f6e2..7f38187 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -8,6 +8,8 @@ Endpoints & Managed Subjects:
     - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
 */
 
+use crate::types::WorkloadResult;
+
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
@@ -55,14 +57,17 @@ impl OrchestratorWorkloadApi {
                     _id: Some(workload_id),
                     ..workload
                 };
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: new_workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: new_workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        workload: None
                     },
-                    None
-                ))
+                    maybe_response_tags: None
+                })
             },
             WorkloadState::Error,
         )
@@ -79,14 +84,17 @@ impl OrchestratorWorkloadApi {
                 let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        workload: None
                     },
-                    None
-                ))
+                    maybe_response_tags: None
+                })
             },
             WorkloadState::Error,
         )
@@ -106,14 +114,17 @@ impl OrchestratorWorkloadApi {
                     "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
                     workload_id
                 );
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Removed,
-                        actual: WorkloadState::Removed,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Removed,
+                            actual: WorkloadState::Removed,
+                        },
+                        workload: None
                     },
-                    None
-                ))
+                    maybe_response_tags: None
+                })
             },
             WorkloadState::Error,
         )
@@ -144,14 +155,18 @@ impl OrchestratorWorkloadApi {
                     for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
                         tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                     }
-                    return Ok(WorkloadApiResult( 
-                        WorkloadStatus {
-                            id: Some(workload_id),
-                            desired: WorkloadState::Assigned,
-                            actual: WorkloadState::Assigned,
+
+                    return Ok(WorkloadApiResult {
+                        result: WorkloadResult {
+                            status: WorkloadStatus {
+                                id: Some(workload_id),
+                                desired: WorkloadState::Assigned,
+                                actual: WorkloadState::Assigned,
+                            },
+                            workload: None
                         },
-                        Some(tag_map)
-                    ));
+                        maybe_response_tags: Some(tag_map)
+                    });
                 }
 
                 // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
@@ -206,14 +221,17 @@ impl OrchestratorWorkloadApi {
                 for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
                     tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                 }
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Assigned,
+                            actual: WorkloadState::Assigned,
+                        },
+                        workload: None
                     },
-                    Some(tag_map)
-                ))
+                    maybe_response_tags: Some(tag_map)
+                })
         },
             WorkloadState::Error,
         )
@@ -225,30 +243,44 @@ impl OrchestratorWorkloadApi {
     pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.modify'");
         
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+        let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
         
         // TODO: ...handle the use case for the update entry change stream 
 
+        // let workload_request_bytes = serde_json::to_vec(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
         let success_status = WorkloadStatus {
-            id: workload._id,
+            id: workload._id.clone(),
             desired: WorkloadState::Running,
             actual: WorkloadState::Running,
         };
-        
-        Ok(WorkloadApiResult(success_status, None))
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: success_status,
+                workload: Some(workload)
+            },
+            maybe_response_tags: None
+        })
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
     pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
-        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
         log::trace!("Workload status to update. Status={:?}", workload_status);
 
         // TODO: ...handle the use case for the workload status update within the orchestrator
-        
-        Ok(WorkloadApiResult(workload_status, None))
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     // Helper function to initialize mongodb collections
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 4229577..0ec4044 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,5 +1,5 @@
 use std::collections::HashMap;
-use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
+use util_libs::{db::schemas::{self, WorkloadStatus}, js_stream_service::{CreateResponse, CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -18,10 +18,28 @@ pub enum WorkloadServiceSubjects {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct WorkloadApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
+pub struct WorkloadResult {
+    pub status: WorkloadStatus,
+    pub workload: Option<schemas::Workload>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadApiResult {
+    pub result: WorkloadResult,
+    pub maybe_response_tags: Option<HashMap<String, String>>
+}
 impl EndpointTraits for WorkloadApiResult {}
 impl CreateTag for WorkloadApiResult {
     fn get_tags(&self) -> HashMap<String, String> {
-        self.1.clone().unwrap_or_default()
+        self.maybe_response_tags.clone().unwrap_or_default()
+    }
+}
+impl CreateResponse for WorkloadApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        let r = self.result.clone();
+        match serde_json::to_vec(&r) {
+            Ok(r) => r.into(),
+            Err(e) => e.to_string().into(),
+        }
     }
 }
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 607e40c..55412a5 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -2,7 +2,6 @@ use super::nats_js_client::EndpointType;
 
 use anyhow::{anyhow, Result};
 use std::any::Any;
-// use async_nats::jetstream::message::Message;
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
@@ -20,11 +19,14 @@ pub trait CreateTag: Send + Sync {
     fn get_tags(&self) -> HashMap<String, String>;
 }
 
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
-{
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
 }
 
+pub trait EndpointTraits:
+    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + CreateResponse + 'static
+{}
+
 #[async_trait]
 pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
     fn get_name(&self) -> &str;
@@ -338,10 +340,7 @@ impl JsStreamService {
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
-                    let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
-                        Ok(r) => r.into(),
-                        Err(e) => e.to_string().into(),
-                    };
+                    let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 },

From 7c9272995d090be2d6e4a5bbbe469b9a6f6af786 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 13 Jan 2025 16:49:10 +0100
Subject: [PATCH 40/91] temporary(flake): bump blueprint for upstreamed fixes

---
 flake.lock                                  |  6 ++--
 nix/checks/holo-agent-integration-nixos.nix | 34 +++++++++++++++++++++
 2 files changed, 37 insertions(+), 3 deletions(-)
 create mode 100644 nix/checks/holo-agent-integration-nixos.nix

diff --git a/flake.lock b/flake.lock
index 2e2f35e..7252249 100644
--- a/flake.lock
+++ b/flake.lock
@@ -8,11 +8,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1733562445,
-        "narHash": "sha256-gLmqbX40Qos+EeBvmlzvntWB3NrdiDaFxhr3VAmhrf4=",
+        "lastModified": 1737561535,
+        "narHash": "sha256-Zkelzjw88We+/w6Ds/F3oyU+BzNJF+8a7YbE5pCcorM=",
         "owner": "numtide",
         "repo": "blueprint",
-        "rev": "97ef7fba3a6eec13e12a108ce4b3473602eb424f",
+        "rev": "c1fd23c89478562298d26bc323ae87921ced80d8",
         "type": "github"
       },
       "original": {
diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
new file mode 100644
index 0000000..2d03ccf
--- /dev/null
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -0,0 +1,34 @@
+{
+  pkgs,
+  flake,
+  ...
+}:
+
+pkgs.testers.runNixOSTest (_: {
+  name = "holo-agent-nixostest-basic";
+
+  nodes.machine =
+    _:
+
+    {
+      imports = [
+        flake.nixosModules.holo-agent
+      ];
+
+      holo.agent = {
+        enable = true;
+        rust = {
+          log = "trace";
+          backtrace = "trace";
+        };
+      };
+    };
+
+  # takes args which are currently removed by deadnix:
+  # { nodes, ... }
+  testScript = _: ''
+    machine.start()
+    # machine.wait_for_unit("holo-agent.service")
+    machine.wait_for_unit("default.target")
+  '';
+})

From d396a5083837fd9223e9da026fe7c9559e065c60 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Mon, 13 Jan 2025 12:34:44 +0100
Subject: [PATCH 41/91] feat(nix/packages/rust-workspace): expose rust binaries

previously it would only expose the target directory as an archive.
---
 nix/packages/rust-workspace.nix | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/nix/packages/rust-workspace.nix b/nix/packages/rust-workspace.nix
index 5203e81..ae42a82 100644
--- a/nix/packages/rust-workspace.nix
+++ b/nix/packages/rust-workspace.nix
@@ -35,7 +35,7 @@ let
   # cache misses when building individual top-level-crates
   cargoArtifacts = craneLib.buildDepsOnly commonArgs;
 in
-craneLib.cargoBuild (
+craneLib.buildPackage (
   commonArgs
   // {
     inherit cargoArtifacts;
@@ -97,8 +97,6 @@ craneLib.cargoBuild (
           partitionType = "count";
         }
       );
-
     };
-
   }
 )

From 5e492351af6c20c9e4b222e052252a21f0e3a6a2 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Wed, 22 Jan 2025 20:23:19 +0100
Subject: [PATCH 42/91] feat(nix): introduce holo-host-agent module with
 integration test

the holo-host agent also pulls in extra-container as that's going to be
the initial vehicle for defining and running host workloads.
---
 flake.lock                                  |  71 +++++-
 flake.nix                                   |   3 +
 nix/checks/holo-agent-integration-nixos.nix | 234 ++++++++++++++++++--
 nix/modules/nixos/holo-agent.nix            |  29 ---
 nix/modules/nixos/holo-host-agent.nix       | 104 +++++++++
 5 files changed, 380 insertions(+), 61 deletions(-)
 delete mode 100644 nix/modules/nixos/holo-agent.nix
 create mode 100644 nix/modules/nixos/holo-host-agent.nix

diff --git a/flake.lock b/flake.lock
index 7252249..57cb8f8 100644
--- a/flake.lock
+++ b/flake.lock
@@ -56,13 +56,37 @@
         "type": "github"
       }
     },
+    "extra-container": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1734542275,
+        "narHash": "sha256-wnRkafo4YrIuvJeRsOmfStxIzi7ty2I0OtGMO9chwJc=",
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "rev": "fa723fb67201c1b4610fd3d608681da362f800eb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "type": "github"
+      }
+    },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1653893745,
-        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
         "type": "github"
       },
       "original": {
@@ -131,9 +155,24 @@
         "type": "github"
       }
     },
+    "flake-utils_6": {
+      "locked": {
+        "lastModified": 1653893745,
+        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "nixago": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "flake-utils": "flake-utils_2",
         "nixago-exts": "nixago-exts",
         "nixpkgs": [
           "nixpkgs"
@@ -155,7 +194,7 @@
     },
     "nixago-exts": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_3",
         "nixago": "nixago_2",
         "nixpkgs": [
           "nixago",
@@ -178,7 +217,7 @@
     },
     "nixago-exts_2": {
       "inputs": {
-        "flake-utils": "flake-utils_4",
+        "flake-utils": "flake-utils_5",
         "nixago": "nixago_3",
         "nixpkgs": [
           "nixago",
@@ -203,7 +242,7 @@
     },
     "nixago_2": {
       "inputs": {
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_4",
         "nixago-exts": "nixago-exts_2",
         "nixpkgs": [
           "nixago",
@@ -228,7 +267,7 @@
     },
     "nixago_3": {
       "inputs": {
-        "flake-utils": "flake-utils_5",
+        "flake-utils": "flake-utils_6",
         "nixpkgs": [
           "nixago",
           "nixago-exts",
@@ -288,6 +327,7 @@
         "blueprint": "blueprint",
         "crane": "crane",
         "disko": "disko",
+        "extra-container": "extra-container",
         "nixago": "nixago",
         "nixpkgs": "nixpkgs",
         "nixpkgs-unstable": "nixpkgs-unstable",
@@ -351,6 +391,21 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "treefmt-nix": {
       "inputs": {
         "nixpkgs": [
diff --git a/flake.nix b/flake.nix
index 9e3a1df..b37911f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -21,6 +21,9 @@
       url = "github:oxalica/rust-overlay";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    extra-container.url = "github:erikarvstedt/extra-container";
+    extra-container.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs =
diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index 2d03ccf..cc3f39e 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -1,34 +1,220 @@
 {
-  pkgs,
   flake,
+  pkgs,
   ...
 }:
 
-pkgs.testers.runNixOSTest (_: {
-  name = "holo-agent-nixostest-basic";
+pkgs.testers.runNixOSTest (
+  { nodes, lib, ... }:
+  let
+    hubIP = (pkgs.lib.head nodes.hub.networking.interfaces.eth1.ipv4.addresses).address;
+    hubJsDomain = "hub";
+
+    hostUseOsNats = false;
+  in
+  {
+    name = "host-agent-integration-nixos";
+    meta.platforms = lib.lists.intersectLists lib.platforms.linux lib.platforms.x86_64;
 
-  nodes.machine =
-    _:
+    nodes.hub =
+      { ... }:
+      {
+        imports = [
+          flake.nixosModules.holo-nats-server
+          # flake.nixosModules.holo-orchestrator
+        ];
 
-    {
-      imports = [
-        flake.nixosModules.holo-agent
-      ];
+        # holo.orchestrator.enable = true;
+        holo.nats-server.enable = true;
+        services.nats.settings = {
+          accounts = {
+            SYS = {
+              users = [
+                {
+                  user = "admin";
+                  "password" = "admin";
+                }
+              ];
+            };
+          };
+          system_account = "SYS";
 
-      holo.agent = {
-        enable = true;
-        rust = {
-          log = "trace";
-          backtrace = "trace";
+          jetstream = {
+            domain = "${hubJsDomain}";
+            enabled = true;
+          };
+
+          # logging options
+          debug = true;
+          trace = false;
+          logtime = false;
         };
       };
-    };
-
-  # takes args which are currently removed by deadnix:
-  # { nodes, ... }
-  testScript = _: ''
-    machine.start()
-    # machine.wait_for_unit("holo-agent.service")
-    machine.wait_for_unit("default.target")
-  '';
-})
+
+    nodes.host =
+      { ... }:
+      {
+        imports = [
+          flake.nixosModules.holo-nats-server
+          flake.nixosModules.holo-host-agent
+        ];
+
+        holo.host-agent = {
+          enable = !hostUseOsNats;
+          rust = {
+            log = "trace";
+            backtrace = "trace";
+          };
+
+          nats = {
+            # url = "agent:${builtins.toString config.services.nats.port}";
+            hubServerUrl = "nats://${hubIP}:${builtins.toString nodes.hub.holo.nats-server.leafnodePort}";
+          };
+        };
+
+        holo.nats-server.enable = hostUseOsNats;
+        services.nats.settings = {
+          accounts = {
+            SYS = {
+              users = [
+                {
+                  user = "admin";
+                  "password" = "admin";
+                }
+              ];
+            };
+          };
+          system_account = "SYS";
+
+          jetstream = {
+            domain = "leaf";
+            enabled = true;
+          };
+
+          leafnodes = {
+            remotes = [
+              { url = "nats://${hubIP}:${builtins.toString nodes.hub.holo.nats-server.leafnodePort}"; }
+            ];
+          };
+
+          # logging options
+          debug = true;
+          trace = false;
+          logtime = false;
+        };
+      };
+
+    # takes args which are currently removed by deadnix:
+    # { nodes, ... }
+    testScript =
+      _:
+      let
+        natsCli = lib.getExe pkgs.natscli;
+        testStreamName = "INTEGRATION";
+
+        _testStreamHubConfig = builtins.toFile "stream.conf" ''
+          {
+              "name": "${testStreamName}",
+              "subjects": [
+                  "${testStreamName}",
+                  "${testStreamName}.\u003e"
+              ],
+              "retention": "limits",
+              "max_consumers": -1,
+              "max_msgs_per_subject": -1,
+              "max_msgs": -1,
+              "max_bytes": -1,
+              "max_age": 0,
+              "max_msg_size": -1,
+              "storage": "memory",
+              "discard": "old",
+              "num_replicas": 1,
+              "duplicate_window": 120000000000,
+              "sealed": false,
+              "deny_delete": false,
+              "deny_purge": false,
+              "allow_rollup_hdrs": false,
+              "allow_direct": true,
+              "mirror_direct": false,
+              "consumer_limits": {}
+          }
+        '';
+        _testStreamLeafConfig = builtins.toFile "stream.conf" ''
+          {
+              "name": "${testStreamName}",
+              "retention": "limits",
+              "max_consumers": -1,
+              "max_msgs_per_subject": -1,
+              "max_msgs": -1,
+              "max_bytes": -1,
+              "max_age": 0,
+              "max_msg_size": -1,
+              "storage": "memory",
+              "discard": "old",
+              "num_replicas": 1,
+              "mirror": {
+                  "name": "${testStreamName}",
+                  "external": {
+                      "api": "$JS.${hubJsDomain}.API",
+                      "deliver": ""
+                  }
+              },
+              "sealed": false,
+              "deny_delete": false,
+              "deny_purge": false,
+              "allow_rollup_hdrs": false,
+              "allow_direct": true,
+              "mirror_direct": false,
+              "consumer_limits": {}
+          }
+        '';
+        hubTestScript =
+          let
+            natsServer = "nats://127.0.0.1:${builtins.toString nodes.hub.holo.nats-server.port}";
+          in
+          pkgs.writeShellScript "cmd" ''
+            set -xe
+
+            ${natsCli} -s "${natsServer}" stream add ${testStreamName} --config ${_testStreamHubConfig}
+            ${natsCli} -s "${natsServer}" pub --count=10 "${testStreamName}.integrate" --js-domain ${hubJsDomain} '{"message":"hello"}'
+            ${natsCli} -s "${natsServer}" stream ls
+            ${natsCli} -s "${natsServer}" sub --stream "${testStreamName}" "${testStreamName}.>" --count=10
+          '';
+
+        hostTestScript =
+          let
+            natsServer =
+              if hostUseOsNats then
+                "nats://127.0.0.1:${builtins.toString nodes.host.holo.nats-server.port}"
+              else
+                "nats://127.0.0.1:${builtins.toString nodes.host.holo.host-agent.nats.listenPort}";
+          in
+          pkgs.writeShellScript "cmd" ''
+            set -xe
+
+            ${natsCli} -s "${natsServer}" stream add ${testStreamName} --config ${_testStreamLeafConfig}
+            ${natsCli} -s "${natsServer}" stream ls
+            ${natsCli} -s "${natsServer}" stream info --json ${testStreamName}
+            ${natsCli} -s '${natsServer}' sub --stream "${testStreamName}" '${testStreamName}.>' --count=10
+          '';
+      in
+      ''
+        with subtest("start the hub and run the testscript"):
+          hub.start()
+          hub.wait_for_unit("nats.service")
+          hub.succeed("${hubTestScript}")
+
+        with subtest("starting the host and waiting for holo-host-agent to be ready"):
+          host.start()
+          ${
+            if hostUseOsNats then
+              "host.wait_for_unit('nats.service')"
+            else
+              "host.wait_for_unit('holo-host-agent')"
+          }
+
+        with subtest("running the host testscript"):
+          host.succeed("${hostTestScript}", timeout = 10)
+      '';
+  }
+)
diff --git a/nix/modules/nixos/holo-agent.nix b/nix/modules/nixos/holo-agent.nix
deleted file mode 100644
index d4fc43b..0000000
--- a/nix/modules/nixos/holo-agent.nix
+++ /dev/null
@@ -1,29 +0,0 @@
-# Module to configure a machine as a holo-agent.
-
-{
-  inputs,
-  lib,
-  config,
-  ...
-}:
-
-let
-  cfg = config.holo.nats-server;
-in
-{
-  imports = [
-    inputs.extra-container.nixosModules.default
-  ];
-
-  options.holo.agent = with lib; {
-    enable = mkOption {
-      description = "enable holo-agent";
-      default = true;
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    # TODO: add holo-agent systemd service
-    # TODO: add nats client here or is it started by the agent?
-  };
-}
diff --git a/nix/modules/nixos/holo-host-agent.nix b/nix/modules/nixos/holo-host-agent.nix
new file mode 100644
index 0000000..383ee14
--- /dev/null
+++ b/nix/modules/nixos/holo-host-agent.nix
@@ -0,0 +1,104 @@
+# Module to configure a machine as a holo-host-agent.
+
+# blueprint specific first level argument that's referred to as "publisherArgs"
+{
+  inputs,
+  ...
+}:
+
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.holo.host-agent;
+in
+{
+  imports = [
+    inputs.extra-container.nixosModules.default
+  ];
+
+  options.holo.host-agent = {
+    enable = lib.mkOption {
+      description = "enable holo-host-agent";
+      default = true;
+    };
+
+    autoStart = lib.mkOption {
+      default = true;
+    };
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = inputs.self.packages.${pkgs.stdenv.system}.rust-workspace;
+    };
+
+    rust = {
+      log = lib.mkOption {
+        type = lib.types.str;
+        default = "debug";
+      };
+
+      backtrace = lib.mkOption {
+        type = lib.types.str;
+        default = "1";
+      };
+    };
+
+    nats = {
+      listenHost = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+      };
+
+      listenPort = lib.mkOption {
+        type = lib.types.int;
+        default = 4222;
+      };
+
+      url = lib.mkOption {
+        type = lib.types.str;
+        default = "${cfg.nats.listenHost}:${builtins.toString cfg.nats.listenPort}";
+      };
+
+      hubServerUrl = lib.mkOption {
+        type = lib.types.str;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.holo-host-agent = {
+      enable = true;
+
+      after = [ "network.target" ];
+      wantedBy = lib.lists.optional cfg.autoStart "multi-user.target";
+
+      environment =
+        {
+          RUST_LOG = cfg.rust.log;
+          RUST_BACKTRACE = cfg.rust.backtrace;
+          NATS_HUB_SERVER_URL = cfg.nats.hubServerUrl;
+          NATS_LISTEN_PORT = builtins.toString cfg.nats.listenPort;
+        }
+        // lib.attrsets.optionalAttrs (cfg.nats.url != null) {
+          NATS_URL = cfg.nats.url;
+        };
+
+      path = [
+        pkgs.nats-server
+      ];
+
+      script = builtins.toString (
+        pkgs.writeShellScript "holo-host-agent" ''
+          ${lib.getExe' cfg.package "host_agent"} daemonize
+        ''
+      );
+    };
+
+    # TODO: add nats server here or is it started by the host-agent?
+  };
+}

From 7f5dc38c31f3245a32ae8a56195e7787df316b03 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Wed, 22 Jan 2025 20:25:55 +0100
Subject: [PATCH 43/91] feat(nix/holo-nats-server): make port and leafnodeport
 configurable

---
 nix/modules/nixos/holo-nats-server.nix | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index e39e1be..a18b268 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -5,11 +5,23 @@ in
 {
   imports = [ ];
 
-  options.holo.nats-server = with lib; {
-    enable = mkOption {
+  options.holo.nats-server = {
+    enable = lib.mkOption {
       description = "enable holo NATS server";
       default = true;
     };
+
+    port = lib.mkOption {
+      description = "enable holo NATS server";
+      type = lib.types.int;
+      default = 4222;
+    };
+
+    leafnodePort = lib.mkOption {
+      description = "enable holo NATS server";
+      type = lib.types.int;
+      default = 7422;
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -24,7 +36,8 @@ in
       jetstream = true;
 
       settings = {
-        leafnodes.port = 7422;
+        inherit (cfg) port;
+        leafnodes.port = cfg.leafnodePort;
       };
     };
   };

From 0a7a572e9adb78aab43464c4cdd98cc0be635366 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Wed, 22 Jan 2025 20:27:15 +0100
Subject: [PATCH 44/91] host-agent: improve resilience and configurability

* turn some hardcoded values into CLI arguments
* wait (with a timeout) for NATS to be ready to serve connections
* pass through NATS stdout/stderr
* provision (techdebt) TODOs
---
 Cargo.lock                                    |   6 +-
 rust/clients/host_agent/Cargo.toml            |   1 +
 rust/clients/host_agent/src/agent_cli.rs      |  21 +-
 .../clients/host_agent/src/gen_leaf_server.rs |  45 +++-
 rust/clients/host_agent/src/main.rs           |  26 ++-
 .../host_agent/src/workload_manager.rs        |  95 ++++----
 rust/services/workload/src/lib.rs             | 211 +++++++++++-------
 rust/services/workload/src/types.rs           |   9 +-
 rust/util_libs/src/db/mongodb.rs              |  14 +-
 rust/util_libs/src/db/schemas.rs              |  19 +-
 rust/util_libs/src/js_stream_service.rs       |  72 +++---
 rust/util_libs/src/nats_js_client.rs          |  68 +++---
 rust/util_libs/src/nats_server.rs             |  60 ++---
 rust/util_libs/src/nats_types.rs              |   2 +-
 14 files changed, 398 insertions(+), 251 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7b76bc2..801c46a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1006,6 +1006,7 @@ dependencies = [
  "rand",
  "serde",
  "serde_json",
+ "tempfile",
  "thiserror 2.0.8",
  "tokio",
  "url",
@@ -2310,12 +2311,13 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tempfile"
-version = "3.14.0"
+version = "3.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
+checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "getrandom",
  "once_cell",
  "rustix",
  "windows-sys 0.59.0",
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 4e8e3b2..5ed82ab 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -25,3 +25,4 @@ rand = "0.8.5"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 hpos-hal = { path = "../../hpos-hal" }
+tempfile = "3.15.0"
diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index e872ec1..04403f9 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -1,6 +1,8 @@
+use std::path::PathBuf;
+
 /// MOdule containing all of the Clap Derive structs/definitions that make up the agent's
 /// command line. To start the agent daemon (usually from systemd), use `host_agent daemonize`.
-use clap::{Parser, Subcommand};
+use clap::{Args, Parser, Subcommand};
 
 #[derive(Parser)]
 #[command(
@@ -17,7 +19,7 @@ pub struct Root {
 #[derive(Subcommand, Clone)]
 pub enum CommandScopes {
     /// Start the Holo Hosting Agent Daemon.
-    Daemonize,
+    Daemonize(DaemonzeArgs),
     /// Commmands for managing this host.
     Host {
         #[command(subcommand)]
@@ -30,6 +32,21 @@ pub enum CommandScopes {
     },
 }
 
+#[derive(Args, Clone, Debug, Default)]
+pub struct DaemonzeArgs {
+    #[arg(help = "directory to contain the NATS persistence")]
+    pub(crate) store_dir: Option<PathBuf>,
+
+    #[arg(help = "path to NATS credentials used for the LeafNode client connection")]
+    pub(crate) nats_leafnode_client_creds_path: Option<PathBuf>,
+
+    #[arg(
+        help = "try to connect to the (internally spawned) Nats instance for the given duration in seconds before giving up",
+        default_value = "30"
+    )]
+    pub(crate) nats_connect_timeout_secs: u64,
+}
+
 /// A set of commands for being able to manage the local host. We may (later) want to gate some
 /// of these behind a global `--advanced` option to deter hosters from certain commands, but in the
 /// meantime, everything is safe to leave open.
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/gen_leaf_server.rs
index 718267b..b28a175 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/gen_leaf_server.rs
@@ -1,25 +1,46 @@
-use util_libs::nats_server::{self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions};
+use std::path::PathBuf;
 
-const LEAF_SERVE_NAME: &str = "test_leaf_server";
-const LEAF_SERVER_CONFIG_PATH: &str = "test_leaf_server.conf";
+use anyhow::Context;
+use tempfile::tempdir;
+use util_libs::nats_server::{
+    self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
+    LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
+};
 
-pub async fn run(user_creds_path: &str) {
+pub async fn run(
+    user_creds_path: &Option<PathBuf>,
+    maybe_store_dir: &Option<PathBuf>,
+) -> anyhow::Result<()> {
     let leaf_server_remote_conn_url = nats_server::get_hub_server_url();
     let leaf_client_conn_domain = "127.0.0.1";
-    let leaf_client_conn_port = 4111;
+    let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
+        .map(|var| var.parse().expect("can't parse into number"))
+        .unwrap_or_else(|_| LEAF_SERVER_DEFAULT_LISTEN_PORT);
 
-    let nsc_path =
-        std::env::var("NSC_PATH").unwrap_or_else(|_| ".local/share/nats/nsc".to_string());
+    let (
+        store_dir,
+        _, // need to prevent the tempdir from dropping
+    ) = if let Some(store_dir) = maybe_store_dir {
+        std::fs::create_dir_all(store_dir).context("creating {store_dir:?}")?;
+        (store_dir.clone(), None)
+    } else {
+        let maybe_tempfile = tempdir()?;
+        (maybe_tempfile.path().to_owned(), Some(tempdir))
+    };
 
     let jetstream_config = JetStreamConfig {
-        store_dir: format!("{}/leaf_store", nsc_path),
+        store_dir,
+        // TODO: make this configurable
         max_memory_store: 1024 * 1024 * 1024, // 1 GB
-        max_file_store: 1024 * 1024 * 1024,   // 1 GB
+        // TODO: make this configurable
+        max_file_store: 1024 * 1024 * 1024, // 1 GB
     };
 
     let logging_options = LoggingOptions {
+        // TODO: make this configurable
         debug: true, // NB: This logging is a blocking action, only run in non-prod
-        trace: true, // NB: This logging is a blocking action, only run in non-prod
+        // TODO: make this configurable
+        trace: false, // NB: This logging is a blocking action, only run in non-prod
         longtime: false,
     };
 
@@ -27,7 +48,7 @@ pub async fn run(user_creds_path: &str) {
     let leaf_node_remotes = vec![LeafNodeRemote {
         // sys account user (automated)
         url: leaf_server_remote_conn_url.to_string(),
-        credentials_path: Some(user_creds_path.to_string()),
+        credentials_path: user_creds_path.clone(),
     }];
 
     // Create a new Leaf Server instance
@@ -53,4 +74,6 @@ pub async fn run(user_creds_path: &str) {
 
     // Await server task termination
     let _ = leaf_server_task.await;
+
+    Ok(())
 }
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index e68fb58..81c60f8 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -11,6 +11,7 @@ This client is responsible for subscribing the host agent to workload stream end
 */
 
 mod workload_manager;
+use agent_cli::DaemonzeArgs;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
@@ -19,7 +20,6 @@ pub mod gen_leaf_server;
 pub mod host_cmds;
 pub mod support_cmds;
 use thiserror::Error;
-use util_libs::nats_js_client;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -36,9 +36,9 @@ async fn main() -> Result<(), AgentCliError> {
 
     let cli = agent_cli::Root::parse();
     match &cli.scope {
-        Some(agent_cli::CommandScopes::Daemonize) => {
+        Some(agent_cli::CommandScopes::Daemonize(daemonize_args)) => {
             log::info!("Spawning host agent.");
-            daemonize().await?;
+            daemonize(daemonize_args).await?;
         }
         Some(agent_cli::CommandScopes::Host { command }) => host_cmds::host_command(command)?,
         Some(agent_cli::CommandScopes::Support { command }) => {
@@ -46,18 +46,26 @@ async fn main() -> Result<(), AgentCliError> {
         }
         None => {
             log::warn!("No arguments given. Spawning host agent.");
-            daemonize().await?;
+            daemonize(&Default::default()).await?;
         }
     }
 
     Ok(())
 }
 
-async fn daemonize() -> Result<(), async_nats::Error> {
+async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let host_creds_path = nats_js_client::get_nats_client_creds("HOLO", "HPOS", "hpos");
-    let host_pubkey = "host_id_placeholder>";
-    gen_leaf_server::run(&host_creds_path).await;
-    workload_manager::run(host_pubkey, &host_creds_path).await?;
+    let _ = gen_leaf_server::run(&args.nats_leafnode_client_creds_path, &args.store_dir).await;
+
+    let _ = workload_manager::run(
+        "host_id_placeholder>",
+        &args.nats_leafnode_client_creds_path,
+        args.nats_connect_timeout_secs,
+    )
+    .await?;
+
+    // Only exit program when explicitly requested
+    tokio::signal::ctrl_c().await?;
+
     Ok(())
 }
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
index 2d61850..8c1414c 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/workload_manager.rs
@@ -12,25 +12,29 @@
 */
 
 use anyhow::{anyhow, Result};
+use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::{sync::Arc, time::Duration};
+use std::{path::PathBuf, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType,  },
+    nats_js_client::{self, EndpointType},
 };
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
-use async_nats::Message;
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
 const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
 
 // TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
-pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_nats::Error> {
+pub async fn run(
+    host_pubkey: &str,
+    host_creds_path: &Option<PathBuf>,
+    nats_connect_timeout_secs: u64,
+) -> Result<nats_js_client::JsClient, async_nats::Error> {
     log::info!("HPOS Agent Client: Connecting to server...");
-    log::info!("host_creds_path : {}", host_creds_path);
+    log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
     // ==================== NATS Setup ====================
@@ -49,21 +53,41 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
     };
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client =
-        nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-            nats_url,
-            name: HOST_AGENT_CLIENT_NAME.to_string(),
-            inbox_prefix: format!(
-                "{}_{}",
-                HOST_AGENT_INBOX_PREFIX, host_pubkey
-            ),
-            service_params: vec![workload_stream_service_params],
-            credentials_path: Some(host_creds_path.to_string()),
-            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
-            ping_interval: Some(Duration::from_secs(10)),
-            request_timeout: Some(Duration::from_secs(5)),
-        })
-        .await?;
+    // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
+    // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
+    let host_workload_client = tokio::select! {
+        client = async {loop {
+                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                    nats_url: nats_url.clone(),
+                    name: HOST_AGENT_CLIENT_NAME.to_string(),
+                    inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
+                    service_params: vec![workload_stream_service_params.clone()],
+                    credentials_path: host_creds_path
+                        .as_ref()
+                        .map(|path| path.to_string_lossy().to_string()),
+                    opts: vec![nats_js_client::with_event_listeners(event_listeners.clone())],
+                    ping_interval: Some(Duration::from_secs(10)),
+                    request_timeout: Some(Duration::from_secs(29)),
+                })
+                .await
+                .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
+
+                match host_workload_client {
+                    Ok(client) => break client,
+                    Err(e) => {
+                        let duration = tokio::time::Duration::from_millis(100);
+                        log::warn!("{}, retrying in {duration:?}", e);
+                        tokio::time::sleep(duration).await;
+                    }
+                }
+            }} => client,
+        _ = {
+            log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
+            tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
+         } => {
+            return Err(format!("timed out waiting for NATS on {nats_url}").into());
+        }
+    };
 
     // ==================== DB Setup ====================
     // Create a new MongoDB Client and connect it to the cluster
@@ -88,11 +112,9 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .add_local_consumer::<workload::types::ApiResult>(
             "start_workload",
             "start",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
-                async move {
-                    api.start_workload(msg).await
-                }
-            })),
+            EndpointType::Async(workload_api.call(
+                |api: WorkloadApi, msg: Arc<Message>| async move { api.start_workload(msg).await },
+            )),
             None,
         )
         .await?;
@@ -101,11 +123,11 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .add_local_consumer::<workload::types::ApiResult>(
             "send_workload_status",
             "send_status",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
-                async move {
+            EndpointType::Async(
+                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
                     api.send_workload_status(msg).await
-                }
-            })),
+                }),
+            ),
             None,
         )
         .await?;
@@ -114,19 +136,14 @@ pub async fn run(host_pubkey: &str, host_creds_path: &str) -> Result<(), async_n
         .add_local_consumer::<workload::types::ApiResult>(
             "uninstall_workload",
             "uninstall",
-            EndpointType::Async(workload_api.call(|api: WorkloadApi, msg: Arc<Message>| {
-                async move {
+            EndpointType::Async(
+                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
-                }
-            })),
+                }),
+            ),
             None,
         )
         .await?;
 
-    // Only exit program when explicitly requested
-    tokio::signal::ctrl_c().await?;
-
-    // Close client and drain internal buffer before exiting to make sure all messages are sent
-    host_workload_client.close().await?;
-    Ok(())
+    Ok(host_workload_client)
 }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 353d803..3bcc279 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -16,20 +16,25 @@ pub mod types;
 
 use anyhow::{anyhow, Result};
 use async_nats::Message;
+use bson::{self, doc, to_document};
 use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use std::{fmt::Debug, sync::Arc};
-use util_libs::{db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}}, nats_js_client};
 use rand::seq::SliceRandom;
-use std::future::Future;
 use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
+use std::future::Future;
+use std::{fmt::Debug, sync::Arc};
+use util_libs::{
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
+    },
+    nats_js_client,
+};
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
 pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
 
-
 #[derive(Debug, Clone)]
 pub struct WorkloadApi {
     pub workload_collection: MongoCollection<schemas::Workload>,
@@ -40,80 +45,101 @@ pub struct WorkloadApi {
 impl WorkloadApi {
     pub async fn new(client: &MongoDBClient) -> Result<Self> {
         Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME).await?,
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
+                .await?,
             host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
             user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
         })
     }
 
-    pub fn call<F, Fut>(
-        &self,
-        handler: F,
-    ) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    pub fn call<F, Fut>(&self, handler: F) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
     where
         F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
     {
-        let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
-            let api_clone = api.clone();
-            Box::pin(handler(api_clone, msg))
-        })
+        let api = self.to_owned();
+        Arc::new(
+            move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+                let api_clone = api.clone();
+                Box::pin(handler(api_clone, msg))
+            },
+        )
     }
 
     /*******************************  For Orchestrator   *********************************/
     pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Reported,
-            |workload: schemas::Workload| async move {
-                let workload_id = self.workload_collection.insert_one_into(workload.clone()).await?;
-                log::info!("Successfully added workload. MongodDB Workload ID={:?}", workload_id);
-                let updated_workload = schemas::Workload {
-                    _id: Some(workload_id),
-                    ..workload
-                };
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: updated_workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
+        Ok(self
+            .process_request(
+                msg,
+                WorkloadState::Reported,
+                |workload: schemas::Workload| async move {
+                    let workload_id = self
+                        .workload_collection
+                        .insert_one_into(workload.clone())
+                        .await?;
+                    log::info!(
+                        "Successfully added workload. MongodDB Workload ID={:?}",
+                        workload_id
+                    );
+                    let updated_workload = schemas::Workload {
+                        _id: Some(workload_id),
+                        ..workload
+                    };
+                    Ok(types::ApiResult(
+                        WorkloadStatus {
+                            id: updated_workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        None,
+                    ))
+                },
+                WorkloadState::Error,
+            )
+            .await)
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Running,
-            |workload: schemas::Workload| async move {
-                let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload = to_document(&workload)?;
-                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload)).await?;
-                log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-
+        Ok(self
+            .process_request(
+                msg,
+                WorkloadState::Running,
+                |workload: schemas::Workload| async move {
+                    let workload_query = doc! { "_id":  workload._id.clone() };
+                    let updated_workload = to_document(&workload)?;
+                    self.workload_collection
+                        .update_one_within(
+                            workload_query,
+                            UpdateModifications::Document(updated_workload),
+                        )
+                        .await?;
+                    log::info!(
+                        "Successfully updated workload. MongodDB Workload ID={:?}",
+                        workload._id
+                    );
+                    Ok(types::ApiResult(
+                        WorkloadStatus {
+                            id: workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        None,
+                    ))
+                },
+                WorkloadState::Error,
+            )
+            .await)
     }
 
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn remove_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
         Ok(self.process_request(
             msg,
@@ -140,7 +166,10 @@ impl WorkloadApi {
     }
 
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_insertion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
         Ok(self.process_request(
             msg,
@@ -159,7 +188,7 @@ impl WorkloadApi {
                 // todo: check for to ensure assigned host *still* has enough capacity for updated workload
                 if !workload.assigned_hosts.is_empty() {
                     log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok(types::ApiResult( 
+                    return Ok(types::ApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
                         desired: WorkloadState::Assigned,
@@ -170,7 +199,7 @@ impl WorkloadApi {
 
                 // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
                 let host_filter = doc! {
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },      
+                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
                     "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
                     "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk }
                 };
@@ -191,7 +220,7 @@ impl WorkloadApi {
                 // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` feild.
                 // Using `unwrap` is therefore safe.
                 let host_id = host._id.to_owned().unwrap();
-                
+
                 // 4. Update the Workload Collection with the assigned Host ID
                 let workload_query = doc! { "_id":  workload_id.clone() };
                 let updated_workload = &Workload {
@@ -204,7 +233,7 @@ impl WorkloadApi {
                     "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
                     updated_workload_result
                 );
-                
+
                 // 5. Update the Host Collection with the assigned Workload ID
                 let host_query = doc! { "_id":  host.clone()._id };
                 let updated_host_doc =  to_document(&Host {
@@ -216,7 +245,7 @@ impl WorkloadApi {
                     "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
                     updated_host_result
                 );
-     
+
                 Ok(types::ApiResult(
                     WorkloadStatus {
                         id: Some(workload_id),
@@ -233,33 +262,39 @@ impl WorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
-        
+
         let payload_buf = msg.payload.to_vec();
         let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-        
-        // TODO: ...handle the use case for the update entry change stream 
+
+        // TODO: ...handle the use case for the update entry change stream
 
         let success_status = WorkloadStatus {
             id: workload._id,
             desired: WorkloadState::Running,
             actual: WorkloadState::Running,
         };
-        
+
         Ok(types::ApiResult(success_status, None))
     }
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_deletion(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_db_deletion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.delete'");
-        
+
         let payload_buf = msg.payload.to_vec();
         let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-        
+
         // TODO: ...handle the use case for the delete entry change stream
 
         let success_status = WorkloadStatus {
@@ -267,12 +302,15 @@ impl WorkloadApi {
             desired: WorkloadState::Removed,
             actual: WorkloadState::Removed,
         };
-        
+
         Ok(types::ApiResult(success_status, None))
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
 
         let payload_buf = msg.payload.to_vec();
@@ -280,12 +318,15 @@ impl WorkloadApi {
         log::trace!("Workload status to update. Status={:?}", workload_status);
 
         // TODO: ...handle the use case for the workload status update
-        
+
         Ok(types::ApiResult(workload_status, None))
-    }    
+    }
 
-     /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    /*******************************   For Host Agent   *********************************/
+    pub async fn start_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
@@ -304,7 +345,10 @@ impl WorkloadApi {
         Ok(types::ApiResult(status, None))
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
 
         let payload_buf = msg.payload.to_vec();
@@ -325,7 +369,10 @@ impl WorkloadApi {
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<types::ApiResult, anyhow::Error> {
         log::debug!(
             "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
             msg
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index cea7d1d..912ffb6 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,10 +1,13 @@
-use util_libs::{db::schemas::WorkloadStatus, js_stream_service::{CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
+use util_libs::{
+    db::schemas::WorkloadStatus,
+    js_stream_service::{CreateTag, EndpointTraits},
+};
 
 pub use String as WorkloadId;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult (pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct ApiResult(pub WorkloadStatus, pub Option<Vec<String>>);
 
 impl CreateTag for ApiResult {
     fn get_tags(&self) -> Option<Vec<String>> {
@@ -12,4 +15,4 @@ impl CreateTag for ApiResult {
     }
 }
 
-impl EndpointTraits for ApiResult {}
\ No newline at end of file
+impl EndpointTraits for ApiResult {}
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index ecb579a..7709901 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -25,7 +25,11 @@ where
     async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
     async fn insert_one_into(&self, item: T) -> Result<String>;
     async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult>;
+    async fn update_one_within(
+        &self,
+        query: Document,
+        updated_doc: UpdateModifications,
+    ) -> Result<UpdateResult>;
     async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
     async fn delete_all_from(&self) -> Result<DeleteResult>;
 }
@@ -134,7 +138,11 @@ where
         Ok(ids)
     }
 
-    async fn update_one_within(&self, query: Document, updated_doc: UpdateModifications) -> Result<UpdateResult> {
+    async fn update_one_within(
+        &self,
+        query: Document,
+        updated_doc: UpdateModifications,
+    ) -> Result<UpdateResult> {
         self.collection
             .update_one(query, updated_doc)
             .await
@@ -273,7 +281,7 @@ mod tests {
                 remaining_capacity: Capacity {
                     memory: 16,
                     disk: 200,
-                    cores: 16
+                    cores: 16,
                 },
                 avg_uptime: 95,
                 avg_network_speed: 500,
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 0a6ff82..6771d26 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -111,8 +111,8 @@ impl IntoIndexes for Hoster {
 // ==================== Host Schema ====================
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct Capacity {
-    pub memory: i64,  // GiB
-    pub disk: i64,  // ssd; GiB 
+    pub memory: i64, // GiB
+    pub disk: i64,   // ssd; GiB
     pub cores: i64,
 }
 
@@ -157,22 +157,21 @@ pub enum WorkloadState {
     Running,
     Removed,
     Uninstalled,
-    Error(String), // String = error message
+    Error(String),   // String = error message
     Unknown(String), // String = context message
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
-    pub id: Option<String>, 
+    pub id: Option<String>,
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SystemSpecs {
-    pub capacity: Capacity
-    // network_speed: i64
-    // uptime: i64
+    pub capacity: Capacity, // network_speed: i64
+                            // uptime: i64
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -185,7 +184,7 @@ pub struct Workload {
     pub min_hosts: u16,
     pub system_specs: SystemSpecs,
     pub assigned_hosts: Vec<String>, // Host Device IDs (eg: assigned nats server id)
-    // pub status: WorkloadStatus,
+                                     // pub status: WorkloadStatus,
 }
 
 impl Default for Workload {
@@ -210,8 +209,8 @@ impl Default for Workload {
                 capacity: Capacity {
                     memory: 64,
                     disk: 400,
-                    cores: 20
-                }
+                    cores: 20,
+                },
             },
             assigned_hosts: Vec::new(),
         }
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index fce50ea..0860c3b 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -3,10 +3,10 @@ use super::nats_js_client::EndpointType;
 use anyhow::{anyhow, Result};
 use std::any::Any;
 // use async_nats::jetstream::message::Message;
-use async_trait::async_trait;
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
+use async_trait::async_trait;
 use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -20,7 +20,10 @@ pub trait CreateTag: Send + Sync {
     fn get_tags(&self) -> Option<Vec<String>>;
 }
 
-pub trait EndpointTraits:  Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static {}
+pub trait EndpointTraits:
+    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
+{
+}
 
 #[async_trait]
 pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
@@ -46,7 +49,7 @@ where
 }
 
 #[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T> 
+pub struct ConsumerExt<T>
 where
     T: EndpointTraits,
 {
@@ -54,11 +57,11 @@ where
     consumer: PullConsumer,
     handler: EndpointType<T>,
     #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>
+    response_subject_fn: Option<ResponseSubjectsGenerator>,
 }
 
 #[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T> 
+impl<T> ConsumerExtTrait for ConsumerExt<T>
 where
     T: EndpointTraits,
 {
@@ -92,7 +95,7 @@ struct LogInfo {
     endpoint_subject: String,
 }
 
-#[derive(Deserialize, Default)]
+#[derive(Clone, Deserialize, Default)]
 pub struct JsServiceParamsPartial {
     pub name: String,
     pub description: String,
@@ -123,9 +126,9 @@ impl JsStreamService {
         description: &str,
         version: &str,
         service_subject: &str,
-    ) -> Result<Self, async_nats::Error> 
+    ) -> Result<Self, async_nats::Error>
     where
-        Self: 'static
+        Self: 'static,
     {
         let stream = context
             .get_or_create_stream(&stream::Config {
@@ -158,7 +161,10 @@ impl JsStreamService {
         }
     }
 
-    pub async fn get_consumer_stream_info(&self, consumer_name: &str) -> Result<Option<consumer::Info>> {
+    pub async fn get_consumer_stream_info(
+        &self,
+        consumer_name: &str,
+    ) -> Result<Option<consumer::Info>> {
         if let Some(consumer_ext) = self
             .to_owned()
             .local_consumers
@@ -191,9 +197,9 @@ impl JsStreamService {
 
         Ok(ConsumerExt {
             name: consumer_ext.get_name().to_string(),
-            consumer:consumer_ext.get_consumer(),
+            consumer: consumer_ext.get_consumer(),
             handler,
-            response_subject_fn: consumer_ext.get_response()
+            response_subject_fn: consumer_ext.get_response(),
         })
     }
 
@@ -203,7 +209,7 @@ impl JsStreamService {
         endpoint_subject: &str,
         endpoint_type: EndpointType<T>,
         response_subject_fn: Option<ResponseSubjectsGenerator>,
-    ) -> Result<ConsumerExt<T>, async_nats::Error>  
+    ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
@@ -251,19 +257,20 @@ impl JsStreamService {
     pub async fn spawn_consumer_handler<T>(
         &self,
         consumer_name: &str,
-    ) -> Result<(), async_nats::Error> 
+    ) -> Result<(), async_nats::Error>
     where
         T: EndpointTraits,
     {
         if let Some(consumer_ext) = self
-        .to_owned()
-        .local_consumers
-        .write()
-        .await
-        .get_mut(&consumer_name.to_string())
+            .to_owned()
+            .local_consumers
+            .write()
+            .await
+            .get_mut(&consumer_name.to_string())
         {
             let consumer_details = consumer_ext.to_owned();
-            let endpoint_handler: EndpointType<T> = EndpointType::try_from(consumer_details.get_endpoint())?;
+            let endpoint_handler: EndpointType<T> =
+                EndpointType::try_from(consumer_details.get_endpoint())?;
             let maybe_response_generator = consumer_ext.get_response();
             let mut consumer = consumer_details.get_consumer();
             let messages = consumer
@@ -271,22 +278,17 @@ impl JsStreamService {
                 .heartbeat(std::time::Duration::from_secs(10))
                 .messages()
                 .await?;
-            
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
                 endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer
-                .info()
-                .await?
-                .config
-                .filter_subject
-                .clone()
+                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
             };
-            
+
             let service_context = self.js_context.clone();
-            
+
             tokio::spawn(async move {
                 Self::process_messages(
                     log_info,
@@ -331,20 +333,18 @@ impl JsStreamService {
 
             let result = match endpoint_handler {
                 EndpointType::Sync(ref handler) => handler(&js_msg.message),
-                EndpointType::Async(ref handler) => {
-                    handler(Arc::new(js_msg.clone().message)).await
-                }
+                EndpointType::Async(ref handler) => handler(Arc::new(js_msg.clone().message)).await,
             };
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
                     let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
                         Ok(r) => r.into(),
-                        Err(e) => e.to_string().into()
+                        Err(e) => e.to_string().into(),
                     };
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
-                },
+                }
                 Err(err) => (err.to_string().into(), None),
             };
 
@@ -355,7 +355,10 @@ impl JsStreamService {
                     .read()
                     .await
                     .publish(
-                        format!("{}.{}.{}", reply, log_info.service_subject, log_info.endpoint_subject),
+                        format!(
+                            "{}.{}.{}",
+                            reply, log_info.service_subject, log_info.endpoint_subject
+                        ),
                         response_bytes.clone(),
                     )
                     .await
@@ -418,7 +421,6 @@ impl JsStreamService {
             }
         }
     }
-
 }
 
 #[cfg(feature = "tests_integration_nats")]
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 7eb43f1..cbd1fed 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -1,17 +1,18 @@
-use super::js_stream_service::{JsServiceParamsPartial, JsStreamService, CreateTag};
+use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
+use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
+
 use anyhow::Result;
-use async_nats::jetstream;
-use async_nats::{Message, ServerInfo};
+use async_nats::{jetstream, Message, ServerInfo};
 use serde::{Deserialize, Serialize};
-use std::future::Future;
 use std::error::Error;
 use std::fmt;
 use std::fmt::Debug;
+use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type EventListener = Box<dyn Fn(&mut JsClient) + Send + Sync>;
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
 pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
 pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
 pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
@@ -22,7 +23,7 @@ pub type AsyncEndpointHandler<T> = Arc<
 >;
 
 #[derive(Clone)]
-pub enum EndpointType<T> 
+pub enum EndpointType<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
 {
@@ -136,7 +137,11 @@ impl JsClient {
             services.push(service);
         }
 
-        let js_services = if services.is_empty() { None } else { Some(services) };
+        let js_services = if services.is_empty() {
+            None
+        } else {
+            Some(services)
+        };
 
         let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
 
@@ -198,7 +203,7 @@ impl JsClient {
         );
         Ok(())
     }
-    
+
     pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
         Ok(())
     }
@@ -250,33 +255,51 @@ impl JsClient {
 
 // Client Options:
 pub fn with_event_listeners(listeners: Vec<EventListener>) -> EventListener {
-    Box::new(move |c: &mut JsClient | {
+    Arc::new(Box::new(move |c: &mut JsClient| {
         for listener in &listeners {
             listener(c);
         }
-    })
+    }))
 }
 
 // Event Listener Options:
 pub fn on_msg_published_event<F>(f: F) -> EventListener
 where
-        F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
+    F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
-    Box::new(move |c: &mut JsClient| {
+    Arc::new(Box::new(move |c: &mut JsClient| {
         c.on_msg_published_event = Some(Box::pin(f.clone()));
-    })
+    }))
 }
 
 pub fn on_msg_failed_event<F>(f: F) -> EventListener
 where
-        F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
+    F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
-    Box::new(move |c: &mut JsClient| {
+    Arc::new(Box::new(move |c: &mut JsClient| {
         c.on_msg_failed_event = Some(Box::pin(f.clone()));
-    })
+    }))
 }
 
 // Helpers:
+// TODO: there's overlap with the NATS_LISTEN_PORT. refactor this to e.g. read NATS_LISTEN_HOST and NATS_LISTEN_PORT
+pub fn get_nats_url() -> String {
+    std::env::var("NATS_URL").unwrap_or_else(|_| {
+        let default = format!("127.0.0.1:{}", LEAF_SERVER_DEFAULT_LISTEN_PORT);
+        log::debug!("using default for NATS_URL: {default}");
+        default
+    })
+}
+
+pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> String {
+    std::env::var("HOST_CREDS_FILE_PATH").unwrap_or_else(|_| {
+        format!(
+            "/.local/share/nats/nsc/keys/creds/{}/{}/{}.creds",
+            operator, account, user
+        )
+    })
+}
+
 pub fn get_event_listeners() -> Vec<EventListener> {
     // TODO: Use duration in handlers..
     let published_msg_handler = move |msg: &str, client_name: &str, _duration: Duration| {
@@ -298,19 +321,6 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     event_listeners
 }
 
-pub fn get_nats_url() -> String {
-    std::env::var("NATS_URL").unwrap_or_else(|_| "127.0.0.1:4111".to_string())
-}
-
-pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> String {
-    std::env::var("HOST_CREDS_FILE_PATH").unwrap_or_else(|_| {
-        format!(
-            "/.local/share/nats/nsc/keys/creds/{}/{}/{}.creds",
-            operator, account, user
-        )
-    })
-}
-
 #[cfg(feature = "tests_integration_nats")]
 #[cfg(test)]
 mod tests {
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index 47a88f8..abab02d 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -6,13 +6,18 @@ use anyhow::Context;
 use std::fmt::Debug;
 use std::fs::File;
 use std::io::Write;
+use std::path::PathBuf;
 use std::process::{Child, Command, Stdio};
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
+pub const LEAF_SERVE_NAME: &str = "test_leaf_server";
+pub const LEAF_SERVER_CONFIG_PATH: &str = "test_leaf_server.conf";
+pub const LEAF_SERVER_DEFAULT_LISTEN_PORT: u16 = 4111;
+
 #[derive(Debug, Clone)]
 pub struct JetStreamConfig {
-    pub store_dir: String,
+    pub store_dir: PathBuf,
     pub max_memory_store: u64,
     pub max_file_store: u64,
 }
@@ -27,7 +32,7 @@ pub struct LoggingOptions {
 #[derive(Debug, Clone)]
 pub struct LeafNodeRemote {
     pub url: String,
-    pub credentials_path: Option<String>,
+    pub credentials_path: Option<PathBuf>,
 }
 
 #[derive(Debug, Clone)]
@@ -83,25 +88,24 @@ impl LeafServer {
             .leaf_node_remotes
             .iter()
             .map(|remote| {
-                if remote.credentials_path.is_some() {
+                let url = &remote.url;
+                if let Some(credentials_path) = &remote.credentials_path {
+                    let credentials = credentials_path.as_path().as_os_str().to_string_lossy();
                     format!(
                         r#"
         {{
-            url: "{}",
-            credentials: "{}",
+            url: "{url}",
+            credentials: "{credentials}",
         }}
                     "#,
-                        remote.url,
-                        remote.credentials_path.as_ref().unwrap() // Unwrap is safe here as the check for `Some()` wraps this condition
                     )
                 } else {
                     format!(
                         r#"
         {{
-            url: "{}",
+            url: "{url}",
         }}
                     "#,
-                        remote.url
                     )
                 }
             })
@@ -109,11 +113,11 @@ impl LeafServer {
             .join(",\n");
 
         // Write the full config file
-        write!(
-            config_file,
+        // TODO: replace this with a struct that's serialized to JSON
+        let config_str = format!(
             r#"
-server_name: {}
-listen: "{}:{}"
+server_name: "{}",
+listen: "{}:{}",
 
 jetstream {{
     domain: "leaf",
@@ -124,29 +128,33 @@ jetstream {{
 
 leafnodes {{
     remotes = [
-        {}
+        {leafnodes_config}
     ]
 }}
 
-{}
+{logging_config}
 "#,
             self.name,
             self.host,
             self.port,
-            self.jetstream_config.store_dir,
+            self.jetstream_config.store_dir.to_string_lossy(),
             self.jetstream_config.max_memory_store,
             self.jetstream_config.max_file_store,
-            leafnodes_config,
-            logging_config
-        )?;
+        );
+
+        log::trace!("NATS leaf config:\n{config_str}");
+
+        config_file
+            .write_all(config_str.as_bytes())
+            .context("writing config to config at {config_path}")?;
 
         // Run the server with the generated config
         let child = Command::new("nats-server")
             .arg("-c")
             .arg(&self.config_path)
-            // TODO: direct these to a file or make it conditional. this is silenced because it was very verbose
-            .stdout(Stdio::null())
-            .stderr(Stdio::null())
+            // TODO: make this configurable and give options to log it to a seperate log file
+            .stdout(Stdio::inherit())
+            .stderr(Stdio::inherit())
             .spawn()
             .expect("Failed to start NATS server");
 
@@ -425,7 +433,7 @@ mod tests {
             .await
             .expect("Failed to publish jetstream message.");
 
-        // Force shut down the Hub Server (note: leaf server run on port 4111)
+        // Force shut down the Hub Server (note: leaf server run on port LEAF_SERVER_DEFAULT_LISTEN_PORT)
         let test_stream_consumer_name = "test_stream_consumer".to_string();
         let consumer = stream
             .get_or_create_consumer(
@@ -470,6 +478,7 @@ mod tests {
             log::error!("Failed to shut down Leaf Server.  Err:{:#?}", err);
 
             // Force the port to close
+            // TODO(techdebt): use the command child handle to terminate the process.
             Command::new("kill")
                 .arg("-9")
                 .arg(format!("`lsof -t -i:{}`", leaf_client_conn_port))
@@ -480,10 +489,11 @@ mod tests {
         }
         log::info!("Leaf Server has shut down successfully");
 
-        // Force shut down the Hub Server (note: leaf server run on port 4111)
+        // Force shut down the Hub Server (note: leaf server run on port LEAF_SERVER_DEFAULT_LISTEN_PORT)
+        // TODO(techdebt): use the command child handle to terminate the process.
         Command::new("kill")
             .arg("-9")
-            .arg("`lsof -t -i:4111`")
+            .arg(format!("`lsof -t -i:{LEAF_SERVER_DEFAULT_LISTEN_PORT}`"))
             .spawn()
             .expect("Error spawning kill command")
             .wait()
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
index 9342371..ece916b 100644
--- a/rust/util_libs/src/nats_types.rs
+++ b/rust/util_libs/src/nats_types.rs
@@ -2,7 +2,7 @@
 NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
 IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
 TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */ 
+-------- */
 
 use serde::{Deserialize, Serialize};
 

From 40e925fa65d1ccf0cf2f845ae836383562b90801 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Thu, 23 Jan 2025 19:50:59 +0100
Subject: [PATCH 45/91] feat(holo-nats-server): use lib.mkDefault for defaults

otherwise users will require `lib.mkForce` or similar to override
---
 nix/modules/nixos/holo-nats-server.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index a18b268..a3949ee 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -32,12 +32,12 @@ in
 
     services.nats = {
       serverName = lib.mkDefault config.networking.hostName;
-      enable = true;
-      jetstream = true;
+      enable = lib.mkDefault true;
+      jetstream = lib.mkDefault true;
 
       settings = {
-        inherit (cfg) port;
-        leafnodes.port = cfg.leafnodePort;
+        port = lib.mkDefault cfg.port;
+        leafnodes.port = lib.mkDefault cfg.leafnodePort;
       };
     };
   };

From 6be974318384eef17308724ef0f9c3af7cee1d80 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Thu, 23 Jan 2025 20:32:46 +0100
Subject: [PATCH 46/91] feat(nix/modules/nixos): expose blueprint's
 publisherArgs

otherwise it uses `flake` from downstream consumers which will not work
as expected.
---
 nix/modules/nixos/hardware-hetzner-cloud-cpx.nix | 4 ++--
 nix/modules/nixos/hardware-hetzner-cloud.nix     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/nix/modules/nixos/hardware-hetzner-cloud-cpx.nix b/nix/modules/nixos/hardware-hetzner-cloud-cpx.nix
index 65da3ed..2a964dd 100644
--- a/nix/modules/nixos/hardware-hetzner-cloud-cpx.nix
+++ b/nix/modules/nixos/hardware-hetzner-cloud-cpx.nix
@@ -1,6 +1,6 @@
 # This is an opinionated module to configure Hetzner Cloud CPX instances.
-
-{ lib, flake, ... }:
+{ flake, ... }:
+{ lib, ... }:
 {
   imports = [
     flake.nixosModules.hardware-hetzner-cloud
diff --git a/nix/modules/nixos/hardware-hetzner-cloud.nix b/nix/modules/nixos/hardware-hetzner-cloud.nix
index 207d026..ab38dfd 100644
--- a/nix/modules/nixos/hardware-hetzner-cloud.nix
+++ b/nix/modules/nixos/hardware-hetzner-cloud.nix
@@ -1,6 +1,6 @@
 # This is an opinionated module to configure Hetzner Cloud instances.
-
-{ inputs, lib, ... }:
+{ inputs, ... }:
+{ lib, ... }:
 {
   imports = [
     inputs.srvos.nixosModules.server

From adde00c3b9b0149786435bbb99b4791754732a27 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Thu, 23 Jan 2025 20:49:31 +0100
Subject: [PATCH 47/91] feat(niox module holo-nats-server): add openFirewall
 cfg and use correct ports

---
 nix/modules/nixos/holo-nats-server.nix | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index a3949ee..681f0fc 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -22,12 +22,16 @@ in
       type = lib.types.int;
       default = 7422;
     };
+
+    openFirewall = lib.mkOption {
+      default = false;
+    };
   };
 
   config = lib.mkIf cfg.enable {
-    networking.firewall.allowedTCPPorts = [
-      config.services.nats.port
-      config.services.nats.settings.leafnodes.port
+    networking.firewall.allowedTCPPorts = lib.optionals cfg.openFirewall [
+      cfg.port
+      cfg.leafnodePort
     ];
 
     services.nats = {

From 2614ca873a548c558f12a98fbb0a505441ec59a4 Mon Sep 17 00:00:00 2001
From: Lisa Jetton <ldjetton21@gmail.com>
Date: Fri, 24 Jan 2025 13:31:16 -0600
Subject: [PATCH 48/91] Update flake.lock

---
 flake.lock | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flake.lock b/flake.lock
index c393b8c..c32e40f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,7 +16,8 @@
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "steveej-forks",
+        "ref": "fix-checks-import",
         "repo": "blueprint",
         "type": "github"
       }
@@ -429,4 +430,4 @@
   },
   "root": "root",
   "version": 7
-}
\ No newline at end of file
+}

From 7fa741e7b3918549124225cc82b66c7aebdb7f01 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Thu, 23 Jan 2025 20:49:31 +0100
Subject: [PATCH 49/91] feat(nixos module holo-nats-server): configure TLS
 websockets via caddy

primarily this is motivated by TLS encryption. websockets are straight
forward to gate via a reverse TLS proxy like caddy.

as a nice side-effect, external clients and leafnodes can now connect
via the a shared port.
---
 nix/modules/nixos/holo-nats-server.nix | 112 +++++++++++++++++++++----
 1 file changed, 98 insertions(+), 14 deletions(-)

diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index a3949ee..5db4af6 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -1,3 +1,7 @@
+/*
+  Opinionated module to configure a NATS server to be used with other holo-host components.
+  The main use-case for this module will be to host a NATS cluster that is reachable by all hosts.
+*/
 { lib, config, ... }:
 let
   cfg = config.holo.nats-server;
@@ -12,33 +16,113 @@ in
     };
 
     port = lib.mkOption {
-      description = "enable holo NATS server";
+      description = "native client port";
       type = lib.types.int;
       default = 4222;
     };
 
     leafnodePort = lib.mkOption {
-      description = "enable holo NATS server";
+      description = "native leafnode port";
       type = lib.types.int;
       default = 7422;
     };
+
+    websocket = {
+      port = lib.mkOption {
+        description = "websocket listen port";
+        type = lib.types.int;
+        default = 4223;
+      };
+
+      externalPort = lib.mkOption {
+        description = "expected external websocket port";
+        type = lib.types.nullOr lib.types.int;
+        default = 443;
+      };
+
+      openFirewall = lib.mkOption {
+        description = "allow incoming TCP connections to the externalWebsocket port";
+        default = false;
+      };
+    };
+
+    caddy = {
+      enable = lib.mkOption {
+        description = "enable holo NATS server";
+        default = true;
+      };
+      staging = lib.mkOption {
+        type = lib.types.bool;
+        description = "use staging acmeCA for testing purposes. change this in production enviornments.";
+        default = true;
+      };
+      logLevel = lib.mkOption {
+        type = lib.types.str;
+        default = "DEBUG";
+      };
+    };
+
+    extraAttrs = lib.mkOption {
+      description = "extra attributes passed to `services.nats`";
+      default = { };
+    };
   };
 
   config = lib.mkIf cfg.enable {
-    networking.firewall.allowedTCPPorts = [
-      config.services.nats.port
-      config.services.nats.settings.leafnodes.port
+    networking.firewall.allowedTCPPorts =
+      # need port 80 to receive well-known ACME requests
+      lib.optional cfg.caddy.enable 80
+      ++ lib.optional (cfg.websocket.openFirewall || cfg.caddy.enable) cfg.websocket.externalPort;
+
+    services.nats = lib.mkMerge [
+      {
+        serverName = lib.mkDefault config.networking.hostName;
+        enable = lib.mkDefault true;
+        jetstream = lib.mkDefault true;
+
+        settings = {
+          port = lib.mkDefault cfg.port;
+          leafnodes.port = lib.mkDefault cfg.leafnodePort;
+          websocket = {
+            inherit (cfg.websocket) port;
+
+            # TLS will be terminated by the reverse-proxy
+            no_tls = true;
+          };
+        };
+      }
+      cfg.extraAttrs
     ];
 
-    services.nats = {
-      serverName = lib.mkDefault config.networking.hostName;
-      enable = lib.mkDefault true;
-      jetstream = lib.mkDefault true;
+    services.caddy = lib.mkIf cfg.caddy.enable (
+      {
+        enable = true;
+        globalConfig = ''
+          auto_https disable_redirects
+        '';
+        logFormat = ''
+          level DEBUG
+        '';
 
-      settings = {
-        port = lib.mkDefault cfg.port;
-        leafnodes.port = lib.mkDefault cfg.leafnodePort;
-      };
-    };
+        virtualHosts =
+          let
+            maybe_fqdn = builtins.tryEval config.networking.fqdn;
+            domain =
+              if maybe_fqdn.success then
+                maybe_fqdn.value
+              else
+                builtins.trace "WARNING: FQDN is not available, this will most likely lead to an invalid caddy configuration. falling back to hostname" config.networking.hostName;
+
+          in
+          {
+            "https://${domain}:${builtins.toString cfg.websocket.externalPort}".extraConfig = ''
+              reverse_proxy http://127.0.0.1:${builtins.toString cfg.websocket.port}
+            '';
+          };
+      }
+      // lib.attrsets.optionalAttrs cfg.caddy.staging {
+        acmeCA = "https://acme-staging-v02.api.letsencrypt.org/directory";
+      }
+    );
   };
 }

From 95fc6a15bd1e82d32a9a9a23fcd5e525fdd26c10 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Fri, 24 Jan 2025 20:21:36 +0100
Subject: [PATCH 50/91] feat,refactor(host-agent): TLS websocket connection,
 CLI args, config serialization

---
 Cargo.lock                                    |   1 +
 rust/clients/host_agent/src/agent_cli.rs      |  17 ++-
 .../clients/host_agent/src/gen_leaf_server.rs |  15 +-
 rust/clients/host_agent/src/main.rs           |   8 +-
 rust/util_libs/Cargo.toml                     |   1 +
 rust/util_libs/src/nats_server.rs             | 130 ++++++++----------
 6 files changed, 88 insertions(+), 84 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 801c46a..5e2cb44 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2701,6 +2701,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
+ "serde_with",
  "strum",
  "tempfile",
  "thiserror 2.0.8",
diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index 04403f9..4b02b68 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -34,13 +34,26 @@ pub enum CommandScopes {
 
 #[derive(Args, Clone, Debug, Default)]
 pub struct DaemonzeArgs {
-    #[arg(help = "directory to contain the NATS persistence")]
+    #[arg(long, help = "directory to contain the NATS persistence")]
     pub(crate) store_dir: Option<PathBuf>,
 
-    #[arg(help = "path to NATS credentials used for the LeafNode client connection")]
+    #[arg(
+        long,
+        help = "path to NATS credentials used for the LeafNode client connection"
+    )]
     pub(crate) nats_leafnode_client_creds_path: Option<PathBuf>,
 
+    #[arg(long, help = "connection URL to the hub")]
+    pub(crate) hub_url: String,
+
+    #[arg(
+        long,
+        help = "whether to tolerate unknown remote TLS certificates for the connection to the hub"
+    )]
+    pub(crate) hub_tls_insecure: bool,
+
     #[arg(
+        long,
         help = "try to connect to the (internally spawned) Nats instance for the given duration in seconds before giving up",
         default_value = "30"
     )]
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/gen_leaf_server.rs
index b28a175..906946c 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/gen_leaf_server.rs
@@ -3,15 +3,16 @@ use std::path::PathBuf;
 use anyhow::Context;
 use tempfile::tempdir;
 use util_libs::nats_server::{
-    self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
-    LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
+    JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
+    LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
 };
 
 pub async fn run(
     user_creds_path: &Option<PathBuf>,
     maybe_store_dir: &Option<PathBuf>,
+    hub_url: String,
+    hub_tls_insecure: bool,
 ) -> anyhow::Result<()> {
-    let leaf_server_remote_conn_url = nats_server::get_hub_server_url();
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
         .map(|var| var.parse().expect("can't parse into number"))
@@ -47,8 +48,12 @@ pub async fn run(
     // Instantiate the Leaf Server with the user cred file
     let leaf_node_remotes = vec![LeafNodeRemote {
         // sys account user (automated)
-        url: leaf_server_remote_conn_url.to_string(),
-        credentials_path: user_creds_path.clone(),
+        url: hub_url,
+        credentials: user_creds_path.clone(),
+        tls: LeafNodeRemoteTlsConfig {
+            insecure: hub_tls_insecure,
+            ..Default::default()
+        },
     }];
 
     // Create a new Leaf Server instance
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 81c60f8..640b31b 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -55,7 +55,13 @@ async fn main() -> Result<(), AgentCliError> {
 
 async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let _ = gen_leaf_server::run(&args.nats_leafnode_client_creds_path, &args.store_dir).await;
+    let _ = gen_leaf_server::run(
+        &args.nats_leafnode_client_creds_path,
+        &args.store_dir,
+        args.hub_url.clone(),
+        args.hub_tls_insecure,
+    )
+    .await;
 
     let _ = workload_manager::run(
         "host_id_placeholder>",
diff --git a/rust/util_libs/Cargo.toml b/rust/util_libs/Cargo.toml
index 6923ccc..5876122 100644
--- a/rust/util_libs/Cargo.toml
+++ b/rust/util_libs/Cargo.toml
@@ -9,6 +9,7 @@ anyhow = { workspace = true }
 nats-jwt = "0.3.0"
 serde = { workspace = true }
 serde_json = { workspace = true }
+serde_with = { version = "3.1", features = ["macros"] }
 semver = "1.0.24"
 futures = { workspace = true }
 tokio = { workspace = true }
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index abab02d..7f71fa3 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -3,6 +3,8 @@
     NB: This setup expects the `nats-server` binary to be locally installed and accessible.
 -------- */
 use anyhow::Context;
+use serde::Serialize;
+use serde_with::skip_serializing_none;
 use std::fmt::Debug;
 use std::fs::File;
 use std::io::Write;
@@ -15,7 +17,7 @@ pub const LEAF_SERVE_NAME: &str = "test_leaf_server";
 pub const LEAF_SERVER_CONFIG_PATH: &str = "test_leaf_server.conf";
 pub const LEAF_SERVER_DEFAULT_LISTEN_PORT: u16 = 4111;
 
-#[derive(Debug, Clone)]
+#[derive(Serialize, Debug, Clone)]
 pub struct JetStreamConfig {
     pub store_dir: PathBuf,
     pub max_memory_store: u64,
@@ -29,10 +31,27 @@ pub struct LoggingOptions {
     pub longtime: bool,
 }
 
-#[derive(Debug, Clone)]
+#[skip_serializing_none]
+#[derive(Debug, Clone, Serialize)]
 pub struct LeafNodeRemote {
     pub url: String,
-    pub credentials_path: Option<PathBuf>,
+    pub credentials: Option<PathBuf>,
+    pub tls: LeafNodeRemoteTlsConfig,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct LeafNodeRemoteTlsConfig {
+    pub insecure: bool,
+    pub handshake_first: bool,
+}
+
+impl Default for LeafNodeRemoteTlsConfig {
+    fn default() -> Self {
+        Self {
+            insecure: false,
+            handshake_first: true,
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -47,6 +66,24 @@ pub struct LeafServer {
     server_handle: Arc<Mutex<Option<Child>>>,
 }
 
+// TODO: consider merging this with the `LeafServer` struct
+#[derive(Serialize)]
+struct NatsConfig {
+    server_name: String,
+    host: String,
+    port: u16,
+    jetstream: JetStreamConfig,
+    leafnodes: LeafNodes,
+    debug: bool,
+    trace: bool,
+    logtime: bool,
+}
+
+#[derive(Serialize)]
+struct LeafNodes {
+    remotes: Vec<LeafNodeRemote>,
+}
+
 impl LeafServer {
     // Instantiate a new leaf server
     #[allow(clippy::too_many_arguments)]
@@ -75,72 +112,21 @@ impl LeafServer {
     pub async fn run(&self) -> Result<(), Box<dyn std::error::Error>> {
         let mut config_file = File::create(&self.config_path)?;
 
-        // Generate logging options
-        let logging_config = format!(
-            "debug: {}\ntrace: {}\nlogtime: {}\n",
-            self.logging.debug, self.logging.trace, self.logging.longtime
-        );
+        let config = NatsConfig {
+            server_name: self.name.clone(),
+            host: self.host.clone(),
+            port: self.port,
+            jetstream: self.jetstream_config.clone(),
+            leafnodes: LeafNodes {
+                remotes: self.leaf_node_remotes.clone(),
+            },
+
+            debug: self.logging.debug,
+            trace: self.logging.trace,
+            logtime: self.logging.longtime,
+        };
 
-        // Generate the "leafnodes" block
-        // ..and only includes the credentials file whenever the username/password auth is *not* being used
-        // NB: Nats does not allow combining auths for same port.
-        let leafnodes_config = self
-            .leaf_node_remotes
-            .iter()
-            .map(|remote| {
-                let url = &remote.url;
-                if let Some(credentials_path) = &remote.credentials_path {
-                    let credentials = credentials_path.as_path().as_os_str().to_string_lossy();
-                    format!(
-                        r#"
-        {{
-            url: "{url}",
-            credentials: "{credentials}",
-        }}
-                    "#,
-                    )
-                } else {
-                    format!(
-                        r#"
-        {{
-            url: "{url}",
-        }}
-                    "#,
-                    )
-                }
-            })
-            .collect::<Vec<String>>()
-            .join(",\n");
-
-        // Write the full config file
-        // TODO: replace this with a struct that's serialized to JSON
-        let config_str = format!(
-            r#"
-server_name: "{}",
-listen: "{}:{}",
-
-jetstream {{
-    domain: "leaf",
-    store_dir: "{}",
-    max_mem: {},
-    max_file: {}
-}}
-
-leafnodes {{
-    remotes = [
-        {leafnodes_config}
-    ]
-}}
-
-{logging_config}
-"#,
-            self.name,
-            self.host,
-            self.port,
-            self.jetstream_config.store_dir.to_string_lossy(),
-            self.jetstream_config.max_memory_store,
-            self.jetstream_config.max_file_store,
-        );
+        let config_str = serde_json::to_string_pretty(&config)?;
 
         log::trace!("NATS leaf config:\n{config_str}");
 
@@ -189,14 +175,6 @@ leafnodes {{
     }
 }
 
-pub fn get_hub_server_url() -> String {
-    const VAR: &str = "NATS_HUB_SERVER_URL";
-    std::env::var(VAR)
-        .context(format!("reading env var {VAR}"))
-        .unwrap()
-        .to_string()
-}
-
 #[cfg(feature = "tests_integration_nats")]
 #[cfg(test)]
 mod tests {

From 0c29820e151037187a9555e364e3d9d0420b02b6 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Fri, 24 Jan 2025 20:28:29 +0100
Subject: [PATCH 51/91] feat(nixos module holo-host-agent): add hub TLS options
 and add extra args option

---
 nix/modules/nixos/holo-host-agent.nix | 54 +++++++++++++++++++++------
 1 file changed, 43 insertions(+), 11 deletions(-)

diff --git a/nix/modules/nixos/holo-host-agent.nix b/nix/modules/nixos/holo-host-agent.nix
index 383ee14..39833c5 100644
--- a/nix/modules/nixos/holo-host-agent.nix
+++ b/nix/modules/nixos/holo-host-agent.nix
@@ -64,8 +64,20 @@ in
         default = "${cfg.nats.listenHost}:${builtins.toString cfg.nats.listenPort}";
       };
 
-      hubServerUrl = lib.mkOption {
-        type = lib.types.str;
+      hub = {
+        url = lib.mkOption {
+          type = lib.types.str;
+        };
+        tlsInsecure = lib.mkOption {
+          type = lib.types.bool;
+        };
+      };
+
+      extraDaemonizeArgs = lib.mkOption {
+        # forcing everything to be a string because the bool -> str conversion is strange (true -> "1" and false -> "")
+        type = lib.types.attrs;
+        default = {
+        };
       };
     };
   };
@@ -74,14 +86,16 @@ in
     systemd.services.holo-host-agent = {
       enable = true;
 
-      after = [ "network.target" ];
+      after = [
+        "network.target"
+        "network-online.target"
+      ];
       wantedBy = lib.lists.optional cfg.autoStart "multi-user.target";
 
       environment =
         {
           RUST_LOG = cfg.rust.log;
           RUST_BACKTRACE = cfg.rust.backtrace;
-          NATS_HUB_SERVER_URL = cfg.nats.hubServerUrl;
           NATS_LISTEN_PORT = builtins.toString cfg.nats.listenPort;
         }
         // lib.attrsets.optionalAttrs (cfg.nats.url != null) {
@@ -92,13 +106,31 @@ in
         pkgs.nats-server
       ];
 
-      script = builtins.toString (
-        pkgs.writeShellScript "holo-host-agent" ''
-          ${lib.getExe' cfg.package "host_agent"} daemonize
-        ''
-      );
+      script =
+        let
+          extraDaemonizeArgsList = lib.attrsets.mapAttrsToList (
+            name: value:
+            let
+              type = lib.typeOf value;
+            in
+            if type == lib.types.str then
+              "--${name}=${value}"
+            else if (type == lib.types.int || type == lib.types.path) then
+              "--${name}=${builtins.toString value}"
+            else if type == lib.types.bool then
+              (lib.optionalString value name)
+            else
+              throw "don't know how to handle type ${type}"
+          ) cfg.nats.extraDaemonizeArgs;
+        in
+        builtins.toString (
+          pkgs.writeShellScript "holo-host-agent" ''
+            ${lib.getExe' cfg.package "host_agent"} daemonize \
+              --hub-url=${cfg.nats.hub.url} \
+              ${lib.optionalString cfg.nats.hub.tlsInsecure "--hub-tls-insecure"} \
+              ${builtins.concatStringsSep " " extraDaemonizeArgsList}
+          ''
+        );
     };
-
-    # TODO: add nats server here or is it started by the host-agent?
   };
 }

From e0cc2c450a306915cfb76a3cd65e27b88f142618 Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Fri, 24 Jan 2025 20:32:42 +0100
Subject: [PATCH 52/91] test(holo-agent-integration-nixos): adapt to TLS via
 websocket

---
 nix/checks/holo-agent-integration-nixos.nix | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index cc3f39e..5454235 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -16,6 +16,10 @@ pkgs.testers.runNixOSTest (
     name = "host-agent-integration-nixos";
     meta.platforms = lib.lists.intersectLists lib.platforms.linux lib.platforms.x86_64;
 
+    defaults.networking.hosts = {
+      hubIP = [ "${nodes.hub.networking.fqdn}" ];
+    };
+
     nodes.hub =
       { ... }:
       {
@@ -24,6 +28,8 @@ pkgs.testers.runNixOSTest (
           # flake.nixosModules.holo-orchestrator
         ];
 
+        networking.domain = "local";
+
         # holo.orchestrator.enable = true;
         holo.nats-server.enable = true;
         services.nats.settings = {
@@ -66,10 +72,8 @@ pkgs.testers.runNixOSTest (
             backtrace = "trace";
           };
 
-          nats = {
-            # url = "agent:${builtins.toString config.services.nats.port}";
-            hubServerUrl = "nats://${hubIP}:${builtins.toString nodes.hub.holo.nats-server.leafnodePort}";
-          };
+          nats.hub.url = "wss://${nodes.hub.networking.fqdn}:${builtins.toString nodes.hub.holo.nats-server.websocket.externalPort}";
+          nats.hub.tlsInsecure = true;
         };
 
         holo.nats-server.enable = hostUseOsNats;
@@ -202,6 +206,11 @@ pkgs.testers.runNixOSTest (
         with subtest("start the hub and run the testscript"):
           hub.start()
           hub.wait_for_unit("nats.service")
+          hub.wait_for_open_port(port = ${builtins.toString nodes.hub.holo.nats-server.websocket.port}, timeout = 1)
+
+          hub.wait_for_unit("caddy.service")
+          hub.wait_for_open_port(port = ${builtins.toString nodes.hub.holo.nats-server.websocket.externalPort}, timeout = 1)
+
           hub.succeed("${hubTestScript}")
 
         with subtest("starting the host and waiting for holo-host-agent to be ready"):
@@ -213,6 +222,9 @@ pkgs.testers.runNixOSTest (
               "host.wait_for_unit('holo-host-agent')"
           }
 
+          host.wait_for_open_port(addr = "${nodes.hub.networking.fqdn}", port = ${builtins.toString nodes.hub.holo.nats-server.websocket.externalPort}, timeout = 10)
+
+
         with subtest("running the host testscript"):
           host.succeed("${hostTestScript}", timeout = 10)
       '';

From 1ffaa47a47709730dc4733ffc33c13729da9d50b Mon Sep 17 00:00:00 2001
From: Stefan Junker <mail@stefanjunker.de>
Date: Fri, 24 Jan 2025 20:39:38 +0100
Subject: [PATCH 53/91] feat(host-agent/cli): require command

---
 rust/clients/host_agent/src/agent_cli.rs |  4 ++--
 rust/clients/host_agent/src/main.rs      | 12 +++---------
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index 4b02b68..8d8aa77 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -13,7 +13,7 @@ use clap::{Args, Parser, Subcommand};
 )]
 pub struct Root {
     #[command(subcommand)]
-    pub scope: Option<CommandScopes>,
+    pub scope: CommandScopes,
 }
 
 #[derive(Subcommand, Clone)]
@@ -32,7 +32,7 @@ pub enum CommandScopes {
     },
 }
 
-#[derive(Args, Clone, Debug, Default)]
+#[derive(Args, Clone, Debug)]
 pub struct DaemonzeArgs {
     #[arg(long, help = "directory to contain the NATS persistence")]
     pub(crate) store_dir: Option<PathBuf>,
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 640b31b..fb88bc6 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -36,18 +36,12 @@ async fn main() -> Result<(), AgentCliError> {
 
     let cli = agent_cli::Root::parse();
     match &cli.scope {
-        Some(agent_cli::CommandScopes::Daemonize(daemonize_args)) => {
+        agent_cli::CommandScopes::Daemonize(daemonize_args) => {
             log::info!("Spawning host agent.");
             daemonize(daemonize_args).await?;
         }
-        Some(agent_cli::CommandScopes::Host { command }) => host_cmds::host_command(command)?,
-        Some(agent_cli::CommandScopes::Support { command }) => {
-            support_cmds::support_command(command)?
-        }
-        None => {
-            log::warn!("No arguments given. Spawning host agent.");
-            daemonize(&Default::default()).await?;
-        }
+        agent_cli::CommandScopes::Host { command } => host_cmds::host_command(command)?,
+        agent_cli::CommandScopes::Support { command } => support_cmds::support_command(command)?,
     }
 
     Ok(())

From 047af97b9f78b0469816f37310fb2ad21762f136 Mon Sep 17 00:00:00 2001
From: Stefan Junker <1181362+steveej@users.noreply.github.com>
Date: Fri, 24 Jan 2025 21:43:50 +0100
Subject: [PATCH 54/91] feat(host-agent): close NATS client connection before
 exiting the process

Co-authored-by: Lisa Jetton <ldjetton21@gmail.com>
---
 rust/clients/host_agent/src/main.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index fb88bc6..39eef31 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -57,7 +57,7 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     )
     .await;
 
-    let _ = workload_manager::run(
+    let host_client = workload_manager::run(
         "host_id_placeholder>",
         &args.nats_leafnode_client_creds_path,
         args.nats_connect_timeout_secs,
@@ -66,6 +66,7 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
 
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
-
+    
+    host_client.close().await?;
     Ok(())
 }

From 2a175b763594878add96937bc2540b46067633ba Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Fri, 24 Jan 2025 16:09:39 -0600
Subject: [PATCH 55/91] fix lock

---
 flake.lock | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index c32e40f..c393b8c 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,8 +16,7 @@
         "type": "github"
       },
       "original": {
-        "owner": "steveej-forks",
-        "ref": "fix-checks-import",
+        "owner": "numtide",
         "repo": "blueprint",
         "type": "github"
       }
@@ -430,4 +429,4 @@
   },
   "root": "root",
   "version": 7
-}
+}
\ No newline at end of file

From afd694b0785695b221a362de4ee0d3486b9cb663 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Fri, 24 Jan 2025 16:14:23 -0600
Subject: [PATCH 56/91] update flake

---
 flake.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 5ebd332..abf97a6 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,7 +4,7 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.11";
     nixpkgs-unstable.url = "github:NixOS/nixpkgs?ref=nixos-unstable";
-    blueprint.url = "github:steveej-forks/blueprint/fix-checks-import";
+    blueprint.url = "github:numtide/blueprint";
     blueprint.inputs.nixpkgs.follows = "nixpkgs";
     treefmt-nix.url = "github:numtide/treefmt-nix";
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
@@ -33,4 +33,4 @@
       prefix = "nix/";
       nixpkgs.config.allowUnfree = true;
     };
-}
+}
\ No newline at end of file

From c040a206c54360d80388d7956ad8611f04eca28f Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Mon, 27 Jan 2025 12:43:42 -0600
Subject: [PATCH 57/91] clean

---
 .env.example                                  |   2 +-
 rust/services/workload/src/host_api.rs        | 161 +++++++++++++-----
 rust/services/workload/src/lib.rs             |  20 ++-
 .../services/workload/src/orchestrator_api.rs | 118 ++++++++-----
 rust/services/workload/src/types.rs           |  27 ++-
 rust/util_libs/src/js_stream_service.rs       |  17 +-
 6 files changed, 232 insertions(+), 113 deletions(-)

diff --git a/.env.example b/.env.example
index 38d15ca..bfe8157 100644
--- a/.env.example
+++ b/.env.example
@@ -3,4 +3,4 @@ HOST_CREDS_FILE_PATH = "ops/admin.creds"
 MONGO_URI = "mongodb://<host>:<port>"
 NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
+LEAF_SERVER_PW = "pw-123456789"
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index 56b8d84..028b581 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -6,6 +6,8 @@ Endpoints & Managed Subjects:
     - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
 */
 
+use crate::types::WorkloadResult;
+
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
@@ -13,7 +15,7 @@ use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
 use util_libs::{
     nats_js_client::ServiceError,
-    db::schemas::{self, WorkloadState, WorkloadStatus}
+    db::schemas::{WorkloadState, WorkloadStatus}
 };
 
 #[derive(Debug, Clone, Default)]
@@ -23,69 +25,134 @@ impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
     pub async fn start_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Unknown("..".to_string()),
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Running,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
         };
-        Ok(WorkloadApiResult(status, None))
+
+        Ok(WorkloadApiResult  {
+            result: WorkloadResult {
+                status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.update_installed' : {:?}", msg);
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id,
-            desired: WorkloadState::Updating,
-            actual: WorkloadState::Unknown("..".to_string()),
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
         };
-        Ok(WorkloadApiResult(status, None))
+            
+        Ok(WorkloadApiResult  {
+            result: WorkloadResult {
+                status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-        let workload_id = Self::convert_to_type::<String>(msg)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
-        // nix_uninstall_with(workload_id)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: Some(workload_id),
-            desired: WorkloadState::Uninstalled,
-            actual: WorkloadState::Unknown("..".to_string()),
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
+            // nix_uninstall_with(workload_id)
+    
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Error(err_msg),
+            }
         };
-        Ok(WorkloadApiResult(status, None))
+
+        Ok(WorkloadApiResult  {
+            result: WorkloadResult {
+                status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
     pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
-        log::debug!(
-            "Incoming message for 'WORKLOAD.send_status' : {:?}",
-            msg
-        );
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
 
-        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
 
         // Send updated status:
         // NB: This will send the update to both the requester (if one exists)
         // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(WorkloadApiResult(workload_status, None))
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
-}
+}
\ No newline at end of file
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 3508061..9d764b0 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -17,7 +17,7 @@ use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
 use std::future::Future;
 use serde::Deserialize;
-use types::WorkloadApiResult;
+use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
     nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
     db::schemas::{WorkloadState, WorkloadStatus}
@@ -50,13 +50,13 @@ where
         })
     }
 
-    fn convert_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
+    fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Send + Sync,
     {
         let payload_buf = msg.payload.to_vec();
         serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject, e);
+            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject.clone().into_string(), e);
             log::error!("{}", err_msg);
             ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
         })
@@ -77,13 +77,13 @@ where
         Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = Self::convert_to_type::<T>(msg.clone())?;
+        let payload: T = Self::convert_msg_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
         Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
-                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
+                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject.clone().into_string(), payload, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
                     id: None,
@@ -92,8 +92,14 @@ where
                 };
 
                 // 3. return response for stream
-                WorkloadApiResult(status, None)
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None
+                    },
+                    maybe_response_tags: None
+                }
             }
         })
     }
-}
+}
\ No newline at end of file
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index 918f6e2..a49db80 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -8,6 +8,8 @@ Endpoints & Managed Subjects:
     - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
 */
 
+use crate::types::WorkloadResult;
+
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
 use core::option::Option::None;
@@ -55,14 +57,17 @@ impl OrchestratorWorkloadApi {
                     _id: Some(workload_id),
                     ..workload
                 };
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: new_workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: new_workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        workload: None
                     },
-                    None
-                ))
+                    maybe_response_tags: None
+                })
             },
             WorkloadState::Error,
         )
@@ -79,14 +84,17 @@ impl OrchestratorWorkloadApi {
                 let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
                 self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
                 log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: workload._id,
-                        desired: WorkloadState::Reported,
-                        actual: WorkloadState::Reported,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: workload._id,
+                            desired: WorkloadState::Reported,
+                            actual: WorkloadState::Reported,
+                        },
+                        workload: None
                     },
-                    None
-                ))
+                    maybe_response_tags: None
+                })
             },
             WorkloadState::Error,
         )
@@ -106,14 +114,17 @@ impl OrchestratorWorkloadApi {
                     "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
                     workload_id
                 );
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Removed,
-                        actual: WorkloadState::Removed,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Removed,
+                            actual: WorkloadState::Removed,
+                        },
+                        workload: None
                     },
-                    None
-                ))
+                    maybe_response_tags: None
+                })
             },
             WorkloadState::Error,
         )
@@ -144,14 +155,18 @@ impl OrchestratorWorkloadApi {
                     for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
                         tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                     }
-                    return Ok(WorkloadApiResult( 
-                        WorkloadStatus {
-                            id: Some(workload_id),
-                            desired: WorkloadState::Assigned,
-                            actual: WorkloadState::Assigned,
+
+                    return Ok(WorkloadApiResult {
+                        result: WorkloadResult {
+                            status: WorkloadStatus {
+                                id: Some(workload_id),
+                                desired: WorkloadState::Assigned,
+                                actual: WorkloadState::Assigned,
+                            },
+                            workload: None
                         },
-                        Some(tag_map)
-                    ));
+                        maybe_response_tags: Some(tag_map)
+                    });
                 }
 
                 // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
@@ -206,14 +221,17 @@ impl OrchestratorWorkloadApi {
                 for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
                     tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
                 }
-                Ok(WorkloadApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Assigned,
+                            actual: WorkloadState::Assigned,
+                        },
+                        workload: None
                     },
-                    Some(tag_map)
-                ))
+                    maybe_response_tags: Some(tag_map)
+                })
         },
             WorkloadState::Error,
         )
@@ -225,30 +243,44 @@ impl OrchestratorWorkloadApi {
     pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.modify'");
         
-        let workload = Self::convert_to_type::<schemas::Workload>(msg)?;
+        let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
         
         // TODO: ...handle the use case for the update entry change stream 
 
+        // let workload_request_bytes = serde_json::to_vec(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
         let success_status = WorkloadStatus {
-            id: workload._id,
+            id: workload._id.clone(),
             desired: WorkloadState::Running,
             actual: WorkloadState::Running,
         };
-        
-        Ok(WorkloadApiResult(success_status, None))
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: success_status,
+                workload: Some(workload)
+            },
+            maybe_response_tags: None
+        })
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
     pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
-        let workload_status = Self::convert_to_type::<WorkloadStatus>(msg)?;
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
         log::trace!("Workload status to update. Status={:?}", workload_status);
 
         // TODO: ...handle the use case for the workload status update within the orchestrator
-        
-        Ok(WorkloadApiResult(workload_status, None))
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None
+            },
+            maybe_response_tags: None
+        })
     }
 
     // Helper function to initialize mongodb collections
@@ -262,4 +294,4 @@ impl OrchestratorWorkloadApi {
         Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
     }
 
-}
+}
\ No newline at end of file
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index e0a32a5..f68d56e 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,9 +1,6 @@
 use std::collections::HashMap;
+use util_libs::{db::schemas::{self, WorkloadStatus}, js_stream_service::{CreateResponse, CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
-use util_libs::{
-    db::schemas::WorkloadStatus,
-    js_stream_service::{CreateTag, EndpointTraits},
-};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "snake_case")]
@@ -21,10 +18,28 @@ pub enum WorkloadServiceSubjects {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct WorkloadApiResult (pub WorkloadStatus, pub Option<HashMap<String, String>>);
+pub struct WorkloadResult {
+    pub status: WorkloadStatus,
+    pub workload: Option<schemas::Workload>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadApiResult {
+    pub result: WorkloadResult,
+    pub maybe_response_tags: Option<HashMap<String, String>>
+}
 impl EndpointTraits for WorkloadApiResult {}
 impl CreateTag for WorkloadApiResult {
     fn get_tags(&self) -> HashMap<String, String> {
-        self.1.clone().unwrap_or_default()
+        self.maybe_response_tags.clone().unwrap_or_default()
     }
 }
+impl CreateResponse for WorkloadApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        let r = self.result.clone();
+        match serde_json::to_vec(&r) {
+            Ok(r) => r.into(),
+            Err(e) => e.to_string().into(),
+        }
+    }
+}
\ No newline at end of file
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 607e40c..94383ab 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -2,7 +2,6 @@ use super::nats_js_client::EndpointType;
 
 use anyhow::{anyhow, Result};
 use std::any::Any;
-// use async_nats::jetstream::message::Message;
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
@@ -20,11 +19,14 @@ pub trait CreateTag: Send + Sync {
     fn get_tags(&self) -> HashMap<String, String>;
 }
 
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
-{
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
 }
 
+pub trait EndpointTraits:
+    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + CreateResponse + 'static
+{}
+
 #[async_trait]
 pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
     fn get_name(&self) -> &str;
@@ -338,10 +340,7 @@ impl JsStreamService {
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
-                    let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
-                        Ok(r) => r.into(),
-                        Err(e) => e.to_string().into(),
-                    };
+                    let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 },
@@ -545,4 +544,4 @@ mod tests {
         let result = service.spawn_consumer_handler(consumer_name).await;
         assert!(result.is_ok(), "Failed to spawn consumer handler");
     }
-}
+}
\ No newline at end of file

From 57060c3e67d217a0cbf09545363e998b6da47623 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 3 Feb 2025 03:18:53 -0600
Subject: [PATCH 58/91] auth update

---
 .env.example                                  |  15 +-
 Cargo.lock                                    | 708 +++++++++++++++++-
 rust/clients/host_agent/Cargo.toml            |   9 +-
 rust/clients/host_agent/src/agent_cli.rs      |   3 +
 rust/clients/host_agent/src/auth/config.rs    |  58 ++
 rust/clients/host_agent/src/auth/init.rs      | 144 ++++
 .../clients/host_agent/src/auth/init_agent.rs | 186 -----
 rust/clients/host_agent/src/auth/mod.rs       |   3 +-
 rust/clients/host_agent/src/auth/utils.rs     |  56 +-
 .../host_agent/src/hostd/gen_leaf_server.rs   |   7 +-
 .../host_agent/src/hostd/workload_manager.rs  |  15 +-
 rust/clients/host_agent/src/keys.rs           | 182 +++++
 rust/clients/host_agent/src/main.rs           |  29 +-
 rust/clients/orchestrator/src/auth.rs         |  68 +-
 rust/clients/orchestrator/src/workloads.rs    |  16 +-
 rust/services/authentication/src/host_api.rs  | 174 -----
 rust/services/authentication/src/lib.rs       | 218 +++++-
 .../authentication/src/orchestrator_api.rs    | 267 -------
 rust/services/authentication/src/types.rs     |  73 +-
 rust/services/authentication/src/utils.rs     |  10 +-
 rust/util_libs/src/nats_js_client.rs          | 116 +--
 scripts/hosting_agent_setup.sh                |  83 ++
 scripts/hub_cluster_config_setup.sh           |  93 +++
 scripts/hub_seed_config_setup.sh              |  90 +++
 scripts/orchestrator_setup.sh                 | 154 ++--
 25 files changed, 1866 insertions(+), 911 deletions(-)
 create mode 100644 rust/clients/host_agent/src/auth/config.rs
 create mode 100644 rust/clients/host_agent/src/auth/init.rs
 delete mode 100644 rust/clients/host_agent/src/auth/init_agent.rs
 create mode 100644 rust/clients/host_agent/src/keys.rs
 delete mode 100644 rust/services/authentication/src/host_api.rs
 delete mode 100644 rust/services/authentication/src/orchestrator_api.rs
 create mode 100644 scripts/hosting_agent_setup.sh
 create mode 100644 scripts/hub_cluster_config_setup.sh
 create mode 100644 scripts/hub_seed_config_setup.sh
 mode change 100644 => 100755 scripts/orchestrator_setup.sh

diff --git a/.env.example b/.env.example
index e4722cb..3e79f0c 100644
--- a/.env.example
+++ b/.env.example
@@ -1,7 +1,8 @@
-NSC_PATH = "<path/to/nsc_dir>"
-HOST_CREDS_FILE_PATH = "ops/admin.creds"
-MONGO_URI = "mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
-LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
-HOST_CREDENTIALS_PATH: &str = "./host_user.creds";
+NSC_PATH="<path/to/nsc_dir>"
+MONGO_URI="mongodb://<host>:<port>"
+NATS_HUB_SERVER_URL="nats://<path/to/leaf>:<port>"
+LEAF_SERVER_USER="test-user"
+LEAF_SERVER_PW="pw-123456789"
+HOST_CREDENTIALS_PATH="./host_user.creds";
+DEVICE_SEED_DEFAULT_PASSWORD="";
+HPOS_CONFIG_PATH="";
diff --git a/Cargo.lock b/Cargo.lock
index 8072b3a..36c29db 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,12 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
 
+[[package]]
+name = "adler32"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
+
 [[package]]
 name = "ahash"
 version = "0.8.11"
@@ -39,6 +45,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "android-tzdata"
 version = "0.1.1"
@@ -110,12 +122,50 @@ version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
 
+[[package]]
+name = "arbitrary"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
+dependencies = [
+ "derive_arbitrary",
+]
+
+[[package]]
+name = "argon2min"
+version = "0.3.0"
+source = "git+https://github.com/Holo-Host/argon2min?rev=28e765e4369e19bc0126bb46acaacadf1303de22#28e765e4369e19bc0126bb46acaacadf1303de22"
+dependencies = [
+ "blake2-rfc",
+]
+
 [[package]]
 name = "array-init"
 version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
 
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
+[[package]]
+name = "arrayvec"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
+dependencies = [
+ "nodrop",
+]
+
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
 [[package]]
 name = "assert_cmd"
 version = "2.0.16"
@@ -203,6 +253,15 @@ dependencies = [
  "util_libs",
 ]
 
+[[package]]
+name = "autocfg"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
+dependencies = [
+ "autocfg 1.4.0",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.4.0"
@@ -224,6 +283,22 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "base-x"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
+
+[[package]]
+name = "base36"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c26bddc1271f7112e5ec797e8eeba6de2de211c1488e506b9500196dbf77c5"
+dependencies = [
+ "base-x",
+ "failure",
+]
+
 [[package]]
 name = "base64"
 version = "0.12.3"
@@ -302,6 +377,27 @@ dependencies = [
  "wyz",
 ]
 
+[[package]]
+name = "blake2-rfc"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400"
+dependencies = [
+ "arrayvec 0.4.12",
+ "constant_time_eq 0.1.5",
+]
+
+[[package]]
+name = "blake2b_simd"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780"
+dependencies = [
+ "arrayref",
+ "arrayvec 0.7.6",
+ "constant_time_eq 0.3.1",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -441,6 +537,15 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
 
+[[package]]
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "colorchoice"
 version = "1.0.3"
@@ -473,6 +578,18 @@ dependencies = [
  "tiny-keccak",
 ]
 
+[[package]]
+name = "constant_time_eq"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+
+[[package]]
+name = "constant_time_eq"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -505,6 +622,15 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "core2"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.16"
@@ -523,6 +649,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
 [[package]]
 name = "crunchy"
 version = "0.2.2"
@@ -552,6 +684,7 @@ dependencies = [
  "fiat-crypto",
  "rustc_version",
  "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -600,6 +733,12 @@ dependencies = [
  "syn 2.0.96",
 ]
 
+[[package]]
+name = "dary_heap"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728"
+
 [[package]]
 name = "data-encoding"
 version = "2.7.0"
@@ -649,6 +788,17 @@ dependencies = [
  "syn 2.0.96",
 ]
 
+[[package]]
+name = "derive_arbitrary"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.18"
@@ -729,6 +879,8 @@ version = "2.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
 dependencies = [
+ "pkcs8",
+ "serde",
  "signature",
 ]
 
@@ -740,9 +892,11 @@ checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
 dependencies = [
  "curve25519-dalek",
  "ed25519",
+ "serde",
  "sha2",
  "signature",
  "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -802,6 +956,28 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "failure"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
+dependencies = [
+ "backtrace",
+ "failure_derive",
+]
+
+[[package]]
+name = "failure_derive"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "synstructure 0.12.6",
+]
+
 [[package]]
 name = "fastrand"
 version = "2.3.0"
@@ -814,6 +990,18 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
 
+[[package]]
+name = "filetime"
+version = "0.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "libredox",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.35"
@@ -839,6 +1027,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -962,8 +1156,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -984,12 +1180,37 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+dependencies = [
+ "ahash",
+ "allocator-api2",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 
+[[package]]
+name = "hc_seed_bundle"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f930e251000e258ff14c36c4d045c9ec1dcbf2a6fff53b1432342b9a34df5ae"
+dependencies = [
+ "futures",
+ "one_err",
+ "rmp-serde",
+ "rmpv",
+ "serde",
+ "serde_bytes",
+ "sodoken",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.3"
@@ -1005,6 +1226,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -1076,16 +1303,23 @@ dependencies = [
  "bytes",
  "chrono",
  "clap",
+ "data-encoding",
  "dotenv",
+ "ed25519-dalek",
  "env_logger",
  "futures",
+ "hpos-config-core",
+ "hpos-config-seed-bundle-explorer",
  "hpos-hal",
+ "jsonwebtoken",
  "log",
  "mongodb",
+ "nats-jwt",
  "nkeys",
  "rand 0.8.5",
  "serde",
  "serde_json",
+ "sha2",
  "textnonce",
  "thiserror 2.0.11",
  "tokio",
@@ -1105,6 +1339,41 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "hpos-config-core"
+version = "0.2.1"
+source = "git+https://github.com/holo-host/hpos-config.git?rev=77d740c83a02e322e670e360eb450076b593b328#77d740c83a02e322e670e360eb450076b593b328"
+dependencies = [
+ "argon2min",
+ "arrayref",
+ "base36",
+ "base64 0.13.1",
+ "blake2b_simd",
+ "ed25519-dalek",
+ "failure",
+ "lazy_static",
+ "rand 0.6.5",
+ "serde",
+ "url",
+]
+
+[[package]]
+name = "hpos-config-seed-bundle-explorer"
+version = "0.2.1"
+source = "git+https://github.com/holo-host/hpos-config.git?rev=77d740c83a02e322e670e360eb450076b593b328#77d740c83a02e322e670e360eb450076b593b328"
+dependencies = [
+ "base36",
+ "base64 0.13.1",
+ "ed25519-dalek",
+ "hc_seed_bundle",
+ "hpos-config-core",
+ "one_err",
+ "rmp-serde",
+ "serde_json",
+ "sodoken",
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "hpos-hal"
 version = "0.1.0"
@@ -1321,7 +1590,7 @@ version = "1.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
- "autocfg",
+ "autocfg 1.4.0",
  "hashbrown 0.12.3",
  "serde",
 ]
@@ -1377,12 +1646,85 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonwebtoken"
+version = "9.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f"
+dependencies = [
+ "base64 0.21.7",
+ "js-sys",
+ "pem",
+ "ring",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "libc"
 version = "0.2.169"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
 
+[[package]]
+name = "libflate"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e"
+dependencies = [
+ "adler32",
+ "core2",
+ "crc32fast",
+ "dary_heap",
+ "libflate_lz77",
+]
+
+[[package]]
+name = "libflate_lz77"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d"
+dependencies = [
+ "core2",
+ "hashbrown 0.14.5",
+ "rle-decode-fast",
+]
+
+[[package]]
+name = "libredox"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
+dependencies = [
+ "bitflags 2.8.0",
+ "libc",
+ "redox_syscall",
+]
+
+[[package]]
+name = "libsodium-sys-stable"
+version = "1.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7717550bb3ec725f7b312848902d1534f332379b1d575d2347ec265c8814566"
+dependencies = [
+ "cc",
+ "libc",
+ "libflate",
+ "minisign-verify",
+ "pkg-config",
+ "tar",
+ "ureq",
+ "vcpkg",
+ "zip",
+]
+
 [[package]]
 name = "linked-hash-map"
 version = "0.5.6"
@@ -1407,10 +1749,16 @@ version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
 dependencies = [
- "autocfg",
+ "autocfg 1.4.0",
  "scopeguard",
 ]
 
+[[package]]
+name = "lockfree-object-pool"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
+
 [[package]]
 name = "log"
 version = "0.4.25"
@@ -1496,6 +1844,12 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "minisign-verify"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6367d84fb54d4242af283086402907277715b8fe46976963af5ebf173f8efba3"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.3"
@@ -1605,6 +1959,12 @@ dependencies = [
  "signatory",
 ]
 
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
 [[package]]
 name = "nuid"
 version = "0.5.0"
@@ -1614,19 +1974,48 @@ dependencies = [
  "rand 0.8.5",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 dependencies = [
- "autocfg",
+ "autocfg 1.4.0",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi",
+ "libc",
 ]
 
 [[package]]
@@ -1644,6 +2033,18 @@ version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
+[[package]]
+name = "one_err"
+version = "0.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e81851974d8bb6cc9a643cca68afdce7f0a3b80e08a4620388836bb99a680554"
+dependencies = [
+ "indexmap 1.9.3",
+ "libc",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
@@ -1705,6 +2106,12 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "pbkdf2"
 version = "0.11.0"
@@ -1714,6 +2121,16 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "pem"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae"
+dependencies = [
+ "base64 0.22.1",
+ "serde",
+]
+
 [[package]]
 name = "pem-rfc7468"
 version = "0.7.0"
@@ -1771,6 +2188,12 @@ dependencies = [
  "spki",
 ]
 
+[[package]]
+name = "pkg-config"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
+
 [[package]]
 name = "portable-atomic"
 version = "1.10.0"
@@ -1874,6 +2297,25 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "rand"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+dependencies = [
+ "autocfg 0.1.8",
+ "libc",
+ "rand_chacha 0.1.1",
+ "rand_core 0.4.2",
+ "rand_hc 0.1.0",
+ "rand_isaac",
+ "rand_jitter",
+ "rand_os",
+ "rand_pcg",
+ "rand_xorshift",
+ "winapi",
+]
+
 [[package]]
 name = "rand"
 version = "0.7.3"
@@ -1884,7 +2326,7 @@ dependencies = [
  "libc",
  "rand_chacha 0.2.2",
  "rand_core 0.5.1",
- "rand_hc",
+ "rand_hc 0.2.0",
 ]
 
 [[package]]
@@ -1898,6 +2340,16 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+dependencies = [
+ "autocfg 0.1.8",
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.2.2"
@@ -1918,6 +2370,21 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
 [[package]]
 name = "rand_core"
 version = "0.5.1"
@@ -1936,6 +2403,15 @@ dependencies = [
  "getrandom 0.2.15",
 ]
 
+[[package]]
+name = "rand_hc"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.2.0"
@@ -1945,6 +2421,68 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_isaac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rand_jitter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
+dependencies = [
+ "libc",
+ "rand_core 0.4.2",
+ "winapi",
+]
+
+[[package]]
+name = "rand_os"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
+dependencies = [
+ "cloudabi",
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.4.2",
+ "rdrand",
+ "winapi",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
+dependencies = [
+ "autocfg 0.1.8",
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.8"
@@ -2008,6 +2546,46 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rle-decode-fast"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
+
+[[package]]
+name = "rmp"
+version = "0.8.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4"
+dependencies = [
+ "byteorder",
+ "num-traits",
+ "paste",
+]
+
+[[package]]
+name = "rmp-serde"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db"
+dependencies = [
+ "byteorder",
+ "rmp",
+ "serde",
+]
+
+[[package]]
+name = "rmpv"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58450723cd9ee93273ce44a20b6ec4efe17f8ed2e3631474387bfdecf18bb2a9"
+dependencies = [
+ "num-traits",
+ "rmp",
+ "serde",
+ "serde_bytes",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.24"
@@ -2385,13 +2963,31 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "simd-adler32"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
+
+[[package]]
+name = "simple_asn1"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
+dependencies = [
+ "num-bigint",
+ "num-traits",
+ "thiserror 2.0.11",
+ "time",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
 dependencies = [
- "autocfg",
+ "autocfg 1.4.0",
 ]
 
 [[package]]
@@ -2410,6 +3006,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "sodoken"
+version = "0.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "907e0ea9699b846c2586ea5685e9abf5963fca64a5179a406e6ac02b94564e30"
+dependencies = [
+ "libc",
+ "libsodium-sys-stable",
+ "num_cpus",
+ "once_cell",
+ "one_err",
+ "parking_lot",
+ "tokio",
+]
+
 [[package]]
 name = "spin"
 version = "0.9.8"
@@ -2483,6 +3094,18 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "unicode-xid",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.13.1"
@@ -2506,6 +3129,17 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
+[[package]]
+name = "tar"
+version = "0.4.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.15.0"
@@ -2866,6 +3500,18 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
+[[package]]
+name = "ureq"
+version = "2.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
+dependencies = [
+ "base64 0.22.1",
+ "log",
+ "once_cell",
+ "url",
+]
+
 [[package]]
 name = "url"
 version = "2.5.4"
@@ -2935,6 +3581,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -3268,6 +3920,17 @@ dependencies = [
  "tap",
 ]
 
+[[package]]
+name = "xattr"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909"
+dependencies = [
+ "libc",
+ "linux-raw-sys",
+ "rustix",
+]
+
 [[package]]
 name = "yoke"
 version = "0.7.5"
@@ -3289,7 +3952,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.96",
- "synstructure",
+ "synstructure 0.13.1",
 ]
 
 [[package]]
@@ -3331,7 +3994,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.96",
- "synstructure",
+ "synstructure 0.13.1",
 ]
 
 [[package]]
@@ -3361,3 +4024,34 @@ dependencies = [
  "quote",
  "syn 2.0.96",
 ]
+
+[[package]]
+name = "zip"
+version = "2.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45"
+dependencies = [
+ "arbitrary",
+ "crc32fast",
+ "crossbeam-utils",
+ "displaydoc",
+ "flate2",
+ "indexmap 2.7.1",
+ "memchr",
+ "thiserror 2.0.11",
+ "zopfli",
+]
+
+[[package]]
+name = "zopfli"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "lockfree-object-pool",
+ "log",
+ "once_cell",
+ "simd-adler32",
+]
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 4893c9b..cffa774 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -17,13 +17,20 @@ thiserror = { workspace = true }
 env_logger = { workspace = true }
 url = { version = "2", features = ["serde"] }
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
+ed25519-dalek = { version = "2.1.1" }
+nkeys = "=0.4.4"
+sha2 = "=0.10.8"
+nats-jwt = "0.3.0"
+data-encoding = "2.7.0"
+jsonwebtoken = "9.3.0"
 textnonce = "1.0.0"
 chrono = "0.4.0"
 mongodb = "3.1"
 bytes = "1.8.0"
-nkeys = "=0.4.4"
 rand = "0.8.5"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 authentication = { path = "../../services/authentication" }
 hpos-hal = { path = "../../hpos-hal" }
+hpos-config-core = { git = "https://github.com/holo-host/hpos-config.git", rev = "77d740c83a02e322e670e360eb450076b593b328" }
+hpos-config-seed-bundle-explorer = { git = "https://github.com/holo-host/hpos-config.git", rev = "77d740c83a02e322e670e360eb450076b593b328" }
diff --git a/rust/clients/host_agent/src/agent_cli.rs b/rust/clients/host_agent/src/agent_cli.rs
index 080480f..04a51c7 100644
--- a/rust/clients/host_agent/src/agent_cli.rs
+++ b/rust/clients/host_agent/src/agent_cli.rs
@@ -37,6 +37,9 @@ pub struct DaemonzeArgs {
     #[arg(help = "path to NATS credentials used for the LeafNode client connection")]
     pub(crate) nats_leafnode_client_creds_path: Option<PathBuf>,
 
+    #[arg(help = "path to NATS credentials used for the LeafNode SYS user management")]
+    pub(crate) nats_leafnode_client_sys_creds_path: Option<PathBuf>,
+
     #[arg(
         help = "try to connect to the (internally spawned) Nats instance for the given duration in seconds before giving up",
         default_value = "10"
diff --git a/rust/clients/host_agent/src/auth/config.rs b/rust/clients/host_agent/src/auth/config.rs
new file mode 100644
index 0000000..6591180
--- /dev/null
+++ b/rust/clients/host_agent/src/auth/config.rs
@@ -0,0 +1,58 @@
+use anyhow::{anyhow, Context, Result};
+use ed25519_dalek::*;
+use hpos_config_core::public_key;
+use hpos_config_core::Config;
+use hpos_config_seed_bundle_explorer::unlock;
+use std::env;
+use std::fs::File;
+
+pub struct HosterConfig {
+    pub email: String,
+    keypair: SigningKey,
+    pub hc_pubkey: String,
+    pub holoport_id: String,
+}
+
+impl HosterConfig {
+    pub async fn new() -> Result<Self> {
+        let (keypair, email) = get_from_config().await?;
+        let hc_pubkey = public_key::to_holochain_encoded_agent_key(&keypair.verifying_key());
+        let holoport_id = public_key::to_base36_id(&keypair.verifying_key());
+
+        Ok(Self {
+            email,
+            keypair,
+            hc_pubkey,
+            holoport_id,
+        })
+    }
+}
+
+async fn get_from_config() -> Result<(SigningKey, String)> {
+    let config_path =
+        env::var("HPOS_CONFIG_PATH").context("Cannot read HPOS_CONFIG_PATH from env var")?;
+
+    let password = env::var("DEVICE_SEED_DEFAULT_PASSWORD")
+        .context("Cannot read bundle password from env var")?;
+
+    let config_file =
+        File::open(&config_path).context(format!("Failed to open config file {}", config_path))?;
+
+    match serde_json::from_reader(config_file)? {
+        Config::V2 {
+            device_bundle,
+            settings,
+            ..
+        } => {
+            // take in password
+            let signing_key = unlock(&device_bundle, Some(password))
+                .await
+                .context(format!(
+                    "unable to unlock the device bundle from {}",
+                    &config_path
+                ))?;
+            Ok((signing_key, settings.admin.email))
+        }
+        _ => Err(anyhow!("Unsupported version of hpos config")),
+    }
+}
diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
new file mode 100644
index 0000000..7296c61
--- /dev/null
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -0,0 +1,144 @@
+/*
+This client is associated with the:
+    - ADMIN account
+    - auth guard user
+
+Nb: Once the host and hoster are validated, and the host creds file is created,
+...this client should close and the hostd workload manager should spin up.
+
+This client is responsible for:
+    - generating new key for host / and accessing hoster key from provided config file
+    - registering with the host auth service to:
+        - get hub operator jwt and hub sys account jwt
+        - send "nkey" version of host pubkey as file to hub
+        - get user jwt from hub and create user creds file with provided file path
+    - publishing to `auth.start` to initilize the auth handshake and validate the host/hoster
+    - returning the host pubkey and closing client cleanly
+*/
+
+use crate::{auth::config::HosterConfig, keys::Keys};
+use anyhow::Result;
+use std::str::FromStr;
+use async_nats::{HeaderMap, HeaderName, HeaderValue};
+use authentication::{
+    types::{AuthApiResult, AuthRequestPayload, AuthGuardPayload}, utils::{handle_internal_err, write_file} // , AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION
+};
+use std::time::Duration;
+use textnonce::TextNonce;
+use util_libs:: nats_js_client::{self, get_nats_creds_by_nsc, get_file_path_buf, Credentials};
+
+pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
+pub const HOST_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX";
+
+pub async fn run(mut host_agent_keys: Keys) -> Result<Keys, async_nats::Error> {
+    log::info!("Host Auth Client: Connecting to server...");
+
+    // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host =============================================
+    // Fetch Hoster Pubkey and email (from config)
+    let config = HosterConfig::new().await?;
+
+    let secret_token = TextNonce::new().to_string();
+    let unique_inbox = format!("{}.{}", HOST_AUTH_CLIENT_INBOX_PREFIX, secret_token);
+    println!(">>> unique_inbox : {}", unique_inbox);
+    let user_unique_auth_subject = format!("AUTH.{}.>", secret_token);
+    println!(">>> user_unique_auth_subject : {}", user_unique_auth_subject);
+
+    let guard_payload = AuthGuardPayload {
+        host_pubkey: host_agent_keys.host_pubkey,
+        email: config.email,
+        hoster_pubkey: config.hc_pubkey,
+        nonce: secret_token
+    };
+
+    let user_auth_json = serde_json::to_string(&guard_payload).expect("Failed to serialize `UserAuthData` into json string");
+    let user_auth_token = crate::utils::json_to_base64(&user_auth_json).expect("Failed to encode user token");
+
+    // Connect to Nats server as auth guard and call NATS AuthCallout
+    let nats_url = nats_js_client::get_nats_url();
+    let event_listeners = nats_js_client::get_event_listeners();
+    let auth_guard_creds = Credentials::Path(get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard")));
+
+    let auth_guard_client = async_nats::ConnectOptions::with_credentials(user_creds)
+        .expect("Failed to parse static creds")
+        .token(user_auth_token)
+        .custom_inbox_prefix(user_unique_inbox)
+        .connect("nats://localhost:4222")
+        .await?;
+
+    println!("User connected to server on port {}.  Connection State: {:#?}", auth_guard_client.server_info().port, auth_guard_client.connection_state());
+
+    let server_node_id = auth_guard_client.server_info().server_id;
+    log::trace!(
+        "Host Auth Client: Retrieved Node ID: {}",
+        server_node_id
+    );
+
+    // ==================== Handle Authenication Results ============================================================
+    let mut auth_inbox_msgs = auth_guard_client.subscribe(user_unique_inbox).await.unwrap();
+
+    tokio::spawn({
+        let auth_inbox_msgs_clone = auth_inbox_msgs.clone();
+        async move {
+            while let Some(msg) = auth_inbox_msgs_clone.next().await {
+                println!("got an AUTH INBOX msg: {:?}", std::str::from_utf8(&msg.clone()).expect("failed to deserialize msg AUTH Inbox msg"));
+                if let AuthApiResult(auth_response) = serde_json::from_slice(msg) {
+                    host_agent_keys = crate::utils::save_host_creds(host_agent_keys, auth_response.host_jwt, auth_response.sys_jwt);
+                    if let Some(reply) = msg.reply {
+                        // Publish the Awk resp to the Orchestrator... (JS)
+                    }
+                    break;
+                };
+            }
+        }
+    });
+
+    let payload = AuthRequestPayload {
+        host_pubkey: host_agent_keys.host_pubkey,
+        sys_pubkey: host_agent_keys.local_sys_pubkey,
+        nonce: secret_token
+    };
+
+    let payload_bytes = serde_json::to_vec(&payload)?;
+    let signature: Vec<u8> = host_user_keys.sign(&payload_bytes)?;
+    let mut headers = HeaderMap::new();
+    headers.insert(HeaderName::from_static("X-Signature"), HeaderValue::from_str(&format!("{:?}",signature))?);
+
+    // let publish_info = nats_js_client::PublishInfo {
+    //     subject: user_unique_auth_subject,
+    //     msg_id: format!("id={}", rand::random::<u8>()),
+    //     data: payload_bytes,
+    //     headers: Some(headers)
+    // };
+    
+    println!(format!("About to send out the {user_unique_auth_subject} message"));
+    let response = auth_guard_client.request_with_headers(user_unique_auth_subject, headers, payload_bytes).await.expect(&format!("Failed to make {user_unique_auth_subject} request"));
+    println!("got an AUTH response: {:?}", std::str::from_utf8(&response.payload).expect("failed to deserialize msg response"));
+    
+    match serde_json::from_slice::<AuthApiResult>(&response.payload) {
+        Ok(r) => {
+            host_agent_keys = crate::utils::save_host_creds(host_agent_keys, auth_response.host_jwt, auth_response.sys_jwt);
+            
+            if let Some(reply) = msg.reply {
+                // Publish the Awk resp to the Orchestrator... (JS)
+            }
+        },
+        Err(e) => {
+            // TODO:
+            // Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject at regular intervals
+            // for a set period of time, then exit loop and initiate auth connection...
+            let payload = "hpos-hal.info()".as_bytes();
+            let mut auth_inbox_msgs = auth_guard_client.publish("DIAGNOSTICS.ERROR", payload).await.unwrap();
+        }
+    };
+
+    // Close and drain internal buffer before exiting to make sure all messages are sent
+    auth_guard_client.close().await?;
+
+    log::trace!(
+        "host_agent_keys: {}", host_agent_keys
+    );
+
+    Ok(host_agent_keys)
+}
+
+
diff --git a/rust/clients/host_agent/src/auth/init_agent.rs b/rust/clients/host_agent/src/auth/init_agent.rs
deleted file mode 100644
index a7c8a50..0000000
--- a/rust/clients/host_agent/src/auth/init_agent.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
-This client is associated with the:
-    - AUTH account
-    - noauth user
-
-Nb: Once the host and hoster are validated, and the host creds file is created,
-...this client should close and the hostd workload manager should spin up.
-
-This client is responsible for:
-    - generating new key for host / and accessing hoster key from provided config file
-    - registering with the host auth service to:
-        - get hub operator jwt and hub sys account jwt
-        - send "nkey" version of host pubkey as file to hub
-        - get user jwt from hub and create user creds file with provided file path
-    - publishing to `auth.start` to initilize the auth handshake and validate the host/hoster
-    - returning the host pubkey and closing client cleanly
-*/
-
-use anyhow::{anyhow, Result};
-use nkeys::KeyPair;
-use std::str::FromStr;
-use async_nats::{HeaderMap, HeaderName, HeaderValue, Message};
-use authentication::{types::{AuthServiceSubjects, AuthRequestPayload, AuthApiResult}, AuthServiceApi, host_api::HostAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
-use core::option::Option::{None, Some};
-use std::{collections::HashMap, sync::Arc, time::Duration};
-use textnonce::TextNonce;
-use util_libs::{
-    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{self, get_nats_client_creds, EndpointType},
-};
-
-pub const HOST_INIT_CLIENT_NAME: &str = "Host Auth";
-pub const HOST_INIT_CLIENT_INBOX_PREFIX: &str = "_host_auth_inbox";
-
-pub fn create_callback_subject_to_orchestrator(sub_subject_name: String) -> ResponseSubjectsGenerator {
-    Arc::new(move |_: HashMap<String, String>| -> Vec<String> {
-        vec![format!("{}", sub_subject_name)]
-    })
-}
-
-pub async fn run() -> Result<String, async_nats::Error> {
-    log::info!("Host Auth Client: Connecting to server...");
-    // ==================== Setup NATS ============================================================
-    // Connect to Nats server
-    let nats_url = nats_js_client::get_nats_url();
-    let event_listeners = nats_js_client::get_event_listeners();
-
-    // Setup JS Stream Service
-    let auth_stream_service_params = JsServiceParamsPartial {
-        name: AUTH_SRV_NAME.to_string(),
-        description: AUTH_SRV_DESC.to_string(),
-        version: AUTH_SRV_VERSION.to_string(),
-        service_subject: AUTH_SRV_SUBJ.to_string(),
-    };
-
-    let host_auth_client =
-        nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-            nats_url,
-            name: HOST_INIT_CLIENT_NAME.to_string(),
-            inbox_prefix: HOST_INIT_CLIENT_INBOX_PREFIX.to_string(),
-            service_params: vec![auth_stream_service_params],
-            credentials_path: None,
-            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
-            ping_interval: Some(Duration::from_secs(10)),
-            request_timeout: Some(Duration::from_secs(5)),
-        })
-        .await?;
-    
-    // ==================== Report Host to Orchestator ============================================
-    // Generate Host Pubkey && Fetch Hoster Pubkey (from config)..
-    // NB: This nkey keypair is a `ed25519_dalek::VerifyingKey` that is `BASE_32` encoded and returned as a String.
-    let host_user_keys = KeyPair::new_user();
-    let host_pubkey = host_user_keys.public_key();
-    // QUESTION: Where is this nkey file saved?
-    
-    // Discover the server Node ID via INFO response
-    let server_node_id = host_auth_client.get_server_info().server_id;
-    log::trace!(
-        "Host Auth Client: Retrieved Node ID: {}",
-        server_node_id
-    );
-    
-    // Publish a message with the Node ID as part of the subject
-    let publish_options = nats_js_client::PublishInfo {
-        subject: format!("HPOS.init.{}", server_node_id),
-        msg_id: format!("hpos_init_mid_{}", rand::random::<u8>()),
-        data: b"Host Auth Connected!".to_vec(),
-        headers: None
-    };
-    
-    match host_auth_client
-    .publish(publish_options)
-    .await
-    {
-        Ok(_r) => {
-            log::trace!("Host Auth Client: Node ID published.");
-        }
-        Err(_e) => {}
-    };
-    
-    // ==================== Setup API & Register Endpoints ===============================================
-    // Generate the Auth API with access to db
-    let auth_api = HostAuthApi::default();
-    
-    // Register Auth Streams for Orchestrator to consume and proceess
-    // NB: The subjects below are published by the Orchestrator
-    
-    let auth_p1_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP1)?;
-    let auth_p2_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP2)?;
-    let auth_end_subject = serde_json::to_string(&AuthServiceSubjects::EndHandshake)?;
-
-    // Call auth service and perform auth handshake
-    let auth_service = host_auth_client
-        .get_js_service(AUTH_SRV_NAME.to_string())
-        .await
-        .ok_or(anyhow!(
-            "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
-        ))?;
-
-    // Register save service for hub auth files (operator and sys)
-    auth_service
-        .add_consumer::<AuthApiResult>(
-            "save_hub_jwts", // consumer name
-            &format!("{}.{}", host_pubkey, auth_p1_subject), // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
-                async move {
-                    api.save_hub_jwts(msg, &get_nats_client_creds("HOLO", "HPOS", "host")).await
-                }
-            })),
-            Some(create_callback_subject_to_orchestrator(auth_p2_subject)),
-        )
-        .await?;
-        
-    // Register save service for signed user jwt file
-    auth_service
-       .add_consumer::<AuthApiResult>(
-            "save_user_jwt", // consumer name
-            &format!("{}.{}", host_pubkey, auth_end_subject), // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: HostAuthApi, msg: Arc<Message>| {
-                async move {
-                    api.save_user_jwt(msg, &get_nats_client_creds("HOLO", "HPOS", "host")).await
-                }
-            })),
-            None,
-        )
-        .await?;
-
-    // ==================== Publish Initial Auth Req =============================================
-    // Initialize auth handshake with Orchestrator
-    // by calling `AUTH.start_handshake` on the Auth Service
-    let payload = AuthRequestPayload {
-        host_pubkey: host_pubkey.clone(),
-        email: "config.test.email@holo.host".to_string(),
-        hoster_pubkey: "test_pubkey_from_config".to_string(),
-        nonce: TextNonce::new().to_string()
-    };
-
-    let payload_bytes = serde_json::to_vec(&payload)?;
-    let signature: Vec<u8> = host_user_keys.sign(&payload_bytes)?;
-
-    let mut headers = HeaderMap::new();
-    headers.insert(HeaderName::from_static("X-Signature"), HeaderValue::from_str(&format!("{:?}",signature))?);
-
-    let publish_info = nats_js_client::PublishInfo {
-        subject: "AUTH.start_handshake".to_string(),
-        msg_id: format!("id={}", rand::random::<u8>()),
-        data: payload_bytes,
-        headers: Some(headers)
-    };
-    host_auth_client
-        .publish(publish_info)
-        .await?;
-
-    log::trace!(
-        "Init Host Agent Service is running. Waiting for requests..."
-    );
-
-    // ==================== Wait for Host Creds File & Safely Exit Auth Client ==================
-    // Register FILE WATCHER and WAIT FOR the Host Creds File to exist
-    // authentication::utils::get_file_path_buf(&host_creds_path).try_exists()?;
-
-    // Close and drain internal buffer before exiting to make sure all messages are sent
-    host_auth_client.close().await?;
-
-    Ok(host_pubkey)
-}
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
index a155dc2..20b0952 100644
--- a/rust/clients/host_agent/src/auth/mod.rs
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -1,3 +1,4 @@
 // pub mod agent_key;
 pub mod utils;
-pub mod init_agent;
+pub mod init;
+pub mod config;
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index c18f322..5dd7ce5 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,15 +1,55 @@
-use std::process::Command;
+use std::{path::PathBuf, process::Command};
+use util_libs::nats_js_client::{ServiceError, get_file_path_buf};
 
-pub fn _get_host_user_pubkey_path() -> String {
-    std::env::var("HOST_USER_PUBKEY").unwrap_or_else(|_| "./host_user.nk".to_string())
+use crate::keys;
+
+// NB: These should match the names of these files when saved locally upon hpos init
+// (should be the same as those in the `orchestrator_setup` file)
+const JWT_DIR_NAME: &str = "jwt";
+const OPERATOR_JWT_FILE_NAME: &str = "holo_operator";
+const SYS_JWT_FILE_NAME: &str = "sys_account";
+const WORKLOAD_JWT_FILE_NAME: &str = "workload_account";
+
+/// Encode a JSON string into a b64-encoded string
+fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
+    // Parse to ensure it's valid JSON
+    let parsed_json: serde_json::Value = serde_json::from_str(json_data)?;
+    // Convert JSON back to a compact string
+    let json_string = serde_json::to_string(&parsed_json)?;
+    // Encode it into b64
+    let encoded = general_purpose::STANDARD.encode(json_string);
+    Ok(encoded)
 }
 
-pub fn _generate_creds_file() -> String {
-    let user_creds_path = "/path/to/host/user.creds".to_string();
+pub async fn save_host_creds(
+    mut host_agent_keys: keys::Keys,
+    host_user_jwt: String,
+    host_sys_user_jwt: String
+) -> Result<keys::Keys, ServiceError> {
+    //  Save user jwt and sys jwt local to hosting agent
+    utils::write_file(host_user_jwt.as_bytes(), output_dir, "host.jwt").await.map_err(|e| {
+        let err_msg = format!("Failed to save operator jwt. Error={}.", e);
+        handle_internal_err(&err_msg)
+    })?;
+    utils::write_file(host_sys_user_jwt.as_bytes(), output_dir, "host_sys.jwt").await.map_err(|e| {
+        let err_msg = format!("Failed to save sys jwt. Error={}.", e);
+        handle_internal_err(&err_msg)
+    })?;
+
+    // Save user creds and sys creds local to hosting agent
+    let host_creds_file_name = "host.creds";
     Command::new("nsc")
-        .arg(format!("... > {}", user_creds_path))
+        .arg(format!("generate creds --name user_host_{} --account {} > {}", host_pubkey, "WORKLOAD", host_creds_file_name))
         .output()
-        .expect("Failed to add user with provided keys");
+        .expect("Failed to add new operator signing key on hosting agent");
+    
+    let host_sys_creds_file_name = "host_sys.creds";
+    Command::new("nsc")
+        .arg(format!("generate creds --name user_host_{} --account {} > {}", host_sys_pubkey, "SYS", host_sys_creds_file_name))
+        .output()
+        .expect("Failed to add new operator signing key on hosting agent");
+
+    host_agent_keys = host_agent_keys.add_creds_paths(utils::get_file_path_buf(host_creds_file_name), utils::get_file_path_buf(host_sys_creds_file_name));
 
-    "placeholder_user.creds".to_string()
+    Ok(host_agent_keys)
 }
diff --git a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index f5367e2..557a11a 100644
--- a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -1,9 +1,9 @@
 use std::path::PathBuf;
 
-use util_libs::nats_server::{
+use util_libs::{nats_js_client, nats_server::{
     self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
     LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
-};
+}};
 
 pub async fn run(user_creds_path: &Option<PathBuf>) {
     let leaf_server_remote_conn_url = nats_server::get_hub_server_url();
@@ -12,8 +12,7 @@ pub async fn run(user_creds_path: &Option<PathBuf>) {
         .map(|var| var.parse().expect("can't parse into number"))
         .unwrap_or_else(|_| LEAF_SERVER_DEFAULT_LISTEN_PORT);
 
-    let nsc_path =
-        std::env::var("NSC_PATH").unwrap_or_else(|_| ".local/share/nats/nsc".to_string());
+    let nsc_path = nats_js_client::get_nsc_root_path();
 
     let jetstream_config = JetStreamConfig {
         store_dir: format!("{}/leaf_store", nsc_path),
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 7117d4e..cea0706 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -15,7 +15,7 @@ use async_nats::Message;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use util_libs::{
     js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType},
+    nats_js_client::{self, Credentials, EndpointType},
 };
 use workload::{
     WorkloadServiceApi, host_api::HostWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
@@ -23,7 +23,7 @@ use workload::{
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
-const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
+const HOST_AGENT_INBOX_PREFIX: &str = "_workload_inbox";
 
 // TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
 pub async fn run(
@@ -53,6 +53,11 @@ pub async fn run(
     // Spin up Nats Client and loaded in the Js Stream Service
     // Nats takes a moment to become responsive, so we try to connect in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
+    let creds = match host_creds_path.to_owned() {
+        Some(p) => Some(Credentials::Path(p)),
+        _ => None
+    };
+
     let host_workload_client = tokio::select! {
         client = async {loop {
                 let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
@@ -60,12 +65,10 @@ pub async fn run(
                     name: HOST_AGENT_CLIENT_NAME.to_string(),
                     inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
                     service_params: vec![workload_stream_service_params.clone()],
-                    credentials_path: host_creds_path
-                        .as_ref()
-                        .map(|path| path.to_string_lossy().to_string()),
-                    opts: vec![nats_js_client::with_event_listeners(event_listeners.clone())],
+                    credentials: creds.clone(),
                     ping_interval: Some(Duration::from_secs(10)),
                     request_timeout: Some(Duration::from_secs(29)),
+                    listeners: vec![nats_js_client::with_event_listeners(event_listeners.clone())],
                 })
                 .await
                 .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
new file mode 100644
index 0000000..bb6f200
--- /dev/null
+++ b/rust/clients/host_agent/src/keys.rs
@@ -0,0 +1,182 @@
+use anyhow::{anyhow, Context, Result};
+use nkeys::KeyPair;
+use data_encoding::BASE64URL_NOPAD;
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::PathBuf;
+use util_libs::nats_js_client;
+
+impl std::fmt::Debug for Keys {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Keys")
+            .field("host_keypair", &"[redacted]")
+            .field("host_pubkey", &self.host_pubkey)
+            .field("maybe_host_creds_path", &self.maybe_host_creds_path.is_some())
+            .field("local_sys_keypair", if &self.local_sys_keypair.is_some(){&"[redacted]"} else { &false })
+            .field("local_sys_pubkey", &self.local_sys_pubkey)
+            .field("maybe_sys_creds_path", &self.maybe_sys_creds_path.is_some())
+            .finish()
+    }
+}
+
+pub struct Keys {
+    host_keypair: KeyPair,
+    pub host_pubkey: String,
+    maybe_host_creds_path: Option<PathBuf>,
+    local_sys_keypair: Option<KeyPair>,
+    pub local_sys_pubkey: Option<String>,
+    maybe_sys_creds_path: Option<PathBuf>,
+}
+
+impl Keys {
+    pub fn new() -> Result<Self> {
+        // let host_key_path = format!("{}/user_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
+        let host_key_path = std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
+        let host_kp = KeyPair::new_user();
+        write_to_file(nats_js_client::get_file_path_buf(&host_key_path), host_kp.clone());
+        let host_pk = host_kp.public_key();
+        
+        // let sys_key_path = format!("{}/user_sys_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
+        let sys_key_path = std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
+        let local_sys_kp = KeyPair::new_user();
+        write_to_file(nats_js_client::get_file_path_buf(&sys_key_path), local_sys_kp.clone());
+        let local_sys_pk = local_sys_kp.public_key();
+
+        Ok(Self {
+            host_keypair: host_kp,
+            host_pubkey: host_pk,
+            maybe_host_creds_path: None, 
+            local_sys_keypair: Some(local_sys_kp),
+            local_sys_pubkey: Some(local_sys_pk),
+            maybe_sys_creds_path: None,
+        })
+    }
+
+    pub fn try_from_storage(maybe_host_creds_path: &Option<PathBuf>, maybe_sys_creds_path: &Option<PathBuf>) -> Result<Option<Self>> {
+        let host_key_path = std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
+        let host_keypair = try_get_from_file(nats_js_client::get_file_path_buf(&host_key_path.clone()))?.ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
+        let host_pk = host_keypair.public_key();
+        let sys_key_path = std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
+        let host_creds_path = maybe_host_creds_path.to_owned().unwrap_or_else(|| nats_js_client::get_file_path_buf(
+            &nats_js_client::get_nats_creds_by_nsc("HOLO", "HPOS", "host")
+        ));
+        let sys_creds_path = maybe_sys_creds_path.to_owned().unwrap_or_else(|| nats_js_client::get_file_path_buf(
+            &nats_js_client::get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
+        ));
+        let keys = match try_get_from_file(nats_js_client::get_file_path_buf(&sys_key_path))? {
+            Some(kp) => {
+                let local_sys_pk = kp.public_key();
+                Self {
+                    host_keypair,
+                    host_pubkey:host_pk,
+                    maybe_host_creds_path: None,
+                    local_sys_keypair: Some(kp),
+                    local_sys_pubkey: Some(local_sys_pk),
+                    maybe_sys_creds_path: None
+                }
+            },
+            None => {
+                Self {
+                    host_keypair,
+                    host_pubkey: host_pk,
+                    maybe_host_creds_path: None,
+                    local_sys_keypair: None,
+                    local_sys_pubkey: None,
+                    maybe_sys_creds_path: None
+                }
+            }
+        };
+        
+        return Ok(Some(keys.add_creds_paths(host_creds_path, sys_creds_path)?));
+    }
+
+    pub fn add_creds_paths(self, host_creds_file_path: PathBuf, sys_creds_file_path: PathBuf) -> Result<Self> {
+        match host_creds_file_path.try_exists() {
+            Ok(is_ok) => {
+                if !is_ok {
+                    return Err(anyhow!("Failed to locate host creds path. Found broken sym link. Path={:?}", host_creds_file_path));
+                }
+                match sys_creds_file_path.try_exists() {
+                    Ok(is_ok) => {
+                        if !is_ok {
+                            return Err(anyhow!("Failed to locate sys creds path. Found broken sym link. Path={:?}", sys_creds_file_path));
+                        }
+
+                        Ok(Self {
+                            maybe_host_creds_path: Some(host_creds_file_path),
+                            maybe_sys_creds_path: Some(sys_creds_file_path),
+                            ..self    
+                        })
+                    },
+                    Err(e) => Err(anyhow!("Failed to locate sys creds path. Path={:?} Err={}", sys_creds_file_path, e))
+                }
+            },
+            Err(e) => Err(anyhow!("Failed to locate host creds path. Path={:?} Err={}", host_creds_file_path, e))
+        }
+    }
+
+    pub fn add_local_sys(self, sys_key_path: Option<PathBuf>) -> Result<Self> {
+        let sys_key_path = sys_key_path.unwrap_or_else(|| nats_js_client::get_file_path_buf(
+            &nats_js_client::get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
+        ));
+        
+        let local_sys_kp = try_get_from_file(sys_key_path.clone())?.unwrap_or_else(|| {
+            let kp = KeyPair::new_user();
+            write_to_file(sys_key_path, kp);
+            KeyPair::new_user()
+        });
+        let local_sys_pk = local_sys_kp.public_key();
+
+        Ok(Self {
+            local_sys_keypair: Some(local_sys_kp),
+            local_sys_pubkey: Some(local_sys_pk),
+            ..self    
+        })
+    }
+
+    pub fn get_host_creds_path(&self) -> Option<PathBuf> {
+        self.maybe_host_creds_path.clone()
+    }
+
+    pub fn get_sys_creds_path(&self) -> Option<PathBuf> {
+        self.maybe_sys_creds_path.clone()
+    }
+
+    pub fn host_sign(&self, payload: &[u8]) -> Result<String> {
+        let signature = self
+            .host_keypair
+            .sign(payload)?;
+
+        Ok(BASE64URL_NOPAD.encode(&signature))
+    }
+}
+
+fn write_to_file(key_file_path: PathBuf, keypair: KeyPair) -> Result<()> {
+    let seed = keypair.seed()?;
+    let mut file = File::create(&key_file_path)?;
+    file.write_all(seed.as_bytes())?;
+    Ok(())
+}
+
+fn try_get_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
+    match key_file_path.try_exists() {
+        Ok(link_is_ok) => {
+            if !link_is_ok {
+                return Err(anyhow!("Failed to read path {:?}. Found broken sym link.", key_file_path));
+            }
+
+            let mut key_file_content =
+                File::open(&key_file_path).context(format!("Failed to open config file {:#?}", key_file_path))?;
+        
+            let mut kps = String::new();
+            key_file_content.read_to_string(&mut kps)?;
+            let kp = KeyPair::from_seed(&kps.trim())?;
+
+            Ok(Some(kp))
+        }
+        Err(_) => {
+            log::debug!("No user file found at {:?}.", key_file_path);
+            Ok(None)
+        }
+    }
+}
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index d4edaa5..efcec85 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -12,6 +12,7 @@ This client is responsible for subscribing the host agent to workload stream end
 
 mod auth;
 mod hostd;
+mod keys;
 pub mod agent_cli;
 pub mod host_cmds;
 pub mod support_cmds;
@@ -20,6 +21,7 @@ use clap::Parser;
 use dotenv::dotenv;
 use thiserror::Error;
 use agent_cli::DaemonzeArgs;
+use util_libs::nats_js_client;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -53,28 +55,21 @@ async fn main() -> Result<(), AgentCliError> {
     Ok(())
 }
 
-async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
-    let creds_path_arg_clone = args.nats_leafnode_client_creds_path.clone();
-    let host_creds_path = creds_path_arg_clone.unwrap_or_else(|| {
-        authentication::utils::get_file_path_buf(
-            &util_libs::nats_js_client::get_nats_client_creds("HOLO", "HPOS", "host")
-        )
-    });
-    let host_pubkey: String = match host_creds_path.try_exists() {
-        Ok(_p) => {
-            // TODO: read creds file and parse out pubkey OR call nsc to read pubkey from file (whichever is cleaner)
-            "host_pubkey_placeholder>".to_string()
-        },
-        Err(_) => {
+async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {    
+    let host_agent_keys = match keys::Keys::try_from_storage(&args.nats_leafnode_client_creds_path, &args.nats_leafnode_client_sys_creds_path)? {
+        Some(k) => k,
+        None => {
             log::debug!("About to run the Hosting Agent Initialization Service");
-            auth::init_agent::run().await?
+            let mut keys = keys::Keys::new()?;
+            keys = auth::init::run(keys).await?;
+            keys
         }
     };
 
-    hostd::gen_leaf_server::run(&args.nats_leafnode_client_creds_path).await;
+    hostd::gen_leaf_server::run(&host_agent_keys.get_host_creds_path()).await;
     hostd::workload_manager::run(
-        &host_pubkey,
-        &args.nats_leafnode_client_creds_path,
+        &host_agent_keys.host_pubkey,
+        &host_agent_keys.get_host_creds_path(),
         args.nats_connect_timeout_secs,
     )
     .await?;
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 3d7a92a..6aa8544 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -1,31 +1,30 @@
 /*
 This client is associated with the:
-    - AUTH account
+    - ADMIN account
     - orchestrator user
 
 This client is responsible for:
     - initalizing connection and handling interface with db
     - registering with the host auth service to:
-        - handling inital auth requests
+        - handling auth requests by:
             - validating user signature
             - validating hoster pubkey
             - validating hoster email
             - bidirectionally pairing hoster and host
-            - sending hub jwt files back to user (once validated) 
-        - handling request to add user pubkey and generate signed user jwt 
             - interfacing with hub nsc resolver and hub credential files
-            - sending user jwt file back to user
+            - adding user to hub
+            - creating signed jwt for user
+            - adding user jwt file to user collection (with ttl)
     - keeping service running until explicitly cancelled out
 */
 
 use crate::utils as local_utils;
 
-use anyhow::{anyhow, Result};
 use std::{collections::HashMap, sync::Arc, time::Duration};
-// use std::process::Command;
+use anyhow::{anyhow, Result};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use authentication::{self, types::{AuthServiceSubjects, AuthApiResult}, AuthServiceApi, orchestrator_api::OrchestratorAuthApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use authentication::{self, AuthServiceApi, types::{self, AuthApiResult}, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
@@ -33,17 +32,7 @@ use util_libs::{
 };
 
 pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Agent";
-pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_orchestrator_auth_inbox";
-
-pub fn create_callback_subject_to_host(tag_name: String, sub_subject_name: String) -> ResponseSubjectsGenerator {
-    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
-        if let Some(tag) = tag_map.get(&tag_name) {
-            return vec![format!("{}.{}", tag, &sub_subject_name)];
-        }
-        log::error!("Auth Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `AUTH.ERROR.INBOX`.", tag_name, sub_subject_name);
-        vec!["AUTH.ERROR.INBOX".to_string()]
-    })
-}
+pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_auth_inbox_orchestrator";
 
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== Setup NATS ====================
@@ -79,15 +68,9 @@ pub async fn run() -> Result<(), async_nats::Error> {
     
     // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
-    let auth_api = OrchestratorAuthApi::new(&client).await?;
+    let auth_api = AuthServiceApi::new(&client).await?;
     
-    // Register Auth Streams for Orchestrator to consume and proceess
-    // NB: The subjects below are published by the Host Agent
-    let auth_start_subject = serde_json::to_string(&AuthServiceSubjects::StartHandshake)?;
-    let auth_p1_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP1)?;
-    let auth_p2_subject = serde_json::to_string(&AuthServiceSubjects::HandleHandshakeP2)?;
-    let auth_end_subject = serde_json::to_string(&AuthServiceSubjects::EndHandshake)?;
-
+    // Register Auth Stream for Orchestrator to consume and proceess
     let auth_service = orchestrator_auth_client
         .get_js_service(AUTH_SRV_NAME.to_string())
         .await
@@ -97,27 +80,14 @@ pub async fn run() -> Result<(), async_nats::Error> {
     
     auth_service
         .add_consumer::<AuthApiResult>(
-            "start_handshake", // consumer name
-            &auth_start_subject, // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
+            "validate", // consumer name
+            &types::AUTH_SERVICE_SUBJECT, // consumer stream subj
+            EndpointType::Async(auth_api.call(|api: AuthServiceApi, msg: Arc<Message>| {
                 async move {
                     api.handle_handshake_request(msg, &local_utils::get_orchestrator_credentials_dir_path()).await
                 }
             })),
-            Some(create_callback_subject_to_host("host_pubkey".to_string(), auth_p1_subject)),
-        )
-        .await?;
-        
-    auth_service
-        .add_consumer::<AuthApiResult>(
-            "add_user_pubkey", // consumer name
-            &auth_p2_subject, // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: OrchestratorAuthApi, msg: Arc<Message>| {
-                async move {
-                    api.add_user_nkey(msg, &local_utils::get_orchestrator_credentials_dir_path()).await
-                }
-            })),
-            Some(create_callback_subject_to_host("host_pubkey".to_string(), auth_end_subject)),
+            Some(create_callback_subject_to_host("host_pubkey".to_string())),
         )
         .await?;
 
@@ -134,3 +104,13 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     Ok(())
 }
+
+pub fn create_callback_subject_to_host(tag_name: String) -> ResponseSubjectsGenerator {
+    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
+        if let Some(tag) = tag_map.get(&tag_name) {
+            return vec![format!("AUTH.{}", tag)];
+        }
+        log::error!("Auth Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject 'AUTH.validate'. Fwding response to `AUTH.ERROR.INBOX`.", tag_name);
+        vec!["AUTH.ERROR.INBOX".to_string()]
+    })
+}
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index c5bd37e..d458112 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -24,11 +24,11 @@ use workload::{
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
+    nats_js_client::{self, Credentials, EndpointType, JsClient, NewJsClientParams, get_nats_url, get_nats_creds_by_nsc, get_event_listeners, get_file_path_buf},
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
-const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_orchestrator_workload_inbox";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_workload_inbox_orchestrator";
 
 pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_subject_name: String) -> ResponseSubjectsGenerator {
     Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
@@ -50,9 +50,9 @@ pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_su
 
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== Setup NATS ====================
-    let nats_url = nats_js_client::get_nats_url();
-    let creds_path = nats_js_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
-    let event_listeners = nats_js_client::get_event_listeners();
+    let nats_url = get_nats_url();
+    let creds_path = Credentials::Path(get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", "orchestrator")));
+    let event_listeners = get_event_listeners();
 
     // Setup JS Stream Service
     let workload_stream_service_params = JsServiceParamsPartial {
@@ -68,10 +68,10 @@ pub async fn run() -> Result<(), async_nats::Error> {
             name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
             inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
             service_params: vec![workload_stream_service_params],
-            credentials_path: Some(creds_path),
-            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
-            ping_interval: Some(Duration::from_secs(10)),
+            credentials: Some(creds_path),
             request_timeout: Some(Duration::from_secs(5)),
+            ping_interval: Some(Duration::from_secs(10)),
+            listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
         })
         .await?;
 
diff --git a/rust/services/authentication/src/host_api.rs b/rust/services/authentication/src/host_api.rs
deleted file mode 100644
index 592f5a7..0000000
--- a/rust/services/authentication/src/host_api.rs
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
-Endpoints & Managed Subjects:
-    - save_hub_jwts: AUTH.<host_pubkey>.handle_handshake_p1
-    - save_user_jwt: AUTH.<host_pubkey>.end_hub_handshake
-*/
-
-use super::{AuthServiceApi, types, utils};
-use utils::handle_internal_err;
-use anyhow::Result;
-use async_nats::Message;
-use types::{AuthApiResult, AuthResult};
-use core::option::Option::None;
-use std::collections::HashMap;
-use std::process::Command;
-use std::sync::Arc;
-use util_libs::nats_js_client::ServiceError;
-
-#[derive(Debug, Clone, Default)]
-pub struct HostAuthApi {}
-
-impl AuthServiceApi for HostAuthApi {}
-
-impl HostAuthApi {
-    pub async fn save_hub_jwts(
-        &self,
-        msg: Arc<Message>,
-        output_dir: &str
-    ) -> Result<AuthApiResult, ServiceError> {
-        let msg_subject = &msg.subject.clone().into_string(); // AUTH.<host_pubkey>.handle_handshake_p1
-        log::trace!("Incoming message for '{}'", msg_subject);
-
-        // 1. Verify expected payload was received
-        let message_payload = Self::convert_msg_to_type::<AuthResult>(msg.clone())?;
-        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
-
-        let operator_jwt_bytes = message_payload.data.inner.get("operator_jwt").ok_or_else(|| {
-            let err_msg = format!("Error: Failed to find operator jwt in message payload. Subject='{}'.", msg_subject);
-            handle_internal_err(&err_msg)
-        })?;
-
-        let sys_account_jwt_bytes = message_payload.data.inner.get("sys_account_jwt").ok_or_else(|| {
-            let err_msg = format!("Error: Failed to find sys jwt in message payload. Subject='{}'.", msg_subject);
-            handle_internal_err(&err_msg)
-        })?;
-        
-        let workload_account_jwt_bytes = message_payload.data.inner.get("workload_account_jwt").ok_or_else(|| {
-            let err_msg = format!("Error: Failed to find sys jwt in message payload. Subject='{}'.", msg_subject);
-            handle_internal_err(&err_msg)
-        })?;
-
-        // 2. Save operator_jwt, sys_account_jwt, and workload_account_jwt local to hosting agent
-        let operator_jwt_file = utils::receive_and_write_file(operator_jwt_bytes.to_owned(), output_dir, "operator.jwt").await.map_err(|e| {
-            let err_msg = format!("Failed to save operator jwt. Subject='{}' Error={}.", msg_subject, e);
-            handle_internal_err(&err_msg)
-        })?;
-
-        let sys_jwt_file = utils::receive_and_write_file(sys_account_jwt_bytes.to_owned(), output_dir, "account_sys.jwt").await.map_err(|e| {
-            let err_msg = format!("Failed to save sys jwt. Subject='{}' Error={}.", msg_subject, e);
-            handle_internal_err(&err_msg)
-        })?;
-
-        let workload_jwt_file = utils::receive_and_write_file(workload_account_jwt_bytes.to_owned(), output_dir, "account_sys.jwt").await.map_err(|e| {
-            let err_msg = format!("Failed to save sys jwt. Subject='{}' Error={}.", msg_subject, e);
-            handle_internal_err(&err_msg)
-        })?;
-
-        Command::new("nsc")
-            .arg(format!("add operator -u {} --force", operator_jwt_file))
-            .output()
-            .expect("Failed to add operator with provided operator jwt file");
-
-        Command::new("nsc")
-            .arg(format!("add import account --file {}", sys_jwt_file))
-            .output()
-            .expect("Failed to add sys with provided sys jwt file");
-
-        Command::new("nsc")
-            .arg(format!("add import account --file {}", workload_jwt_file))
-            .output()
-            .expect("Failed to add workload account with provided workload jwt file");
-
-        // Command::new("nsc")
-        //     .arg(format!("generate nkey -o --store > operator_sk.nk"))
-        //     .output()
-        //     .expect("Failed to add new operator signing key on hosting agent");
-
-        let host_sys_user_file_name = format!("{}/user_sys_host_{}.nk", output_dir, message_payload.status.host_pubkey);
-        Command::new("nsc")
-            .arg(format!("generate nkey -u --store > {}", host_sys_user_file_name))
-            .output()
-            .expect("Failed to add new sys user key on hosting agent");
-
-        // 3. Prepare to send over user pubkey(to trigger the user jwt gen on hub)
-        let sys_user_nkey_path = utils::get_file_path_buf(&host_sys_user_file_name);
-        let sys_user_nkey: Vec<u8> = std::fs::read(sys_user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let host_user_file_name = format!("{}/user_host_{}.nk", output_dir, message_payload.status.host_pubkey);
-        let host_user_nkey_path = utils::get_file_path_buf(&host_user_file_name);
-        let host_user_nkey: Vec<u8> = std::fs::read(host_user_nkey_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-    
-        // let host_pubkey = serde_json::to_string(&user_nkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        let mut tag_map: HashMap<String, String> = HashMap::new();
-        tag_map.insert("host_pubkey".to_string(), message_payload.status.host_pubkey.clone());
-
-        let mut result_hash_map: HashMap<String, Vec<u8>> = HashMap::new();
-        result_hash_map.insert("sys_user_nkey".to_string(), sys_user_nkey);
-        result_hash_map.insert("host_user_nkey".to_string(), host_user_nkey);
-        
-        // 4. Respond to endpoint request
-        Ok(AuthApiResult {
-            result: AuthResult {
-                status: types::AuthStatus { 
-                    host_pubkey: message_payload.status.host_pubkey,
-                    status: types::AuthState::Requested
-                },
-                data: types::AuthResultData { inner: result_hash_map }
-            },
-            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
-        })
-    }
-
-    pub async fn save_user_jwt(
-        &self,
-        msg: Arc<Message>,
-        output_dir: &str,
-    ) -> Result<AuthApiResult, ServiceError> {
-        let msg_subject = &msg.subject.clone().into_string(); // AUTH.<host_pubkey>.end_handshake
-        log::trace!("Incoming message for '{}'", msg_subject);
-
-        // 1. Verify expected payload was received
-        let message_payload = Self::convert_msg_to_type::<AuthResult>(msg.clone())?;
-        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
-
-        let host_sys_user_jwt_bytes = message_payload.data.inner.get("host_sys_user_jwt").ok_or_else(|| {
-            let err_msg = format!("Error: . Subject='{}'.", msg_subject);
-            handle_internal_err(&err_msg)
-        })?;
-
-        let host_user_jwt_bytes = message_payload.data.inner.get("host_user_jwt").ok_or_else(|| {
-            let err_msg = format!("Error: Failed to find sys jwt in message payload. Subject='{}'.", msg_subject);
-            handle_internal_err(&err_msg)
-        })?;
-
-        // 2. Save user_jwt and sys_jwt local to hosting agent
-        utils::receive_and_write_file(host_sys_user_jwt_bytes.to_owned(), output_dir, "operator.jwt").await.map_err(|e| {
-            let err_msg = format!("Failed to save operator jwt. Subject='{}' Error={}.", msg_subject, e);
-            handle_internal_err(&err_msg)
-        })?;
-
-        utils::receive_and_write_file(host_user_jwt_bytes.to_owned(), output_dir, "account_sys.jwt").await.map_err(|e| {
-            let err_msg = format!("Failed to save sys jwt. Subject='{}' Error={}.", msg_subject, e);
-            handle_internal_err(&err_msg)
-        })?;
-
-        let host_user_log = Command::new("nsc") 
-            .arg(format!("describe user -a WORKLOAD -n user_host_{} --json", message_payload.status.host_pubkey))
-            .output()
-            .expect("Failed to add user with provided keys");    
-
-        log::debug!("HOST USER JWT: {:?}", host_user_log);
-
-        // 3. Respond to endpoint request
-        Ok(AuthApiResult {
-            result: AuthResult {
-                status: types::AuthStatus { 
-                    host_pubkey: message_payload.status.host_pubkey,
-                    status: types::AuthState::Authenticated
-                },
-                data: types::AuthResultData { inner: HashMap::new() }
-            },
-            maybe_response_tags: None
-        })
-    }
-}
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 88a81a6..8e07547 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -1,25 +1,41 @@
 /*
 Service Name: AUTH
 Subject: "AUTH.>"
-Provisioning Account: AUTH Account (ie: This service is exclusively permissioned to the AUTH account.)
+Provisioning Account: ADMIN Account (ie: This service is exclusively permissioned to the ADMIN account.)
 Users: orchestrator & noauth
-
+Endpoints & Managed Subjects:
+    - handle_handshake_request: AUTH.validate
 */
 
-pub mod orchestrator_api;
-pub mod host_api;
 pub mod types;
 pub mod utils;
-
 use anyhow::Result;
 use async_nats::Message;
 use async_nats::jetstream::ErrorCode;
-use async_trait::async_trait;
 use std::sync::Arc;
 use std::future::Future;
-use serde::Deserialize;
-use types::AuthApiResult;
+use types::{WORKLOAD_SK_ROLE, AuthApiResult};
 use util_libs::nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse};
+use async_nats::HeaderValue;
+use nkeys::KeyPair;
+use utils::handle_internal_err;
+use core::option::Option::None;
+use std::collections::HashMap;
+use std::process::Command;
+use serde::{Deserialize, Serialize};
+use bson::{self, doc, to_document};
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use util_libs::db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, 
+    schemas::{
+        self,
+        User,
+        Hoster,
+        Host,
+        Role,
+        RoleInfo
+    },
+};
+
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";
@@ -27,12 +43,174 @@ pub const AUTH_SRV_VERSION: &str = "0.0.1";
 pub const AUTH_SRV_DESC: &str =
     "This service handles the Authentication flow the Host and the Orchestrator.";
 
-#[async_trait]
-pub trait AuthServiceApi
-where
-    Self: std::fmt::Debug + Clone + 'static,
-{
-    fn call<F, Fut>(
+#[derive(Clone, Debug)]
+pub struct AuthServiceApi {
+    pub user_collection: MongoCollection<User>,
+    pub hoster_collection: MongoCollection<Hoster>,
+    pub host_collection: MongoCollection<Host>,
+}
+
+impl AuthServiceApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        Ok(Self {
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME).await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
+        })
+    }
+
+    pub async fn handle_handshake_request(
+        &self,
+        msg: Arc<Message>,
+        creds_dir_path: &str,
+    ) -> Result<AuthApiResult, ServiceError> {
+        log::warn!("INCOMING Message for 'AUTH.validate' : {:?}", msg);
+
+        let mut status = types::AuthState::Unauthenticated;
+
+        // 1. Verify expected data was received
+        let signature: &[u8] = match &msg.headers {
+            Some(h) => {
+                HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
+                    log::error!(
+                        "Error: Missing x-signature header. Subject='AUTH.authorize'"
+                    );
+                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
+                })?)
+            },
+            None => {
+                log::error!(
+                    "Error: Missing message headers. Subject='AUTH.authorize'"
+                );
+                return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+            }
+        };
+
+        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, sys_pubkey, nonce: _ } = Self::convert_msg_to_type::<types::AuthRequestPayload>(msg.clone())?;
+
+        // 2. Validate signature
+        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
+            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
+            return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+        };
+
+        // 3. Authenticate the Hosting Agent (via email and host id info?)
+        let hoster_pubkey_as_holo_hash = "convert_hoster_pubkey_to_raw_value_and_then_into_holo_hash";
+        match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_pubkey_as_holo_hash.clone() }).await? {
+            Some(u) => {
+                // If hoster exists with pubkey, verify email
+                if u.email != email {
+                    log::error!("Error: Failed to validate user email. Subject='{}'.", msg.subject);
+                    return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+                }
+
+                // ...then find the host collection that contains the provided host pubkey
+                match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
+                    Some(h) => {
+                        // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
+                        if h.assigned_hoster != hoster_pubkey {
+                            let host_query: bson::Document = doc! { "_id":  h._id.clone() };
+                            let updated_host_doc = to_document(& Host{
+                                assigned_hoster: hoster_pubkey,
+                                ..h
+                            }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                            self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
+                        }
+                    },
+                    None => {
+                        let err_msg = format!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
+                        return Err(handle_internal_err(&err_msg));
+                    }
+                }
+
+                // Find the mongo_id ref for the hoster associated with this user
+                let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
+                    let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
+                    handle_internal_err(&err_msg)
+                })?;
+                
+                // Finally, find the hoster collection
+                match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
+                    Some(hr) => {
+                        // ...and pair the hoster with host (if the host is not already assiged to the hoster)
+                        let mut updated_assigned_hosts = hr.assigned_hosts;
+                        if !updated_assigned_hosts.contains(&host_pubkey) {
+                        let hoster_query: bson::Document = doc! { "_id":  hr._id.clone() };
+                        updated_assigned_hosts.push(host_pubkey.clone());
+                        let updated_hoster_doc = to_document(& Hoster {
+                            assigned_hosts: updated_assigned_hosts,
+                            ..hr
+                        }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                        self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
+                        }
+                    },
+                    None => {
+                        let err_msg = format!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
+                        return Err(handle_internal_err(&err_msg));
+                    }
+                }
+            },
+            None => {
+                let err_msg = format!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
+                return Err(handle_internal_err(&err_msg));
+            }
+        };
+
+        // 4. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
+        Command::new("nsc")
+            .arg(format!("add user -a SYS -n user_sys_host_{} -k {}", host_pubkey, sys_pubkey))
+            .output()
+            .expect("Failed to add host sys user with provided keys");
+
+        Command::new("nsc")
+            .arg(format!("add user -a WORKLOAD -n user_host_{} -k {} -K {} --tag pubkey:{}", host_pubkey, host_pubkey, WORKLOAD_SK_ROLE, host_pubkey))
+            .output()
+            .expect("Failed to add host user with provided keys");
+
+        // ..and push auth updates to hub server
+        Command::new("nsc")
+            .arg("push -A")
+            .output()
+            .expect("Failed to update resolver config file");    
+
+        // 3. Create User JWT files (automatically signed with respective account key)
+        let host_sys_user_file_name = format!("{}/user_sys_host_{}.jwt", creds_dir_path, host_pubkey);
+        Command::new("nsc") 
+            .arg(format!("describe user -a SYS -n user_sys_host_{} --raw --output-file {}", host_pubkey, host_sys_user_file_name))
+            .output()
+            .expect("Failed to generate host sys user jwt file");
+
+        let host_user_file_name = format!("{}/user_host_{}.jwt", creds_dir_path, host_pubkey);
+        Command::new("nsc") 
+            .arg(format!("describe user -a WORKLOAD -n user_host_{} --raw --output-file {}", host_pubkey, host_user_file_name))
+            .output()
+            .expect("Failed to generate host user jwt file");        
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
+
+        status = types::AuthState::Authenticated;
+        
+        Ok(AuthApiResult {
+            host_pubkey: host_pubkey.clone(),
+            status,
+            maybe_response_tags: Some(tag_map)
+        })
+    }
+
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+
+    pub fn call<F, Fut>(
         &self,
         handler: F,
     ) -> AsyncEndpointHandler<AuthApiResult>
@@ -48,18 +226,6 @@ where
         })
     }
 
-    fn convert_to_type<T>(data: Vec<u8>, msg_subject: &str) -> Result<T, ServiceError>
-    where
-        T: for<'de> Deserialize<'de> + Send + Sync,
-    {
-        serde_json::from_slice::<T>(&data).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload data. Subject='{}' Err={}", msg_subject, e);
-            log::error!("{}", err_msg);
-            ServiceError::Internal(err_msg.to_string())
-        })
-        
-    }
-
     fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Send + Sync,
diff --git a/rust/services/authentication/src/orchestrator_api.rs b/rust/services/authentication/src/orchestrator_api.rs
deleted file mode 100644
index bfcdda0..0000000
--- a/rust/services/authentication/src/orchestrator_api.rs
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
-Endpoints & Managed Subjects:
-    - handle_handshake_request: AUTH.start_handshake
-    - add_user_pubkey: AUTH.handle_handshake_p2
-*/
-
-use super::{AuthServiceApi, types, utils};
-use anyhow::Result;
-use async_nats::{Message, HeaderValue};
-use async_nats::jetstream::ErrorCode;
-use nkeys::KeyPair;
-use types::{AuthApiResult, AuthResult};
-use utils::handle_internal_err;
-use core::option::Option::None;
-use std::collections::HashMap;
-use std::process::Command;
-use std::sync::Arc;
-use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use util_libs::{
-    nats_js_client::ServiceError,
-    db::{
-        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{
-            self,
-            User,
-            Hoster,
-            Host,
-            Role,
-            RoleInfo,
-        }
-    },
-};
-
-#[derive(Debug, Clone)]
-pub struct OrchestratorAuthApi {
-    pub user_collection: MongoCollection<User>,
-    pub hoster_collection: MongoCollection<Hoster>,
-    pub host_collection: MongoCollection<Host>,
-}
-
-impl AuthServiceApi for OrchestratorAuthApi {}
-
-impl OrchestratorAuthApi {
-    pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        Ok(Self {
-            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME).await?,
-            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
-        })
-    }
-
-    /*******************************  For Orchestrator   *********************************/
-    // nb: returns to the `save_hub_files` subject
-    pub async fn handle_handshake_request(
-        &self,
-        msg: Arc<Message>,
-        creds_dir_path: &str,
-    ) -> Result<AuthApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.start_handshake' : {:?}", msg);
-
-        // 1. Verify expected data was received
-        let signature: &[u8] = match &msg.headers {
-            Some(h) => {
-                HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
-                    log::error!(
-                        "Error: Missing x-signature header. Subject='AUTH.authorize'"
-                    );
-                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
-                })?)
-            },
-            None => {
-                log::error!(
-                    "Error: Missing message headers. Subject='AUTH.authorize'"
-                );
-                return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-            }
-        };
-
-        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, nonce: _ } = Self::convert_msg_to_type::<types::AuthRequestPayload>(msg.clone())?;
-
-        // 2. Validate signature
-        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
-            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
-            return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-        };
-
-        // 3. Authenticate the Hosting Agent (via email and host id info?)
-        match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_pubkey.clone() }).await? {
-            Some(u) => {
-                // If hoster exists with pubkey, verify email
-                if u.email != email {
-                    log::error!("Error: Failed to validate user email. Subject='{}'.", msg.subject);
-                    return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-                }
-
-                // ...then find the host collection that contains the provided host pubkey
-                match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
-                    Some(h) => {
-                        // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
-                        if h.assigned_hoster != hoster_pubkey {
-                            let host_query: bson::Document = doc! { "_id":  h._id.clone() };
-                            let updated_host_doc = to_document(& Host{
-                                assigned_hoster: hoster_pubkey,
-                                ..h
-                            }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                            self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
-                        }
-                    },
-                    None => {
-                        let err_msg = format!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
-                        return Err(handle_internal_err(&err_msg));
-                    }
-                }
-
-                // Find the mongo_id ref for the hoster associated with this user
-                let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
-                    let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
-                    handle_internal_err(&err_msg)
-                })?;
-                
-                // Finally, find the hoster collection
-                match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
-                    Some(hr) => {
-                        // ...and pair the hoster with host (if the host is not already assiged to the hoster)
-                        let mut updated_assigned_hosts = hr.assigned_hosts;
-                        if !updated_assigned_hosts.contains(&host_pubkey) {
-                        let hoster_query: bson::Document = doc! { "_id":  hr._id.clone() };
-                        updated_assigned_hosts.push(host_pubkey.clone());
-                        let updated_hoster_doc = to_document(& Hoster {
-                            assigned_hosts: updated_assigned_hosts,
-                            ..hr
-                        }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                        self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
-                        }
-                    },
-                    None => {
-                        let err_msg = format!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
-                        return Err(handle_internal_err(&err_msg));
-                    }
-                }
-            },
-            None => {
-                let err_msg = format!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
-                return Err(handle_internal_err(&err_msg));
-            }
-        };
-
-        // 4. Read operator and sys account jwts and prepare them to be sent as a payload in the publication callback
-        let operator_path = utils::get_file_path_buf(&format!("{}/operator.creds", creds_dir_path));
-        let hub_operator_jwt: Vec<u8> = std::fs::read(operator_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let sys_account_path = utils::get_file_path_buf(&format!("{}/account_sys.creds", creds_dir_path));
-        let hub_sys_account_jwt: Vec<u8> = std::fs::read(sys_account_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let workload_account_path = utils::get_file_path_buf(&format!("{}/account_workload.creds", creds_dir_path));
-        let hub_workload_account_jwt: Vec<u8> = std::fs::read(workload_account_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let mut tag_map: HashMap<String, String> = HashMap::new();
-        tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
-
-        let mut result_hash_map: HashMap<String, Vec<u8>> = HashMap::new();
-        result_hash_map.insert("operator_jwt".to_string(), hub_operator_jwt);
-        result_hash_map.insert("sys_account_jwt".to_string(), hub_sys_account_jwt);
-        result_hash_map.insert("workload_account_jwt".to_string(), hub_workload_account_jwt);
-
-        Ok(AuthApiResult {
-            result: AuthResult {
-                status: types::AuthStatus { 
-                    host_pubkey: host_pubkey.clone(),
-                    status: types::AuthState::Requested
-                },
-                data: types::AuthResultData { inner: result_hash_map }
-            },
-            maybe_response_tags: Some(tag_map) // used to inject as tag in response subject
-        })
-    }
-
-    pub async fn add_user_nkey(&self,
-        msg: Arc<Message>,
-        creds_dir_path: &str,
-    ) -> Result<AuthApiResult, ServiceError> {
-        let msg_subject = &msg.subject.clone().into_string(); // AUTH.handle_handshake_p2
-        log::trace!("Incoming message for '{}'", msg_subject);
-
-        // 1. Verify expected payload was received
-        let message_payload = Self::convert_msg_to_type::<AuthResult>(msg.clone())?;
-        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
-
-        let host_user_nkey_bytes = message_payload.data.inner.get("host_user_nkey").ok_or_else(|| {
-            let err_msg = format!("Error: . Subject='{}'.", msg_subject);
-            handle_internal_err(&err_msg)
-        })?;
-        let host_user_nkey = Self::convert_to_type::<String>(host_user_nkey_bytes.to_owned(), msg_subject)?;
-
-        let host_pubkey = &message_payload.status.host_pubkey;
-
-        // 2. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
-        Command::new("nsc")
-            .arg(format!("add user -a SYS -n user_sys_host_{} -k {}", host_pubkey, host_user_nkey))
-            .output()
-            .expect("Failed to add host sys user with provided keys");
-
-        Command::new("nsc")
-            .arg(format!("add user -a WORKLOAD -n user_host_{} -k {}", host_pubkey, host_user_nkey))
-            .output()
-            .expect("Failed to add host user with provided keys");
-
-        // ..and push auth updates to hub server
-        Command::new("nsc")
-            .arg("push -A")
-            .output()
-            .expect("Failed to update resolver config file");    
-
-        // 3. Create User JWT files (automatically signed with respective account key)
-        let host_sys_user_file_name = format!("{}/user_sys_host_{}.jwt", creds_dir_path, host_pubkey);
-        Command::new("nsc") 
-            .arg(format!("describe user -a SYS -n user_sys_host_{} --raw --output-file {}", host_pubkey, host_sys_user_file_name))
-            .output()
-            .expect("Failed to generate host sys user jwt file");
-
-        let host_user_file_name = format!("{}/user_host_{}.jwt", creds_dir_path, host_pubkey);
-        Command::new("nsc") 
-            .arg(format!("describe user -a WORKLOAD -n user_host_{} --raw --output-file {}", host_pubkey, host_user_file_name))
-            .output()
-            .expect("Failed to generate host user jwt file");        
-    
-        // let account_signing_key = utils::get_account_signing_key();
-        // utils::generate_user_jwt(&user_nkey, &account_signing_key);
-
-        // 4. Prepare User JWT to be sent as a payload in the publication callback
-        let host_sys_user_jwt_path = utils::get_file_path_buf(&host_sys_user_file_name);
-        let host_sys_user_jwt: Vec<u8> = std::fs::read(host_sys_user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let host_user_jwt_path = utils::get_file_path_buf(&host_user_file_name);
-        let host_user_jwt: Vec<u8> = std::fs::read(host_user_jwt_path).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        
-        let mut result_hash_map: HashMap<String, Vec<u8>> = HashMap::new();
-        result_hash_map.insert("host_sys_user_jwt".to_string(), host_sys_user_jwt);
-        result_hash_map.insert("host_user_jwt".to_string(), host_user_jwt);
-
-        // 5. Respond to endpoint request
-        Ok(AuthApiResult {
-            result: AuthResult {
-                status: types::AuthStatus { 
-                    host_pubkey: message_payload.status.host_pubkey,
-                    status: types::AuthState::ValidatedAgent
-                },
-                data: types::AuthResultData { inner: result_hash_map }
-            },
-            maybe_response_tags: None
-        })
-    }
-
-    // Helper function to initialize mongodb collections
-    async fn init_collection<T>(
-        client: &MongoDBClient,
-        collection_name: &str,
-    ) -> Result<MongoCollection<T>>
-    where
-        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
-    {
-        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
-    }
-}
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index e2a89dd..5574698 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -3,70 +3,39 @@ use std::collections::HashMap;
 use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
-#[serde(rename_all = "snake_case")]
-pub enum AuthServiceSubjects {
-    StartHandshake,
-    HandleHandshakeP1,
-    HandleHandshakeP2,
-    EndHandshake,
-}
+pub const AUTH_SERVICE_SUBJECT: &str = "validate";
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub enum AuthState {
-    Requested,
-    ValidatedAgent, //    AddedHostPubkey
-    SignedJWT,
-    Authenticated
-}
+// The workload_sk_role is assigned when the host agent is created during the auth flow.
+// NB: This role name *must* match the `ROLE_NAME_WORKLOAD` in the `orchestrator_setup.sh` script file.
+pub const WORKLOAD_SK_ROLE: &str = "workload-role";
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct AuthStatus {
-    pub host_pubkey: String,
-    pub status: AuthState
+pub enum AuthState {
+    Unauthenticated,
+    Authenticated,
+    Forbidden,
+    Error(String)
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
-pub struct AuthHeaders {
-    signature: String,
+pub struct AuthGuardPayload {
+    pub host_pubkey: String, // nkey
+    pub hoster_pubkey: String, // nkey
+    pub email: String,
+    pub nonce: String
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct AuthRequestPayload {
-    pub hoster_pubkey: String,
-    pub email: String,
-    pub host_pubkey: String,
+    pub host_pubkey: String, // nkey
+    pub sys_pubkey: Option<String>, // nkey
     pub nonce: String
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct AuthResultData {
-    pub inner: HashMap<String,Vec<u8>>
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct AuthResult {
-    pub status: AuthStatus,
-    pub data: AuthResultData,
-}
-
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthApiResult {
-    pub result: AuthResult,
-    pub maybe_response_tags: Option<HashMap<String, String>>
-}
-impl EndpointTraits for AuthApiResult {}
-impl CreateTag for AuthApiResult {
-    fn get_tags(&self) -> HashMap<String, String> {
-        self.maybe_response_tags.clone().unwrap_or_default()
-    }
-}
-impl CreateResponse for AuthApiResult {
-    fn get_response(&self) -> bytes::Bytes {
-        let r = self.result.clone();
-        match serde_json::to_vec(&r) {
-            Ok(r) => r.into(),
-            Err(e) => e.to_string().into(),
-        }
-    }
-}
+    pub host_pubkey: String,
+    pub status: AuthState,
+    pub host_jwt: String,
+    pub sys_jwt: String
+} 
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index 79dbc5c..0416708 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -2,21 +2,13 @@ use anyhow::Result;
 use async_nats::jetstream::Context;
 use util_libs::nats_js_client::ServiceError;
 use std::io::Write;
-use std::path::PathBuf;
 
 pub fn handle_internal_err(err_msg: &str) -> ServiceError {
     log::error!("{}", err_msg);
     ServiceError::Internal(err_msg.to_string())
 }
 
-pub fn get_file_path_buf(
-    file_name: &str,
-) -> PathBuf {
-    let root_path = std::env::current_dir().expect("Failed to locate root directory.");
-    root_path.join(file_name)
-}
-
-pub async fn receive_and_write_file(
+pub async fn write_file(
     data: Vec<u8>,
     output_dir: &str,
     file_name: &str,
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index ce2a0ee..69334aa 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -2,8 +2,9 @@ use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamServic
 use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
 
 use anyhow::Result;
-use core::option::Option::None;
-use async_nats::{jetstream, HeaderMap, Message, ServerInfo};
+use std::path::PathBuf;
+use core::marker::Sync;
+use async_nats::{jetstream, AuthError, HeaderMap, Message, ServerInfo};
 use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::fmt;
@@ -19,6 +20,8 @@ pub enum ServiceError {
     Request(String),
     #[error(transparent)]
     Database(#[from] mongodb::error::Error),
+    #[error(transparent)]
+    Authentication(#[from] AuthError),
     #[error("Nats Error: {0}")]
     NATS(String),
     #[error("Internal Error: {0}")]
@@ -99,21 +102,24 @@ pub struct JsClient {
     service_log_prefix: String,
 }
 
+#[derive(Clone)]
+pub enum Credentials {
+    Path(std::path::PathBuf), // String = pathbuf as string
+    Password(String, String)
+}   
+
 #[derive(Deserialize, Default)]
 pub struct NewJsClientParams {
     pub nats_url: String,
     pub name: String,
     pub inbox_prefix: String,
-    #[serde(default)]
     pub service_params: Vec<JsServiceParamsPartial>,
     #[serde(skip_deserializing)]
-    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
-    #[serde(default)]
-    pub credentials_path: Option<String>,
-    #[serde(default)]
+    pub credentials: Option<Credentials>,
     pub ping_interval: Option<Duration>,
-    #[serde(default)]
     pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
 }
 
 impl JsClient {
@@ -125,14 +131,22 @@ impl JsClient {
             .request_timeout(Some(p.request_timeout.unwrap_or(Duration::from_secs(10))))
             .custom_inbox_prefix(&p.inbox_prefix);
 
-        let client = match p.credentials_path {
-            Some(cp) => {
-                let path = std::path::Path::new(&cp);
-                connect_options
-                    .credentials_file(path)
-                    .await?
-                    .connect(&p.nats_url)
-                    .await?
+        let client = match p.credentials {
+            Some(c) => match c {
+                Credentials::Password(user, pw) => {
+                    connect_options
+                        .user_and_password(user, pw)
+                        .connect(&p.nats_url)
+                        .await?
+                },
+                Credentials::Path(cp) => {
+                    let path = std::path::Path::new(&cp);
+                    connect_options
+                        .credentials_file(path)
+                        .await?
+                        .connect(&p.nats_url)
+                        .await?
+                }
             }
             None => connect_options.connect(&p.nats_url).await?,
         };
@@ -159,7 +173,7 @@ impl JsClient {
 
         let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
 
-        let mut default_client = JsClient {
+        let mut js_client = JsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
@@ -170,43 +184,23 @@ impl JsClient {
             service_log_prefix: service_log_prefix.clone(),
         };
 
-        for opt in p.opts {
-            opt(&mut default_client);
+        for listener in p.listeners {
+            listener(&mut js_client);
         }
 
         log::info!(
             "{}Connected to NATS server at {}",
             service_log_prefix,
-            default_client.url
+            js_client.url
         );
-        Ok(default_client)
-    }
-
-    pub fn name(&self) -> &str {
-        &self.name
+        Ok(js_client)
     }
 
     pub fn get_server_info(&self) -> ServerInfo {
         self.client.server_info()
     }
 
-    pub async fn monitor(&self) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            Err(Box::new(ErrClientDisconnected))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub async fn close(&self) -> Result<(), async_nats::Error> {
-        self.client.drain().await?;
-        Ok(())
-    }
-
-    pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            return Err(Box::new(ErrClientDisconnected));
-        }
+    pub async fn get_stream_info(&self, stream_name: &str) -> Result<(), async_nats::Error> {
         let stream = &self.js.get_stream(stream_name).await?;
         let info = stream.get_info().await?;
         log::debug!(
@@ -218,6 +212,15 @@ impl JsClient {
         Ok(())
     }
 
+    pub async fn check_connection(&self) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state =self.client.connection_state();
+        if let async_nats::connection::State::Disconnected = conn_state {
+            Err(Box::new(ErrClientDisconnected))
+        } else {
+            Ok(conn_state)
+        }
+    }
+
     pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
         let now = Instant::now();
         let result = match payload.headers {
@@ -267,6 +270,11 @@ impl JsClient {
         }
         None
     }
+
+    pub async fn close(&self) -> Result<(), async_nats::Error> {
+        self.client.drain().await?;
+        Ok(())
+    }
 }
 
 // Client Options:
@@ -307,13 +315,23 @@ pub fn get_nats_url() -> String {
     })
 }
 
-pub fn get_nats_client_creds(operator: &str, account: &str, user: &str) -> String {
-    std::env::var("HOST_CREDS_FILE_PATH").unwrap_or_else(|_| {
-        format!(
-            "/.local/share/nats/nsc/keys/creds/{}/{}/{}.creds",
-            operator, account, user
-        )
-    })
+pub fn get_nsc_root_path() -> String {
+    let nsc_path = std::env::var("NSC_PATH").unwrap_or_else(|_| "/.local/share/nats/nsc".to_string());
+    nsc_path
+}
+
+pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {
+    format!(
+        "{}/keys/creds/{}/{}/{}.creds",
+        get_nsc_root_path(), operator, account, user
+    )
+}
+
+pub fn get_file_path_buf(
+    file_name: &str,
+) -> PathBuf {
+    let current_dir_path = std::env::current_dir().expect("Failed to locate current directory.");
+    current_dir_path.join(file_name)
 }
 
 pub fn get_event_listeners() -> Vec<EventListener> {
diff --git a/scripts/hosting_agent_setup.sh b/scripts/hosting_agent_setup.sh
new file mode 100644
index 0000000..4c02f69
--- /dev/null
+++ b/scripts/hosting_agent_setup.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2005,SC2086
+
+# --------
+# NB: This setup expects the `nats` and the `nsc` binarys to be locally installed and accessible.  This script will verify that they both exist locally before running setup commnds.
+
+# Script Overview:
+# This script is responsible for setting up the "Operator Chain of Trust" (eg: O/A/U) authentication pattern that is associated with the Orchestrator Hub on the Hosting Agent.
+
+# Input Vars:
+#   - SHARED_CREDS_DIR
+#   - OPERATOR_JWT_PATH
+#   - SYS_ACCOUNT_JWT_PATH
+#   - AUTH_ACCOUNT_JWT_PATH
+
+# --------
+
+set -e # Exit on any error
+
+# Check for required commands
+for cmd in nsc nats; do
+  echo "Executing command: $cmd --version"
+  if command -v "$cmd" &>/dev/null; then
+    $cmd --version
+  else
+    echo "Command '$cmd' not found."
+  fi
+done
+
+# Variables
+OPERATOR_NAME="HOLO"
+SYS_ACCOUNT_NAME="SYS"
+AUTH_ACCOUNT_NAME="AUTH"
+SHARED_CREDS_DIR="shared_creds_output"
+OPERATOR_JWT_PATH="$SHARED_CREDS_DIR/$OPERATOR_NAME.jwt"
+SYS_ACCOUNT_JWT_PATH="$SHARED_CREDS_DIR/$SYS_ACCOUNT_NAME.jwt"
+AUTH_GUARD_USER_NAME="auth-guard"
+AUTH_GUARD_USER_PATH="$SHARED_CREDS_DIR/$AUTH_GUARD_USER_NAME.creds"
+
+if [ ! -d "$SHARED_CREDS_DIR" ]; then
+    echo "Shared output dir not found. Unable to set up local chain of trust."
+    exit 1
+else
+    if [ ! -d "$OPERATOR_JWT_PATH" ]; then
+        echo "Operator JWT not found. Unable to set up local chain of trust."
+        exit 1
+    else
+        echo "Found the $OPERATOR_JWT_PATH. Adding Operator to local chain reference."
+        # Add Operator
+        nsc add operator -u $OPERATOR_JWT_PATH --force
+        echo "Operator added to local nsc successfully."
+
+        if [ ! -d "$SYS_ACCOUNT_JWT_PATH" ]; then
+            echo "SYS account JWT not found. Unable to add SYS ACCOUNT to the local chain of trust."
+            exit 1
+        else
+            echo "Found the $SYS_ACCOUNT_JWT_PATH. Adding SYS Account to local chain reference."
+            # Add SYS Account
+            nsc import account --file $SYS_ACCOUNT_JWT_PATH
+            echo "SYS account added to local nsc successfully."
+
+            # TODO: For if/when add local sys user (that's) associated the Orchestrator SYS Account
+            # if [ ! -d "$SYS_USER_PATH" ]; then
+            #     echo "WARNING: SYS user JWT not found. Unable to add the SYS user as a locally trusted user."
+            # else
+            #     echo "Found the $SYS_USER_PATH usr to local chain reference."
+            #     # Add SYS user
+            #     nsc import user --file $SYS_USER_PATH
+            #     # Create SYS user cred file and add to shared creds dir
+            #     nsc generate creds --name $SYS_USER_NAME --account $SYS_ACCOUNT > $SHARED_CREDS_DIR/$SYS_USER_NAME.creds
+            #     echo "SYS user added to local nsc successfully."
+            # fi
+        fi
+
+        if [ ! -d "$AUTH_GUARD_USER_PATH" ]; then
+            echo "WARNING: AUTH_GUARD user credentials not found. Unable to add the complete Hosting Agent set-up."
+        else
+            echo "Found the $AUTH_GUARD_USER_NAME credentials file."
+            echo "Set-up complete. Credential files are in the $SHARED_CREDS_DIR/ directory."
+        fi
+    fi
+fi
+
diff --git a/scripts/hub_cluster_config_setup.sh b/scripts/hub_cluster_config_setup.sh
new file mode 100644
index 0000000..7dc84c4
--- /dev/null
+++ b/scripts/hub_cluster_config_setup.sh
@@ -0,0 +1,93 @@
+#!/bin/sh
+
+# Ensure all required environment variables are set
+: "${SERVER_NAME:?Environment variable SERVER_NAME is required}"
+: "${SERVER_ADDRESS:?Environment variable SERVER_ADDRESS is required}"
+: "${HTTP_ADDRESS:?Environment variable HTTP_ADDRESS is required}"
+: "${JS_DOMAIN:?Environment variable JS_DOMAIN is required}"
+: "${STORE_PATH:?Environment variable STORE_PATH is required}"
+: "${CLUSTER_PORT:?Environment variable CLUSTER_PORT is required}"
+: "${CLUSTER_SEED_ADDRESSES:?Environment variable CLUSTER_SEED_ADDRESSES is required}"
+: "${CLUSTER_USER_NAME:?Environment variable CLUSTER_USER_NAME is required}"
+: "${CLUSTER_USER_PW:?Environment variable CLUSTER_USER_PW is required}"
+: "${RESOLVER_PATH:?Environment variable RESOLVER_PATH is required}"
+
+# Define the output config file
+CONFIG_FILE="nats-cluster-server.conf"
+
+# Create the configuration file
+cat > "$CONFIG_FILE" <<EOL
+# Cluster Node Configuration
+server_name: ${SERVER_NAME}
+listen: ${SERVER_ADDRESS}
+http: ${HTTP_ADDRESS}
+
+system_account: SYS
+
+jetstream: {
+  enabled: true
+  domain: ${JS_DOMAIN}
+  store_dir: "${STORE_PATH}/${JS_DOMAIN}"
+}
+
+# Cluster configuration
+cluster {
+  name: nats_cluster
+  listen: 0.0.0.0:${CLUSTER_PORT}
+
+  # Route to connect to the seed server(s)
+  routes: [
+    ${CLUSTER_SEED_ADDRESSES}
+  ]
+
+  authorization {
+    user: ${CLUSTER_USER_NAME}
+    password: ${CLUSTER_USER_PW}
+  }
+}
+
+# Leaf node connection
+leafnodes {
+  port: 7422
+}
+
+include ${RESOLVER_PATH}
+
+# logging options
+debug:   true
+trace:   true
+logtime: false
+
+# max_connections
+max_connections: 100
+
+# max_subscriptions (per connection)
+max_subscriptions: 1000
+
+# max_pending
+max_pending: 10000000
+
+# maximum control line
+max_control_line: 2048
+
+# maximum payload
+max_payload: 65536
+
+# ping interval and no pong threshold
+ping_interval: "60s"
+ping_max: 3
+
+# how long server can block on a socket write to a client
+write_deadline: "3s"
+
+lame_duck_duration: "4m"
+
+# report repeated failed route/gateway/leafNode connection
+# every 24hour (24*60*60)
+connect_error_reports: 86400
+
+# report failed reconnect events every 5 attempts
+reconnect_error_reports: 5
+EOL
+
+echo "NATS configuration file '$CONFIG_FILE' has been created successfully."
diff --git a/scripts/hub_seed_config_setup.sh b/scripts/hub_seed_config_setup.sh
new file mode 100644
index 0000000..b0ba277
--- /dev/null
+++ b/scripts/hub_seed_config_setup.sh
@@ -0,0 +1,90 @@
+#!/bin/sh
+
+# Ensure all required environment variables are set
+: "${SERVER_NAME:?Environment variable SERVER_NAME is required}"
+: "${SERVER_ADDRESS:?Environment variable SERVER_ADDRESS is required}"
+: "${HTTP_ADDRESS:?Environment variable HTTP_ADDRESS is required}"
+: "${JS_DOMAIN:?Environment variable JS_DOMAIN is required}"
+: "${STORE_PATH:?Environment variable STORE_PATH is required}"
+: "${CLUSTER_PORT:?Environment variable CLUSTER_PORT is required}"
+: "${CLUSTER_USER_NAME:?Environment variable CLUSTER_USER_NAME is required}"
+: "${CLUSTER_USER_PW:?Environment variable CLUSTER_USER_PW is required}"
+: "${RESOLVER_PATH:?Environment variable RESOLVER_PATH is required}"
+
+# Define the output config file
+CONFIG_FILE="nats-seed-server.conf"
+
+# Create the configuration file
+cat > "$CONFIG_FILE" <<EOL
+# Cluster Node Configuration
+server_name: ${SERVER_NAME}
+listen: ${SERVER_ADDRESS}
+http: ${HTTP_ADDRESS}
+
+system_account: SYS
+
+jetstream: {
+  enabled: true
+  domain: ${JS_DOMAIN}
+  store_dir: "${STORE_PATH}/${JS_DOMAIN}"
+}
+
+# Cluster configuration
+cluster {
+  name: nats_cluster
+  listen: 0.0.0.0:${CLUSTER_PORT}
+
+  # No routes should be listed here as this is the seed node
+
+  authorization {
+    user: ${CLUSTER_USER_NAME}
+    password: ${CLUSTER_USER_PW}
+    timeout: 10
+  }
+}
+
+# Leaf node connection
+leafnodes {
+  port: 7422
+}
+
+include ${RESOLVER_PATH}
+
+# logging options
+debug:   true
+trace:   true
+logtime: false
+
+# max_connections
+max_connections: 100
+
+# max_subscriptions (per connection)
+max_subscriptions: 1000
+
+# max_pending
+max_pending: 10000000
+
+# maximum control line
+max_control_line: 2048
+
+# maximum payload
+max_payload: 65536
+
+# ping interval and no pong threshold
+ping_interval: "60s"
+ping_max: 3
+
+# how long server can block on a socket write to a client
+write_deadline: "3s"
+
+lame_duck_duration: "4m"
+
+# report repeated failed route/gateway/leafNode connection
+# every 24hour (24*60*60)
+connect_error_reports: 86400
+
+# report failed reconnect events every 5 attempts
+reconnect_error_reports: 5
+EOL
+
+echo "NATS configuration file '$CONFIG_FILE' has been created successfully."
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
old mode 100644
new mode 100755
index 347ea6c..2919167
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -2,39 +2,50 @@
 # shellcheck disable=SC2005,SC2086
 
 # --------
-# NB: This setup expects the `nats` and the `nsc` binarys to be locally installed and accessible.  This script will verify that they both exist locally before running setup commnds.
+# NB: This setup expects the `nats` and the `nsc` binaries to be locally installed and accessible.  This script will verify that they both exist locally before running setup commnds.
 
 # Script Overview:
-# This script is responsible ser setting up the "Operator Chain of Trust" (eg: O/A/U) authentication pattern on the Orchestrator Hub.
+# This script is responsible ser setting up the "Operator Chain of Trust" (eg: O/A/U) authentication pattern on the Orchestrator Hub.  It also instantiates the distributed auth-callout on the orchestrator NATS server hub for the AUTH account.
 
 # Operator Creation:
-# The operator is generated and named "HOLO" and a system account is generated and named SYS.  Both are assigned two signing keys and are associated with the JWT server.  The JWT server URL set to nats://0.0.0.0:4222.
+# The operator is generated and named "HOLO".  The JWT server URL set to nats://0.0.0.0:4222.
 
 # Account Creation:
-# Two accounts, named "ADMIN" and "WORKLOAD", are created with JetStream enabled.  Both are associated with the HOLO Operator.
-# Each account has a signing key with a randomly generated role name, which is assigned scoped permissions to allow only users assigned to the signing key to publish and subscribe to their respective streams.
+# Four accounts, named "ADMIN", "SYS", "AUTH", and "WORKLOAD" are all associated with the HOLO Operator.  The "ADMIN", "AUTH" and "WORKLOAD" accounts are created with JetStream enabled.
+# Each account has a signing key, of which both the ADMIN and WORKLOAD signing keys are scoped by an assigned role name.  This ensures that only the users assigned to the signing key role inheirt their scoped permissions.
 
 # User Creation:
-# One user named "admin" is created under the "ADMIN" account.
-# One user named "orchestrator" is created under the "WORKLOAD" account.
+# Four users:
+#  - user named "admin" is created under the "ADMIN" account.
+#  - user named "orchestrator_auth" is created under the "AUTH" account.
+#  - user named "sys" is created under the "SYS" account.
+#  - user named "auth_guard" is created under the "AUTH" account.
 
 # JWT Generation:
-# JWT files are generated for the operator and both accounts, saved in the jwt_output/ directory.
+# JWT files are generated for the operator and sys account and are saved in the shared_creds/ directory.
 
 # Input Vars:
-#   - OPERATOR_NAME
+#   - OPERATOR
 #   - SYS_ACCOUNT
+#   - NATS_PORT
 #   - ACCOUNT_JWT_SERVER
-#   - WORKLOAD_ACCOUNT
+#   - OPERATOR_SERVICE_URL
 #   - ADMIN_ACCOUNT
-#   - JWT_OUTPUT_DIR
+#   - ADMIN_USER
+#   - AUTH_ACCOUNT
+#   - ORCHESTRATOR_AUTH_USER
+#   - AUTH_GUARD_USER
+#   - WORKLOAD_ACCOUNT 
+#   - SHARED_CREDS_DIR
+#   - LOCAL_CREDS_DIR 
 #   - RESOLVER_FILE
 
 # Output:
 # One Operator: HOLO
-# Two accounts: HOLO/ADMIN & `HOLO/WORKLOAD`
-# Two Users: /HOLO/ADMIN/admin & HOLO/WORKLOAD/orchestrator
-# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and workload_account.jwt in the `jwt_output` directory.
+# Four Accounts: HOLO/ADMIN & HOLO/SYS (SYS is automated) & HOLO/AUTH & HOLO/WORKLOAD
+# Four Users: /HOLO/ADMIN/admin & HOLO/SYS/sys (sys is automated) & /HOLO/AUTH/orchestrator_auth & HOLO/AUTH/auth_guard
+# Private Files (to only be stored local to Orchestrator): `orchestrator_auth.creds` in the `local_creds/` directory.
+# Shared Files (to be exported to HPOSs): HOLO.jwt, SYS.jwt, `auth_guard.creds` in the `shared_creds/` directory.
 # --------
 
 set -e # Exit on any error
@@ -50,53 +61,106 @@ for cmd in nsc nats; do
 done
 
 # Variables
-OPERATOR_NAME="HOLO"
+OPERATOR="HOLO"
 SYS_ACCOUNT="SYS"
-ACCOUNT_JWT_SERVER="nats://143.244.144.52:4222"
-OPERATOR_SERVICE_URL="nats://143.244.144.52:4222"
-WORKLOAD_ACCOUNT="WORKLOAD"
+NATS_PORT="4222"
+ACCOUNT_JWT_SERVER="nats://192.168.1.96:$NATS_PORT" # "nats://143.244.144.52:$NATS_PORT"
+OPERATOR_SERVICE_URL="nats://192.168.1.96:$NATS_PORT" # "nats://143.244.144.52:$NATS_PORT"
 ADMIN_ACCOUNT="ADMIN"
-JWT_OUTPUT_DIR="jwt_output"
+ADMIN_USER="admin"
+AUTH_ACCOUNT="AUTH"
+ORCHESTRATOR_AUTH_USER="orchestrator_auth"
+AUTH_GUARD_USER="auth_guard"
+WORKLOAD_ACCOUNT="WORKLOAD"
+SHARED_CREDS_DIR="shared_creds"
+LOCAL_CREDS_DIR="local_creds"
 RESOLVER_FILE="main-resolver.conf"
 
-# Create output directory
-mkdir -p $JWT_OUTPUT_DIR
+# Create output directory when it doesn't already exist
+if [ ! -d "$SHARED_CREDS_DIR" ]; then
+    echo "The shared output dir does not exist. Creating $SHARED_CREDS_DIR."
+    mkdir -p $SHARED_CREDS_DIR
+    echo "Shared output dir created successfully."
+else
+    echo "Shared output dir exists."
+fi
+
+if [ ! -d "$LOCAL_CREDS_DIR" ]; then
+    echo "The local output dir does not exist. Creating $LOCAL_CREDS_DIR."
+    mkdir -p $LOCAL_CREDS_DIR
+    echo "Local output dir created successfully."
+else
+    echo "Local output dir exists."
+fi
+
+function extract_signing_key() {
+  sk=$2
+  name=$1
+  seed_file_path="$HOME/.local/share/nats/nsc/keys/keys/${sk:0:1}/${sk:1:2}/${sk}.nk"
+  echo "coping file over to '$LOCAL_CREDS_DIR/${name}_SK.nk'"
+  cp "$seed_file_path" "$LOCAL_CREDS_DIR/${name}_SK.nk"
+}
 
 # Step 1: Create Operator with SYS account and two signing keys
-nsc add operator --name $OPERATOR_NAME --sys --generate-signing-key
+nsc add operator --name $OPERATOR --sys --generate-signing-key
 nsc edit operator --require-signing-keys --account-jwt-server-url $ACCOUNT_JWT_SERVER --service-url $OPERATOR_SERVICE_URL
 nsc edit operator --sk generate
 
 # Step 2: Create ADMIN_Account with JetStream and scoped signing key
 nsc add account --name $ADMIN_ACCOUNT
-nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_ADMIN="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_ADMIN="admin_role"
-nsc edit signing-key --sk $SIGNING_KEY_ADMIN --role $ROLE_NAME_ADMIN --allow-pub "ADMIN_>" --allow-sub "ADMIN_>" --allow-pub-response
+nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G --conns -1 --leaf-conns -1
+
+ADMIN_SK="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+ADMIN_ROLE_NAME="admin_role"
+nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_workload_inbox_*.>","_auth_inbox_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_workload_inbox_orchestrator.>","_auth_inbox_orchestrator.>" --allow-pub-response
+
+# Step 3: Create AUTH with JetStream with non-scoped signing key
+nsc add account --name $AUTH_ACCOUNT
+nsc edit account --name $AUTH_ACCOUNT --sk generate --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G --conns -1 --leaf-conns -1
+AUTH_ACCOUNT_PUBKEY=$(nsc describe account $AUTH_ACCOUNT --field sub | jq -r)
+AUTH_SK_ACCOUNT_PUBKEY=$(nsc describe account $AUTH_ACCOUNT --field 'nats.signing_keys[0]' | tr -d '"')
+
+# Step 4: Create "Sentinel" User in AUTH Account
+nsc add user --name $AUTH_GUARD_USER --account $AUTH_ACCOUNT --deny-pubsub ">"
 
-# Step 3: Create WORKLOAD Account with JetStream and scoped signing key
+# Step 5: Create WORKLOAD Account with JetStream and scoped signing keys
 nsc add account --name $WORKLOAD_ACCOUNT
-nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_WORKLOAD="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_WORKLOAD="workload-role"
-nsc edit signing-key --sk $SIGNING_KEY_WORKLOAD --role $ROLE_NAME_WORKLOAD --allow-pub "WORKLOAD.>" --allow-sub "WORKLOAD.>" --allow-pub-response
+nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G --conns -1 --leaf-conns -1
+WORKLOAD_SK="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+WORKLOAD_ROLE_NAME="workload_role"
+nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","_INBOX_{{tag(pubkey)}}.>","_workload_inbox_{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","_INBOX_{{tag(pubkey)}}.>","_workload_inbox_{{tag(pubkey)}}.>" --allow-pub-response
 
-# Step 4: Create User "orchestrator" in ADMIN Account // noauth
-nsc add user --name admin --account $ADMIN_ACCOUNT
+# Step 6: Create Operator User in ADMIN Account (for use in Orchestrator)
+nsc add user --name $ADMIN_USER --account $ADMIN_ACCOUNT -K $ADMIN_ROLE_NAME
 
-# Step 5: Create User "orchestrator" in WORKLOAD Account
-nsc add user --name orchestrator --account $WORKLOAD_ACCOUNT
+# Step 7: Create Operator User in AUTH Account (used in auth service)
+nsc add user --name $ORCHESTRATOR_AUTH_USER --account $AUTH_ACCOUNT --allow-pubsub ">"
+AUTH_USER_PUBKEY=$(nsc describe user --name $ORCHESTRATOR_AUTH_USER --account $AUTH_ACCOUNT --field sub | jq -r)
+echo "assigned auth user pubkey: $AUTH_USER_PUBKEY"
 
-# Step 6: Generate JWT files
-nsc describe operator --raw --output-file $JWT_OUTPUT_DIR/holo_operator.jwt
-nsc describe account --name SYS --raw --output-file $JWT_OUTPUT_DIR/sys_account.jwt
-nsc describe account --name $WORKLOAD_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/workload_account.jwt
-nsc describe account --name $ADMIN_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/admin_account.jwt
+# Step 8: Configure Auth Callout
+echo $AUTH_ACCOUNT_PUBKEY
+echo $AUTH_SK_ACCOUNT_PUBKEY
+nsc edit authcallout --account $AUTH_ACCOUNT --allowed-account "\"$AUTH_ACCOUNT_PUBKEY\",\"$AUTH_SK_ACCOUNT_PUBKEY\"" --auth-user $AUTH_USER_PUBKEY
 
-# Step 7: Generate Resolver Config
-nsc generate config --nats-resolver --sys-account $SYS_ACCOUNT --force --config-file $RESOLVER_FILE
+# Step 9: Generate JWT files
+nsc generate creds --name $ORCHESTRATOR_AUTH_USER --account $AUTH_ACCOUNT > $LOCAL_CREDS_DIR/$ORCHESTRATOR_AUTH_USER.creds # --> local to hub exclusively
+nsc describe operator --raw --output-file $SHARED_CREDS_DIR/$OPERATOR.jwt
+nsc describe account --name SYS --raw --output-file $SHARED_CREDS_DIR/$SYS_ACCOUNT.jwt
+nsc generate creds --name $AUTH_GUARD_USER --account $AUTH_ACCOUNT --output-file $SHARED_CREDS_DIR/$AUTH_GUARD_USER.creds
+
+# ADMIN_SK=$(nsc describe account ADMIN --field 'nats.signing_keys[0].key' | tr -d '"')
+extract_signing_key ADMIN $ADMIN_SK
+echo "extracted ADMIN signing key"
+
+extract_signing_key AUTH $AUTH_SK_ACCOUNT_PUBKEY
+echo "extracted AUTH signing key"
 
-# Step 8: Push credentials to NATS server
-nsc push -A
+extract_signing_key AUTH_ROOT $AUTH_ACCOUNT_PUBKEY
+echo "extracted AUTH root key"
+
+# Step 10: Generate Resolver Config
+nsc generate config --nats-resolver --sys-account $SYS_ACCOUNT --force --config-file $RESOLVER_FILE
 
-echo "Setup complete. JWTs and resolver file are in the $JWT_OUTPUT_DIR/ directory."
+echo "Setup complete. Shared JWTs and resolver file are in the $SHARED_CREDS_DIR/ directory. Private creds are in the $LOCAL_CREDS_DIR/ directory."
+echo "!! Don't forget to start the NATS server and push the credentials to the server with 'nsc push -A' !!"

From 28491f4428afa5911275fa562b149093a7565ba2 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 4 Feb 2025 18:12:06 -0600
Subject: [PATCH 59/91] auth clean-up

---
 Cargo.lock                                    |   4 +
 rust/clients/host_agent/src/auth/config.rs    |   2 +
 rust/clients/host_agent/src/auth/init.rs      | 150 +++++----
 rust/clients/host_agent/src/auth/utils.rs     |  53 +--
 rust/clients/host_agent/src/hostd/mod.rs      |   2 +-
 .../{workload_manager.rs => workloads.rs}     |   5 +-
 rust/clients/host_agent/src/keys.rs           | 225 +++++++++----
 rust/clients/host_agent/src/main.rs           |  67 +++-
 rust/clients/orchestrator/src/auth.rs         |  10 +-
 rust/clients/orchestrator/src/utils.rs        |   2 +-
 rust/services/authentication/Cargo.toml       |   4 +
 rust/services/authentication/src/lib.rs       | 311 ++++++++++++------
 rust/services/authentication/src/types.rs     | 247 ++++++++++++--
 rust/services/authentication/src/utils.rs     | 223 ++++++++++++-
 .../services/workload/src/orchestrator_api.rs |   7 +-
 rust/util_libs/src/nats_js_client.rs          |   3 +-
 rust/util_libs/src/nats_server.rs             |   2 +-
 17 files changed, 976 insertions(+), 341 deletions(-)
 rename rust/clients/host_agent/src/hostd/{workload_manager.rs => workloads.rs} (98%)

diff --git a/Cargo.lock b/Cargo.lock
index 36c29db..e5dc82c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -239,14 +239,18 @@ dependencies = [
  "bson",
  "bytes",
  "chrono",
+ "data-encoding",
  "dotenv",
  "env_logger",
  "futures",
+ "jsonwebtoken",
  "log",
  "mongodb",
+ "nats-jwt",
  "nkeys",
  "serde",
  "serde_json",
+ "sha2",
  "thiserror 2.0.11",
  "tokio",
  "url",
diff --git a/rust/clients/host_agent/src/auth/config.rs b/rust/clients/host_agent/src/auth/config.rs
index 6591180..15d8a1d 100644
--- a/rust/clients/host_agent/src/auth/config.rs
+++ b/rust/clients/host_agent/src/auth/config.rs
@@ -8,8 +8,10 @@ use std::fs::File;
 
 pub struct HosterConfig {
     pub email: String,
+    #[allow(dead_code)]
     keypair: SigningKey,
     pub hc_pubkey: String,
+    #[allow(dead_code)]
     pub holoport_id: String,
 }
 
diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index 7296c61..669bdad 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -16,53 +16,59 @@ This client is responsible for:
     - returning the host pubkey and closing client cleanly
 */
 
-use crate::{auth::config::HosterConfig, keys::Keys};
+use super::utils::json_to_base64;
+use crate::{auth::config::HosterConfig, keys::{AuthCredType, Keys}};
 use anyhow::Result;
 use std::str::FromStr;
+use futures::StreamExt;
 use async_nats::{HeaderMap, HeaderName, HeaderValue};
-use authentication::{
-    types::{AuthApiResult, AuthRequestPayload, AuthGuardPayload}, utils::{handle_internal_err, write_file} // , AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION
-};
-use std::time::Duration;
+use authentication::types::{AuthApiResult, AuthGuardPayload, AuthJWTPayload, AuthResult}; // , AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION
 use textnonce::TextNonce;
-use util_libs:: nats_js_client::{self, get_nats_creds_by_nsc, get_file_path_buf, Credentials};
+use hpos_hal::inventory::HoloInventory;
+use util_libs:: nats_js_client;
 
-pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
+// pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
 pub const HOST_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX";
 
-pub async fn run(mut host_agent_keys: Keys) -> Result<Keys, async_nats::Error> {
+pub async fn run(mut host_agent_keys: Keys) -> Result<(Keys, async_nats::Client), async_nats::Error> {
     log::info!("Host Auth Client: Connecting to server...");
 
-    // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host =============================================
-    // Fetch Hoster Pubkey and email (from config)
-    let config = HosterConfig::new().await?;
-
-    let secret_token = TextNonce::new().to_string();
-    let unique_inbox = format!("{}.{}", HOST_AUTH_CLIENT_INBOX_PREFIX, secret_token);
+    // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host & Hoster =============================================
+    let nonce = TextNonce::new().to_string();
+    let unique_inbox = &format!("{}.{}", HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey);
     println!(">>> unique_inbox : {}", unique_inbox);
-    let user_unique_auth_subject = format!("AUTH.{}.>", secret_token);
+    let user_unique_auth_subject = &format!("AUTH.{}.>", host_agent_keys.host_pubkey);
     println!(">>> user_unique_auth_subject : {}", user_unique_auth_subject);
 
-    let guard_payload = AuthGuardPayload {
-        host_pubkey: host_agent_keys.host_pubkey,
-        email: config.email,
-        hoster_pubkey: config.hc_pubkey,
-        nonce: secret_token
+    // Fetch Hoster Pubkey and email (from config)
+    let mut auth_guard_payload = AuthGuardPayload::default();
+    match HosterConfig::new().await {
+        Ok(config) => {
+            auth_guard_payload.host_pubkey = host_agent_keys.host_pubkey.to_string();
+            auth_guard_payload.hoster_hc_pubkey = Some(config.hc_pubkey);
+            auth_guard_payload.email = Some(config.email);
+            auth_guard_payload.nonce = nonce;
+        },
+        Err(e) => {
+            log::error!("Failed to locate Hoster config. Err={e}");
+            auth_guard_payload.host_pubkey = host_agent_keys.host_pubkey.to_string();
+            auth_guard_payload.nonce = nonce;
+        }
     };
+    auth_guard_payload = auth_guard_payload.try_add_signature( |p| host_agent_keys.host_sign(p))?;
 
-    let user_auth_json = serde_json::to_string(&guard_payload).expect("Failed to serialize `UserAuthData` into json string");
-    let user_auth_token = crate::utils::json_to_base64(&user_auth_json).expect("Failed to encode user token");
+    let user_auth_json = serde_json::to_string(&auth_guard_payload)?;
+    let user_auth_token = json_to_base64(&user_auth_json)?;
+    let user_creds = if let AuthCredType::Guard(creds) = host_agent_keys.creds.clone() { creds } else {
+        return Err(async_nats::Error::from("Failed to locate Auth Guard credentials"));
+    };
 
     // Connect to Nats server as auth guard and call NATS AuthCallout
     let nats_url = nats_js_client::get_nats_url();
-    let event_listeners = nats_js_client::get_event_listeners();
-    let auth_guard_creds = Credentials::Path(get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard")));
-
-    let auth_guard_client = async_nats::ConnectOptions::with_credentials(user_creds)
-        .expect("Failed to parse static creds")
+    let auth_guard_client = async_nats::ConnectOptions::with_credentials(&user_creds.to_string_lossy())?
         .token(user_auth_token)
-        .custom_inbox_prefix(user_unique_inbox)
-        .connect("nats://localhost:4222")
+        .custom_inbox_prefix(unique_inbox.to_string())
+        .connect(nats_url)
         .await?;
 
     println!("User connected to server on port {}.  Connection State: {:#?}", auth_guard_client.server_info().port, auth_guard_client.connection_state());
@@ -73,72 +79,72 @@ pub async fn run(mut host_agent_keys: Keys) -> Result<Keys, async_nats::Error> {
         server_node_id
     );
 
-    // ==================== Handle Authenication Results ============================================================
-    let mut auth_inbox_msgs = auth_guard_client.subscribe(user_unique_inbox).await.unwrap();
-
+    // ==================== Handle Host User and SYS Authoriation ============================================================
+    let auth_guard_client_clone = auth_guard_client.clone();
     tokio::spawn({
-        let auth_inbox_msgs_clone = auth_inbox_msgs.clone();
+        let mut auth_inbox_msgs = auth_guard_client_clone.subscribe(unique_inbox.to_string()).await?;
         async move {
-            while let Some(msg) = auth_inbox_msgs_clone.next().await {
-                println!("got an AUTH INBOX msg: {:?}", std::str::from_utf8(&msg.clone()).expect("failed to deserialize msg AUTH Inbox msg"));
-                if let AuthApiResult(auth_response) = serde_json::from_slice(msg) {
-                    host_agent_keys = crate::utils::save_host_creds(host_agent_keys, auth_response.host_jwt, auth_response.sys_jwt);
-                    if let Some(reply) = msg.reply {
-                        // Publish the Awk resp to the Orchestrator... (JS)
-                    }
-                    break;
-                };
+            while let Some(msg) = auth_inbox_msgs.next().await {
+                println!("got an AUTH INBOX msg: {:?}", &msg);
             }
         }
     });
 
-    let payload = AuthRequestPayload {
-        host_pubkey: host_agent_keys.host_pubkey,
-        sys_pubkey: host_agent_keys.local_sys_pubkey,
-        nonce: secret_token
+    let payload = AuthJWTPayload {
+        host_pubkey: host_agent_keys.host_pubkey.to_string(),
+        maybe_sys_pubkey: host_agent_keys.local_sys_pubkey.clone(),
+        nonce: TextNonce::new().to_string()
     };
 
     let payload_bytes = serde_json::to_vec(&payload)?;
-    let signature: Vec<u8> = host_user_keys.sign(&payload_bytes)?;
+    let signature = host_agent_keys.host_sign(&payload_bytes)?;
     let mut headers = HeaderMap::new();
-    headers.insert(HeaderName::from_static("X-Signature"), HeaderValue::from_str(&format!("{:?}",signature))?);
+    headers.insert(HeaderName::from_static("X-Signature"), HeaderValue::from_str(&format!("{:?}",signature.as_bytes()))?);
+
+    println!("About to send out the {} message", user_unique_auth_subject);
+    let response = auth_guard_client.request_with_headers(
+            user_unique_auth_subject.to_string(),
+            headers,
+            payload_bytes.into()
+        ).await?;
 
-    // let publish_info = nats_js_client::PublishInfo {
-    //     subject: user_unique_auth_subject,
-    //     msg_id: format!("id={}", rand::random::<u8>()),
-    //     data: payload_bytes,
-    //     headers: Some(headers)
-    // };
-    
-    println!(format!("About to send out the {user_unique_auth_subject} message"));
-    let response = auth_guard_client.request_with_headers(user_unique_auth_subject, headers, payload_bytes).await.expect(&format!("Failed to make {user_unique_auth_subject} request"));
     println!("got an AUTH response: {:?}", std::str::from_utf8(&response.payload).expect("failed to deserialize msg response"));
     
     match serde_json::from_slice::<AuthApiResult>(&response.payload) {
-        Ok(r) => {
-            host_agent_keys = crate::utils::save_host_creds(host_agent_keys, auth_response.host_jwt, auth_response.sys_jwt);
-            
-            if let Some(reply) = msg.reply {
-                // Publish the Awk resp to the Orchestrator... (JS)
+        Ok(auth_response) => match auth_response.result {
+            AuthResult::Authorization(r)=>{
+                host_agent_keys = host_agent_keys.save_host_creds(r.host_jwt, r.sys_jwt).await?;
+                
+                if let Some(_reply) = response.reply {
+                    // Publish the Awk resp to the Orchestrator... (JS)
+                }
+            },
+            _ => {
+                log::error!("got unexpected AUTH RESPONSE : {:?}", auth_response);
             }
         },
         Err(e) => {
-            // TODO:
-            // Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject at regular intervals
-            // for a set period of time, then exit loop and initiate auth connection...
-            let payload = "hpos-hal.info()".as_bytes();
-            let mut auth_inbox_msgs = auth_guard_client.publish("DIAGNOSTICS.ERROR", payload).await.unwrap();
+            // TODO: Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject to ensure has correct permissions
+            println!("got an AUTH RES ERROR: {:?}", e);
+
+            let unauthenticated_user_diagnostics_subject = format!("DIAGNOSTICS.unauthenticated.{}", host_agent_keys.host_pubkey);
+            let diganostics = HoloInventory::from_host();
+            let payload_bytes = serde_json::to_vec(&diganostics)?;
+            auth_guard_client.publish(unauthenticated_user_diagnostics_subject, payload_bytes.into()).await?;
         }
     };
 
-    // Close and drain internal buffer before exiting to make sure all messages are sent
-    auth_guard_client.close().await?;
-
     log::trace!(
-        "host_agent_keys: {}", host_agent_keys
+        "host_agent_keys: {:#?}", host_agent_keys
     );
 
-    Ok(host_agent_keys)
+    Ok((host_agent_keys, auth_guard_client))
 }
 
+    // let publish_info = nats_js_client::PublishInfo {
+    //     subject: user_unique_auth_subject,
+    //     msg_id: format!("id={}", rand::random::<u8>()),
+    //     data: payload_bytes,
+    //     headers: Some(headers)
+    // };
 
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index 5dd7ce5..cdeaa35 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,55 +1,20 @@
-use std::{path::PathBuf, process::Command};
-use util_libs::nats_js_client::{ServiceError, get_file_path_buf};
+use data_encoding::BASE64URL_NOPAD;
 
-use crate::keys;
-
-// NB: These should match the names of these files when saved locally upon hpos init
-// (should be the same as those in the `orchestrator_setup` file)
-const JWT_DIR_NAME: &str = "jwt";
-const OPERATOR_JWT_FILE_NAME: &str = "holo_operator";
-const SYS_JWT_FILE_NAME: &str = "sys_account";
-const WORKLOAD_JWT_FILE_NAME: &str = "workload_account";
+// // NB: These should match the names of these files when saved locally upon hpos init
+// // (should be the same as those in the `orchestrator_setup` file)
+// const JWT_DIR_NAME: &str = "jwt";
+// const OPERATOR_JWT_FILE_NAME: &str = "holo_operator";
+// const SYS_JWT_FILE_NAME: &str = "sys_account";
+// const WORKLOAD_JWT_FILE_NAME: &str = "workload_account";
 
 /// Encode a JSON string into a b64-encoded string
-fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
+pub fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
     // Parse to ensure it's valid JSON
     let parsed_json: serde_json::Value = serde_json::from_str(json_data)?;
     // Convert JSON back to a compact string
     let json_string = serde_json::to_string(&parsed_json)?;
     // Encode it into b64
-    let encoded = general_purpose::STANDARD.encode(json_string);
+    let encoded = BASE64URL_NOPAD.encode(json_string.as_bytes());
     Ok(encoded)
 }
 
-pub async fn save_host_creds(
-    mut host_agent_keys: keys::Keys,
-    host_user_jwt: String,
-    host_sys_user_jwt: String
-) -> Result<keys::Keys, ServiceError> {
-    //  Save user jwt and sys jwt local to hosting agent
-    utils::write_file(host_user_jwt.as_bytes(), output_dir, "host.jwt").await.map_err(|e| {
-        let err_msg = format!("Failed to save operator jwt. Error={}.", e);
-        handle_internal_err(&err_msg)
-    })?;
-    utils::write_file(host_sys_user_jwt.as_bytes(), output_dir, "host_sys.jwt").await.map_err(|e| {
-        let err_msg = format!("Failed to save sys jwt. Error={}.", e);
-        handle_internal_err(&err_msg)
-    })?;
-
-    // Save user creds and sys creds local to hosting agent
-    let host_creds_file_name = "host.creds";
-    Command::new("nsc")
-        .arg(format!("generate creds --name user_host_{} --account {} > {}", host_pubkey, "WORKLOAD", host_creds_file_name))
-        .output()
-        .expect("Failed to add new operator signing key on hosting agent");
-    
-    let host_sys_creds_file_name = "host_sys.creds";
-    Command::new("nsc")
-        .arg(format!("generate creds --name user_host_{} --account {} > {}", host_sys_pubkey, "SYS", host_sys_creds_file_name))
-        .output()
-        .expect("Failed to add new operator signing key on hosting agent");
-
-    host_agent_keys = host_agent_keys.add_creds_paths(utils::get_file_path_buf(host_creds_file_name), utils::get_file_path_buf(host_sys_creds_file_name));
-
-    Ok(host_agent_keys)
-}
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
index 6a971dc..b3629fc 100644
--- a/rust/clients/host_agent/src/hostd/mod.rs
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -1,2 +1,2 @@
-pub mod workload_manager;
+pub mod workloads;
 pub mod gen_leaf_server;
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workloads.rs
similarity index 98%
rename from rust/clients/host_agent/src/hostd/workload_manager.rs
rename to rust/clients/host_agent/src/hostd/workloads.rs
index cea0706..5927d5d 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -53,10 +53,7 @@ pub async fn run(
     // Spin up Nats Client and loaded in the Js Stream Service
     // Nats takes a moment to become responsive, so we try to connect in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let creds = match host_creds_path.to_owned() {
-        Some(p) => Some(Credentials::Path(p)),
-        _ => None
-    };
+    let creds = host_creds_path.to_owned().map(Credentials::Path);
 
     let host_workload_client = tokio::select! {
         client = async {loop {
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index bb6f200..3d02708 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -4,28 +4,45 @@ use data_encoding::BASE64URL_NOPAD;
 use std::fs::File;
 use std::io::{Read, Write};
 use std::path::PathBuf;
-use util_libs::nats_js_client;
+use std::process::Command;
+use util_libs:: nats_js_client::{get_nats_creds_by_nsc, get_file_path_buf};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let creds_type = match self.creds {
+            AuthCredType::Guard(_) => "Guard",
+            AuthCredType::Authenticated(_) => "Authenticated",
+        };
         f.debug_struct("Keys")
             .field("host_keypair", &"[redacted]")
             .field("host_pubkey", &self.host_pubkey)
-            .field("maybe_host_creds_path", &self.maybe_host_creds_path.is_some())
-            .field("local_sys_keypair", if &self.local_sys_keypair.is_some(){&"[redacted]"} else { &false })
+            .field("local_sys_keypair", if self.local_sys_keypair.is_some(){&"[redacted]"} else { &false })
             .field("local_sys_pubkey", &self.local_sys_pubkey)
-            .field("maybe_sys_creds_path", &self.maybe_sys_creds_path.is_some())
+            .field("creds", &creds_type)
             .finish()
     }
 }
 
+#[derive(Clone)]
+pub struct CredPaths {
+    host_creds_path: PathBuf,
+    #[allow(dead_code)]
+    sys_creds_path: Option<PathBuf>,
+}
+
+#[derive(Clone)]
+pub enum AuthCredType {
+    Guard(PathBuf), // Default
+    Authenticated(CredPaths) // only assiged after successful hoster authentication
+}
+
+#[derive(Clone)]
 pub struct Keys {
     host_keypair: KeyPair,
     pub host_pubkey: String,
-    maybe_host_creds_path: Option<PathBuf>,
     local_sys_keypair: Option<KeyPair>,
     pub local_sys_pubkey: Option<String>,
-    maybe_sys_creds_path: Option<PathBuf>,
+    pub creds: AuthCredType,
 }
 
 impl Keys {
@@ -33,113 +50,174 @@ impl Keys {
         // let host_key_path = format!("{}/user_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
         let host_key_path = std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
         let host_kp = KeyPair::new_user();
-        write_to_file(nats_js_client::get_file_path_buf(&host_key_path), host_kp.clone());
+        write_keypair_to_file(get_file_path_buf(&host_key_path), host_kp.clone())?;
         let host_pk = host_kp.public_key();
         
         // let sys_key_path = format!("{}/user_sys_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
         let sys_key_path = std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
         let local_sys_kp = KeyPair::new_user();
-        write_to_file(nats_js_client::get_file_path_buf(&sys_key_path), local_sys_kp.clone());
+        write_keypair_to_file(get_file_path_buf(&sys_key_path), local_sys_kp.clone())?;
         let local_sys_pk = local_sys_kp.public_key();
 
+        let auth_guard_creds = get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
+
         Ok(Self {
             host_keypair: host_kp,
             host_pubkey: host_pk,
-            maybe_host_creds_path: None, 
             local_sys_keypair: Some(local_sys_kp),
             local_sys_pubkey: Some(local_sys_pk),
-            maybe_sys_creds_path: None,
+            creds: AuthCredType::Guard(auth_guard_creds),
         })
     }
 
-    pub fn try_from_storage(maybe_host_creds_path: &Option<PathBuf>, maybe_sys_creds_path: &Option<PathBuf>) -> Result<Option<Self>> {
+    // NB: Only call when trying to load an already authenticated Host and Sys User
+    pub fn try_from_storage(maybe_host_creds_path: &Option<PathBuf>, maybe_sys_creds_path: &Option<PathBuf>) -> Result<Self> {
         let host_key_path = std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
-        let host_keypair = try_get_from_file(nats_js_client::get_file_path_buf(&host_key_path.clone()))?.ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
+        let host_keypair = try_read_keypair_from_file(get_file_path_buf(&host_key_path.clone()))?.ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
         let host_pk = host_keypair.public_key();
         let sys_key_path = std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
-        let host_creds_path = maybe_host_creds_path.to_owned().unwrap_or_else(|| nats_js_client::get_file_path_buf(
-            &nats_js_client::get_nats_creds_by_nsc("HOLO", "HPOS", "host")
+        let host_creds_path = maybe_host_creds_path.to_owned().unwrap_or_else(|| get_file_path_buf(
+            &get_nats_creds_by_nsc("HOLO", "HPOS", "host")
         ));
-        let sys_creds_path = maybe_sys_creds_path.to_owned().unwrap_or_else(|| nats_js_client::get_file_path_buf(
-            &nats_js_client::get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
+        let sys_creds_path = maybe_sys_creds_path.to_owned().unwrap_or_else(|| get_file_path_buf(
+            &get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
         ));
-        let keys = match try_get_from_file(nats_js_client::get_file_path_buf(&sys_key_path))? {
+
+        // Set auth_guard_creds as default:
+        let auth_guard_creds = get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
+
+        let keys = match try_read_keypair_from_file(get_file_path_buf(&sys_key_path))? {
             Some(kp) => {
                 let local_sys_pk = kp.public_key();
                 Self {
                     host_keypair,
                     host_pubkey:host_pk,
-                    maybe_host_creds_path: None,
                     local_sys_keypair: Some(kp),
                     local_sys_pubkey: Some(local_sys_pk),
-                    maybe_sys_creds_path: None
+                    creds: AuthCredType::Guard(auth_guard_creds)
                 }
             },
             None => {
                 Self {
                     host_keypair,
                     host_pubkey: host_pk,
-                    maybe_host_creds_path: None,
                     local_sys_keypair: None,
                     local_sys_pubkey: None,
-                    maybe_sys_creds_path: None
+                    creds: AuthCredType::Guard(auth_guard_creds)
                 }
             }
         };
         
-        return Ok(Some(keys.add_creds_paths(host_creds_path, sys_creds_path)?));
+        Ok(keys.clone().add_creds_paths(host_creds_path, Some(sys_creds_path)).unwrap_or_else(move |e| {
+            log::error!("Error: Cannot locate authenticated cred files. Defaulting to auth_guard_creds. Err={}",e);
+            keys
+        }))
+    }
+
+    pub fn _add_local_sys(mut self, sys_key_path: Option<PathBuf>) -> Result<Self> {
+        let sys_key_path = sys_key_path.unwrap_or_else(|| get_file_path_buf(
+            &get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
+        ));
+        
+        let mut is_new_key = false;
+
+        let local_sys_kp = try_read_keypair_from_file(sys_key_path.clone())?.unwrap_or_else(|| {
+            is_new_key = true;
+            KeyPair::new_user()
+        });
+
+        if is_new_key {
+            write_keypair_to_file(sys_key_path, local_sys_kp.clone())?;
+        }
+
+        let local_sys_pk = local_sys_kp.public_key();
+
+        self.local_sys_keypair = Some(local_sys_kp);
+        self.local_sys_pubkey = Some(local_sys_pk);
+
+        Ok(self)
     }
 
-    pub fn add_creds_paths(self, host_creds_file_path: PathBuf, sys_creds_file_path: PathBuf) -> Result<Self> {
+    pub fn add_creds_paths(mut self, host_creds_file_path: PathBuf, sys_creds_file_path: Option<PathBuf>) -> Result<Self> {
         match host_creds_file_path.try_exists() {
             Ok(is_ok) => {
                 if !is_ok {
                     return Err(anyhow!("Failed to locate host creds path. Found broken sym link. Path={:?}", host_creds_file_path));
                 }
-                match sys_creds_file_path.try_exists() {
-                    Ok(is_ok) => {
-                        if !is_ok {
-                            return Err(anyhow!("Failed to locate sys creds path. Found broken sym link. Path={:?}", sys_creds_file_path));
-                        }
 
-                        Ok(Self {
-                            maybe_host_creds_path: Some(host_creds_file_path),
-                            maybe_sys_creds_path: Some(sys_creds_file_path),
-                            ..self    
-                        })
+                let creds = match sys_creds_file_path {
+                    Some(sys_path) => match sys_path.try_exists() {
+                        Ok(is_ok) => {
+                            if !is_ok {
+                                return Err(anyhow!("Failed to locate sys creds path. Found broken sym link. Path={:?}", sys_path));
+                            }
+                            CredPaths {
+                                host_creds_path: host_creds_file_path,
+                                sys_creds_path: Some(sys_path),
+                            }
+                        },
+                        Err(e) => {
+                            return Err(anyhow!("Failed to locate sys creds path. Path={:?} Err={}", sys_path, e));
+                        }
                     },
-                    Err(e) => Err(anyhow!("Failed to locate sys creds path. Path={:?} Err={}", sys_creds_file_path, e))
-                }
+                    None => CredPaths {
+                        host_creds_path: host_creds_file_path,
+                        sys_creds_path: None,
+                    }
+                };
+                self.creds = AuthCredType::Authenticated(creds);
+                Ok(self)
             },
             Err(e) => Err(anyhow!("Failed to locate host creds path. Path={:?} Err={}", host_creds_file_path, e))
         }
     }
 
-    pub fn add_local_sys(self, sys_key_path: Option<PathBuf>) -> Result<Self> {
-        let sys_key_path = sys_key_path.unwrap_or_else(|| nats_js_client::get_file_path_buf(
-            &nats_js_client::get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
-        ));
+    pub async fn save_host_creds(
+        &self,
+        host_user_jwt: String,
+        host_sys_user_jwt: String
+    ) -> Result<Self> {
+        //  Save user jwt and sys jwt local to hosting agent
+        let host_path = get_file_path_buf(&format!("{}.{}","output_dir", "host.jwt"));
+        write_to_file(host_path, host_user_jwt.as_bytes())?;
+        let sys_path = get_file_path_buf(&format!("{}.{}","output_dir", "host_sys.jwt"));
+        write_to_file(sys_path, host_sys_user_jwt.as_bytes())?;
+    
+        // Save user creds and sys creds local to hosting agent
+        let host_creds_file_name = "host.creds";
+        Command::new("nsc")
+            .arg(format!("generate creds --name user_host_{} --account {} > {}", self.host_pubkey, "WORKLOAD", host_creds_file_name))
+            .output()
+            .context("Failed to add new operator signing key on hosting agent")?;
         
-        let local_sys_kp = try_get_from_file(sys_key_path.clone())?.unwrap_or_else(|| {
-            let kp = KeyPair::new_user();
-            write_to_file(sys_key_path, kp);
-            KeyPair::new_user()
-        });
-        let local_sys_pk = local_sys_kp.public_key();
-
-        Ok(Self {
-            local_sys_keypair: Some(local_sys_kp),
-            local_sys_pubkey: Some(local_sys_pk),
-            ..self    
-        })
+        let mut sys_creds_file_name = None;
+        if let Some(sys_pubkey) = self.local_sys_pubkey.as_ref() {
+            let file_name = "host_sys.creds";
+            sys_creds_file_name = Some(get_file_path_buf(file_name));
+            Command::new("nsc")
+                .arg(format!("generate creds --name user_host_{} --account {} > {}", sys_pubkey, "SYS", file_name))
+                .output()
+                .context("Failed to add new operator signing key on hosting agent")?;
+        }
+    
+        self.to_owned().add_creds_paths(
+            get_file_path_buf(host_creds_file_name),
+            sys_creds_file_name
+        )
     }
 
     pub fn get_host_creds_path(&self) -> Option<PathBuf> {
-        self.maybe_host_creds_path.clone()
+        if let AuthCredType::Authenticated(creds) = self.to_owned().creds {
+            return Some(creds.host_creds_path);
+        };
+        None
     }
 
-    pub fn get_sys_creds_path(&self) -> Option<PathBuf> {
-        self.maybe_sys_creds_path.clone()
+    pub fn _get_sys_creds_path(&self) -> Option<PathBuf> {
+        if let AuthCredType::Authenticated(creds) = self.to_owned().creds {
+            return creds.sys_creds_path;
+        };
+        None
     }
 
     pub fn host_sign(&self, payload: &[u8]) -> Result<String> {
@@ -151,32 +229,41 @@ impl Keys {
     }
 }
 
-fn write_to_file(key_file_path: PathBuf, keypair: KeyPair) -> Result<()> {
+fn write_keypair_to_file(key_file_path: PathBuf, keypair: KeyPair) -> Result<()> {
     let seed = keypair.seed()?;
-    let mut file = File::create(&key_file_path)?;
-    file.write_all(seed.as_bytes())?;
+    write_to_file(key_file_path, seed.as_bytes())
+}
+
+fn write_to_file(file_path: PathBuf, data: &[u8]) -> Result<()> {
+    let mut file = File::create(&file_path)?;
+    file.write_all(data)?;
     Ok(())
 }
 
-fn try_get_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
-    match key_file_path.try_exists() {
+fn try_read_keypair_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
+    match try_read_from_file(key_file_path)? {
+        Some(kps) => Ok(Some(KeyPair::from_seed(&kps)?)),
+        None => Ok(None)
+    }
+}
+
+fn try_read_from_file(file_path: PathBuf) -> Result<Option<String>> {
+    match file_path.try_exists() {
         Ok(link_is_ok) => {
             if !link_is_ok {
-                return Err(anyhow!("Failed to read path {:?}. Found broken sym link.", key_file_path));
+                return Err(anyhow!("Failed to read path {:?}. Found broken sym link.", file_path));
             }
 
-            let mut key_file_content =
-                File::open(&key_file_path).context(format!("Failed to open config file {:#?}", key_file_path))?;
+            let mut file_content =
+                File::open(&file_path).context(format!("Failed to open config file {:#?}", file_path))?;
         
-            let mut kps = String::new();
-            key_file_content.read_to_string(&mut kps)?;
-            let kp = KeyPair::from_seed(&kps.trim())?;
-
-            Ok(Some(kp))
+            let mut s = String::new();
+            file_content.read_to_string(&mut s)?;
+            Ok(Some(s.trim().to_string()))
         }
         Err(_) => {
-            log::debug!("No user file found at {:?}.", key_file_path);
+            log::debug!("No user file found at {:?}.", file_path);
             Ok(None)
         }
     }
-}
+}
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index efcec85..2b83e06 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -21,7 +21,7 @@ use clap::Parser;
 use dotenv::dotenv;
 use thiserror::Error;
 use agent_cli::DaemonzeArgs;
-use util_libs::nats_js_client;
+use hpos_hal::inventory::HoloInventory;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -55,19 +55,23 @@ async fn main() -> Result<(), AgentCliError> {
     Ok(())
 }
 
-async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {    
-    let host_agent_keys = match keys::Keys::try_from_storage(&args.nats_leafnode_client_creds_path, &args.nats_leafnode_client_sys_creds_path)? {
-        Some(k) => k,
-        None => {
-            log::debug!("About to run the Hosting Agent Initialization Service");
-            let mut keys = keys::Keys::new()?;
-            keys = auth::init::run(keys).await?;
-            keys
-        }
-    };
+async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
+    let mut host_agent_keys = keys::Keys::try_from_storage(
+        &args.nats_leafnode_client_creds_path,
+        &args.nats_leafnode_client_sys_creds_path
+    ).or_else(|_| keys::Keys::new().map_err(|e| {
+        eprintln!("Failed to create new keys: {:?}", e);
+        async_nats::Error::from(e)
+    }))?;
+
+    // If user cred file is for the auth_guard user, run loop to authenticate host & hoster...
+    if let keys::AuthCredType::Guard(_) = host_agent_keys.creds {
+        host_agent_keys = run_auth_loop(host_agent_keys).await?;
+    }
 
+    // Once authenticated, start leaf server and run workload api calls.
     hostd::gen_leaf_server::run(&host_agent_keys.get_host_creds_path()).await;
-    hostd::workload_manager::run(
+    hostd::workloads::run(
         &host_agent_keys.host_pubkey,
         &host_agent_keys.get_host_creds_path(),
         args.nats_connect_timeout_secs,
@@ -75,3 +79,42 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     .await?;
     Ok(())
 }
+
+async fn run_auth_loop(mut keys: keys::Keys) ->  Result<keys::Keys, async_nats::Error> {
+    let mut start = chrono::Utc::now();
+    
+    // while let keys::AuthCredType::Guard(auth_creds) = keys.creds {
+        loop {
+        log::debug!("About to run the Hosting Agent Authentication Service");
+        let auth_guard_client: async_nats::Client;
+        (keys, auth_guard_client) = auth::init::run(keys).await?;
+        
+        // If authenicated creds exist, then auth call was successful.
+        // Close buffer, exit loop, and return.
+        if let keys::AuthCredType::Authenticated(_) = keys.creds {
+            auth_guard_client.drain().await?;
+            break;
+        }
+
+        // Otherwise, send diagonostics every 1hr for the next 24hrs, then exit while loop and retry auth.
+        // TODO: Discuss interval for sending diagnostic reports and wait duration before retrying auth with team.
+        let now = chrono::Utc::now();
+        let max_time_interval = chrono::TimeDelta::days(1);
+
+        while max_time_interval < start.signed_duration_since(now) {
+            let unauthenticated_user_diagnostics_subject = format!("DIAGNOSTICS.unauthenticated.{}", keys.host_pubkey);
+            let diganostics = HoloInventory::from_host();
+            let payload_bytes = serde_json::to_vec(&diganostics)?;
+            if let Err(e) = auth_guard_client.publish(unauthenticated_user_diagnostics_subject, payload_bytes.into()).await {
+                log::error!("Encountered error when sending diganostics. Err={:#?}", e);
+            };
+            tokio::time::sleep(chrono::TimeDelta::hours(1).to_std()?).await;
+        }
+
+        // Close and drain internal buffer before exiting to make sure all messages are sent.
+        auth_guard_client.drain().await?;
+        start = chrono::Utc::now();
+    }
+
+    Ok(keys)
+}
\ No newline at end of file
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 6aa8544..660825d 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -18,8 +18,6 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
-use crate::utils as local_utils;
-
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use anyhow::{anyhow, Result};
 use async_nats::Message;
@@ -53,8 +51,8 @@ pub async fn run() -> Result<(), async_nats::Error> {
             name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
             inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
             service_params: vec![auth_stream_service_params],
-            credentials_path: None,
-            opts: vec![nats_js_client::with_event_listeners(event_listeners)],
+            credentials: None,
+            listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
             ping_interval: Some(Duration::from_secs(10)),
             request_timeout: Some(Duration::from_secs(5)),
         })
@@ -81,10 +79,10 @@ pub async fn run() -> Result<(), async_nats::Error> {
     auth_service
         .add_consumer::<AuthApiResult>(
             "validate", // consumer name
-            &types::AUTH_SERVICE_SUBJECT, // consumer stream subj
+            types::AUTH_SERVICE_SUBJECT, // consumer stream subj
             EndpointType::Async(auth_api.call(|api: AuthServiceApi, msg: Arc<Message>| {
                 async move {
-                    api.handle_handshake_request(msg, &local_utils::get_orchestrator_credentials_dir_path()).await
+                    api.handle_handshake_request(msg).await
                 }
             })),
             Some(create_callback_subject_to_host("host_pubkey".to_string())),
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
index 8262114..bd68b56 100644
--- a/rust/clients/orchestrator/src/utils.rs
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -6,7 +6,7 @@ pub fn _get_resolver_path() -> String {
     std::env::var("RESOLVER_FILE_PATH").unwrap_or_else(|_| "./resolver.conf".to_string())
 }
 
-pub fn get_orchestrator_credentials_dir_path() -> String {
+pub fn _get_orchestrator_credentials_dir_path() -> String {
     std::env::var("ORCHESTRATOR_CREDENTIALS_DIR_PATH").unwrap_or_else(|e| panic!("Failed to locate 'ORCHESTRATOR_CREDENTIALS_DIR_PATH' env var.  Was it set?  Error={}", e))
 }
 
diff --git a/rust/services/authentication/Cargo.toml b/rust/services/authentication/Cargo.toml
index fba30e9..2bdc024 100644
--- a/rust/services/authentication/Cargo.toml
+++ b/rust/services/authentication/Cargo.toml
@@ -19,6 +19,10 @@ mongodb = "3.1"
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
 url = { version = "2", features = ["serde"] }
 nkeys = "=0.4.4"
+sha2 = "=0.10.8"
+nats-jwt = "0.3.0"
+data-encoding = "2.7.0"
+jsonwebtoken = "9.3.0"
 bytes = "1.8.0"
 chrono = "0.4.0"
 util_libs = { path = "../../util_libs" }
\ No newline at end of file
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 8e07547..2d57342 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -9,12 +9,12 @@ Endpoints & Managed Subjects:
 
 pub mod types;
 pub mod utils;
-use anyhow::Result;
-use async_nats::Message;
+use anyhow::{Result, Context};
+use async_nats::{AuthError, Message};
 use async_nats::jetstream::ErrorCode;
 use std::sync::Arc;
 use std::future::Future;
-use types::{WORKLOAD_SK_ROLE, AuthApiResult};
+use types::{AuthApiResult, WORKLOAD_SK_ROLE};
 use util_libs::nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse};
 use async_nats::HeaderValue;
 use nkeys::KeyPair;
@@ -59,15 +59,189 @@ impl AuthServiceApi {
         })
     }
 
+    pub async fn handle_auth_callout(
+        &self,
+        msg: Arc<Message>,
+        auth_signing_account_keypair: KeyPair,
+        auth_signing_account_pubkey: String,
+        auth_root_account_keypair: KeyPair,
+        auth_root_account_pubkey: String,
+    ) -> Result<AuthApiResult, ServiceError> {
+        // 1. Verify expected data was received
+        let auth_request_token = String::from_utf8_lossy(&msg.payload).to_string();
+        println!("auth_request_token  : {:?}", auth_request_token);
+
+        let auth_request_claim = utils::decode_jwt::<types::NatsAuthorizationRequestClaim>(&auth_request_token).map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
+        println!("\nauth REQUEST - main claim : {}", serde_json::to_string_pretty(&auth_request_claim).unwrap());
+
+        let auth_request_user_claim = utils::decode_jwt::<types::UserClaim>(&auth_request_claim.auth_request.connect_opts.user_jwt).map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
+        println!("\nauth REQUEST - user claim : {}", serde_json::to_string_pretty(&auth_request_user_claim).unwrap());
+
+        let user_data: types::AuthGuardPayload = utils::base64_to_data::<types::AuthGuardPayload>(&auth_request_claim.auth_request.connect_opts.user_auth_token).map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
+        println!("user_data TO VALIDATE  : {:#?}", user_data); 
+        
+        // 2. Validate Host signature, returning validation error if not successful
+        let host_pubkey = user_data.host_pubkey.as_ref();
+        let host_signature = user_data.get_host_signature();
+        let user_verifying_keypair = KeyPair::from_public_key(host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let raw_payload = serde_json::to_vec(&user_data.clone().without_signature()).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &host_signature) {
+            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
+            return Err(ServiceError::Authentication(AuthError::new(e)));
+        };
+
+        // 3. If provided, authenticate the Hoster pubkey and email and assign full permissions if successful
+        let is_hoster_valid = if user_data.email.is_some() && user_data.hoster_hc_pubkey.is_some() {
+            let hoster_hc_pubkey = user_data.hoster_hc_pubkey.unwrap(); // unwrap is safe here as checked above
+            let hoster_email = user_data.email.unwrap(); // unwrap is safe here as checked above
+
+            let is_valid: bool = match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_hc_pubkey.clone() }).await? {
+                Some(u) => {
+                    let mut is_valid = true;
+                    // If hoster exists with pubkey, verify email
+                    if u.email != hoster_email {
+                        log::error!("Error: Failed to validate hoster email. Email='{}'.", hoster_email);
+                        is_valid = false;
+                    }
+
+                    // ...then find the host collection that contains the provided host pubkey
+                    match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey }).await? {
+                        Some(host) => {
+                            // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
+                            if host.assigned_hoster != hoster_hc_pubkey {
+                                let host_query: bson::Document = doc! { "_id":  host._id.clone() };
+                                let updated_host_doc = to_document(& Host{
+                                    assigned_hoster: hoster_hc_pubkey,
+                                    ..host
+                                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                                self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
+                            }
+                        },
+                        None => {
+                            log::error!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
+                            is_valid = false;
+                        }
+                    }
+
+                    // Find the mongo_id ref for the hoster associated with this user
+                    let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
+                        let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
+                        handle_internal_err(&err_msg)
+                    })?;
+                    
+                    // Finally, find the hoster collection
+                    match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
+                        Some(hoster) => {
+                            // ...and pair the hoster with host (if the host is not already assiged to the hoster)
+                            let mut updated_assigned_hosts = hoster.assigned_hosts;
+                            if !updated_assigned_hosts.contains(&host_pubkey.to_string()) {
+                                let hoster_query: bson::Document = doc! { "_id":  hoster._id.clone() };
+                                updated_assigned_hosts.push(host_pubkey.to_string());
+                                let updated_hoster_doc = to_document(& Hoster {
+                                    assigned_hosts: updated_assigned_hosts,
+                                    ..hoster
+                                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                                self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
+                            }
+                        },
+                        None => {
+                            log::error!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
+                            is_valid = false;
+                        }
+                    }
+                    is_valid
+                },
+                None => {
+                    log::error!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
+                    false
+                }
+            };
+            is_valid
+        } else { false };
+
+
+        // 4. Assign permissions based on whether the hoster was successfully validated
+        let permissions = if is_hoster_valid {
+            // If successful, assign personalized inbox and auth permissions
+            let user_unique_auth_subject = &format!("AUTH.{}.>", host_pubkey);
+            println!(">>> user_unique_auth_subject : {user_unique_auth_subject}");    
+
+            let user_unique_inbox = &format!("_INBOX.{}.>", host_pubkey);
+            println!(">>> user_unique_inbox : {user_unique_inbox}");    
+                    
+            let authenticated_user_diagnostics_subject = &format!("DIAGNOSTICS.authenticated.{}.>", host_pubkey);
+            println!(">>> authenticated_user_diagnostics_subject : {authenticated_user_diagnostics_subject}");    
+
+            types::Permissions {
+                publish: types::PermissionLimits {
+                    allow: Some(vec![
+                        "AUTH.validate".to_string(),
+                        user_unique_auth_subject.to_string(),
+                        user_unique_inbox.to_string(),
+                        authenticated_user_diagnostics_subject.to_string()
+                ]),
+                    deny: None,
+                },
+                subscribe: types::PermissionLimits {
+                    allow: Some(vec![
+                        user_unique_auth_subject.to_string(),
+                        user_unique_inbox.to_string(),
+                        authenticated_user_diagnostics_subject.to_string()
+                    ]),
+                    deny: None,
+                },
+            }
+        } else {
+            // Otherwise, exclusively grant publication permissions for the unauthenticated diagnostics subj
+            // ...to allow the host device to still send diganostic reports
+            let unauthenticated_user_diagnostics_subject = format!("DIAGNOSTICS.unauthenticated.{}.>", host_pubkey);
+            types::Permissions {
+                publish: types::PermissionLimits {
+                    allow: Some(vec![unauthenticated_user_diagnostics_subject]),
+                    deny: None,
+                },
+                subscribe: types::PermissionLimits {
+                    allow: None,
+                    deny: Some(vec![">".to_string()]),
+                },
+            }
+        };
+
+        let auth_response_claim = utils::generate_auth_response_claim(
+            auth_signing_account_keypair,
+            auth_signing_account_pubkey,
+            auth_root_account_pubkey,
+            permissions,
+            auth_request_claim
+        ).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+
+        let claim_str = serde_json::to_string(&auth_response_claim).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let token = utils::encode_jwt(&claim_str, &auth_root_account_keypair)
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        println!("\n\n\n\nencoded_jwt: {:#?}", token);
+
+        // DONE BY JS HANDLER
+        // let res = token.into_bytes();
+        // if let Some(reply) = msg.reply {
+        //     client.publish(reply, res.into()).await?;
+        // }
+
+        Ok(types::AuthApiResult {
+            result: types::AuthResult::Callout(token),
+            maybe_response_tags: None
+        })
+    }
+
     pub async fn handle_handshake_request(
         &self,
         msg: Arc<Message>,
-        creds_dir_path: &str,
     ) -> Result<AuthApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.validate' : {:?}", msg);
 
-        let mut status = types::AuthState::Unauthenticated;
-
         // 1. Verify expected data was received
         let signature: &[u8] = match &msg.headers {
             Some(h) => {
@@ -86,7 +260,7 @@ impl AuthServiceApi {
             }
         };
 
-        let types::AuthRequestPayload { host_pubkey, email, hoster_pubkey, sys_pubkey, nonce: _ } = Self::convert_msg_to_type::<types::AuthRequestPayload>(msg.clone())?;
+        let types::AuthJWTPayload { host_pubkey, maybe_sys_pubkey, nonce: _ } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
 
         // 2. Validate signature
         let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
@@ -95,106 +269,59 @@ impl AuthServiceApi {
             return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
         };
 
-        // 3. Authenticate the Hosting Agent (via email and host id info?)
-        let hoster_pubkey_as_holo_hash = "convert_hoster_pubkey_to_raw_value_and_then_into_holo_hash";
-        match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_pubkey_as_holo_hash.clone() }).await? {
-            Some(u) => {
-                // If hoster exists with pubkey, verify email
-                if u.email != email {
-                    log::error!("Error: Failed to validate user email. Subject='{}'.", msg.subject);
-                    return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
-                }
-
-                // ...then find the host collection that contains the provided host pubkey
-                match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey.clone() }).await? {
-                    Some(h) => {
-                        // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
-                        if h.assigned_hoster != hoster_pubkey {
-                            let host_query: bson::Document = doc! { "_id":  h._id.clone() };
-                            let updated_host_doc = to_document(& Host{
-                                assigned_hoster: hoster_pubkey,
-                                ..h
-                            }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                            self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
-                        }
-                    },
-                    None => {
-                        let err_msg = format!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
-                        return Err(handle_internal_err(&err_msg));
-                    }
-                }
-
-                // Find the mongo_id ref for the hoster associated with this user
-                let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
-                    let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
-                    handle_internal_err(&err_msg)
-                })?;
-                
-                // Finally, find the hoster collection
-                match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
-                    Some(hr) => {
-                        // ...and pair the hoster with host (if the host is not already assiged to the hoster)
-                        let mut updated_assigned_hosts = hr.assigned_hosts;
-                        if !updated_assigned_hosts.contains(&host_pubkey) {
-                        let hoster_query: bson::Document = doc! { "_id":  hr._id.clone() };
-                        updated_assigned_hosts.push(host_pubkey.clone());
-                        let updated_hoster_doc = to_document(& Hoster {
-                            assigned_hosts: updated_assigned_hosts,
-                            ..hr
-                        }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                        self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
-                        }
-                    },
-                    None => {
-                        let err_msg = format!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
-                        return Err(handle_internal_err(&err_msg));
-                    }
-                }
-            },
-            None => {
-                let err_msg = format!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
-                return Err(handle_internal_err(&err_msg));
-            }
-        };
-
         // 4. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
-        Command::new("nsc")
-            .arg(format!("add user -a SYS -n user_sys_host_{} -k {}", host_pubkey, sys_pubkey))
-            .output()
-            .expect("Failed to add host sys user with provided keys");
-
-        Command::new("nsc")
-            .arg(format!("add user -a WORKLOAD -n user_host_{} -k {} -K {} --tag pubkey:{}", host_pubkey, host_pubkey, WORKLOAD_SK_ROLE, host_pubkey))
-            .output()
-            .expect("Failed to add host user with provided keys");
+        if let Some(sys_pubkey) = maybe_sys_pubkey {
+            Command::new("nsc")
+                .arg(format!("add user -a SYS -n user_sys_host_{} -k {}", host_pubkey, sys_pubkey))
+                .output()
+                .context("Failed to add host sys user with provided keys")
+                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+    
+            Command::new("nsc")
+                .arg(format!("add user -a WORKLOAD -n user_host_{} -k {} -K {} --tag pubkey:{}", host_pubkey, host_pubkey, WORKLOAD_SK_ROLE, host_pubkey))
+                .output()
+                .context("Failed to add host user with provided keys")
+                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        }
 
         // ..and push auth updates to hub server
         Command::new("nsc")
             .arg("push -A")
             .output()
-            .expect("Failed to update resolver config file");    
+            .context("Failed to update resolver config file")
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         // 3. Create User JWT files (automatically signed with respective account key)
-        let host_sys_user_file_name = format!("{}/user_sys_host_{}.jwt", creds_dir_path, host_pubkey);
-        Command::new("nsc") 
-            .arg(format!("describe user -a SYS -n user_sys_host_{} --raw --output-file {}", host_pubkey, host_sys_user_file_name))
+        let sys_jwt_output = Command::new("nsc") 
+            .arg(format!("describe user -n user_sys_host_{} -a SYS --raw", host_pubkey))
             .output()
-            .expect("Failed to generate host sys user jwt file");
-
-        let host_user_file_name = format!("{}/user_host_{}.jwt", creds_dir_path, host_pubkey);
-        Command::new("nsc") 
-            .arg(format!("describe user -a WORKLOAD -n user_host_{} --raw --output-file {}", host_pubkey, host_user_file_name))
+            .context("Failed to generate host sys user jwt file")
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+    
+        let sys_jwt = String::from_utf8(sys_jwt_output.stdout)
+            .context("Command returned invalid UTF-8 output")
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        
+        let host_jwt_output= Command::new("nsc") 
+            .arg(format!("describe user -n user_host_{} -a WORKLOAD --raw", host_pubkey))
             .output()
-            .expect("Failed to generate host user jwt file");        
+            .context("Failed to generate host user jwt file")
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+        let host_jwt = String::from_utf8(host_jwt_output.stdout)
+            .context("Command returned invalid UTF-8 output")
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
-
-        status = types::AuthState::Authenticated;
         
         Ok(AuthApiResult {
-            host_pubkey: host_pubkey.clone(),
-            status,
+            result: types::AuthResult::Authorization(types::AuthJWTResult {
+                host_pubkey: host_pubkey.clone(),
+                status: types::AuthState::Authorized,
+                host_jwt,
+                sys_jwt
+            }),
             maybe_response_tags: Some(tag_map)
         })
     }
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 5574698..539c453 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -1,7 +1,7 @@
-use std::collections::HashMap;
-
-use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
+use anyhow::Result;
 use serde::{Deserialize, Serialize};
+use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
+use std::collections::HashMap;
 
 pub const AUTH_SERVICE_SUBJECT: &str = "validate";
 
@@ -11,31 +11,240 @@ pub const WORKLOAD_SK_ROLE: &str = "workload-role";
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthState {
-    Unauthenticated,
-    Authenticated,
-    Forbidden,
-    Error(String)
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, Default)]
-pub struct AuthGuardPayload {
-    pub host_pubkey: String, // nkey
-    pub hoster_pubkey: String, // nkey
-    pub email: String,
-    pub nonce: String
+    Unauthenticated, // step 0
+    Authenticated, // step 1
+    Authorized, // step 2
+    Forbidden, // failure to auth
+    Error(String) // internal error
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
-pub struct AuthRequestPayload {
+pub struct AuthJWTPayload {
     pub host_pubkey: String, // nkey
-    pub sys_pubkey: Option<String>, // nkey
+    pub maybe_sys_pubkey: Option<String>, // nkey
     pub nonce: String
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct AuthApiResult {
-    pub host_pubkey: String,
+pub struct AuthJWTResult {
     pub status: AuthState,
+    pub host_pubkey: String,
     pub host_jwt: String,
     pub sys_jwt: String
 } 
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum AuthResult {
+    Callout(String), // stringifiedAuthResponseClaim
+    Authorization(AuthJWTResult)
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct AuthApiResult {
+    pub result: AuthResult,
+    // NB: `maybe_response_tags` optionally return endpoint scoped vars to be available for use as a response subject in JS Service Endpoint handler
+    pub maybe_response_tags: Option<HashMap<String, String>> 
+}
+// NB: The following Traits make API Service compatible as a JS Service Endpoint
+impl EndpointTraits for AuthApiResult {}
+impl CreateTag for AuthApiResult {
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.maybe_response_tags.clone().unwrap_or_default()
+    }
+}
+impl CreateResponse for AuthApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        match self.clone().result {
+            AuthResult::Authorization(r) => {
+                match serde_json::to_vec(&r) {
+                    Ok(r) => r.into(),
+                    Err(e) => e.to_string().into(),
+                }
+            },
+            AuthResult::Callout(token) => token
+                .clone()
+                .into_bytes()
+                .into()
+        }
+    }
+}
+
+//////////////////////////
+// Auth Callout Types
+//////////////////////////
+// Callout Request Types:
+#[derive(Serialize, Deserialize, Clone, Debug, Default)]
+pub struct AuthGuardPayload {
+    pub host_pubkey: String, // nkey pubkey
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hoster_hc_pubkey: Option<String>, // holochain encoded hoster pubkey
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub email: Option<String>,
+    pub nonce: String,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    host_signature: Vec<u8>, // used to verify the host keypair
+}
+// NB: Currently there is no way to pass headers in the auth callout.
+// Therefore the host_signature is passed within the b64 encoded `AuthGuardPayload` token
+impl AuthGuardPayload {
+    pub fn try_add_signature<T>(mut self, sign_handler: T) -> Result<Self>
+    where
+     T: Fn(&[u8]) -> Result<String>
+    {
+        let payload_bytes = serde_json::to_vec(&self)?;
+        let signature = sign_handler(&payload_bytes)?;
+        self.host_signature = signature.as_bytes().to_vec();
+        Ok(self)
+    }
+
+    pub fn without_signature(mut self) -> Self {
+        self.host_signature = vec![];
+        self
+    }
+
+    pub fn get_host_signature(&self) -> Vec<u8> {
+        self.host_signature.clone()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct NatsAuthorizationRequestClaim {
+    #[serde(flatten)]
+    pub generic_claim_data: ClaimData,
+    #[serde(rename = "nats")]
+    pub auth_request: NatsAuthorizationRequest,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct NatsAuthorizationRequest {
+    pub server_id: NatsServerId,
+    pub user_nkey: String,
+    pub client_info: NatsClientInfo,
+    pub connect_opts: ConnectOptions,
+    pub r#type: String, // should be authorization_request
+    pub version: u8, // should be 2
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct NatsServerId {
+    pub name: String, // Server name
+    pub host: String, // Server host address
+    pub id: String, // Server connection ID
+    pub version: String, // Version of server (current stable = 2.10.22)
+    pub cluster: String, // Server cluster name
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct NatsClientInfo {
+    pub host: String, // client host address
+    pub id: u64, // client connection ID (I think...)
+    pub user: String, // the user pubkey (the passed-in key)
+    pub name_tag: String, // The user pubkey name
+    pub kind: String, // should be "Client"
+    pub nonce: String,
+    pub r#type: String, // should be "nats"
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct ConnectOptions {
+    #[serde(rename = "auth_token")]
+    pub user_auth_token: String, // This is the b64 encoding of the `AuthGuardPayload` -- used to validate user
+    #[serde(rename = "jwt")]
+    pub user_jwt: String, // This is the jwt string of the `UserClaim`
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sig: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub lang: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub version: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub protocol: Option<u16>
+}
+
+// Callout Response Types:
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AuthResponseClaim {
+    #[serde(flatten)]
+    pub generic_claim_data: ClaimData,
+    #[serde(rename = "nats")]
+    pub auth_response: AuthGuardResponse,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct ClaimData {
+    #[serde(rename = "iat")] 
+    pub issued_at: i64,  // Issued At (Unix timestamp)
+    #[serde(rename = "iss")]
+    pub issuer: String,  // Issuer -- head account (from which any signing keys were created)
+    #[serde(default, rename = "aud", skip_serializing_if = "Option::is_none")]
+    pub audience: Option<String>,  // Audience for whom the token is intended
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+    #[serde(default, rename = "exp", skip_serializing_if = "Option::is_none")]
+    pub expires_at: Option<i64>,  // Expiry (Optional, Unix timestamp)
+    #[serde(default, rename = "jti", skip_serializing_if = "Option::is_none")]
+    pub jwt_id: Option<String>, // Base32 hash of the claims
+    #[serde(default, rename = "nbf", skip_serializing_if = "Option::is_none")]
+    pub not_before: Option<i64>,  // Issued At (Unix timestamp)
+    #[serde(default, rename = "sub")]
+    pub subcriber: String,  // Public key of the account or user to which the JWT is being issued
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct NatsGenericData {
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub tags: Vec<String>,
+    #[serde(rename = "type")]
+	pub claim_type: String, // should be "user"
+    pub version: u8, // should be 2
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct AuthGuardResponse {
+    #[serde(flatten)]
+    pub generic_data: NatsGenericData,
+    #[serde(default, rename = "jwt", skip_serializing_if = "Option::is_none")]
+	pub user_jwt: Option<String>, // This is the jwt string of the `UserClaim`
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub issuer_account: Option<String>, // Issuer Account === the signing nkey. Should set when the claim is issued by a signing key.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+	pub error: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default)]
+pub struct UserClaim {
+    #[serde(flatten)]
+    pub generic_claim_data: ClaimData,
+    #[serde(rename = "nats")]
+	pub user_claim_data: UserClaimData,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default)]
+pub struct UserClaimData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub issuer_account: Option<String>,
+    #[serde(flatten)]
+    pub permissions: Permissions,
+    #[serde(flatten)]
+    pub generic_data: NatsGenericData,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default)]
+pub struct Permissions {
+    #[serde(rename = "pub")]
+    pub publish: PermissionLimits,
+    #[serde(rename = "sub")]
+    pub subscribe: PermissionLimits,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default)]
+pub struct PermissionLimits {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub allow: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub deny: Option<Vec<String>>,
+}
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index 0416708..17b7e2e 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -1,7 +1,14 @@
-use anyhow::Result;
-use async_nats::jetstream::Context;
-use util_libs::nats_js_client::ServiceError;
+use super::types;
+use anyhow::{anyhow, Result};
+use nkeys::KeyPair;
 use std::io::Write;
+use util_libs::nats_js_client::ServiceError;
+use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
+use data_encoding::{BASE32HEX_NOPAD, BASE64URL_NOPAD};
+use sha2::{Digest, Sha256};
+use serde_json::Value;
+use std::time::SystemTime;
+use serde::Deserialize;
 
 pub fn handle_internal_err(err_msg: &str) -> ServiceError {
     log::error!("{}", err_msg);
@@ -24,18 +31,6 @@ pub async fn write_file(
     Ok(output_path)
 }
 
-pub async fn publish_chunks(js: &Context, subject: &str, file_name: &str, data: Vec<u8>) -> Result<()> {
-    // let data: Vec<u8> = std::fs::read(file_path)?;
-    js.publish(format!("{}.{} ", subject, file_name), data.into()).await?;
-    Ok(())
-}
-
-// Placeholder functions for the missing implementations
-pub fn get_account_signing_key() -> String {
-    // Implementation here
-    String::new()
-}
-
 pub fn generate_user_jwt(_user_public_key: &str, _account_signing_key: &str) -> Option<String> {
     // Implementation here
 
@@ -49,6 +44,12 @@ pub fn generate_user_jwt(_user_public_key: &str, _account_signing_key: &str) ->
 
         Some(String::new())
 }
+// pub async fn publish_chunks(js: &Context, subject: &str, file_name: &str, data: Vec<u8>) -> Result<()> {
+//     // let data: Vec<u8> = std::fs::read(file_path)?;
+//     js.publish(format!("{}.{} ", subject, file_name), data.into()).await?;
+//     Ok(())
+// }
+
 
 // pub async fn chunk_file_and_publish(_js: &Context, _subject: &str, _file_path: &str) -> Result<()> {
     // let mut file = std::fs::File::open(file_path)?;
@@ -69,3 +70,195 @@ pub fn generate_user_jwt(_user_public_key: &str, _account_signing_key: &str) ->
 
 //     Ok(())
 // }
+
+
+/// Decode a Base64-encoded string back into a JSON string
+pub fn base64_to_data<T>(base64_data: &str) -> Result<T>
+where
+    T: for<'de> Deserialize<'de>,
+{
+    let decoded_bytes = BASE64URL_NOPAD.decode(base64_data.as_bytes())?;
+    let json_string = String::from_utf8(decoded_bytes)?;
+    let parsed_json: T = serde_json::from_str(&json_string)?;
+    Ok(parsed_json)
+}
+
+pub fn hash_claim(claims_str: &str) -> Vec<u8> {
+    let mut hasher = Sha256::new();
+    hasher.update(claims_str);
+    let claims_hash = hasher.finalize();
+    claims_hash.as_slice().into()
+}
+
+// Convert claims to JWT/Token
+pub fn encode_jwt(claims_str: &str, signing_kp: &KeyPair) -> Result<String> {    
+    const JWT_HEADER: &str = r#"{"typ":"JWT","alg":"ed25519-nkey"}"#;
+    let b64_header: String =  BASE64URL_NOPAD.encode(JWT_HEADER.as_bytes());
+    println!("encoded b64 header: {:?}", b64_header);
+    let b64_body =  BASE64URL_NOPAD.encode(claims_str.as_bytes());
+    println!("encoded header: {:?}", b64_body);
+
+    let jwt_half = format!("{b64_header}.{b64_body}");
+    let sig = signing_kp.sign(jwt_half.as_bytes())?;
+    let b64_sig = BASE64URL_NOPAD.encode(&sig);
+
+    let token = format!("{jwt_half}.{b64_sig}");
+    Ok(token)
+}
+
+/// Convert token into the 
+pub fn decode_jwt<T>(token: &str) -> Result<T> 
+where
+    T: for<'de> Deserialize<'de> + std::fmt::Debug,
+{
+    // Decode and replace custom `ed25519-nkey` to `EdDSA`
+    let parts: Vec<&str> = token.split('.').collect();
+    println!("parts: {:?}", parts);
+    println!("parts.len() : {:?}", parts.len()); 
+
+    if parts.len() != 3 {
+        return Err(anyhow!("Invalid JWT format"));
+    }
+
+    // Decode base64 JWT header and fix the algorithm field
+    let header_json = BASE64URL_NOPAD.decode(parts[0].as_bytes())?;
+    let mut header: Value = serde_json::from_slice(&header_json).expect("failed to create header");
+    println!("header: {:?}", header);
+
+    // Manually fix the algorithm name
+    if let Some(alg) = header.get_mut("alg") {
+        if alg == "ed25519-nkey" {
+            *alg = serde_json::Value::String("EdDSA".to_string());
+        }
+    }
+    println!("after header: {:?}", header);
+
+    let modified_header = BASE64URL_NOPAD.encode(&serde_json::to_vec(&header)?);
+    println!("modified_header: {:?}", modified_header);
+
+    let part_1_json = BASE64URL_NOPAD.decode(parts[1].as_bytes())?;
+    let mut part_1: Value = serde_json::from_slice(&part_1_json)?;
+    if part_1.get("exp").is_none() {
+        let one_week = std::time::Duration::from_secs(7 * 24 * 60 * 60);
+        let one_week_from_now = SystemTime::now() + one_week;
+        let expires_at: i64 =  one_week_from_now.duration_since(SystemTime::UNIX_EPOCH)?
+            .as_secs()
+            .try_into()?;
+
+        let mut b: types::UserClaim = serde_json::from_value(part_1)?;
+        b.generic_claim_data.expires_at = Some(expires_at);
+        part_1 = serde_json::to_value(b)?;
+    }
+    let modified_part_1 = BASE64URL_NOPAD.encode(&serde_json::to_vec(&part_1)?);
+
+    let modified_token = format!("{}.{}.{}", modified_header, modified_part_1, parts[2]);
+    println!("modified_token: {:?}", modified_token);
+
+    let account_kp = KeyPair::from_public_key("ABYGJO6B2OJTXL7DLL7EGR45RQ4I2CKM4D5XYYUSUBZJ7HJJF67E54VC")?;
+
+    let public_key_b32 = account_kp.public_key();
+    println!("Public Key (Base32): {}", public_key_b32);
+
+    // Decode from Base32 to raw bytes using Rfc4648 (compatible with NATS keys)
+    let public_key_bytes =  Some(BASE32HEX_NOPAD
+        .decode(public_key_b32.as_bytes()))
+        .ok_or(anyhow!("Failed to convert public key to bytes"))??;
+    println!("Decoded Public Key Bytes: {:?}", public_key_bytes);
+
+    // Use the decoded key to create a DecodingKey
+    let decoding_key = DecodingKey::from_ed_der(&public_key_bytes);
+    println!(">>>>>>> decoded key");
+
+    // Validate the token with the correct algorithm
+    let mut validation = Validation::new(Algorithm::EdDSA);
+    validation.insecure_disable_signature_validation();
+    validation.validate_aud = false;  // Disable audience validation
+    println!("passed validation");
+
+    let token_data = decode::<T>(&modified_token, &decoding_key, &validation)?;
+    // println!("token_data: {:#?}", token_data);
+
+    Ok(token_data.claims)
+}
+
+pub fn generate_auth_response_claim (
+    auth_signing_account_keypair: KeyPair,
+    auth_signing_account_pubkey: String,
+    auth_root_account_pubkey: String,
+    permissions: types::Permissions,
+    auth_request_claim: types::NatsAuthorizationRequestClaim
+) -> Result<types::AuthResponseClaim> {
+        let now = SystemTime::now();
+        let issued_at = now
+            .duration_since(SystemTime::UNIX_EPOCH)?
+            .as_secs()
+            .try_into()?;
+        let one_week = std::time::Duration::from_secs(7 * 24 * 60 * 60);
+        let one_week_from_now = now + one_week;
+        let expires_at: i64 =  one_week_from_now.duration_since(SystemTime::UNIX_EPOCH)?
+            .as_secs()
+            .try_into()?;    
+        let inner_generic_data = types::NatsGenericData {
+            claim_type: "user".to_string(),
+            tags: vec![],
+            version: 2,
+        };
+        let user_claim_data = types::UserClaimData {
+            permissions,
+            generic_data: inner_generic_data,
+            issuer_account: Some(auth_root_account_pubkey.clone()), // must be the root account pubkey or the issuer account that signs the claim AND must be listed "allowed-account" 
+        };
+        let inner_nats_claim = types::ClaimData {
+            issuer: auth_signing_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
+            subcriber: auth_request_claim.auth_request.user_nkey.clone(),
+            issued_at,
+            audience: None, // Inner claim should have no `audience` when using the operator-auth mode
+            expires_at: Some(expires_at),
+            not_before: None,
+            name: Some("allowed_auth_user".to_string()),
+            jwt_id: None,
+        };
+        let mut user_claim = types::UserClaim {
+            generic_claim_data: inner_nats_claim,
+            user_claim_data
+        };
+        
+        let mut user_claim_str = serde_json::to_string(&user_claim)?;
+        let hashed_user_claim_bytes = hash_claim(&user_claim_str);
+        user_claim.generic_claim_data.jwt_id = Some(BASE32HEX_NOPAD.encode(&hashed_user_claim_bytes));
+        user_claim_str = serde_json::to_string(&user_claim)?;
+        let user_token = encode_jwt(&user_claim_str, &auth_signing_account_keypair)?;
+        println!("user_token: {:#?}", user_token);
+
+        let outer_nats_claim = types::ClaimData {
+            issuer: auth_root_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
+            subcriber: auth_request_claim.auth_request.user_nkey.clone(),
+            issued_at,
+            audience: Some(auth_request_claim.auth_request.server_id.id),
+            expires_at: None, // Some(expires_at),
+            not_before: None,
+            name: None,
+            jwt_id: None,
+        };
+        let outer_generic_data =  types::NatsGenericData {
+            claim_type: "authorization_response".to_string(),
+            tags: vec![],
+            version: 2,
+        };
+        let auth_response = types::AuthGuardResponse {
+            generic_data: outer_generic_data,
+            user_jwt: Some(user_token),
+            issuer_account: None,
+            error: None,
+        };
+        let mut auth_response_claim = types::AuthResponseClaim {
+            generic_claim_data: outer_nats_claim,
+            auth_response,
+        };
+
+        let claim_str = serde_json::to_string(&auth_response_claim)?;
+        let hashed_claim_bytes = hash_claim(&claim_str);
+        auth_response_claim.generic_claim_data.jwt_id = Some(BASE32HEX_NOPAD.encode(&hashed_claim_bytes));
+        
+        Ok(auth_response_claim)
+}
\ No newline at end of file
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index 7f38187..f2731ca 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -189,9 +189,10 @@ impl OrchestratorWorkloadApi {
                 };
 
                 // Note: The `_id` is an option because it is only generated upon the intial insertion of a record in
-                // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` feild.
-                // Using `unwrap` is therefore safe.
-                let host_id = host._id.to_owned().unwrap();
+                // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` field.
+                let host_id = host._id
+                    .to_owned()
+                    .ok_or_else(|| ServiceError::Internal("Failed to read ._id from record".to_string()))?;
                 
                 // 4. Update the Workload Collection with the assigned Host ID
                 let workload_query = doc! { "_id":  workload_id.clone() };
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 69334aa..656a076 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -316,8 +316,7 @@ pub fn get_nats_url() -> String {
 }
 
 pub fn get_nsc_root_path() -> String {
-    let nsc_path = std::env::var("NSC_PATH").unwrap_or_else(|_| "/.local/share/nats/nsc".to_string());
-    nsc_path
+    std::env::var("NSC_PATH").unwrap_or_else(|_| "/.local/share/nats/nsc".to_string())
 }
 
 pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index 601cd75..6e45efa 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -151,7 +151,7 @@ leafnodes {{
             .stdout(Stdio::null())
             .stderr(Stdio::null())
             .spawn()
-            .expect("Failed to start NATS server");
+            .context("Failed to start NATS server")?;
 
         // TODO: wait for a readiness indicator
         std::thread::sleep(std::time::Duration::from_millis(100));

From bf93fd984e789c8b77b9d0a5c2e7fd57f72e54da Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 4 Feb 2025 18:29:51 -0600
Subject: [PATCH 60/91] rust fmt

---
 rust/clients/host_agent/src/auth/init.rs      | 123 +++++---
 rust/clients/host_agent/src/auth/mod.rs       |   5 +-
 rust/clients/host_agent/src/auth/utils.rs     |   8 -
 .../host_agent/src/hostd/gen_leaf_server.rs   |  11 +-
 rust/clients/host_agent/src/hostd/mod.rs      |   2 +-
 .../clients/host_agent/src/hostd/workloads.rs |  37 ++-
 rust/clients/host_agent/src/keys.rs           | 163 ++++++----
 rust/clients/host_agent/src/main.rs           |  44 +--
 rust/clients/orchestrator/src/auth.rs         |  55 ++--
 rust/clients/orchestrator/src/main.rs         |   5 +-
 rust/clients/orchestrator/src/utils.rs        |  48 ---
 rust/clients/orchestrator/src/workloads.rs    | 158 ++++++----
 rust/services/authentication/src/lib.rs       | 296 +++++++++++-------
 rust/services/authentication/src/types.rs     |  77 +++--
 rust/services/authentication/src/utils.rs     | 236 ++++++--------
 rust/services/workload/src/host_api.rs        |  51 +--
 rust/services/workload/src/lib.rs             |  45 +--
 .../services/workload/src/orchestrator_api.rs |  93 ++++--
 rust/services/workload/src/types.rs           |  11 +-
 rust/util_libs/src/db/mongodb.rs              |   6 +-
 rust/util_libs/src/db/schemas.rs              |   2 +-
 rust/util_libs/src/js_stream_service.rs       |  22 +-
 rust/util_libs/src/nats_js_client.rs          |  47 +--
 23 files changed, 838 insertions(+), 707 deletions(-)
 delete mode 100644 rust/clients/orchestrator/src/utils.rs

diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index 669bdad..b566d52 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -17,28 +17,39 @@ This client is responsible for:
 */
 
 use super::utils::json_to_base64;
-use crate::{auth::config::HosterConfig, keys::{AuthCredType, Keys}};
+use crate::{
+    auth::config::HosterConfig,
+    keys::{AuthCredType, Keys},
+};
 use anyhow::Result;
-use std::str::FromStr;
-use futures::StreamExt;
 use async_nats::{HeaderMap, HeaderName, HeaderValue};
 use authentication::types::{AuthApiResult, AuthGuardPayload, AuthJWTPayload, AuthResult}; // , AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION
-use textnonce::TextNonce;
+use futures::StreamExt;
 use hpos_hal::inventory::HoloInventory;
-use util_libs:: nats_js_client;
+use std::str::FromStr;
+use textnonce::TextNonce;
+use util_libs::nats_js_client;
 
 // pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
 pub const HOST_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX";
 
-pub async fn run(mut host_agent_keys: Keys) -> Result<(Keys, async_nats::Client), async_nats::Error> {
+pub async fn run(
+    mut host_agent_keys: Keys,
+) -> Result<(Keys, async_nats::Client), async_nats::Error> {
     log::info!("Host Auth Client: Connecting to server...");
 
     // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host & Hoster =============================================
     let nonce = TextNonce::new().to_string();
-    let unique_inbox = &format!("{}.{}", HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey);
+    let unique_inbox = &format!(
+        "{}.{}",
+        HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey
+    );
     println!(">>> unique_inbox : {}", unique_inbox);
     let user_unique_auth_subject = &format!("AUTH.{}.>", host_agent_keys.host_pubkey);
-    println!(">>> user_unique_auth_subject : {}", user_unique_auth_subject);
+    println!(
+        ">>> user_unique_auth_subject : {}",
+        user_unique_auth_subject
+    );
 
     // Fetch Hoster Pubkey and email (from config)
     let mut auth_guard_payload = AuthGuardPayload::default();
@@ -48,41 +59,49 @@ pub async fn run(mut host_agent_keys: Keys) -> Result<(Keys, async_nats::Client)
             auth_guard_payload.hoster_hc_pubkey = Some(config.hc_pubkey);
             auth_guard_payload.email = Some(config.email);
             auth_guard_payload.nonce = nonce;
-        },
+        }
         Err(e) => {
             log::error!("Failed to locate Hoster config. Err={e}");
             auth_guard_payload.host_pubkey = host_agent_keys.host_pubkey.to_string();
             auth_guard_payload.nonce = nonce;
         }
     };
-    auth_guard_payload = auth_guard_payload.try_add_signature( |p| host_agent_keys.host_sign(p))?;
+    auth_guard_payload = auth_guard_payload.try_add_signature(|p| host_agent_keys.host_sign(p))?;
 
     let user_auth_json = serde_json::to_string(&auth_guard_payload)?;
     let user_auth_token = json_to_base64(&user_auth_json)?;
-    let user_creds = if let AuthCredType::Guard(creds) = host_agent_keys.creds.clone() { creds } else {
-        return Err(async_nats::Error::from("Failed to locate Auth Guard credentials"));
+    let user_creds = if let AuthCredType::Guard(creds) = host_agent_keys.creds.clone() {
+        creds
+    } else {
+        return Err(async_nats::Error::from(
+            "Failed to locate Auth Guard credentials",
+        ));
     };
 
     // Connect to Nats server as auth guard and call NATS AuthCallout
     let nats_url = nats_js_client::get_nats_url();
-    let auth_guard_client = async_nats::ConnectOptions::with_credentials(&user_creds.to_string_lossy())?
-        .token(user_auth_token)
-        .custom_inbox_prefix(unique_inbox.to_string())
-        .connect(nats_url)
-        .await?;
-
-    println!("User connected to server on port {}.  Connection State: {:#?}", auth_guard_client.server_info().port, auth_guard_client.connection_state());
+    let auth_guard_client =
+        async_nats::ConnectOptions::with_credentials(&user_creds.to_string_lossy())?
+            .token(user_auth_token)
+            .custom_inbox_prefix(unique_inbox.to_string())
+            .connect(nats_url)
+            .await?;
+
+    println!(
+        "User connected to server on port {}.  Connection State: {:#?}",
+        auth_guard_client.server_info().port,
+        auth_guard_client.connection_state()
+    );
 
     let server_node_id = auth_guard_client.server_info().server_id;
-    log::trace!(
-        "Host Auth Client: Retrieved Node ID: {}",
-        server_node_id
-    );
+    log::trace!("Host Auth Client: Retrieved Node ID: {}", server_node_id);
 
     // ==================== Handle Host User and SYS Authoriation ============================================================
     let auth_guard_client_clone = auth_guard_client.clone();
     tokio::spawn({
-        let mut auth_inbox_msgs = auth_guard_client_clone.subscribe(unique_inbox.to_string()).await?;
+        let mut auth_inbox_msgs = auth_guard_client_clone
+            .subscribe(unique_inbox.to_string())
+            .await?;
         async move {
             while let Some(msg) = auth_inbox_msgs.next().await {
                 println!("got an AUTH INBOX msg: {:?}", &msg);
@@ -93,32 +112,42 @@ pub async fn run(mut host_agent_keys: Keys) -> Result<(Keys, async_nats::Client)
     let payload = AuthJWTPayload {
         host_pubkey: host_agent_keys.host_pubkey.to_string(),
         maybe_sys_pubkey: host_agent_keys.local_sys_pubkey.clone(),
-        nonce: TextNonce::new().to_string()
+        nonce: TextNonce::new().to_string(),
     };
 
     let payload_bytes = serde_json::to_vec(&payload)?;
     let signature = host_agent_keys.host_sign(&payload_bytes)?;
     let mut headers = HeaderMap::new();
-    headers.insert(HeaderName::from_static("X-Signature"), HeaderValue::from_str(&format!("{:?}",signature.as_bytes()))?);
+    headers.insert(
+        HeaderName::from_static("X-Signature"),
+        HeaderValue::from_str(&format!("{:?}", signature.as_bytes()))?,
+    );
 
     println!("About to send out the {} message", user_unique_auth_subject);
-    let response = auth_guard_client.request_with_headers(
+    let response = auth_guard_client
+        .request_with_headers(
             user_unique_auth_subject.to_string(),
             headers,
-            payload_bytes.into()
-        ).await?;
+            payload_bytes.into(),
+        )
+        .await?;
+
+    println!(
+        "got an AUTH response: {:?}",
+        std::str::from_utf8(&response.payload).expect("failed to deserialize msg response")
+    );
 
-    println!("got an AUTH response: {:?}", std::str::from_utf8(&response.payload).expect("failed to deserialize msg response"));
-    
     match serde_json::from_slice::<AuthApiResult>(&response.payload) {
         Ok(auth_response) => match auth_response.result {
-            AuthResult::Authorization(r)=>{
-                host_agent_keys = host_agent_keys.save_host_creds(r.host_jwt, r.sys_jwt).await?;
-                
+            AuthResult::Authorization(r) => {
+                host_agent_keys = host_agent_keys
+                    .save_host_creds(r.host_jwt, r.sys_jwt)
+                    .await?;
+
                 if let Some(_reply) = response.reply {
                     // Publish the Awk resp to the Orchestrator... (JS)
                 }
-            },
+            }
             _ => {
                 log::error!("got unexpected AUTH RESPONSE : {:?}", auth_response);
             }
@@ -127,24 +156,22 @@ pub async fn run(mut host_agent_keys: Keys) -> Result<(Keys, async_nats::Client)
             // TODO: Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject to ensure has correct permissions
             println!("got an AUTH RES ERROR: {:?}", e);
 
-            let unauthenticated_user_diagnostics_subject = format!("DIAGNOSTICS.unauthenticated.{}", host_agent_keys.host_pubkey);
+            let unauthenticated_user_diagnostics_subject = format!(
+                "DIAGNOSTICS.unauthenticated.{}",
+                host_agent_keys.host_pubkey
+            );
             let diganostics = HoloInventory::from_host();
             let payload_bytes = serde_json::to_vec(&diganostics)?;
-            auth_guard_client.publish(unauthenticated_user_diagnostics_subject, payload_bytes.into()).await?;
+            auth_guard_client
+                .publish(
+                    unauthenticated_user_diagnostics_subject,
+                    payload_bytes.into(),
+                )
+                .await?;
         }
     };
 
-    log::trace!(
-        "host_agent_keys: {:#?}", host_agent_keys
-    );
+    log::trace!("host_agent_keys: {:#?}", host_agent_keys);
 
     Ok((host_agent_keys, auth_guard_client))
 }
-
-    // let publish_info = nats_js_client::PublishInfo {
-    //     subject: user_unique_auth_subject,
-    //     msg_id: format!("id={}", rand::random::<u8>()),
-    //     data: payload_bytes,
-    //     headers: Some(headers)
-    // };
-
diff --git a/rust/clients/host_agent/src/auth/mod.rs b/rust/clients/host_agent/src/auth/mod.rs
index 20b0952..2bc32b9 100644
--- a/rust/clients/host_agent/src/auth/mod.rs
+++ b/rust/clients/host_agent/src/auth/mod.rs
@@ -1,4 +1,3 @@
-// pub mod agent_key;
-pub mod utils;
-pub mod init;
 pub mod config;
+pub mod init;
+pub mod utils;
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index cdeaa35..510f354 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,12 +1,5 @@
 use data_encoding::BASE64URL_NOPAD;
 
-// // NB: These should match the names of these files when saved locally upon hpos init
-// // (should be the same as those in the `orchestrator_setup` file)
-// const JWT_DIR_NAME: &str = "jwt";
-// const OPERATOR_JWT_FILE_NAME: &str = "holo_operator";
-// const SYS_JWT_FILE_NAME: &str = "sys_account";
-// const WORKLOAD_JWT_FILE_NAME: &str = "workload_account";
-
 /// Encode a JSON string into a b64-encoded string
 pub fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
     // Parse to ensure it's valid JSON
@@ -17,4 +10,3 @@ pub fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
     let encoded = BASE64URL_NOPAD.encode(json_string.as_bytes());
     Ok(encoded)
 }
-
diff --git a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 557a11a..f467341 100644
--- a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -1,9 +1,12 @@
 use std::path::PathBuf;
 
-use util_libs::{nats_js_client, nats_server::{
-    self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
-    LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
-}};
+use util_libs::{
+    nats_js_client,
+    nats_server::{
+        self, JetStreamConfig, LeafNodeRemote, LeafServer, LoggingOptions, LEAF_SERVER_CONFIG_PATH,
+        LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
+    },
+};
 
 pub async fn run(user_creds_path: &Option<PathBuf>) {
     let leaf_server_remote_conn_url = nats_server::get_hub_server_url();
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
index b3629fc..21ed86c 100644
--- a/rust/clients/host_agent/src/hostd/mod.rs
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -1,2 +1,2 @@
-pub mod workloads;
 pub mod gen_leaf_server;
+pub mod workloads;
diff --git a/rust/clients/host_agent/src/hostd/workloads.rs b/rust/clients/host_agent/src/hostd/workloads.rs
index 5927d5d..a2e7979 100644
--- a/rust/clients/host_agent/src/hostd/workloads.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -18,8 +18,10 @@ use util_libs::{
     nats_js_client::{self, Credentials, EndpointType},
 };
 use workload::{
-    WorkloadServiceApi, host_api::HostWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
-    types::{WorkloadServiceSubjects, WorkloadApiResult}
+    host_api::HostWorkloadApi,
+    types::{WorkloadApiResult, WorkloadServiceSubjects},
+    WorkloadServiceApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ,
+    WORKLOAD_SRV_VERSION,
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
@@ -87,13 +89,14 @@ pub async fn run(
     // ==================== Setup API & Register Endpoints ====================
     // Instantiate the Workload API
     let workload_api = HostWorkloadApi::default();
-    
+
     // Register Workload Streams for Host Agent to consume and process
     // NB: Subjects are published by orchestrator
     let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
     let workload_send_status_subject = serde_json::to_string(&WorkloadServiceSubjects::SendStatus)?;
     let workload_uninstall_subject = serde_json::to_string(&WorkloadServiceSubjects::Uninstall)?;
-    let workload_update_installed_subject = serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
+    let workload_update_installed_subject =
+        serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
 
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
@@ -104,12 +107,12 @@ pub async fn run(
 
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "start_workload", // consumer name
+            "start_workload",                                       // consumer name
             &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
             EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.start_workload(msg).await
-                })
+                }),
             ),
             None,
         )
@@ -122,7 +125,7 @@ pub async fn run(
             EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.update_workload(msg).await
-                })
+                }),
             ),
             None,
         )
@@ -130,26 +133,26 @@ pub async fn run(
 
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "uninstall_workload", // consumer name
+            "uninstall_workload",                                       // consumer name
             &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
-            EndpointType::Async(
-                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+            EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
-                }),
-            ),
+                },
+            )),
             None,
         )
         .await?;
 
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "send_workload_status", // consumer name
+            "send_workload_status",                                       // consumer name
             &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
-            EndpointType::Async(
-                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+            EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.send_workload_status(msg).await
-                })
-            ),
+                },
+            )),
             None,
         )
         .await?;
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 3d02708..b273a04 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -1,11 +1,11 @@
 use anyhow::{anyhow, Context, Result};
-use nkeys::KeyPair;
 use data_encoding::BASE64URL_NOPAD;
+use nkeys::KeyPair;
 use std::fs::File;
 use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Command;
-use util_libs:: nats_js_client::{get_nats_creds_by_nsc, get_file_path_buf};
+use util_libs::nats_js_client::{get_file_path_buf, get_nats_creds_by_nsc};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -16,7 +16,14 @@ impl std::fmt::Debug for Keys {
         f.debug_struct("Keys")
             .field("host_keypair", &"[redacted]")
             .field("host_pubkey", &self.host_pubkey)
-            .field("local_sys_keypair", if self.local_sys_keypair.is_some(){&"[redacted]"} else { &false })
+            .field(
+                "local_sys_keypair",
+                if self.local_sys_keypair.is_some() {
+                    &"[redacted]"
+                } else {
+                    &false
+                },
+            )
             .field("local_sys_pubkey", &self.local_sys_pubkey)
             .field("creds", &creds_type)
             .finish()
@@ -32,8 +39,8 @@ pub struct CredPaths {
 
 #[derive(Clone)]
 pub enum AuthCredType {
-    Guard(PathBuf), // Default
-    Authenticated(CredPaths) // only assiged after successful hoster authentication
+    Guard(PathBuf),           // Default
+    Authenticated(CredPaths), // only assiged after successful hoster authentication
 }
 
 #[derive(Clone)]
@@ -48,18 +55,21 @@ pub struct Keys {
 impl Keys {
     pub fn new() -> Result<Self> {
         // let host_key_path = format!("{}/user_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
-        let host_key_path = std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
+        let host_key_path =
+            std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
         let host_kp = KeyPair::new_user();
         write_keypair_to_file(get_file_path_buf(&host_key_path), host_kp.clone())?;
         let host_pk = host_kp.public_key();
-        
+
         // let sys_key_path = format!("{}/user_sys_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
-        let sys_key_path = std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
+        let sys_key_path =
+            std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
         let local_sys_kp = KeyPair::new_user();
         write_keypair_to_file(get_file_path_buf(&sys_key_path), local_sys_kp.clone())?;
         let local_sys_pk = local_sys_kp.public_key();
 
-        let auth_guard_creds = get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
+        let auth_guard_creds =
+            get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
 
         Ok(Self {
             host_keypair: host_kp,
@@ -71,43 +81,49 @@ impl Keys {
     }
 
     // NB: Only call when trying to load an already authenticated Host and Sys User
-    pub fn try_from_storage(maybe_host_creds_path: &Option<PathBuf>, maybe_sys_creds_path: &Option<PathBuf>) -> Result<Self> {
-        let host_key_path = std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
-        let host_keypair = try_read_keypair_from_file(get_file_path_buf(&host_key_path.clone()))?.ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
+    pub fn try_from_storage(
+        maybe_host_creds_path: &Option<PathBuf>,
+        maybe_sys_creds_path: &Option<PathBuf>,
+    ) -> Result<Self> {
+        let host_key_path =
+            std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
+        let host_keypair =
+            try_read_keypair_from_file(get_file_path_buf(&host_key_path.clone()))?
+                .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
         let host_pk = host_keypair.public_key();
-        let sys_key_path = std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
-        let host_creds_path = maybe_host_creds_path.to_owned().unwrap_or_else(|| get_file_path_buf(
-            &get_nats_creds_by_nsc("HOLO", "HPOS", "host")
-        ));
-        let sys_creds_path = maybe_sys_creds_path.to_owned().unwrap_or_else(|| get_file_path_buf(
-            &get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
-        ));
+        let sys_key_path =
+            std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
+        let host_creds_path = maybe_host_creds_path
+            .to_owned()
+            .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "host")));
+        let sys_creds_path = maybe_sys_creds_path
+            .to_owned()
+            .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")));
 
         // Set auth_guard_creds as default:
-        let auth_guard_creds = get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
+        let auth_guard_creds =
+            get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
 
         let keys = match try_read_keypair_from_file(get_file_path_buf(&sys_key_path))? {
             Some(kp) => {
                 let local_sys_pk = kp.public_key();
                 Self {
                     host_keypair,
-                    host_pubkey:host_pk,
+                    host_pubkey: host_pk,
                     local_sys_keypair: Some(kp),
                     local_sys_pubkey: Some(local_sys_pk),
-                    creds: AuthCredType::Guard(auth_guard_creds)
-                }
-            },
-            None => {
-                Self {
-                    host_keypair,
-                    host_pubkey: host_pk,
-                    local_sys_keypair: None,
-                    local_sys_pubkey: None,
-                    creds: AuthCredType::Guard(auth_guard_creds)
+                    creds: AuthCredType::Guard(auth_guard_creds),
                 }
             }
+            None => Self {
+                host_keypair,
+                host_pubkey: host_pk,
+                local_sys_keypair: None,
+                local_sys_pubkey: None,
+                creds: AuthCredType::Guard(auth_guard_creds),
+            },
         };
-        
+
         Ok(keys.clone().add_creds_paths(host_creds_path, Some(sys_creds_path)).unwrap_or_else(move |e| {
             log::error!("Error: Cannot locate authenticated cred files. Defaulting to auth_guard_creds. Err={}",e);
             keys
@@ -115,10 +131,9 @@ impl Keys {
     }
 
     pub fn _add_local_sys(mut self, sys_key_path: Option<PathBuf>) -> Result<Self> {
-        let sys_key_path = sys_key_path.unwrap_or_else(|| get_file_path_buf(
-            &get_nats_creds_by_nsc("HOLO", "HPOS", "sys")
-        ));
-        
+        let sys_key_path = sys_key_path
+            .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")));
+
         let mut is_new_key = false;
 
         let local_sys_kp = try_read_keypair_from_file(sys_key_path.clone())?.unwrap_or_else(|| {
@@ -138,11 +153,18 @@ impl Keys {
         Ok(self)
     }
 
-    pub fn add_creds_paths(mut self, host_creds_file_path: PathBuf, sys_creds_file_path: Option<PathBuf>) -> Result<Self> {
+    pub fn add_creds_paths(
+        mut self,
+        host_creds_file_path: PathBuf,
+        sys_creds_file_path: Option<PathBuf>,
+    ) -> Result<Self> {
         match host_creds_file_path.try_exists() {
             Ok(is_ok) => {
                 if !is_ok {
-                    return Err(anyhow!("Failed to locate host creds path. Found broken sym link. Path={:?}", host_creds_file_path));
+                    return Err(anyhow!(
+                        "Failed to locate host creds path. Found broken sym link. Path={:?}",
+                        host_creds_file_path
+                    ));
                 }
 
                 let creds = match sys_creds_file_path {
@@ -155,55 +177,67 @@ impl Keys {
                                 host_creds_path: host_creds_file_path,
                                 sys_creds_path: Some(sys_path),
                             }
-                        },
+                        }
                         Err(e) => {
-                            return Err(anyhow!("Failed to locate sys creds path. Path={:?} Err={}", sys_path, e));
+                            return Err(anyhow!(
+                                "Failed to locate sys creds path. Path={:?} Err={}",
+                                sys_path,
+                                e
+                            ));
                         }
                     },
                     None => CredPaths {
                         host_creds_path: host_creds_file_path,
                         sys_creds_path: None,
-                    }
+                    },
                 };
                 self.creds = AuthCredType::Authenticated(creds);
                 Ok(self)
-            },
-            Err(e) => Err(anyhow!("Failed to locate host creds path. Path={:?} Err={}", host_creds_file_path, e))
+            }
+            Err(e) => Err(anyhow!(
+                "Failed to locate host creds path. Path={:?} Err={}",
+                host_creds_file_path,
+                e
+            )),
         }
     }
 
     pub async fn save_host_creds(
         &self,
         host_user_jwt: String,
-        host_sys_user_jwt: String
+        host_sys_user_jwt: String,
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
-        let host_path = get_file_path_buf(&format!("{}.{}","output_dir", "host.jwt"));
+        let host_path = get_file_path_buf(&format!("{}.{}", "output_dir", "host.jwt"));
         write_to_file(host_path, host_user_jwt.as_bytes())?;
-        let sys_path = get_file_path_buf(&format!("{}.{}","output_dir", "host_sys.jwt"));
+        let sys_path = get_file_path_buf(&format!("{}.{}", "output_dir", "host_sys.jwt"));
         write_to_file(sys_path, host_sys_user_jwt.as_bytes())?;
-    
+
         // Save user creds and sys creds local to hosting agent
         let host_creds_file_name = "host.creds";
         Command::new("nsc")
-            .arg(format!("generate creds --name user_host_{} --account {} > {}", self.host_pubkey, "WORKLOAD", host_creds_file_name))
+            .arg(format!(
+                "generate creds --name user_host_{} --account {} > {}",
+                self.host_pubkey, "WORKLOAD", host_creds_file_name
+            ))
             .output()
             .context("Failed to add new operator signing key on hosting agent")?;
-        
+
         let mut sys_creds_file_name = None;
         if let Some(sys_pubkey) = self.local_sys_pubkey.as_ref() {
             let file_name = "host_sys.creds";
             sys_creds_file_name = Some(get_file_path_buf(file_name));
             Command::new("nsc")
-                .arg(format!("generate creds --name user_host_{} --account {} > {}", sys_pubkey, "SYS", file_name))
+                .arg(format!(
+                    "generate creds --name user_host_{} --account {} > {}",
+                    sys_pubkey, "SYS", file_name
+                ))
                 .output()
                 .context("Failed to add new operator signing key on hosting agent")?;
         }
-    
-        self.to_owned().add_creds_paths(
-            get_file_path_buf(host_creds_file_name),
-            sys_creds_file_name
-        )
+
+        self.to_owned()
+            .add_creds_paths(get_file_path_buf(host_creds_file_name), sys_creds_file_name)
     }
 
     pub fn get_host_creds_path(&self) -> Option<PathBuf> {
@@ -221,9 +255,7 @@ impl Keys {
     }
 
     pub fn host_sign(&self, payload: &[u8]) -> Result<String> {
-        let signature = self
-            .host_keypair
-            .sign(payload)?;
+        let signature = self.host_keypair.sign(payload)?;
 
         Ok(BASE64URL_NOPAD.encode(&signature))
     }
@@ -243,7 +275,7 @@ fn write_to_file(file_path: PathBuf, data: &[u8]) -> Result<()> {
 fn try_read_keypair_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
     match try_read_from_file(key_file_path)? {
         Some(kps) => Ok(Some(KeyPair::from_seed(&kps)?)),
-        None => Ok(None)
+        None => Ok(None),
     }
 }
 
@@ -251,12 +283,15 @@ fn try_read_from_file(file_path: PathBuf) -> Result<Option<String>> {
     match file_path.try_exists() {
         Ok(link_is_ok) => {
             if !link_is_ok {
-                return Err(anyhow!("Failed to read path {:?}. Found broken sym link.", file_path));
+                return Err(anyhow!(
+                    "Failed to read path {:?}. Found broken sym link.",
+                    file_path
+                ));
             }
 
-            let mut file_content =
-                File::open(&file_path).context(format!("Failed to open config file {:#?}", file_path))?;
-        
+            let mut file_content = File::open(&file_path)
+                .context(format!("Failed to open config file {:#?}", file_path))?;
+
             let mut s = String::new();
             file_content.read_to_string(&mut s)?;
             Ok(Some(s.trim().to_string()))
@@ -266,4 +301,4 @@ fn try_read_from_file(file_path: PathBuf) -> Result<Option<String>> {
             Ok(None)
         }
     }
-}
\ No newline at end of file
+}
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 2b83e06..5d59cf7 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -10,18 +10,18 @@ This client is responsible for subscribing the host agent to workload stream end
   - sending workload status upon request
 */
 
+pub mod agent_cli;
 mod auth;
+pub mod host_cmds;
 mod hostd;
 mod keys;
-pub mod agent_cli;
-pub mod host_cmds;
 pub mod support_cmds;
+use agent_cli::DaemonzeArgs;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
-use thiserror::Error;
-use agent_cli::DaemonzeArgs;
 use hpos_hal::inventory::HoloInventory;
+use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -58,11 +58,14 @@ async fn main() -> Result<(), AgentCliError> {
 async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     let mut host_agent_keys = keys::Keys::try_from_storage(
         &args.nats_leafnode_client_creds_path,
-        &args.nats_leafnode_client_sys_creds_path
-    ).or_else(|_| keys::Keys::new().map_err(|e| {
-        eprintln!("Failed to create new keys: {:?}", e);
-        async_nats::Error::from(e)
-    }))?;
+        &args.nats_leafnode_client_sys_creds_path,
+    )
+    .or_else(|_| {
+        keys::Keys::new().map_err(|e| {
+            eprintln!("Failed to create new keys: {:?}", e);
+            async_nats::Error::from(e)
+        })
+    })?;
 
     // If user cred file is for the auth_guard user, run loop to authenticate host & hoster...
     if let keys::AuthCredType::Guard(_) = host_agent_keys.creds {
@@ -80,15 +83,13 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     Ok(())
 }
 
-async fn run_auth_loop(mut keys: keys::Keys) ->  Result<keys::Keys, async_nats::Error> {
+async fn run_auth_loop(mut keys: keys::Keys) -> Result<keys::Keys, async_nats::Error> {
     let mut start = chrono::Utc::now();
-    
-    // while let keys::AuthCredType::Guard(auth_creds) = keys.creds {
-        loop {
+    loop {
         log::debug!("About to run the Hosting Agent Authentication Service");
         let auth_guard_client: async_nats::Client;
         (keys, auth_guard_client) = auth::init::run(keys).await?;
-        
+
         // If authenicated creds exist, then auth call was successful.
         // Close buffer, exit loop, and return.
         if let keys::AuthCredType::Authenticated(_) = keys.creds {
@@ -101,11 +102,18 @@ async fn run_auth_loop(mut keys: keys::Keys) ->  Result<keys::Keys, async_nats::
         let now = chrono::Utc::now();
         let max_time_interval = chrono::TimeDelta::days(1);
 
-        while max_time_interval < start.signed_duration_since(now) {
-            let unauthenticated_user_diagnostics_subject = format!("DIAGNOSTICS.unauthenticated.{}", keys.host_pubkey);
+        while max_time_interval > now.signed_duration_since(start) {
+            let unauthenticated_user_diagnostics_subject =
+                format!("DIAGNOSTICS.unauthenticated.{}", keys.host_pubkey);
             let diganostics = HoloInventory::from_host();
             let payload_bytes = serde_json::to_vec(&diganostics)?;
-            if let Err(e) = auth_guard_client.publish(unauthenticated_user_diagnostics_subject, payload_bytes.into()).await {
+            if let Err(e) = auth_guard_client
+                .publish(
+                    unauthenticated_user_diagnostics_subject,
+                    payload_bytes.into(),
+                )
+                .await
+            {
                 log::error!("Encountered error when sending diganostics. Err={:#?}", e);
             };
             tokio::time::sleep(chrono::TimeDelta::hours(1).to_std()?).await;
@@ -117,4 +125,4 @@ async fn run_auth_loop(mut keys: keys::Keys) ->  Result<keys::Keys, async_nats::
     }
 
     Ok(keys)
-}
\ No newline at end of file
+}
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 660825d..345a84f 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -18,11 +18,15 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
-use std::{collections::HashMap, sync::Arc, time::Duration};
 use anyhow::{anyhow, Result};
 use async_nats::Message;
+use authentication::{
+    self,
+    types::{self, AuthApiResult},
+    AuthServiceApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION,
+};
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use authentication::{self, AuthServiceApi, types::{self, AuthApiResult}, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION};
+use std::{collections::HashMap, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
@@ -44,54 +48,51 @@ pub async fn run() -> Result<(), async_nats::Error> {
         version: AUTH_SRV_VERSION.to_string(),
         service_subject: AUTH_SRV_SUBJ.to_string(),
     };
-    
-    let orchestrator_auth_client =
-        JsClient::new(NewJsClientParams {
-            nats_url,
-            name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
-            inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
-            service_params: vec![auth_stream_service_params],
-            credentials: None,
-            listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
-            ping_interval: Some(Duration::from_secs(10)),
-            request_timeout: Some(Duration::from_secs(5)),
-        })
-        .await?;
+
+    let orchestrator_auth_client = JsClient::new(NewJsClientParams {
+        nats_url,
+        name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
+        inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
+        service_params: vec![auth_stream_service_params],
+        credentials: None,
+        listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(5)),
+    })
+    .await?;
 
     // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
     let client = MongoDBClient::with_options(client_options)?;
-    
+
     // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
     let auth_api = AuthServiceApi::new(&client).await?;
-    
+
     // Register Auth Stream for Orchestrator to consume and proceess
     let auth_service = orchestrator_auth_client
         .get_js_service(AUTH_SRV_NAME.to_string())
         .await
         .ok_or(anyhow!(
             "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
-        ))?;    
-    
+        ))?;
+
     auth_service
         .add_consumer::<AuthApiResult>(
-            "validate", // consumer name
+            "validate",                  // consumer name
             types::AUTH_SERVICE_SUBJECT, // consumer stream subj
-            EndpointType::Async(auth_api.call(|api: AuthServiceApi, msg: Arc<Message>| {
-                async move {
+            EndpointType::Async(auth_api.call(
+                |api: AuthServiceApi, msg: Arc<Message>| async move {
                     api.handle_handshake_request(msg).await
-                }
-            })),
+                },
+            )),
             Some(create_callback_subject_to_host("host_pubkey".to_string())),
         )
         .await?;
 
-    log::trace!(
-        "Orchestrator Auth Service is running. Waiting for requests..."
-    );
+    log::trace!("Orchestrator Auth Service is running. Waiting for requests...");
 
     // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index b389b58..6d25650 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -1,5 +1,4 @@
 mod auth;
-mod utils;
 mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
@@ -9,7 +8,7 @@ use tokio::task::spawn;
 async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
-    spawn(async move { 
+    spawn(async move {
         if let Err(e) = auth::run().await {
             log::error!("{}", e)
         }
@@ -17,7 +16,7 @@ async fn main() -> Result<(), async_nats::Error> {
     spawn(async move {
         if let Err(e) = workloads::run().await {
             log::error!("{}", e)
-        } 
+        }
     });
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
deleted file mode 100644
index bd68b56..0000000
--- a/rust/clients/orchestrator/src/utils.rs
+++ /dev/null
@@ -1,48 +0,0 @@
-// use anyhow::Result;
-// use std::io::Read;
-// use util_libs::nats_js_client::{JsClient, PublishInfo};
-
-pub fn _get_resolver_path() -> String {
-    std::env::var("RESOLVER_FILE_PATH").unwrap_or_else(|_| "./resolver.conf".to_string())
-}
-
-pub fn _get_orchestrator_credentials_dir_path() -> String {
-    std::env::var("ORCHESTRATOR_CREDENTIALS_DIR_PATH").unwrap_or_else(|e| panic!("Failed to locate 'ORCHESTRATOR_CREDENTIALS_DIR_PATH' env var.  Was it set?  Error={}", e))
-}
-
-// const CHUNK_SIZE: usize = 1024; // 1 KB chunk size
-// pub async fn chunk_file_and_publish(
-//     auth_client: &JsClient,
-//     subject: &str,
-//     host_id: &str,
-// ) -> Result<()> {
-//     let file_path = format!("{}/{}.jwt", get_host_user_pubkey_path(), host_id);
-//     let mut file = std::fs::File::open(file_path)?;
-//     let mut buffer = vec![0; CHUNK_SIZE];
-//     let mut chunk_id = 0;
-
-//     while let Ok(bytes_read) = file.read(&mut buffer) {
-//         if bytes_read == 0 {
-//             break;
-//         }
-//         let chunk_data = &buffer[..bytes_read];
-
-//         let send_user_jwt_publish_options = PublishInfo {
-//             subject: subject.to_string(),
-//             msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
-//             data: chunk_data.into(),
-//         };
-//         let _ = auth_client.publish(&send_user_jwt_publish_options).await;
-//         chunk_id += 1;
-//     }
-
-//     // Send an EOF marker
-//     let send_user_jwt_publish_options = PublishInfo {
-//         subject: subject.to_string(),
-//         msg_id: format!("hpos_init_msg_id_{}", rand::random::<u8>()),
-//         data: "EOF".into(),
-//     };
-//     let _ = auth_client.publish(&send_user_jwt_publish_options).await;
-
-//     Ok(())
-// }
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index d458112..97462f6 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -15,22 +15,32 @@ This client is responsible for:
 */
 
 use anyhow::{anyhow, Result};
-use std::{collections::HashMap, sync::Arc, time::Duration};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use workload::{
-    WorkloadServiceApi, orchestrator_api::OrchestratorWorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION, types::{WorkloadServiceSubjects, WorkloadApiResult}
-};
+use std::{collections::HashMap, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{self, Credentials, EndpointType, JsClient, NewJsClientParams, get_nats_url, get_nats_creds_by_nsc, get_event_listeners, get_file_path_buf},
+    nats_js_client::{
+        self, get_event_listeners, get_file_path_buf, get_nats_creds_by_nsc, get_nats_url,
+        Credentials, EndpointType, JsClient, NewJsClientParams,
+    },
+};
+use workload::{
+    orchestrator_api::OrchestratorWorkloadApi,
+    types::{WorkloadApiResult, WorkloadServiceSubjects},
+    WorkloadServiceApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ,
+    WORKLOAD_SRV_VERSION,
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
 const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_workload_inbox_orchestrator";
 
-pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_subject_name: String) -> ResponseSubjectsGenerator {
+pub fn create_callback_subject_to_host(
+    is_prefix: bool,
+    tag_name: String,
+    sub_subject_name: String,
+) -> ResponseSubjectsGenerator {
     Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
         if is_prefix {
             let matching_tags = tag_map.into_iter().fold(vec![], |mut acc, (k, v)| {
@@ -51,7 +61,11 @@ pub fn create_callback_subject_to_host(is_prefix: bool, tag_name: String, sub_su
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== Setup NATS ====================
     let nats_url = get_nats_url();
-    let creds_path = Credentials::Path(get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", "orchestrator")));
+    let creds_path = Credentials::Path(get_file_path_buf(&get_nats_creds_by_nsc(
+        "HOLO",
+        "WORKLOAD",
+        "orchestrator",
+    )));
     let event_listeners = get_event_listeners();
 
     // Setup JS Stream Service
@@ -62,25 +76,24 @@ pub async fn run() -> Result<(), async_nats::Error> {
         service_subject: WORKLOAD_SRV_SUBJ.to_string(),
     };
 
-    let orchestrator_workload_client =
-        JsClient::new(NewJsClientParams {
-            nats_url,
-            name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
-            inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
-            service_params: vec![workload_stream_service_params],
-            credentials: Some(creds_path),
-            request_timeout: Some(Duration::from_secs(5)),
-            ping_interval: Some(Duration::from_secs(10)),
-            listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
-        })
-        .await?;
+    let orchestrator_workload_client = JsClient::new(NewJsClientParams {
+        nats_url,
+        name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
+        inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
+        service_params: vec![workload_stream_service_params],
+        credentials: Some(creds_path),
+        request_timeout: Some(Duration::from_secs(5)),
+        ping_interval: Some(Duration::from_secs(10)),
+        listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
+    })
+    .await?;
 
     // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
     let client = MongoDBClient::with_options(client_options)?;
-    
+
     // ==================== Setup API & Register Endpoints ====================
     // Instantiate the Workload API (requires access to db client)
     let workload_api = OrchestratorWorkloadApi::new(&client).await?;
@@ -92,9 +105,11 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let workload_remove_subject = serde_json::to_string(&WorkloadServiceSubjects::Remove)?;
     let workload_db_insert_subject = serde_json::to_string(&WorkloadServiceSubjects::Insert)?;
     let workload_db_modification_subject = serde_json::to_string(&WorkloadServiceSubjects::Modify)?;
-    let workload_handle_status_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
+    let workload_handle_status_subject =
+        serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
     let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
-    let workload_update_installed_subject = serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
+    let workload_update_installed_subject =
+        serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
 
     let workload_service = orchestrator_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
@@ -106,84 +121,91 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Published by Developer
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "add_workload", // consumer name
-             &workload_add_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
-                async move {
+            "add_workload",        // consumer name
+            &workload_add_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.add_workload(msg).await
-                }
-            })),
+                },
+            )),
             None,
         )
         .await?;
 
-        workload_service
+    workload_service
         .add_consumer::<WorkloadApiResult>(
-            "update_workload", // consumer name
-             &workload_update_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
-                async move {
+            "update_workload",        // consumer name
+            &workload_update_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.update_workload(msg).await
-                }
-            })),
+                },
+            )),
             None,
         )
         .await?;
 
-    
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "remove_workload", // consumer name
-             &workload_remove_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
-                async move {
+            "remove_workload",        // consumer name
+            &workload_remove_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.remove_workload(msg).await
-                }
-            })),
+                },
+            )),
             None,
         )
         .await?;
-    
+
     // Automatically published by the Nats-DB-Connector
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "handle_db_insertion", // consumer name
-             &workload_db_insert_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
-                async move {
+            "handle_db_insertion",       // consumer name
+            &workload_db_insert_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.handle_db_insertion(msg).await
-                }
-            })),
-            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_start_subject)),
+                },
+            )),
+            Some(create_callback_subject_to_host(
+                true,
+                "assigned_hosts".to_string(),
+                workload_start_subject,
+            )),
         )
         .await?;
 
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "handle_db_modification", // consumer name
-                &workload_db_modification_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
-                async move {
+            "handle_db_modification",          // consumer name
+            &workload_db_modification_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.handle_db_modification(msg).await
-                }
-            })),
-            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_update_installed_subject)),
+                },
+            )),
+            Some(create_callback_subject_to_host(
+                true,
+                "assigned_hosts".to_string(),
+                workload_update_installed_subject,
+            )),
         )
         .await?;
 
     // Published by the Host Agent
     workload_service
-    .add_consumer::<WorkloadApiResult>(
-        "handle_status_update", // consumer name
-        &workload_handle_status_subject, // consumer stream subj
-        EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
-            async move {
-                api.handle_status_update(msg).await
-            }
-        })),
-        None,
-    )
-    .await?;
+        .add_consumer::<WorkloadApiResult>(
+            "handle_status_update",          // consumer name
+            &workload_handle_status_subject, // consumer stream subj
+            EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.handle_status_update(msg).await
+                },
+            )),
+            None,
+        )
+        .await?;
 
     // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 2d57342..39918ce 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -9,33 +9,26 @@ Endpoints & Managed Subjects:
 
 pub mod types;
 pub mod utils;
-use anyhow::{Result, Context};
-use async_nats::{AuthError, Message};
+use anyhow::{Context, Result};
 use async_nats::jetstream::ErrorCode;
-use std::sync::Arc;
-use std::future::Future;
-use types::{AuthApiResult, WORKLOAD_SK_ROLE};
-use util_libs::nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse};
 use async_nats::HeaderValue;
-use nkeys::KeyPair;
-use utils::handle_internal_err;
+use async_nats::{AuthError, Message};
+use bson::{self, doc, to_document};
 use core::option::Option::None;
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use nkeys::KeyPair;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::future::Future;
 use std::process::Command;
-use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use util_libs::db::{mongodb::{IntoIndexes, MongoCollection, MongoDbAPI}, 
-    schemas::{
-        self,
-        User,
-        Hoster,
-        Host,
-        Role,
-        RoleInfo
-    },
+use std::sync::Arc;
+use types::{AuthApiResult, WORKLOAD_SK_ROLE};
+use util_libs::db::{
+    mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+    schemas::{self, Host, Hoster, Role, RoleInfo, User},
 };
-
+use util_libs::nats_js_client::{AsyncEndpointHandler, JsServiceResponse, ServiceError};
+use utils::handle_internal_err;
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";
@@ -54,7 +47,8 @@ impl AuthServiceApi {
     pub async fn new(client: &MongoDBClient) -> Result<Self> {
         Ok(Self {
             user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME).await?,
+            hoster_collection: Self::init_collection(client, schemas::HOSTER_COLLECTION_NAME)
+                .await?,
             host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
         })
     }
@@ -71,22 +65,42 @@ impl AuthServiceApi {
         let auth_request_token = String::from_utf8_lossy(&msg.payload).to_string();
         println!("auth_request_token  : {:?}", auth_request_token);
 
-        let auth_request_claim = utils::decode_jwt::<types::NatsAuthorizationRequestClaim>(&auth_request_token).map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
-        println!("\nauth REQUEST - main claim : {}", serde_json::to_string_pretty(&auth_request_claim).unwrap());
+        let auth_request_claim =
+            utils::decode_jwt::<types::NatsAuthorizationRequestClaim>(&auth_request_token)
+                .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
+        println!(
+            "\nauth REQUEST - main claim : {}",
+            serde_json::to_string_pretty(&auth_request_claim).unwrap()
+        );
+
+        let auth_request_user_claim = utils::decode_jwt::<types::UserClaim>(
+            &auth_request_claim.auth_request.connect_opts.user_jwt,
+        )
+        .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
+        println!(
+            "\nauth REQUEST - user claim : {}",
+            serde_json::to_string_pretty(&auth_request_user_claim).unwrap()
+        );
+
+        let user_data: types::AuthGuardPayload = utils::base64_to_data::<types::AuthGuardPayload>(
+            &auth_request_claim.auth_request.connect_opts.user_auth_token,
+        )
+        .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
+        println!("user_data TO VALIDATE  : {:#?}", user_data);
 
-        let auth_request_user_claim = utils::decode_jwt::<types::UserClaim>(&auth_request_claim.auth_request.connect_opts.user_jwt).map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
-        println!("\nauth REQUEST - user claim : {}", serde_json::to_string_pretty(&auth_request_user_claim).unwrap());
-
-        let user_data: types::AuthGuardPayload = utils::base64_to_data::<types::AuthGuardPayload>(&auth_request_claim.auth_request.connect_opts.user_auth_token).map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
-        println!("user_data TO VALIDATE  : {:#?}", user_data); 
-        
         // 2. Validate Host signature, returning validation error if not successful
         let host_pubkey = user_data.host_pubkey.as_ref();
         let host_signature = user_data.get_host_signature();
-        let user_verifying_keypair = KeyPair::from_public_key(host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
-        let raw_payload = serde_json::to_vec(&user_data.clone().without_signature()).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let user_verifying_keypair = KeyPair::from_public_key(host_pubkey)
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let raw_payload = serde_json::to_vec(&user_data.clone().without_signature())
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
         if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &host_signature) {
-            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
+            log::error!(
+                "Error: Failed to validate Signature. Subject='{}'. Err={}",
+                msg.subject,
+                e
+            );
             return Err(ServiceError::Authentication(AuthError::new(e)));
         };
 
@@ -95,31 +109,51 @@ impl AuthServiceApi {
             let hoster_hc_pubkey = user_data.hoster_hc_pubkey.unwrap(); // unwrap is safe here as checked above
             let hoster_email = user_data.email.unwrap(); // unwrap is safe here as checked above
 
-            let is_valid: bool = match self.user_collection.get_one_from(doc! { "roles.role.Hoster": hoster_hc_pubkey.clone() }).await? {
+            let is_valid: bool = match self
+                .user_collection
+                .get_one_from(doc! { "roles.role.Hoster": hoster_hc_pubkey.clone() })
+                .await?
+            {
                 Some(u) => {
                     let mut is_valid = true;
                     // If hoster exists with pubkey, verify email
                     if u.email != hoster_email {
-                        log::error!("Error: Failed to validate hoster email. Email='{}'.", hoster_email);
+                        log::error!(
+                            "Error: Failed to validate hoster email. Email='{}'.",
+                            hoster_email
+                        );
                         is_valid = false;
                     }
 
                     // ...then find the host collection that contains the provided host pubkey
-                    match self.host_collection.get_one_from(doc! { "pubkey": host_pubkey }).await? {
+                    match self
+                        .host_collection
+                        .get_one_from(doc! { "pubkey": host_pubkey })
+                        .await?
+                    {
                         Some(host) => {
                             // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
                             if host.assigned_hoster != hoster_hc_pubkey {
                                 let host_query: bson::Document = doc! { "_id":  host._id.clone() };
-                                let updated_host_doc = to_document(& Host{
+                                let updated_host_doc = to_document(&Host {
                                     assigned_hoster: hoster_hc_pubkey,
                                     ..host
-                                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-                                self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;                
+                                })
+                                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                                self.host_collection
+                                    .update_one_within(
+                                        host_query,
+                                        UpdateModifications::Document(updated_host_doc),
+                                    )
+                                    .await?;
                             }
-                        },
+                        }
                         None => {
-                            log::error!("Error: Failed to locate Host record. Subject='{}'.", msg.subject);
+                            log::error!(
+                                "Error: Failed to locate Host record. Subject='{}'.",
+                                msg.subject
+                            );
                             is_valid = false;
                         }
                     }
@@ -129,50 +163,69 @@ impl AuthServiceApi {
                         let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
                         handle_internal_err(&err_msg)
                     })?;
-                    
+
                     // Finally, find the hoster collection
-                    match self.hoster_collection.get_one_from(doc! { "_id":  ref_id.clone() }).await? {
+                    match self
+                        .hoster_collection
+                        .get_one_from(doc! { "_id":  ref_id.clone() })
+                        .await?
+                    {
                         Some(hoster) => {
                             // ...and pair the hoster with host (if the host is not already assiged to the hoster)
                             let mut updated_assigned_hosts = hoster.assigned_hosts;
                             if !updated_assigned_hosts.contains(&host_pubkey.to_string()) {
-                                let hoster_query: bson::Document = doc! { "_id":  hoster._id.clone() };
+                                let hoster_query: bson::Document =
+                                    doc! { "_id":  hoster._id.clone() };
                                 updated_assigned_hosts.push(host_pubkey.to_string());
-                                let updated_hoster_doc = to_document(& Hoster {
+                                let updated_hoster_doc = to_document(&Hoster {
                                     assigned_hosts: updated_assigned_hosts,
                                     ..hoster
-                                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-                                self.host_collection.update_one_within(hoster_query, UpdateModifications::Document(updated_hoster_doc)).await?;                
+                                })
+                                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                                self.host_collection
+                                    .update_one_within(
+                                        hoster_query,
+                                        UpdateModifications::Document(updated_hoster_doc),
+                                    )
+                                    .await?;
                             }
-                        },
+                        }
                         None => {
-                            log::error!("Error: Failed to locate Hoster record. Subject='{}'.", msg.subject);
+                            log::error!(
+                                "Error: Failed to locate Hoster record. Subject='{}'.",
+                                msg.subject
+                            );
                             is_valid = false;
                         }
                     }
                     is_valid
-                },
+                }
                 None => {
-                    log::error!("Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.", msg.subject);
+                    log::error!(
+                        "Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.",
+                        msg.subject
+                    );
                     false
                 }
             };
             is_valid
-        } else { false };
-
+        } else {
+            false
+        };
 
         // 4. Assign permissions based on whether the hoster was successfully validated
         let permissions = if is_hoster_valid {
             // If successful, assign personalized inbox and auth permissions
             let user_unique_auth_subject = &format!("AUTH.{}.>", host_pubkey);
-            println!(">>> user_unique_auth_subject : {user_unique_auth_subject}");    
+            println!(">>> user_unique_auth_subject : {user_unique_auth_subject}");
 
             let user_unique_inbox = &format!("_INBOX.{}.>", host_pubkey);
-            println!(">>> user_unique_inbox : {user_unique_inbox}");    
-                    
-            let authenticated_user_diagnostics_subject = &format!("DIAGNOSTICS.authenticated.{}.>", host_pubkey);
-            println!(">>> authenticated_user_diagnostics_subject : {authenticated_user_diagnostics_subject}");    
+            println!(">>> user_unique_inbox : {user_unique_inbox}");
+
+            let authenticated_user_diagnostics_subject =
+                &format!("DIAGNOSTICS.authenticated.{}.>", host_pubkey);
+            println!(">>> authenticated_user_diagnostics_subject : {authenticated_user_diagnostics_subject}");
 
             types::Permissions {
                 publish: types::PermissionLimits {
@@ -180,15 +233,15 @@ impl AuthServiceApi {
                         "AUTH.validate".to_string(),
                         user_unique_auth_subject.to_string(),
                         user_unique_inbox.to_string(),
-                        authenticated_user_diagnostics_subject.to_string()
-                ]),
+                        authenticated_user_diagnostics_subject.to_string(),
+                    ]),
                     deny: None,
                 },
                 subscribe: types::PermissionLimits {
                     allow: Some(vec![
                         user_unique_auth_subject.to_string(),
                         user_unique_inbox.to_string(),
-                        authenticated_user_diagnostics_subject.to_string()
+                        authenticated_user_diagnostics_subject.to_string(),
                     ]),
                     deny: None,
                 },
@@ -196,7 +249,8 @@ impl AuthServiceApi {
         } else {
             // Otherwise, exclusively grant publication permissions for the unauthenticated diagnostics subj
             // ...to allow the host device to still send diganostic reports
-            let unauthenticated_user_diagnostics_subject = format!("DIAGNOSTICS.unauthenticated.{}.>", host_pubkey);
+            let unauthenticated_user_diagnostics_subject =
+                format!("DIAGNOSTICS.unauthenticated.{}.>", host_pubkey);
             types::Permissions {
                 publish: types::PermissionLimits {
                     allow: Some(vec![unauthenticated_user_diagnostics_subject]),
@@ -214,11 +268,12 @@ impl AuthServiceApi {
             auth_signing_account_pubkey,
             auth_root_account_pubkey,
             permissions,
-            auth_request_claim
-        ).map_err(|e| ServiceError::Internal(e.to_string()))?;
-
+            auth_request_claim,
+        )
+        .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        let claim_str = serde_json::to_string(&auth_response_claim).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let claim_str = serde_json::to_string(&auth_response_claim)
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
         let token = utils::encode_jwt(&claim_str, &auth_root_account_keypair)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
@@ -232,7 +287,7 @@ impl AuthServiceApi {
 
         Ok(types::AuthApiResult {
             result: types::AuthResult::Callout(token),
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
@@ -244,41 +299,56 @@ impl AuthServiceApi {
 
         // 1. Verify expected data was received
         let signature: &[u8] = match &msg.headers {
-            Some(h) => {
-                HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
-                    log::error!(
-                        "Error: Missing x-signature header. Subject='AUTH.authorize'"
-                    );
-                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
-                })?)
-            },
+            Some(h) => HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
+                log::error!("Error: Missing x-signature header. Subject='AUTH.authorize'");
+                ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
+            })?),
             None => {
-                log::error!(
-                    "Error: Missing message headers. Subject='AUTH.authorize'"
-                );
-                return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+                log::error!("Error: Missing message headers. Subject='AUTH.authorize'");
+                return Err(ServiceError::Request(format!(
+                    "{:?}",
+                    ErrorCode::BAD_REQUEST
+                )));
             }
         };
 
-        let types::AuthJWTPayload { host_pubkey, maybe_sys_pubkey, nonce: _ } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
+        let types::AuthJWTPayload {
+            host_pubkey,
+            maybe_sys_pubkey,
+            nonce: _,
+        } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
 
         // 2. Validate signature
-        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey)
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
         if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
-            log::error!("Error: Failed to validate Signature. Subject='{}'. Err={}", msg.subject, e);
-            return Err(ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST)));
+            log::error!(
+                "Error: Failed to validate Signature. Subject='{}'. Err={}",
+                msg.subject,
+                e
+            );
+            return Err(ServiceError::Request(format!(
+                "{:?}",
+                ErrorCode::BAD_REQUEST
+            )));
         };
 
         // 4. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
         if let Some(sys_pubkey) = maybe_sys_pubkey {
             Command::new("nsc")
-                .arg(format!("add user -a SYS -n user_sys_host_{} -k {}", host_pubkey, sys_pubkey))
+                .arg(format!(
+                    "add user -a SYS -n user_sys_host_{} -k {}",
+                    host_pubkey, sys_pubkey
+                ))
                 .output()
                 .context("Failed to add host sys user with provided keys")
                 .map_err(|e| ServiceError::Internal(e.to_string()))?;
-    
+
             Command::new("nsc")
-                .arg(format!("add user -a WORKLOAD -n user_host_{} -k {} -K {} --tag pubkey:{}", host_pubkey, host_pubkey, WORKLOAD_SK_ROLE, host_pubkey))
+                .arg(format!(
+                    "add user -a WORKLOAD -n user_host_{} -k {} -K {} --tag pubkey:{}",
+                    host_pubkey, host_pubkey, WORKLOAD_SK_ROLE, host_pubkey
+                ))
                 .output()
                 .context("Failed to add host user with provided keys")
                 .map_err(|e| ServiceError::Internal(e.to_string()))?;
@@ -292,18 +362,24 @@ impl AuthServiceApi {
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         // 3. Create User JWT files (automatically signed with respective account key)
-        let sys_jwt_output = Command::new("nsc") 
-            .arg(format!("describe user -n user_sys_host_{} -a SYS --raw", host_pubkey))
+        let sys_jwt_output = Command::new("nsc")
+            .arg(format!(
+                "describe user -n user_sys_host_{} -a SYS --raw",
+                host_pubkey
+            ))
             .output()
             .context("Failed to generate host sys user jwt file")
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-    
+
         let sys_jwt = String::from_utf8(sys_jwt_output.stdout)
             .context("Command returned invalid UTF-8 output")
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        
-        let host_jwt_output= Command::new("nsc") 
-            .arg(format!("describe user -n user_host_{} -a WORKLOAD --raw", host_pubkey))
+
+        let host_jwt_output = Command::new("nsc")
+            .arg(format!(
+                "describe user -n user_host_{} -a WORKLOAD --raw",
+                host_pubkey
+            ))
             .output()
             .context("Failed to generate host user jwt file")
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
@@ -314,15 +390,15 @@ impl AuthServiceApi {
 
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
-        
+
         Ok(AuthApiResult {
             result: types::AuthResult::Authorization(types::AuthJWTResult {
                 host_pubkey: host_pubkey.clone(),
                 status: types::AuthState::Authorized,
                 host_jwt,
-                sys_jwt
+                sys_jwt,
             }),
-            maybe_response_tags: Some(tag_map)
+            maybe_response_tags: Some(tag_map),
         })
     }
 
@@ -337,20 +413,19 @@ impl AuthServiceApi {
         Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
     }
 
-    pub fn call<F, Fut>(
-        &self,
-        handler: F,
-    ) -> AsyncEndpointHandler<AuthApiResult>
+    pub fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<AuthApiResult>
     where
         F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<AuthApiResult, ServiceError>> + Send + 'static,
-        Self: Send + Sync
+        Self: Send + Sync,
     {
-        let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<AuthApiResult> {
-            let api_clone = api.clone();
-            Box::pin(handler(api_clone, msg))
-        })
+        let api = self.to_owned();
+        Arc::new(
+            move |msg: Arc<Message>| -> JsServiceResponse<AuthApiResult> {
+                let api_clone = api.clone();
+                Box::pin(handler(api_clone, msg))
+            },
+        )
     }
 
     fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
@@ -359,10 +434,13 @@ impl AuthServiceApi {
     {
         let payload_buf = msg.payload.to_vec();
         serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject.clone().into_string(), e);
+            let err_msg = format!(
+                "Error: Failed to deserialize payload. Subject='{}' Err={}",
+                msg.subject.clone().into_string(),
+                e
+            );
             log::error!("{}", err_msg);
             ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
         })
     }
-
 }
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 539c453..fa89c23 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -1,7 +1,7 @@
 use anyhow::Result;
 use serde::{Deserialize, Serialize};
-use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 use std::collections::HashMap;
+use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 
 pub const AUTH_SERVICE_SUBJECT: &str = "validate";
 
@@ -12,17 +12,17 @@ pub const WORKLOAD_SK_ROLE: &str = "workload-role";
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthState {
     Unauthenticated, // step 0
-    Authenticated, // step 1
-    Authorized, // step 2
-    Forbidden, // failure to auth
-    Error(String) // internal error
+    Authenticated,   // step 1
+    Authorized,      // step 2
+    Forbidden,       // failure to auth
+    Error(String),   // internal error
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct AuthJWTPayload {
-    pub host_pubkey: String, // nkey
+    pub host_pubkey: String,              // nkey
     pub maybe_sys_pubkey: Option<String>, // nkey
-    pub nonce: String
+    pub nonce: String,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -30,20 +30,20 @@ pub struct AuthJWTResult {
     pub status: AuthState,
     pub host_pubkey: String,
     pub host_jwt: String,
-    pub sys_jwt: String
-} 
+    pub sys_jwt: String,
+}
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthResult {
     Callout(String), // stringifiedAuthResponseClaim
-    Authorization(AuthJWTResult)
+    Authorization(AuthJWTResult),
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthApiResult {
     pub result: AuthResult,
     // NB: `maybe_response_tags` optionally return endpoint scoped vars to be available for use as a response subject in JS Service Endpoint handler
-    pub maybe_response_tags: Option<HashMap<String, String>> 
+    pub maybe_response_tags: Option<HashMap<String, String>>,
 }
 // NB: The following Traits make API Service compatible as a JS Service Endpoint
 impl EndpointTraits for AuthApiResult {}
@@ -55,16 +55,11 @@ impl CreateTag for AuthApiResult {
 impl CreateResponse for AuthApiResult {
     fn get_response(&self) -> bytes::Bytes {
         match self.clone().result {
-            AuthResult::Authorization(r) => {
-                match serde_json::to_vec(&r) {
-                    Ok(r) => r.into(),
-                    Err(e) => e.to_string().into(),
-                }
+            AuthResult::Authorization(r) => match serde_json::to_vec(&r) {
+                Ok(r) => r.into(),
+                Err(e) => e.to_string().into(),
             },
-            AuthResult::Callout(token) => token
-                .clone()
-                .into_bytes()
-                .into()
+            AuthResult::Callout(token) => token.clone().into_bytes().into(),
         }
     }
 }
@@ -89,7 +84,7 @@ pub struct AuthGuardPayload {
 impl AuthGuardPayload {
     pub fn try_add_signature<T>(mut self, sign_handler: T) -> Result<Self>
     where
-     T: Fn(&[u8]) -> Result<String>
+        T: Fn(&[u8]) -> Result<String>,
     {
         let payload_bytes = serde_json::to_vec(&self)?;
         let signature = sign_handler(&payload_bytes)?;
@@ -122,25 +117,25 @@ pub struct NatsAuthorizationRequest {
     pub client_info: NatsClientInfo,
     pub connect_opts: ConnectOptions,
     pub r#type: String, // should be authorization_request
-    pub version: u8, // should be 2
+    pub version: u8,    // should be 2
 }
 
 #[derive(Debug, Serialize, Deserialize, Default, Clone)]
 pub struct NatsServerId {
-    pub name: String, // Server name
-    pub host: String, // Server host address
-    pub id: String, // Server connection ID
+    pub name: String,    // Server name
+    pub host: String,    // Server host address
+    pub id: String,      // Server connection ID
     pub version: String, // Version of server (current stable = 2.10.22)
     pub cluster: String, // Server cluster name
 }
 
 #[derive(Debug, Serialize, Deserialize, Default, Clone)]
 pub struct NatsClientInfo {
-    pub host: String, // client host address
-    pub id: u64, // client connection ID (I think...)
-    pub user: String, // the user pubkey (the passed-in key)
+    pub host: String,     // client host address
+    pub id: u64,          // client connection ID (I think...)
+    pub user: String,     // the user pubkey (the passed-in key)
     pub name_tag: String, // The user pubkey name
-    pub kind: String, // should be "Client"
+    pub kind: String,     // should be "Client"
     pub nonce: String,
     pub r#type: String, // should be "nats"
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -162,7 +157,7 @@ pub struct ConnectOptions {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub version: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub protocol: Option<u16>
+    pub protocol: Option<u16>,
 }
 
 // Callout Response Types:
@@ -176,22 +171,22 @@ pub struct AuthResponseClaim {
 
 #[derive(Debug, Serialize, Deserialize, Default, Clone)]
 pub struct ClaimData {
-    #[serde(rename = "iat")] 
-    pub issued_at: i64,  // Issued At (Unix timestamp)
+    #[serde(rename = "iat")]
+    pub issued_at: i64, // Issued At (Unix timestamp)
     #[serde(rename = "iss")]
-    pub issuer: String,  // Issuer -- head account (from which any signing keys were created)
+    pub issuer: String, // Issuer -- head account (from which any signing keys were created)
     #[serde(default, rename = "aud", skip_serializing_if = "Option::is_none")]
-    pub audience: Option<String>,  // Audience for whom the token is intended
+    pub audience: Option<String>, // Audience for whom the token is intended
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub name: Option<String>,
     #[serde(default, rename = "exp", skip_serializing_if = "Option::is_none")]
-    pub expires_at: Option<i64>,  // Expiry (Optional, Unix timestamp)
+    pub expires_at: Option<i64>, // Expiry (Optional, Unix timestamp)
     #[serde(default, rename = "jti", skip_serializing_if = "Option::is_none")]
     pub jwt_id: Option<String>, // Base32 hash of the claims
     #[serde(default, rename = "nbf", skip_serializing_if = "Option::is_none")]
-    pub not_before: Option<i64>,  // Issued At (Unix timestamp)
+    pub not_before: Option<i64>, // Issued At (Unix timestamp)
     #[serde(default, rename = "sub")]
-    pub subcriber: String,  // Public key of the account or user to which the JWT is being issued
+    pub subcriber: String, // Public key of the account or user to which the JWT is being issued
 }
 
 #[derive(Debug, Serialize, Deserialize, Default, Clone)]
@@ -199,7 +194,7 @@ pub struct NatsGenericData {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub tags: Vec<String>,
     #[serde(rename = "type")]
-	pub claim_type: String, // should be "user"
+    pub claim_type: String, // should be "user"
     pub version: u8, // should be 2
 }
 
@@ -208,11 +203,11 @@ pub struct AuthGuardResponse {
     #[serde(flatten)]
     pub generic_data: NatsGenericData,
     #[serde(default, rename = "jwt", skip_serializing_if = "Option::is_none")]
-	pub user_jwt: Option<String>, // This is the jwt string of the `UserClaim`
+    pub user_jwt: Option<String>, // This is the jwt string of the `UserClaim`
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub issuer_account: Option<String>, // Issuer Account === the signing nkey. Should set when the claim is issued by a signing key.
     #[serde(default, skip_serializing_if = "Option::is_none")]
-	pub error: Option<String>,
+    pub error: Option<String>,
 }
 
 #[derive(Debug, Serialize, Deserialize, Default)]
@@ -220,7 +215,7 @@ pub struct UserClaim {
     #[serde(flatten)]
     pub generic_claim_data: ClaimData,
     #[serde(rename = "nats")]
-	pub user_claim_data: UserClaimData,
+    pub user_claim_data: UserClaimData,
 }
 
 #[derive(Debug, Serialize, Deserialize, Default)]
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index 17b7e2e..0f8d1ad 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -1,25 +1,21 @@
 use super::types;
 use anyhow::{anyhow, Result};
-use nkeys::KeyPair;
-use std::io::Write;
-use util_libs::nats_js_client::ServiceError;
-use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
 use data_encoding::{BASE32HEX_NOPAD, BASE64URL_NOPAD};
-use sha2::{Digest, Sha256};
+use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
+use nkeys::KeyPair;
+use serde::Deserialize;
 use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::io::Write;
 use std::time::SystemTime;
-use serde::Deserialize;
+use util_libs::nats_js_client::ServiceError;
 
 pub fn handle_internal_err(err_msg: &str) -> ServiceError {
     log::error!("{}", err_msg);
     ServiceError::Internal(err_msg.to_string())
 }
 
-pub async fn write_file(
-    data: Vec<u8>,
-    output_dir: &str,
-    file_name: &str,
-) -> Result<String> {
+pub async fn write_file(data: Vec<u8>, output_dir: &str, file_name: &str) -> Result<String> {
     let output_path = format!("{}/{}", output_dir, file_name);
     let mut file = std::fs::OpenOptions::new()
         .create(true)
@@ -31,47 +27,6 @@ pub async fn write_file(
     Ok(output_path)
 }
 
-pub fn generate_user_jwt(_user_public_key: &str, _account_signing_key: &str) -> Option<String> {
-    // Implementation here
-
-    // // Output jwt with nsc
-    // let user_jwt_path = Command::new("nsc")
-    //     .arg("...")
-    //     // .arg(format!("> {}", output_dir))
-    //     .output()
-    //     .expect("Failed to output user jwt to file")
-    //     .stdout;
-
-        Some(String::new())
-}
-// pub async fn publish_chunks(js: &Context, subject: &str, file_name: &str, data: Vec<u8>) -> Result<()> {
-//     // let data: Vec<u8> = std::fs::read(file_path)?;
-//     js.publish(format!("{}.{} ", subject, file_name), data.into()).await?;
-//     Ok(())
-// }
-
-
-// pub async fn chunk_file_and_publish(_js: &Context, _subject: &str, _file_path: &str) -> Result<()> {
-    // let mut file = std::fs::File::open(file_path)?;
-    // let mut buffer = vec![0; CHUNK_SIZE];
-    // let mut chunk_id = 0;
-
-    // while let Ok(bytes_read) = file.read(mut buffer) {
-    //     if bytes_read == 0 {
-    //         break;
-    //     }
-    //     let chunk_data = &buffer[..bytes_read];
-    //     js.publish(subject.to_string(), chunk_data.into()).await.unwrap();
-    //     chunk_id += 1;
-    // }
-
-    // // Send an EOF marker
-    // js.publish(subject.to_string(), "EOF".into()).await.unwrap();
-
-//     Ok(())
-// }
-
-
 /// Decode a Base64-encoded string back into a JSON string
 pub fn base64_to_data<T>(base64_data: &str) -> Result<T>
 where
@@ -91,11 +46,11 @@ pub fn hash_claim(claims_str: &str) -> Vec<u8> {
 }
 
 // Convert claims to JWT/Token
-pub fn encode_jwt(claims_str: &str, signing_kp: &KeyPair) -> Result<String> {    
+pub fn encode_jwt(claims_str: &str, signing_kp: &KeyPair) -> Result<String> {
     const JWT_HEADER: &str = r#"{"typ":"JWT","alg":"ed25519-nkey"}"#;
-    let b64_header: String =  BASE64URL_NOPAD.encode(JWT_HEADER.as_bytes());
+    let b64_header: String = BASE64URL_NOPAD.encode(JWT_HEADER.as_bytes());
     println!("encoded b64 header: {:?}", b64_header);
-    let b64_body =  BASE64URL_NOPAD.encode(claims_str.as_bytes());
+    let b64_body = BASE64URL_NOPAD.encode(claims_str.as_bytes());
     println!("encoded header: {:?}", b64_body);
 
     let jwt_half = format!("{b64_header}.{b64_body}");
@@ -106,15 +61,15 @@ pub fn encode_jwt(claims_str: &str, signing_kp: &KeyPair) -> Result<String> {
     Ok(token)
 }
 
-/// Convert token into the 
-pub fn decode_jwt<T>(token: &str) -> Result<T> 
+/// Convert token into the
+pub fn decode_jwt<T>(token: &str) -> Result<T>
 where
     T: for<'de> Deserialize<'de> + std::fmt::Debug,
 {
     // Decode and replace custom `ed25519-nkey` to `EdDSA`
     let parts: Vec<&str> = token.split('.').collect();
     println!("parts: {:?}", parts);
-    println!("parts.len() : {:?}", parts.len()); 
+    println!("parts.len() : {:?}", parts.len());
 
     if parts.len() != 3 {
         return Err(anyhow!("Invalid JWT format"));
@@ -141,7 +96,8 @@ where
     if part_1.get("exp").is_none() {
         let one_week = std::time::Duration::from_secs(7 * 24 * 60 * 60);
         let one_week_from_now = SystemTime::now() + one_week;
-        let expires_at: i64 =  one_week_from_now.duration_since(SystemTime::UNIX_EPOCH)?
+        let expires_at: i64 = one_week_from_now
+            .duration_since(SystemTime::UNIX_EPOCH)?
             .as_secs()
             .try_into()?;
 
@@ -154,14 +110,14 @@ where
     let modified_token = format!("{}.{}.{}", modified_header, modified_part_1, parts[2]);
     println!("modified_token: {:?}", modified_token);
 
-    let account_kp = KeyPair::from_public_key("ABYGJO6B2OJTXL7DLL7EGR45RQ4I2CKM4D5XYYUSUBZJ7HJJF67E54VC")?;
+    let account_kp =
+        KeyPair::from_public_key("ABYGJO6B2OJTXL7DLL7EGR45RQ4I2CKM4D5XYYUSUBZJ7HJJF67E54VC")?;
 
     let public_key_b32 = account_kp.public_key();
     println!("Public Key (Base32): {}", public_key_b32);
 
     // Decode from Base32 to raw bytes using Rfc4648 (compatible with NATS keys)
-    let public_key_bytes =  Some(BASE32HEX_NOPAD
-        .decode(public_key_b32.as_bytes()))
+    let public_key_bytes = Some(BASE32HEX_NOPAD.decode(public_key_b32.as_bytes()))
         .ok_or(anyhow!("Failed to convert public key to bytes"))??;
     println!("Decoded Public Key Bytes: {:?}", public_key_bytes);
 
@@ -172,7 +128,7 @@ where
     // Validate the token with the correct algorithm
     let mut validation = Validation::new(Algorithm::EdDSA);
     validation.insecure_disable_signature_validation();
-    validation.validate_aud = false;  // Disable audience validation
+    validation.validate_aud = false; // Disable audience validation
     println!("passed validation");
 
     let token_data = decode::<T>(&modified_token, &decoding_key, &validation)?;
@@ -181,84 +137,86 @@ where
     Ok(token_data.claims)
 }
 
-pub fn generate_auth_response_claim (
+pub fn generate_auth_response_claim(
     auth_signing_account_keypair: KeyPair,
     auth_signing_account_pubkey: String,
     auth_root_account_pubkey: String,
     permissions: types::Permissions,
-    auth_request_claim: types::NatsAuthorizationRequestClaim
+    auth_request_claim: types::NatsAuthorizationRequestClaim,
 ) -> Result<types::AuthResponseClaim> {
-        let now = SystemTime::now();
-        let issued_at = now
-            .duration_since(SystemTime::UNIX_EPOCH)?
-            .as_secs()
-            .try_into()?;
-        let one_week = std::time::Duration::from_secs(7 * 24 * 60 * 60);
-        let one_week_from_now = now + one_week;
-        let expires_at: i64 =  one_week_from_now.duration_since(SystemTime::UNIX_EPOCH)?
-            .as_secs()
-            .try_into()?;    
-        let inner_generic_data = types::NatsGenericData {
-            claim_type: "user".to_string(),
-            tags: vec![],
-            version: 2,
-        };
-        let user_claim_data = types::UserClaimData {
-            permissions,
-            generic_data: inner_generic_data,
-            issuer_account: Some(auth_root_account_pubkey.clone()), // must be the root account pubkey or the issuer account that signs the claim AND must be listed "allowed-account" 
-        };
-        let inner_nats_claim = types::ClaimData {
-            issuer: auth_signing_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
-            subcriber: auth_request_claim.auth_request.user_nkey.clone(),
-            issued_at,
-            audience: None, // Inner claim should have no `audience` when using the operator-auth mode
-            expires_at: Some(expires_at),
-            not_before: None,
-            name: Some("allowed_auth_user".to_string()),
-            jwt_id: None,
-        };
-        let mut user_claim = types::UserClaim {
-            generic_claim_data: inner_nats_claim,
-            user_claim_data
-        };
-        
-        let mut user_claim_str = serde_json::to_string(&user_claim)?;
-        let hashed_user_claim_bytes = hash_claim(&user_claim_str);
-        user_claim.generic_claim_data.jwt_id = Some(BASE32HEX_NOPAD.encode(&hashed_user_claim_bytes));
-        user_claim_str = serde_json::to_string(&user_claim)?;
-        let user_token = encode_jwt(&user_claim_str, &auth_signing_account_keypair)?;
-        println!("user_token: {:#?}", user_token);
-
-        let outer_nats_claim = types::ClaimData {
-            issuer: auth_root_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
-            subcriber: auth_request_claim.auth_request.user_nkey.clone(),
-            issued_at,
-            audience: Some(auth_request_claim.auth_request.server_id.id),
-            expires_at: None, // Some(expires_at),
-            not_before: None,
-            name: None,
-            jwt_id: None,
-        };
-        let outer_generic_data =  types::NatsGenericData {
-            claim_type: "authorization_response".to_string(),
-            tags: vec![],
-            version: 2,
-        };
-        let auth_response = types::AuthGuardResponse {
-            generic_data: outer_generic_data,
-            user_jwt: Some(user_token),
-            issuer_account: None,
-            error: None,
-        };
-        let mut auth_response_claim = types::AuthResponseClaim {
-            generic_claim_data: outer_nats_claim,
-            auth_response,
-        };
-
-        let claim_str = serde_json::to_string(&auth_response_claim)?;
-        let hashed_claim_bytes = hash_claim(&claim_str);
-        auth_response_claim.generic_claim_data.jwt_id = Some(BASE32HEX_NOPAD.encode(&hashed_claim_bytes));
-        
-        Ok(auth_response_claim)
-}
\ No newline at end of file
+    let now = SystemTime::now();
+    let issued_at = now
+        .duration_since(SystemTime::UNIX_EPOCH)?
+        .as_secs()
+        .try_into()?;
+    let one_week = std::time::Duration::from_secs(7 * 24 * 60 * 60);
+    let one_week_from_now = now + one_week;
+    let expires_at: i64 = one_week_from_now
+        .duration_since(SystemTime::UNIX_EPOCH)?
+        .as_secs()
+        .try_into()?;
+    let inner_generic_data = types::NatsGenericData {
+        claim_type: "user".to_string(),
+        tags: vec![],
+        version: 2,
+    };
+    let user_claim_data = types::UserClaimData {
+        permissions,
+        generic_data: inner_generic_data,
+        issuer_account: Some(auth_root_account_pubkey.clone()), // must be the root account pubkey or the issuer account that signs the claim AND must be listed "allowed-account"
+    };
+    let inner_nats_claim = types::ClaimData {
+        issuer: auth_signing_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
+        subcriber: auth_request_claim.auth_request.user_nkey.clone(),
+        issued_at,
+        audience: None, // Inner claim should have no `audience` when using the operator-auth mode
+        expires_at: Some(expires_at),
+        not_before: None,
+        name: Some("allowed_auth_user".to_string()),
+        jwt_id: None,
+    };
+    let mut user_claim = types::UserClaim {
+        generic_claim_data: inner_nats_claim,
+        user_claim_data,
+    };
+
+    let mut user_claim_str = serde_json::to_string(&user_claim)?;
+    let hashed_user_claim_bytes = hash_claim(&user_claim_str);
+    user_claim.generic_claim_data.jwt_id = Some(BASE32HEX_NOPAD.encode(&hashed_user_claim_bytes));
+    user_claim_str = serde_json::to_string(&user_claim)?;
+    let user_token = encode_jwt(&user_claim_str, &auth_signing_account_keypair)?;
+    println!("user_token: {:#?}", user_token);
+
+    let outer_nats_claim = types::ClaimData {
+        issuer: auth_root_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
+        subcriber: auth_request_claim.auth_request.user_nkey.clone(),
+        issued_at,
+        audience: Some(auth_request_claim.auth_request.server_id.id),
+        expires_at: None, // Some(expires_at),
+        not_before: None,
+        name: None,
+        jwt_id: None,
+    };
+    let outer_generic_data = types::NatsGenericData {
+        claim_type: "authorization_response".to_string(),
+        tags: vec![],
+        version: 2,
+    };
+    let auth_response = types::AuthGuardResponse {
+        generic_data: outer_generic_data,
+        user_jwt: Some(user_token),
+        issuer_account: None,
+        error: None,
+    };
+    let mut auth_response_claim = types::AuthResponseClaim {
+        generic_claim_data: outer_nats_claim,
+        auth_response,
+    };
+
+    let claim_str = serde_json::to_string(&auth_response_claim)?;
+    let hashed_claim_bytes = hash_claim(&claim_str);
+    auth_response_claim.generic_claim_data.jwt_id =
+        Some(BASE32HEX_NOPAD.encode(&hashed_claim_bytes));
+
+    Ok(auth_response_claim)
+}
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index a24fdc4..f44b338 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -10,12 +10,12 @@ use crate::types::WorkloadResult;
 
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
+use async_nats::Message;
 use core::option::Option::None;
 use std::{fmt::Debug, sync::Arc};
-use async_nats::Message;
 use util_libs::{
+    db::schemas::{WorkloadState, WorkloadStatus},
     nats_js_client::ServiceError,
-    db::schemas::{WorkloadState, WorkloadStatus}
 };
 
 #[derive(Debug, Clone, Default)]
@@ -24,7 +24,10 @@ pub struct HostWorkloadApi {}
 impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn start_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -52,16 +55,19 @@ impl HostWorkloadApi {
             }
         };
 
-        Ok(WorkloadApiResult  {
+        Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -69,7 +75,6 @@ impl HostWorkloadApi {
         log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
 
         let status = if let Some(workload) = message_payload.workload {
-
             // TODO: Talk through with Stefan
             // 1. Connect to interface for Nix and instruct systemd to install workload...
             // eg: nix_install_with(workload)
@@ -89,17 +94,20 @@ impl HostWorkloadApi {
                 actual: WorkloadState::Error(err_msg),
             }
         };
-            
-        Ok(WorkloadApiResult  {
+
+        Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -110,7 +118,7 @@ impl HostWorkloadApi {
             // TODO: Talk through with Stefan
             // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
             // nix_uninstall_with(workload_id)
-    
+
             // 2. Respond to endpoint request
             WorkloadStatus {
                 id: workload._id,
@@ -127,18 +135,21 @@ impl HostWorkloadApi {
             }
         };
 
-        Ok(WorkloadApiResult  {
+        Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -150,9 +161,9 @@ impl HostWorkloadApi {
         Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status: workload_status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 }
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 46aeeab..51c1807 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -5,22 +5,22 @@ Provisioning Account: WORKLOAD
 Users: orchestrator & host
 */
 
-pub mod orchestrator_api;
 pub mod host_api;
+pub mod orchestrator_api;
 pub mod types;
 
 use anyhow::Result;
-use core::option::Option::None;
 use async_nats::jetstream::ErrorCode;
-use async_trait::async_trait;
-use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
-use std::future::Future;
+use async_trait::async_trait;
+use core::option::Option::None;
 use serde::Deserialize;
+use std::future::Future;
+use std::{fmt::Debug, sync::Arc};
 use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
-    nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
-    db::schemas::{WorkloadState, WorkloadStatus}
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats_js_client::{AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
@@ -28,26 +28,24 @@ pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
 pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
 
-
 #[async_trait]
 pub trait WorkloadServiceApi
 where
     Self: std::fmt::Debug + Clone + 'static,
 {
-    fn call<F, Fut>(
-        &self,
-        handler: F,
-    ) -> AsyncEndpointHandler<WorkloadApiResult>
+    fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<WorkloadApiResult>
     where
         F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
-        Self: Send + Sync
+        Self: Send + Sync,
     {
-        let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
-            let api_clone = api.clone();
-            Box::pin(handler(api_clone, msg))
-        })
+        let api = self.to_owned();
+        Arc::new(
+            move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
+                let api_clone = api.clone();
+                Box::pin(handler(api_clone, msg))
+            },
+        )
     }
 
     fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
@@ -56,11 +54,14 @@ where
     {
         let payload_buf = msg.payload.to_vec();
         serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject.clone().into_string(), e);
+            let err_msg = format!(
+                "Error: Failed to deserialize payload. Subject='{}' Err={}",
+                msg.subject.clone().into_string(),
+                e
+            );
             log::error!("{}", err_msg);
             ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
         })
-        
     }
 
     // Helper function to streamline the processing of incoming workload messages
@@ -95,9 +96,9 @@ where
                 WorkloadApiResult {
                     result: WorkloadResult {
                         status,
-                        workload: None
+                        workload: None,
                     },
-                    maybe_response_tags: None
+                    maybe_response_tags: None,
                 }
             }
         })
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index f2731ca..6bb0e54 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -12,19 +12,19 @@ use crate::types::WorkloadResult;
 
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
-use core::option::Option::None;
-use std::{collections::HashMap, fmt::Debug, sync::Arc};
 use async_nats::Message;
+use bson::{self, doc, to_document};
+use core::option::Option::None;
 use mongodb::{options::UpdateModifications, Client as MongoDBClient};
 use rand::seq::SliceRandom;
 use serde::{Deserialize, Serialize};
-use bson::{self, doc, to_document};
+use std::{collections::HashMap, fmt::Debug, sync::Arc};
 use util_libs::{
-    nats_js_client::ServiceError,
     db::{
         mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus}
-    }
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
+    },
+    nats_js_client::ServiceError,
 };
 
 #[derive(Debug, Clone)]
@@ -39,7 +39,8 @@ impl WorkloadServiceApi for OrchestratorWorkloadApi {}
 impl OrchestratorWorkloadApi {
     pub async fn new(client: &MongoDBClient) -> Result<Self> {
         Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME).await?,
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
+                .await?,
             host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
             user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
         })
@@ -51,8 +52,14 @@ impl OrchestratorWorkloadApi {
             msg,
             WorkloadState::Reported,
             |workload: schemas::Workload| async move {
-                let workload_id = self.workload_collection.insert_one_into(workload.clone()).await?;
-                log::info!("Successfully added workload. MongodDB Workload ID={:?}", workload_id);
+                let workload_id = self
+                    .workload_collection
+                    .insert_one_into(workload.clone())
+                    .await?;
+                log::info!(
+                    "Successfully added workload. MongodDB Workload ID={:?}",
+                    workload_id
+                );
                 let new_workload = schemas::Workload {
                     _id: Some(workload_id),
                     ..workload
@@ -64,9 +71,9 @@ impl OrchestratorWorkloadApi {
                             desired: WorkloadState::Reported,
                             actual: WorkloadState::Reported,
                         },
-                        workload: None
+                        workload: None,
                     },
-                    maybe_response_tags: None
+                    maybe_response_tags: None,
                 })
             },
             WorkloadState::Error,
@@ -74,16 +81,28 @@ impl OrchestratorWorkloadApi {
         .await
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.update'");
         self.process_request(
             msg,
             WorkloadState::Running,
             |workload: schemas::Workload| async move {
                 let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
-                log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
+                let updated_workload_doc =
+                    to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+                self.workload_collection
+                    .update_one_within(
+                        workload_query,
+                        UpdateModifications::Document(updated_workload_doc),
+                    )
+                    .await?;
+                log::info!(
+                    "Successfully updated workload. MongodDB Workload ID={:?}",
+                    workload._id
+                );
                 Ok(WorkloadApiResult {
                     result: WorkloadResult {
                         status: WorkloadStatus {
@@ -91,18 +110,20 @@ impl OrchestratorWorkloadApi {
                             desired: WorkloadState::Reported,
                             actual: WorkloadState::Reported,
                         },
-                        workload: None
+                        workload: None,
                     },
-                    maybe_response_tags: None
+                    maybe_response_tags: None,
                 })
             },
             WorkloadState::Error,
         )
         .await
-
     }
 
-    pub async fn remove_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn remove_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.remove'");
         self.process_request(
             msg,
@@ -132,7 +153,10 @@ impl OrchestratorWorkloadApi {
     }
 
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_insertion(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn handle_db_insertion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.insert'");
         self.process_request(
             msg,
@@ -193,7 +217,7 @@ impl OrchestratorWorkloadApi {
                 let host_id = host._id
                     .to_owned()
                     .ok_or_else(|| ServiceError::Internal("Failed to read ._id from record".to_string()))?;
-                
+
                 // 4. Update the Workload Collection with the assigned Host ID
                 let workload_query = doc! { "_id":  workload_id.clone() };
                 let updated_workload = &Workload {
@@ -206,7 +230,7 @@ impl OrchestratorWorkloadApi {
                     "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
                     updated_workload_result
                 );
-                
+
                 // 5. Update the Host Collection with the assigned Workload ID
                 let host_query = doc! { "_id":  host.clone()._id };
                 let updated_host_doc =  to_document(&Host {
@@ -241,13 +265,16 @@ impl OrchestratorWorkloadApi {
 
     // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn handle_db_modification(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.modify'");
-        
+
         let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-        
-        // TODO: ...handle the use case for the update entry change stream 
+
+        // TODO: ...handle the use case for the update entry change stream
 
         // let workload_request_bytes = serde_json::to_vec(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
 
@@ -260,14 +287,17 @@ impl OrchestratorWorkloadApi {
         Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status: success_status,
-                workload: Some(workload)
+                workload: Some(workload),
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
         let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
@@ -278,9 +308,9 @@ impl OrchestratorWorkloadApi {
         Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status: workload_status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
@@ -294,5 +324,4 @@ impl OrchestratorWorkloadApi {
     {
         Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
     }
-
 }
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 0ec4044..c608bd1 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,6 +1,9 @@
-use std::collections::HashMap;
-use util_libs::{db::schemas::{self, WorkloadStatus}, js_stream_service::{CreateResponse, CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use util_libs::{
+    db::schemas::{self, WorkloadStatus},
+    js_stream_service::{CreateResponse, CreateTag, EndpointTraits},
+};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "snake_case")]
@@ -14,7 +17,7 @@ pub enum WorkloadServiceSubjects {
     SendStatus,
     Start,
     Uninstall,
-    UpdateInstalled
+    UpdateInstalled,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -26,7 +29,7 @@ pub struct WorkloadResult {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadApiResult {
     pub result: WorkloadResult,
-    pub maybe_response_tags: Option<HashMap<String, String>>
+    pub maybe_response_tags: Option<HashMap<String, String>>,
 }
 impl EndpointTraits for WorkloadApiResult {}
 impl CreateTag for WorkloadApiResult {
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index 1204912..0219876 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,3 +1,4 @@
+use crate::nats_js_client::ServiceError;
 use anyhow::Result;
 use async_trait::async_trait;
 use bson::{self, doc, Document};
@@ -7,7 +8,6 @@ use mongodb::results::{DeleteResult, UpdateResult};
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
-use crate::nats_js_client::ServiceError;
 
 #[async_trait]
 pub trait MongoDbAPI<T>
@@ -23,7 +23,7 @@ where
     async fn update_one_within(
         &self,
         query: Document,
-        updated_doc: UpdateModifications
+        updated_doc: UpdateModifications,
     ) -> Result<UpdateResult, Self::Error>;
     async fn delete_one_from(&self, query: Document) -> Result<DeleteResult, Self::Error>;
     async fn delete_all_from(&self) -> Result<DeleteResult, Self::Error>;
@@ -138,7 +138,7 @@ where
     async fn update_one_within(
         &self,
         query: Document,
-        updated_doc: UpdateModifications
+        updated_doc: UpdateModifications,
     ) -> Result<UpdateResult, Self::Error> {
         self.collection
             .update_one(query, updated_doc)
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 1239448..76f2141 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -31,7 +31,7 @@ pub use String as MongoDbId;
 #[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
 pub enum Role {
     Developer(DeveloperJWT), // jwt string
-    Hoster(HosterPubKey),      // host pubkey
+    Hoster(HosterPubKey),    // host pubkey
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 55412a5..0db52d9 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -1,19 +1,20 @@
 use super::nats_js_client::EndpointType;
 
 use anyhow::{anyhow, Result};
-use std::any::Any;
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
 use async_trait::async_trait;
 use futures::StreamExt;
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-pub type ResponseSubjectsGenerator = Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
+pub type ResponseSubjectsGenerator =
+    Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
 
 pub trait CreateTag: Send + Sync {
     fn get_tags(&self) -> HashMap<String, String>;
@@ -24,8 +25,17 @@ pub trait CreateResponse: Send + Sync {
 }
 
 pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + CreateResponse + 'static
-{}
+    Serialize
+    + for<'de> Deserialize<'de>
+    + Send
+    + Sync
+    + Clone
+    + Debug
+    + CreateTag
+    + CreateResponse
+    + 'static
+{
+}
 
 #[async_trait]
 pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
@@ -343,8 +353,8 @@ impl JsStreamService {
                     let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
-                },
-                Err(err) => (err.to_string().into(), HashMap::new())
+                }
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 656a076..4c6e694 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -2,14 +2,14 @@ use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamServic
 use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
 
 use anyhow::Result;
-use std::path::PathBuf;
-use core::marker::Sync;
 use async_nats::{jetstream, AuthError, HeaderMap, Message, ServerInfo};
+use core::marker::Sync;
 use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::fmt;
 use std::fmt::Debug;
 use std::future::Future;
+use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -66,7 +66,7 @@ pub struct PublishInfo {
     pub subject: String,
     pub msg_id: String,
     pub data: Vec<u8>,
-    pub headers: Option<HeaderMap> 
+    pub headers: Option<HeaderMap>,
 }
 
 #[derive(Debug)]
@@ -105,8 +105,8 @@ pub struct JsClient {
 #[derive(Clone)]
 pub enum Credentials {
     Path(std::path::PathBuf), // String = pathbuf as string
-    Password(String, String)
-}   
+    Password(String, String),
+}
 
 #[derive(Deserialize, Default)]
 pub struct NewJsClientParams {
@@ -138,7 +138,7 @@ impl JsClient {
                         .user_and_password(user, pw)
                         .connect(&p.nats_url)
                         .await?
-                },
+                }
                 Credentials::Path(cp) => {
                     let path = std::path::Path::new(&cp);
                     connect_options
@@ -147,7 +147,7 @@ impl JsClient {
                         .connect(&p.nats_url)
                         .await?
                 }
-            }
+            },
             None => connect_options.connect(&p.nats_url).await?,
         };
 
@@ -212,8 +212,10 @@ impl JsClient {
         Ok(())
     }
 
-    pub async fn check_connection(&self) -> Result<async_nats::connection::State, async_nats::Error> {
-        let conn_state =self.client.connection_state();
+    pub async fn check_connection(
+        &self,
+    ) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state = self.client.connection_state();
         if let async_nats::connection::State::Disconnected = conn_state {
             Err(Box::new(ErrClientDisconnected))
         } else {
@@ -224,14 +226,16 @@ impl JsClient {
     pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
         let now = Instant::now();
         let result = match payload.headers {
-            Some(h) => self
-                .js
-                .publish_with_headers(payload.subject.clone(), h, payload.data.clone().into())
-                .await,
-            None => self
-                .js
-                .publish(payload.subject.clone(), payload.data.clone().into())
-                .await
+            Some(h) => {
+                self.js
+                    .publish_with_headers(payload.subject.clone(), h, payload.data.clone().into())
+                    .await
+            }
+            None => {
+                self.js
+                    .publish(payload.subject.clone(), payload.data.clone().into())
+                    .await
+            }
         };
 
         let duration = now.elapsed();
@@ -322,13 +326,14 @@ pub fn get_nsc_root_path() -> String {
 pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {
     format!(
         "{}/keys/creds/{}/{}/{}.creds",
-        get_nsc_root_path(), operator, account, user
+        get_nsc_root_path(),
+        operator,
+        account,
+        user
     )
 }
 
-pub fn get_file_path_buf(
-    file_name: &str,
-) -> PathBuf {
+pub fn get_file_path_buf(file_name: &str) -> PathBuf {
     let current_dir_path = std::env::current_dir().expect("Failed to locate current directory.");
     current_dir_path.join(file_name)
 }

From 0ed68c9644a75a7e50e4b609bc89c8d2c934a4fa Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 4 Feb 2025 18:47:29 -0600
Subject: [PATCH 61/91] lint

---
 rust/clients/host_agent/src/hostd/gen_leaf_server.rs | 3 +--
 rust/clients/host_agent/src/main.rs                  | 5 +++--
 rust/services/workload/src/host_api.rs               | 2 +-
 rust/services/workload/src/lib.rs                    | 2 +-
 rust/services/workload/src/types.rs                  | 2 +-
 rust/util_libs/src/js_stream_service.rs              | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 9c53bc7..906946c 100644
--- a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -2,8 +2,7 @@ use std::path::PathBuf;
 
 use anyhow::Context;
 use tempfile::tempdir;
-use util_libs::
-    nats_server::{
+use util_libs::nats_server::{
     JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
     LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
 };
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 10a54d9..e642d9d 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -72,8 +72,9 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
         &args.store_dir,
         args.hub_url.clone(),
         args.hub_tls_insecure,
-    ).await;
-    
+    )
+    .await;
+
     let host_workload_client = hostd::workloads::run(
         &host_agent_keys.host_pubkey,
         &host_agent_keys.get_host_creds_path(),
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index bfa8d75..f44b338 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -166,4 +166,4 @@ impl HostWorkloadApi {
             maybe_response_tags: None,
         })
     }
-}
\ No newline at end of file
+}
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 800eddf..51c1807 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -103,4 +103,4 @@ where
             }
         })
     }
-}
\ No newline at end of file
+}
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 3d1f431..c608bd1 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -45,4 +45,4 @@ impl CreateResponse for WorkloadApiResult {
             Err(e) => e.to_string().into(),
         }
     }
-}
\ No newline at end of file
+}
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 462a355..0db52d9 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -554,4 +554,4 @@ mod tests {
         let result = service.spawn_consumer_handler(consumer_name).await;
         assert!(result.is_ok(), "Failed to spawn consumer handler");
     }
-}
\ No newline at end of file
+}

From e9203ab2b941c92952fcd759cb04f7c89c3b2a27 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 4 Feb 2025 21:06:16 -0600
Subject: [PATCH 62/91] tidy env vars

---
 .env.example                              | 13 ++---
 .gitignore                                |  2 +-
 rust/clients/host_agent/src/auth/utils.rs |  5 +-
 rust/clients/host_agent/src/keys.rs       |  8 +--
 rust/clients/orchestrator/src/auth.rs     | 66 +++++++++++++++++++----
 rust/services/authentication/src/lib.rs   |  4 +-
 rust/services/authentication/src/types.rs |  3 +-
 rust/services/authentication/src/utils.rs |  5 +-
 scripts/orchestrator_setup.sh             |  5 +-
 9 files changed, 79 insertions(+), 32 deletions(-)

diff --git a/.env.example b/.env.example
index 3e79f0c..7ad33bc 100644
--- a/.env.example
+++ b/.env.example
@@ -1,8 +1,9 @@
 NSC_PATH="<path/to/nsc_dir>"
+HOST_NKEY_PATH="<path>/host.nk"
+SYS_NKEY_PATH="<path>/sys.nk"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="<port>"
 MONGO_URI="mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL="nats://<path/to/leaf>:<port>"
-LEAF_SERVER_USER="test-user"
-LEAF_SERVER_PW="pw-123456789"
-HOST_CREDENTIALS_PATH="./host_user.creds";
-DEVICE_SEED_DEFAULT_PASSWORD="";
-HPOS_CONFIG_PATH="";
+HPOS_CONFIG_PATH="path/to/file.config";
+DEVICE_SEED_DEFAULT_PASSWORD="device_pw_1234"
diff --git a/.gitignore b/.gitignore
index 41cbad9..2196367 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,4 +22,4 @@ rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf
 .local
-
+rust/*/*/*/test_admin.creds
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index 510f354..cef4ffe 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,12 +1,9 @@
 use data_encoding::BASE64URL_NOPAD;
 
-/// Encode a JSON string into a b64-encoded string
+/// Encode a json string into a b64 string
 pub fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
-    // Parse to ensure it's valid JSON
     let parsed_json: serde_json::Value = serde_json::from_str(json_data)?;
-    // Convert JSON back to a compact string
     let json_string = serde_json::to_string(&parsed_json)?;
-    // Encode it into b64
     let encoded = BASE64URL_NOPAD.encode(json_string.as_bytes());
     Ok(encoded)
 }
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index b273a04..4242b4d 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -56,14 +56,14 @@ impl Keys {
     pub fn new() -> Result<Self> {
         // let host_key_path = format!("{}/user_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
         let host_key_path =
-            std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
+            std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
         let host_kp = KeyPair::new_user();
         write_keypair_to_file(get_file_path_buf(&host_key_path), host_kp.clone())?;
         let host_pk = host_kp.public_key();
 
         // let sys_key_path = format!("{}/user_sys_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
         let sys_key_path =
-            std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
+            std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
         let local_sys_kp = KeyPair::new_user();
         write_keypair_to_file(get_file_path_buf(&sys_key_path), local_sys_kp.clone())?;
         let local_sys_pk = local_sys_kp.public_key();
@@ -86,13 +86,13 @@ impl Keys {
         maybe_sys_creds_path: &Option<PathBuf>,
     ) -> Result<Self> {
         let host_key_path =
-            std::env::var("HOST_KEY_PATH").context("Cannot read HOST_KEY_PATH from env var")?;
+            std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
         let host_keypair =
             try_read_keypair_from_file(get_file_path_buf(&host_key_path.clone()))?
                 .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
         let host_pk = host_keypair.public_key();
         let sys_key_path =
-            std::env::var("SYS_KEY_PATH").context("Cannot read SYS_KEY_PATH from env var")?;
+            std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
         let host_creds_path = maybe_host_creds_path
             .to_owned()
             .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "host")));
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 345a84f..5228c1e 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -26,21 +26,40 @@ use authentication::{
     AuthServiceApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION,
 };
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use nkeys::KeyPair;
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
+    nats_js_client::{
+        get_event_listeners, get_file_path_buf, get_nats_url, with_event_listeners, Credentials,
+        EndpointType, JsClient, NewJsClientParams,
+    },
 };
 
 pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Agent";
 pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_auth_inbox_orchestrator";
 
 pub async fn run() -> Result<(), async_nats::Error> {
-    // ==================== Setup NATS ====================
-    let nats_url = nats_js_client::get_nats_url();
-    let event_listeners = nats_js_client::get_event_listeners();
+    let admin_account_creds_path = get_file_path_buf("test_admin.creds");
+    println!(
+        " >>>> admin_account_creds_path: {:#?} ",
+        admin_account_creds_path
+    );
+
+    // Root Keypair associated with AUTH account
+    let root_account_keypair = Arc::new(KeyPair::from_seed("<TEST_SK_SEED>")?);
+    let root_account_pubkey = root_account_keypair.public_key().clone();
+
+    // AUTH Account Signing Keypair associated with the `auth` user
+    let signing_account_keypair = Arc::new(KeyPair::from_seed("<TEST_SK_SEED>")?);
+    let signing_account_pubkey = signing_account_keypair.public_key().clone();
+    println!(
+        ">>>>>>>>> signing_account pubkey: {:?}",
+        signing_account_pubkey
+    );
 
+    // ==================== Setup NATS ====================
     // Setup JS Stream Service
     let auth_stream_service_params = JsServiceParamsPartial {
         name: AUTH_SRV_NAME.to_string(),
@@ -50,12 +69,12 @@ pub async fn run() -> Result<(), async_nats::Error> {
     };
 
     let orchestrator_auth_client = JsClient::new(NewJsClientParams {
-        nats_url,
+        nats_url: get_nats_url(),
         name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
         inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
         service_params: vec![auth_stream_service_params],
-        credentials: None,
-        listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
+        credentials: Some(Credentials::Path(admin_account_creds_path)),
+        listeners: vec![with_event_listeners(get_event_listeners())],
         ping_interval: Some(Duration::from_secs(10)),
         request_timeout: Some(Duration::from_secs(5)),
     })
@@ -71,7 +90,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Generate the Auth API with access to db
     let auth_api = AuthServiceApi::new(&client).await?;
 
-    // Register Auth Stream for Orchestrator to consume and proceess
+    // Register Auth Stream for Orchestrator to consume and process
     let auth_service = orchestrator_auth_client
         .get_js_service(AUTH_SRV_NAME.to_string())
         .await
@@ -81,8 +100,35 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     auth_service
         .add_consumer::<AuthApiResult>(
-            "validate",                  // consumer name
-            types::AUTH_SERVICE_SUBJECT, // consumer stream subj
+            "auth_callout",
+            types::AUTH_CALLOUT_SUBJECT, // consumer stream subj
+            EndpointType::Async(auth_api.call({
+                move |api: AuthServiceApi, msg: Arc<Message>| {
+                    let signing_account_kp = Arc::clone(&signing_account_keypair);
+                    let signing_account_pk = signing_account_pubkey.clone();
+                    let root_account_kp = Arc::clone(&root_account_keypair);
+                    let root_account_pk = root_account_pubkey.clone();
+
+                    async move {
+                        api.handle_auth_callout(
+                            msg,
+                            signing_account_kp,
+                            signing_account_pk,
+                            root_account_kp,
+                            root_account_pk,
+                        )
+                        .await
+                    }
+                }
+            })),
+            None,
+        )
+        .await?;
+
+    auth_service
+        .add_consumer::<AuthApiResult>(
+            "authorize_host_and_sys",
+            types::AUTHORIZE_SUBJECT, // consumer stream subj
             EndpointType::Async(auth_api.call(
                 |api: AuthServiceApi, msg: Arc<Message>| async move {
                     api.handle_handshake_request(msg).await
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 39918ce..738e4e4 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -56,9 +56,9 @@ impl AuthServiceApi {
     pub async fn handle_auth_callout(
         &self,
         msg: Arc<Message>,
-        auth_signing_account_keypair: KeyPair,
+        auth_signing_account_keypair: Arc<KeyPair>,
         auth_signing_account_pubkey: String,
-        auth_root_account_keypair: KeyPair,
+        auth_root_account_keypair: Arc<KeyPair>,
         auth_root_account_pubkey: String,
     ) -> Result<AuthApiResult, ServiceError> {
         // 1. Verify expected data was received
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index fa89c23..2cfaa3a 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -3,7 +3,8 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 
-pub const AUTH_SERVICE_SUBJECT: &str = "validate";
+pub const AUTH_CALLOUT_SUBJECT: &str = "$SYS.REQ.USER.AUTH";
+pub const AUTHORIZE_SUBJECT: &str = "validate";
 
 // The workload_sk_role is assigned when the host agent is created during the auth flow.
 // NB: This role name *must* match the `ROLE_NAME_WORKLOAD` in the `orchestrator_setup.sh` script file.
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index 0f8d1ad..e3001ea 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -7,6 +7,7 @@ use serde::Deserialize;
 use serde_json::Value;
 use sha2::{Digest, Sha256};
 use std::io::Write;
+use std::sync::Arc;
 use std::time::SystemTime;
 use util_libs::nats_js_client::ServiceError;
 
@@ -46,7 +47,7 @@ pub fn hash_claim(claims_str: &str) -> Vec<u8> {
 }
 
 // Convert claims to JWT/Token
-pub fn encode_jwt(claims_str: &str, signing_kp: &KeyPair) -> Result<String> {
+pub fn encode_jwt(claims_str: &str, signing_kp: &Arc<KeyPair>) -> Result<String> {
     const JWT_HEADER: &str = r#"{"typ":"JWT","alg":"ed25519-nkey"}"#;
     let b64_header: String = BASE64URL_NOPAD.encode(JWT_HEADER.as_bytes());
     println!("encoded b64 header: {:?}", b64_header);
@@ -138,7 +139,7 @@ where
 }
 
 pub fn generate_auth_response_claim(
-    auth_signing_account_keypair: KeyPair,
+    auth_signing_account_keypair: Arc<KeyPair>,
     auth_signing_account_pubkey: String,
     auth_root_account_pubkey: String,
     permissions: types::Permissions,
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index 2919167..cca39d8 100755
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -61,11 +61,12 @@ for cmd in nsc nats; do
 done
 
 # Variables
+NATS_SERVER_DOMAIN=$1
+OPERATOR_SERVICE_URL="nats://{$NATS_SERVER_DOMAIN}:$NATS_PORT"
+ACCOUNT_JWT_SERVER="nats://{$NATS_SERVER_DOMAIN}:$NATS_PORT"
 OPERATOR="HOLO"
 SYS_ACCOUNT="SYS"
 NATS_PORT="4222"
-ACCOUNT_JWT_SERVER="nats://192.168.1.96:$NATS_PORT" # "nats://143.244.144.52:$NATS_PORT"
-OPERATOR_SERVICE_URL="nats://192.168.1.96:$NATS_PORT" # "nats://143.244.144.52:$NATS_PORT"
 ADMIN_ACCOUNT="ADMIN"
 ADMIN_USER="admin"
 AUTH_ACCOUNT="AUTH"

From fce6d28aa8851ca809bdea5294a06af9ffd7ef3e Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 4 Feb 2025 22:04:05 -0600
Subject: [PATCH 63/91] use env vars for callut auth keys

---
 .gitignore                                    |  2 +-
 .../clients/host_agent/src/hostd/workloads.rs |  1 -
 rust/clients/host_agent/src/keys.rs           |  3 +-
 rust/clients/orchestrator/src/auth.rs         | 69 ++++++++++++++++++-
 rust/util_libs/src/nats_js_client.rs          |  2 +-
 scripts/hosting_agent_setup.sh                | 12 ----
 scripts/orchestrator_setup.sh                 |  6 +-
 7 files changed, 72 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2196367..19765ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,4 +22,4 @@ rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf
 .local
-rust/*/*/*/test_admin.creds
+rust/*/*/tmp/
diff --git a/rust/clients/host_agent/src/hostd/workloads.rs b/rust/clients/host_agent/src/hostd/workloads.rs
index 3bca8a8..a9f1760 100644
--- a/rust/clients/host_agent/src/hostd/workloads.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -27,7 +27,6 @@ use workload::{
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
 const HOST_AGENT_INBOX_PREFIX: &str = "_workload_inbox";
 
-// TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
 pub async fn run(
     host_pubkey: &str,
     host_creds_path: &Option<PathBuf>,
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 4242b4d..0657349 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -54,14 +54,12 @@ pub struct Keys {
 
 impl Keys {
     pub fn new() -> Result<Self> {
-        // let host_key_path = format!("{}/user_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
         let host_key_path =
             std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
         let host_kp = KeyPair::new_user();
         write_keypair_to_file(get_file_path_buf(&host_key_path), host_kp.clone())?;
         let host_pk = host_kp.public_key();
 
-        // let sys_key_path = format!("{}/user_sys_host_{}.nk", &get_nats_creds_by_nsc("HOLO", "HPOS", "host"), host_pubkey);
         let sys_key_path =
             std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
         let local_sys_kp = KeyPair::new_user();
@@ -267,6 +265,7 @@ fn write_keypair_to_file(key_file_path: PathBuf, keypair: KeyPair) -> Result<()>
 }
 
 fn write_to_file(file_path: PathBuf, data: &[u8]) -> Result<()> {
+    // TODO: ensure dirs already exist and create them if not...
     let mut file = File::create(&file_path)?;
     file.write_all(data)?;
     Ok(())
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 5228c1e..e0d55c6 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -18,7 +18,7 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
 use async_nats::Message;
 use authentication::{
     self,
@@ -27,6 +27,9 @@ use authentication::{
 };
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use nkeys::KeyPair;
+use std::fs::File;
+use std::io::Read;
+use std::path::PathBuf;
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
@@ -48,11 +51,40 @@ pub async fn run() -> Result<(), async_nats::Error> {
     );
 
     // Root Keypair associated with AUTH account
-    let root_account_keypair = Arc::new(KeyPair::from_seed("<TEST_SK_SEED>")?);
+    let root_account_key_path = std::env::var("ROOT_AUTH_NKEY_PATH")
+        .context("Cannot read ROOT_AUTH_NKEY_PATH from env var")?;
+    let root_account_keypair = Arc::new(
+        try_read_keypair_from_file(get_file_path_buf(&root_account_key_path.clone()))?.ok_or_else(
+            || {
+                anyhow!(
+                    "Root AUTH Account keypair not found at path {:?}",
+                    root_account_key_path
+                )
+            },
+        )?,
+    );
+    // TODO: REMOVE
+    // let root_account_keypair = Arc::new(KeyPair::from_seed(
+    //     "<>",
+    // )?);
     let root_account_pubkey = root_account_keypair.public_key().clone();
 
     // AUTH Account Signing Keypair associated with the `auth` user
-    let signing_account_keypair = Arc::new(KeyPair::from_seed("<TEST_SK_SEED>")?);
+    let signing_account_key_path = std::env::var("SIGNING_AUTH_NKEY_PATH")
+        .context("Cannot read SIGNING_AUTH_NKEY_PATH from env var")?;
+    let signing_account_keypair = Arc::new(
+        try_read_keypair_from_file(get_file_path_buf(&signing_account_key_path.clone()))?
+            .ok_or_else(|| {
+                anyhow!(
+                    "Signing AUTH Account keypair not found at path {:?}",
+                    signing_account_key_path
+                )
+            })?,
+    );
+    // TODO: REMOVE
+    // let signing_account_keypair = Arc::new(KeyPair::from_seed(
+    //     "<>",
+    // )?);
     let signing_account_pubkey = signing_account_keypair.public_key().clone();
     println!(
         ">>>>>>>>> signing_account pubkey: {:?}",
@@ -159,3 +191,34 @@ pub fn create_callback_subject_to_host(tag_name: String) -> ResponseSubjectsGene
         vec!["AUTH.ERROR.INBOX".to_string()]
     })
 }
+
+fn try_read_keypair_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
+    match try_read_from_file(key_file_path)? {
+        Some(kps) => Ok(Some(KeyPair::from_seed(&kps)?)),
+        None => Ok(None),
+    }
+}
+
+fn try_read_from_file(file_path: PathBuf) -> Result<Option<String>> {
+    match file_path.try_exists() {
+        Ok(link_is_ok) => {
+            if !link_is_ok {
+                return Err(anyhow!(
+                    "Failed to read path {:?}. Found broken sym link.",
+                    file_path
+                ));
+            }
+
+            let mut file_content = File::open(&file_path)
+                .context(format!("Failed to open config file {:#?}", file_path))?;
+
+            let mut s = String::new();
+            file_content.read_to_string(&mut s)?;
+            Ok(Some(s.trim().to_string()))
+        }
+        Err(_) => {
+            log::debug!("No user file found at {:?}.", file_path);
+            Ok(None)
+        }
+    }
+}
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 4c6e694..eef4ad8 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -313,7 +313,7 @@ where
 // TODO: there's overlap with the NATS_LISTEN_PORT. refactor this to e.g. read NATS_LISTEN_HOST and NATS_LISTEN_PORT
 pub fn get_nats_url() -> String {
     std::env::var("NATS_URL").unwrap_or_else(|_| {
-        let default = format!("127.0.0.1:{}", LEAF_SERVER_DEFAULT_LISTEN_PORT);
+        let default = format!("127.0.0.1:{}", LEAF_SERVER_DEFAULT_LISTEN_PORT); // Shouldn't this be the 'NATS_LISTEN_PORT'?
         log::debug!("using default for NATS_URL: {default}");
         default
     })
diff --git a/scripts/hosting_agent_setup.sh b/scripts/hosting_agent_setup.sh
index 4c02f69..94e36f1 100644
--- a/scripts/hosting_agent_setup.sh
+++ b/scripts/hosting_agent_setup.sh
@@ -58,18 +58,6 @@ else
             # Add SYS Account
             nsc import account --file $SYS_ACCOUNT_JWT_PATH
             echo "SYS account added to local nsc successfully."
-
-            # TODO: For if/when add local sys user (that's) associated the Orchestrator SYS Account
-            # if [ ! -d "$SYS_USER_PATH" ]; then
-            #     echo "WARNING: SYS user JWT not found. Unable to add the SYS user as a locally trusted user."
-            # else
-            #     echo "Found the $SYS_USER_PATH usr to local chain reference."
-            #     # Add SYS user
-            #     nsc import user --file $SYS_USER_PATH
-            #     # Create SYS user cred file and add to shared creds dir
-            #     nsc generate creds --name $SYS_USER_NAME --account $SYS_ACCOUNT > $SHARED_CREDS_DIR/$SYS_USER_NAME.creds
-            #     echo "SYS user added to local nsc successfully."
-            # fi
         fi
 
         if [ ! -d "$AUTH_GUARD_USER_PATH" ]; then
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index cca39d8..eed4271 100755
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -61,9 +61,9 @@ for cmd in nsc nats; do
 done
 
 # Variables
-NATS_SERVER_DOMAIN=$1
-OPERATOR_SERVICE_URL="nats://{$NATS_SERVER_DOMAIN}:$NATS_PORT"
-ACCOUNT_JWT_SERVER="nats://{$NATS_SERVER_DOMAIN}:$NATS_PORT"
+NATS_SERVER_HOST=$1
+OPERATOR_SERVICE_URL="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
+ACCOUNT_JWT_SERVER="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
 OPERATOR="HOLO"
 SYS_ACCOUNT="SYS"
 NATS_PORT="4222"

From 45b30d0046f067e26d48dea4ca73322c8fd0997a Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Wed, 5 Feb 2025 00:53:14 -0600
Subject: [PATCH 64/91] update link paths

---
 rust/clients/host_agent/src/keys.rs        | 30 ++++----
 rust/clients/orchestrator/src/auth.rs      | 83 +++++++++++++++++++---
 rust/clients/orchestrator/src/main.rs      | 23 +++---
 rust/clients/orchestrator/src/workloads.rs | 14 ++--
 rust/util_libs/src/nats_js_client.rs       |  2 +-
 5 files changed, 110 insertions(+), 42 deletions(-)

diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 0657349..b92dc41 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -5,7 +5,8 @@ use std::fs::File;
 use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Command;
-use util_libs::nats_js_client::{get_file_path_buf, get_nats_creds_by_nsc};
+use std::str::FromStr;
+use util_libs::nats_js_client::{get_path_buf_from_current_dir, get_nats_creds_by_nsc};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -57,17 +58,16 @@ impl Keys {
         let host_key_path =
             std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
         let host_kp = KeyPair::new_user();
-        write_keypair_to_file(get_file_path_buf(&host_key_path), host_kp.clone())?;
+        write_keypair_to_file(PathBuf::from_str(&host_key_path)?, host_kp.clone())?;
         let host_pk = host_kp.public_key();
 
         let sys_key_path =
             std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
         let local_sys_kp = KeyPair::new_user();
-        write_keypair_to_file(get_file_path_buf(&sys_key_path), local_sys_kp.clone())?;
+        write_keypair_to_file(PathBuf::from_str(&sys_key_path)?, local_sys_kp.clone())?;
         let local_sys_pk = local_sys_kp.public_key();
 
-        let auth_guard_creds =
-            get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
+        let auth_guard_creds = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
 
         Ok(Self {
             host_keypair: host_kp,
@@ -86,23 +86,23 @@ impl Keys {
         let host_key_path =
             std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
         let host_keypair =
-            try_read_keypair_from_file(get_file_path_buf(&host_key_path.clone()))?
+            try_read_keypair_from_file(PathBuf::from_str(&host_key_path.clone())?)?
                 .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
         let host_pk = host_keypair.public_key();
         let sys_key_path =
             std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
         let host_creds_path = maybe_host_creds_path
             .to_owned()
-            .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "host")));
+            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "host")), Ok)?;
         let sys_creds_path = maybe_sys_creds_path
             .to_owned()
-            .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")));
+            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")), Ok)?;
 
         // Set auth_guard_creds as default:
         let auth_guard_creds =
-            get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"));
+        PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
 
-        let keys = match try_read_keypair_from_file(get_file_path_buf(&sys_key_path))? {
+        let keys = match try_read_keypair_from_file(PathBuf::from_str(&sys_key_path)?)? {
             Some(kp) => {
                 let local_sys_pk = kp.public_key();
                 Self {
@@ -130,7 +130,7 @@ impl Keys {
 
     pub fn _add_local_sys(mut self, sys_key_path: Option<PathBuf>) -> Result<Self> {
         let sys_key_path = sys_key_path
-            .unwrap_or_else(|| get_file_path_buf(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")));
+            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")), Ok)?;
 
         let mut is_new_key = false;
 
@@ -206,9 +206,9 @@ impl Keys {
         host_sys_user_jwt: String,
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
-        let host_path = get_file_path_buf(&format!("{}.{}", "output_dir", "host.jwt"));
+        let host_path = PathBuf::from_str(&format!("{}.{}", "output_dir", "host.jwt"))?;
         write_to_file(host_path, host_user_jwt.as_bytes())?;
-        let sys_path = get_file_path_buf(&format!("{}.{}", "output_dir", "host_sys.jwt"));
+        let sys_path = PathBuf::from_str(&format!("{}.{}", "output_dir", "host_sys.jwt"))?;
         write_to_file(sys_path, host_sys_user_jwt.as_bytes())?;
 
         // Save user creds and sys creds local to hosting agent
@@ -224,7 +224,7 @@ impl Keys {
         let mut sys_creds_file_name = None;
         if let Some(sys_pubkey) = self.local_sys_pubkey.as_ref() {
             let file_name = "host_sys.creds";
-            sys_creds_file_name = Some(get_file_path_buf(file_name));
+            sys_creds_file_name = Some(get_path_buf_from_current_dir(file_name));
             Command::new("nsc")
                 .arg(format!(
                     "generate creds --name user_host_{} --account {} > {}",
@@ -235,7 +235,7 @@ impl Keys {
         }
 
         self.to_owned()
-            .add_creds_paths(get_file_path_buf(host_creds_file_name), sys_creds_file_name)
+            .add_creds_paths(get_path_buf_from_current_dir(host_creds_file_name), sys_creds_file_name)
     }
 
     pub fn get_host_creds_path(&self) -> Option<PathBuf> {
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index e0d55c6..e6249b9 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -31,30 +31,40 @@ use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
 use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::str::FromStr;
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
     nats_js_client::{
-        get_event_listeners, get_file_path_buf, get_nats_url, with_event_listeners, Credentials,
+        get_event_listeners, get_nats_url, with_event_listeners, get_nats_creds_by_nsc, Credentials,
         EndpointType, JsClient, NewJsClientParams,
     },
 };
 
-pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Agent";
+pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Manager";
 pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_auth_inbox_orchestrator";
 
 pub async fn run() -> Result<(), async_nats::Error> {
-    let admin_account_creds_path = get_file_path_buf("test_admin.creds");
+    println!("inside auth... 0");
+
+    // let admin_account_creds_path = PathBuf::from_str("/home/za/Documents/holo-v2/holo-host/rust/clients/orchestrator/src/tmp/test_admin.creds")?;
+    let admin_account_creds_path = PathBuf::from_str(&get_nats_creds_by_nsc(
+        "HOLO",
+        "AUTH",
+        "auth",
+    ))?;
     println!(
         " >>>> admin_account_creds_path: {:#?} ",
         admin_account_creds_path
     );
 
+    println!("inside auth... 1");
+
     // Root Keypair associated with AUTH account
     let root_account_key_path = std::env::var("ROOT_AUTH_NKEY_PATH")
         .context("Cannot read ROOT_AUTH_NKEY_PATH from env var")?;
     let root_account_keypair = Arc::new(
-        try_read_keypair_from_file(get_file_path_buf(&root_account_key_path.clone()))?.ok_or_else(
+        try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
             || {
                 anyhow!(
                     "Root AUTH Account keypair not found at path {:?}",
@@ -63,17 +73,23 @@ pub async fn run() -> Result<(), async_nats::Error> {
             },
         )?,
     );
+
+    println!("inside auth... 2");
+
     // TODO: REMOVE
     // let root_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "<>",
+    //     "SAAINFLMRAAE6GYKTQ4SCXNPPCZQTSSSWB3BU3PDKK7CHZDEDYXHL5IP4E",
     // )?);
     let root_account_pubkey = root_account_keypair.public_key().clone();
+    println!("inside auth... 3");
 
     // AUTH Account Signing Keypair associated with the `auth` user
     let signing_account_key_path = std::env::var("SIGNING_AUTH_NKEY_PATH")
         .context("Cannot read SIGNING_AUTH_NKEY_PATH from env var")?;
+    println!("inside auth... 4");
+
     let signing_account_keypair = Arc::new(
-        try_read_keypair_from_file(get_file_path_buf(&signing_account_key_path.clone()))?
+        try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
             .ok_or_else(|| {
                 anyhow!(
                     "Signing AUTH Account keypair not found at path {:?}",
@@ -81,15 +97,19 @@ pub async fn run() -> Result<(), async_nats::Error> {
                 )
             })?,
     );
+    println!("inside auth... 5");
+
     // TODO: REMOVE
     // let signing_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "<>",
+    //     "SAAL7ULQELTAX5VHVYDDZZ3636AY2AO2O25CRVOPPRFS2KOMVEZV6HTLXI",
     // )?);
     let signing_account_pubkey = signing_account_keypair.public_key().clone();
     println!(
         ">>>>>>>>> signing_account pubkey: {:?}",
         signing_account_pubkey
     );
+    println!("inside auth... 6");
+
 
     // ==================== Setup NATS ====================
     // Setup JS Stream Service
@@ -99,6 +119,9 @@ pub async fn run() -> Result<(), async_nats::Error> {
         version: AUTH_SRV_VERSION.to_string(),
         service_subject: AUTH_SRV_SUBJ.to_string(),
     };
+    println!("inside auth... 7");
+    let nats_url = get_nats_url();
+    let nats_connect_timeout_secs: u64 = 180; 
 
     let orchestrator_auth_client = JsClient::new(NewJsClientParams {
         nats_url: get_nats_url(),
@@ -112,15 +135,51 @@ pub async fn run() -> Result<(), async_nats::Error> {
     })
     .await?;
 
+    // let orchestrator_auth_client = tokio::select! {
+    //     client = async {loop {
+    //             let orchestrator_auth_client = JsClient::new(NewJsClientParams {
+    //                 nats_url: nats_url.clone(),
+    //                 name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
+    //                 inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
+    //                 service_params: vec![auth_stream_service_params.clone()],
+    //                 credentials: Some(Credentials::Path(admin_account_creds_path.clone())),
+    //                 listeners: vec![with_event_listeners(get_event_listeners())],
+    //                 ping_interval: Some(Duration::from_secs(10)),
+    //                 request_timeout: Some(Duration::from_secs(5)),
+    //             })
+    //             .await
+    //             .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
+
+    //             match orchestrator_auth_client {
+    //                 Ok(client) => break client,
+    //                 Err(e) => {
+    //                     let duration = tokio::time::Duration::from_millis(100);
+    //                     log::warn!("{}, retrying in {duration:?}", e);
+    //                     tokio::time::sleep(duration).await;
+    //                 }
+    //             }
+    //         }} => client,
+    //     _ = {
+    //         log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
+    //         tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
+    //      } => {
+    //         return Err(format!("timed out waiting for NATS on {nats_url}").into());
+    //     }
+    // };
+
+    println!("inside auth... 8");
+
     // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
     let client = MongoDBClient::with_options(client_options)?;
-
+    println!("inside auth... 9");
+    
     // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
     let auth_api = AuthServiceApi::new(&client).await?;
+    println!("inside auth... 10");
 
     // Register Auth Stream for Orchestrator to consume and process
     let auth_service = orchestrator_auth_client
@@ -129,6 +188,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .ok_or(anyhow!(
             "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
         ))?;
+    println!("inside auth... 11");
 
     auth_service
         .add_consumer::<AuthApiResult>(
@@ -156,6 +216,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
             None,
         )
         .await?;
+    println!("inside auth... 12");
 
     auth_service
         .add_consumer::<AuthApiResult>(
@@ -169,15 +230,19 @@ pub async fn run() -> Result<(), async_nats::Error> {
             Some(create_callback_subject_to_host("host_pubkey".to_string())),
         )
         .await?;
+    println!("inside auth... 13");
 
-    log::trace!("Orchestrator Auth Service is running. Waiting for requests...");
+    println!("Orchestrator Auth Service is running. Waiting for requests...");
 
     // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
+    println!("inside auth... 14... closing");
+
     // Close client and drain internal buffer before exiting to make sure all messages are sent
     orchestrator_auth_client.close().await?;
+    println!("inside auth... 15... closed");
 
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index 6d25650..286e0dc 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -1,5 +1,5 @@
 mod auth;
-mod workloads;
+// mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
 use tokio::task::spawn;
@@ -8,15 +8,16 @@ use tokio::task::spawn;
 async fn main() -> Result<(), async_nats::Error> {
     dotenv().ok();
     env_logger::init();
-    spawn(async move {
-        if let Err(e) = auth::run().await {
-            log::error!("{}", e)
-        }
-    });
-    spawn(async move {
-        if let Err(e) = workloads::run().await {
-            log::error!("{}", e)
-        }
-    });
+    println!("starting auth...");
+
+    auth::run().await?;
+
+    println!("finished auth...");
+
+    // spawn(async move {
+    //     if let Err(e) = workloads::run().await {
+    //         log::error!("{}", e)
+    //     }
+    // });
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 97462f6..291eadc 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -17,12 +17,14 @@ This client is responsible for:
 use anyhow::{anyhow, Result};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use std::path::PathBuf;
 use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::str::FromStr;
 use util_libs::{
     db::mongodb::get_mongodb_url,
     js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
     nats_js_client::{
-        self, get_event_listeners, get_file_path_buf, get_nats_creds_by_nsc, get_nats_url,
+        self, get_event_listeners, get_nats_creds_by_nsc, get_nats_url,
         Credentials, EndpointType, JsClient, NewJsClientParams,
     },
 };
@@ -33,7 +35,7 @@ use workload::{
     WORKLOAD_SRV_VERSION,
 };
 
-const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
+const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Manager";
 const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_workload_inbox_orchestrator";
 
 pub fn create_callback_subject_to_host(
@@ -61,11 +63,11 @@ pub fn create_callback_subject_to_host(
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== Setup NATS ====================
     let nats_url = get_nats_url();
-    let creds_path = Credentials::Path(get_file_path_buf(&get_nats_creds_by_nsc(
+    let creds_path = Credentials::Path(PathBuf::from_str(&get_nats_creds_by_nsc(
         "HOLO",
-        "WORKLOAD",
-        "orchestrator",
-    )));
+        "ADMIN",
+        "admin",
+    ))?);
     let event_listeners = get_event_listeners();
 
     // Setup JS Stream Service
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index eef4ad8..0820a77 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -333,7 +333,7 @@ pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> Strin
     )
 }
 
-pub fn get_file_path_buf(file_name: &str) -> PathBuf {
+pub fn get_path_buf_from_current_dir(file_name: &str) -> PathBuf {
     let current_dir_path = std::env::current_dir().expect("Failed to locate current directory.");
     current_dir_path.join(file_name)
 }

From 74dd850222a6814073e636e0ff3131beb9bccb6a Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Wed, 5 Feb 2025 23:53:20 -0600
Subject: [PATCH 65/91] test

---
 .gitignore                              |   3 +-
 rust/clients/orchestrator/src/auth.rs   |   4 +-
 rust/services/authentication/src/lib.rs | 222 +++++++++++++-----------
 rust/util_libs/src/js_stream_service.rs |  14 +-
 scripts/hosting_agent_setup.sh          |   6 +-
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/.gitignore b/.gitignore
index 19765ef..1812acf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,4 +22,5 @@ rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf
 .local
-rust/*/*/tmp/
+rust/*/*/*/tmp/
+rust/*/*/*/*/tmp/
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index e6249b9..96e8385 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -78,7 +78,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // TODO: REMOVE
     // let root_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "SAAINFLMRAAE6GYKTQ4SCXNPPCZQTSSSWB3BU3PDKK7CHZDEDYXHL5IP4E",
+    //     "<>",
     // )?);
     let root_account_pubkey = root_account_keypair.public_key().clone();
     println!("inside auth... 3");
@@ -101,7 +101,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // TODO: REMOVE
     // let signing_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "SAAL7ULQELTAX5VHVYDDZZ3636AY2AO2O25CRVOPPRFS2KOMVEZV6HTLXI",
+    //     "<>",
     // )?);
     let signing_account_pubkey = signing_account_keypair.public_key().clone();
     println!(
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 738e4e4..86d9b42 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -106,110 +106,111 @@ impl AuthServiceApi {
 
         // 3. If provided, authenticate the Hoster pubkey and email and assign full permissions if successful
         let is_hoster_valid = if user_data.email.is_some() && user_data.hoster_hc_pubkey.is_some() {
-            let hoster_hc_pubkey = user_data.hoster_hc_pubkey.unwrap(); // unwrap is safe here as checked above
-            let hoster_email = user_data.email.unwrap(); // unwrap is safe here as checked above
-
-            let is_valid: bool = match self
-                .user_collection
-                .get_one_from(doc! { "roles.role.Hoster": hoster_hc_pubkey.clone() })
-                .await?
-            {
-                Some(u) => {
-                    let mut is_valid = true;
-                    // If hoster exists with pubkey, verify email
-                    if u.email != hoster_email {
-                        log::error!(
-                            "Error: Failed to validate hoster email. Email='{}'.",
-                            hoster_email
-                        );
-                        is_valid = false;
-                    }
-
-                    // ...then find the host collection that contains the provided host pubkey
-                    match self
-                        .host_collection
-                        .get_one_from(doc! { "pubkey": host_pubkey })
-                        .await?
-                    {
-                        Some(host) => {
-                            // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
-                            if host.assigned_hoster != hoster_hc_pubkey {
-                                let host_query: bson::Document = doc! { "_id":  host._id.clone() };
-                                let updated_host_doc = to_document(&Host {
-                                    assigned_hoster: hoster_hc_pubkey,
-                                    ..host
-                                })
-                                .map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-                                self.host_collection
-                                    .update_one_within(
-                                        host_query,
-                                        UpdateModifications::Document(updated_host_doc),
-                                    )
-                                    .await?;
-                            }
-                        }
-                        None => {
-                            log::error!(
-                                "Error: Failed to locate Host record. Subject='{}'.",
-                                msg.subject
-                            );
-                            is_valid = false;
-                        }
-                    }
-
-                    // Find the mongo_id ref for the hoster associated with this user
-                    let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
-                        let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
-                        handle_internal_err(&err_msg)
-                    })?;
-
-                    // Finally, find the hoster collection
-                    match self
-                        .hoster_collection
-                        .get_one_from(doc! { "_id":  ref_id.clone() })
-                        .await?
-                    {
-                        Some(hoster) => {
-                            // ...and pair the hoster with host (if the host is not already assiged to the hoster)
-                            let mut updated_assigned_hosts = hoster.assigned_hosts;
-                            if !updated_assigned_hosts.contains(&host_pubkey.to_string()) {
-                                let hoster_query: bson::Document =
-                                    doc! { "_id":  hoster._id.clone() };
-                                updated_assigned_hosts.push(host_pubkey.to_string());
-                                let updated_hoster_doc = to_document(&Hoster {
-                                    assigned_hosts: updated_assigned_hosts,
-                                    ..hoster
-                                })
-                                .map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-                                self.host_collection
-                                    .update_one_within(
-                                        hoster_query,
-                                        UpdateModifications::Document(updated_hoster_doc),
-                                    )
-                                    .await?;
-                            }
-                        }
-                        None => {
-                            log::error!(
-                                "Error: Failed to locate Hoster record. Subject='{}'.",
-                                msg.subject
-                            );
-                            is_valid = false;
-                        }
-                    }
-                    is_valid
-                }
-                None => {
-                    log::error!(
-                        "Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.",
-                        msg.subject
-                    );
-                    false
-                }
-            };
-            is_valid
+            true 
+            // let hoster_hc_pubkey = user_data.hoster_hc_pubkey.unwrap(); // unwrap is safe here as checked above
+            // let hoster_email = user_data.email.unwrap(); // unwrap is safe here as checked above
+
+            // let is_valid: bool = match self
+            //     .user_collection
+            //     .get_one_from(doc! { "roles.role.Hoster": hoster_hc_pubkey.clone() })
+            //     .await?
+            // {
+            //     Some(u) => {
+            //         let mut is_valid = true;
+            //         // If hoster exists with pubkey, verify email
+            //         if u.email != hoster_email {
+            //             log::error!(
+            //                 "Error: Failed to validate hoster email. Email='{}'.",
+            //                 hoster_email
+            //             );
+            //             is_valid = false;
+            //         }
+
+            //         // ...then find the host collection that contains the provided host pubkey
+            //         match self
+            //             .host_collection
+            //             .get_one_from(doc! { "pubkey": host_pubkey })
+            //             .await?
+            //         {
+            //             Some(host) => {
+            //                 // ...and pair the host with hoster pubkey (if the hoster is not already assiged to host)
+            //                 if host.assigned_hoster != hoster_hc_pubkey {
+            //                     let host_query: bson::Document = doc! { "_id":  host._id.clone() };
+            //                     let updated_host_doc = to_document(&Host {
+            //                         assigned_hoster: hoster_hc_pubkey,
+            //                         ..host
+            //                     })
+            //                     .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+            //                     self.host_collection
+            //                         .update_one_within(
+            //                             host_query,
+            //                             UpdateModifications::Document(updated_host_doc),
+            //                         )
+            //                         .await?;
+            //                 }
+            //             }
+            //             None => {
+            //                 log::error!(
+            //                     "Error: Failed to locate Host record. Subject='{}'.",
+            //                     msg.subject
+            //                 );
+            //                 is_valid = false;
+            //             }
+            //         }
+
+            //         // Find the mongo_id ref for the hoster associated with this user
+            //         let RoleInfo { ref_id, role: _ } = u.roles.into_iter().find(|r| matches!(r.role, Role::Hoster(_))).ok_or_else(|| {
+            //             let err_msg = format!("Error: Failed to locate Hoster record id in User collection. Subject='{}'.", msg.subject);
+            //             handle_internal_err(&err_msg)
+            //         })?;
+
+            //         // Finally, find the hoster collection
+            //         match self
+            //             .hoster_collection
+            //             .get_one_from(doc! { "_id":  ref_id.clone() })
+            //             .await?
+            //         {
+            //             Some(hoster) => {
+            //                 // ...and pair the hoster with host (if the host is not already assiged to the hoster)
+            //                 let mut updated_assigned_hosts = hoster.assigned_hosts;
+            //                 if !updated_assigned_hosts.contains(&host_pubkey.to_string()) {
+            //                     let hoster_query: bson::Document =
+            //                         doc! { "_id":  hoster._id.clone() };
+            //                     updated_assigned_hosts.push(host_pubkey.to_string());
+            //                     let updated_hoster_doc = to_document(&Hoster {
+            //                         assigned_hosts: updated_assigned_hosts,
+            //                         ..hoster
+            //                     })
+            //                     .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+            //                     self.host_collection
+            //                         .update_one_within(
+            //                             hoster_query,
+            //                             UpdateModifications::Document(updated_hoster_doc),
+            //                         )
+            //                         .await?;
+            //                 }
+            //             }
+            //             None => {
+            //                 log::error!(
+            //                     "Error: Failed to locate Hoster record. Subject='{}'.",
+            //                     msg.subject
+            //                 );
+            //                 is_valid = false;
+            //             }
+            //         }
+            //         is_valid
+            //     }
+            //     None => {
+            //         log::error!(
+            //             "Error: Failed to find User Collection with Hoster pubkey. Subject='{}'.",
+            //             msg.subject
+            //         );
+            //         false
+            //     }
+            // };
+            // is_valid
         } else {
             false
         };
@@ -224,7 +225,7 @@ impl AuthServiceApi {
             println!(">>> user_unique_inbox : {user_unique_inbox}");
 
             let authenticated_user_diagnostics_subject =
-                &format!("DIAGNOSTICS.authenticated.{}.>", host_pubkey);
+                &format!("DIAGNOSTICS.{}.>", host_pubkey);
             println!(">>> authenticated_user_diagnostics_subject : {authenticated_user_diagnostics_subject}");
 
             types::Permissions {
@@ -250,7 +251,7 @@ impl AuthServiceApi {
             // Otherwise, exclusively grant publication permissions for the unauthenticated diagnostics subj
             // ...to allow the host device to still send diganostic reports
             let unauthenticated_user_diagnostics_subject =
-                format!("DIAGNOSTICS.unauthenticated.{}.>", host_pubkey);
+                format!("DIAGNOSTICS.{}.unauthenticated.>", host_pubkey);
             types::Permissions {
                 publish: types::PermissionLimits {
                     allow: Some(vec![unauthenticated_user_diagnostics_subject]),
@@ -444,3 +445,12 @@ impl AuthServiceApi {
         })
     }
 }
+
+
+// example:
+// [1] Subject: AUTH.UDS2A7I4BCECURHE64C52ORK6IDSOSE4ILZ7RJM4IO4EAYF33B67EWEF.> Received: 2025-02-05T21:19:52-06:00
+//   X-Signature: [80, 71, 109, 80, 76, 99, 48, 122, 56, 113, 112, 48, 101, 95, 57, 107, 45, 105, 78, 75, 72, 67, 66, 97, 120, 117, 102, 110, 100, 72, 110, 53, 101, 74, 82, 77, 52, 121, 65, 66, 85, 53, 48, 109, 101, 51, 107, 54, 50, 65, 89, 81, 85, 51, 52, 50, 80, 81, 74, 49, 119, 90, 118, 104, 112, 100, 68, 109, 99, 105, 49, 69, 101, 85, 116, 67, 48, 118, 68, 89, 74, 86, 56, 86, 65, 103]
+// {"host_pubkey":"UDS2A7I4BCECURHE64C52ORK6IDSOSE4ILZ7RJM4IO4EAYF33B67EWEF","maybe_sys_pubkey":"UACJZQOQK2Y2JFQVNV4CJORAEZGV3GYTCK7UOSCLNEZJRKMOW4ATUZZG","nonce":"zq7bDlgqpGcAAAAA3ItWHoUsldKNZg7/"}
+
+// - subject: _INBOX.Uwce1Uabie65ojhlucmyhB.vy24bmby
+// - subject: _INBOX.5RgE68PiQieODvqbf4Yn1s.6f14XeRJ
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 0db52d9..f044dbe 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -225,13 +225,18 @@ impl JsStreamService {
     where
         T: EndpointTraits,
     {
-        let full_subject = format!("{}.{}", self.service_subject, endpoint_subject);
+        // Avoid adding the Service Subject prefix if the Endpoint Subject name starts with global keywords $SYS or $JS
+        let consumer_subject = if endpoint_subject.starts_with("$SYS") || endpoint_subject.starts_with("$JS") {
+            endpoint_subject.to_string()
+        } else {
+            format!("{}.{}", self.service_subject, endpoint_subject)
+        };
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
             durable_name: Some(consumer_name.to_string()),
             ack_policy: AckPolicy::Explicit,
-            filter_subject: full_subject,
+            filter_subject: consumer_subject,
             ..Default::default()
         };
 
@@ -288,6 +293,8 @@ impl JsStreamService {
             let messages = consumer
                 .stream()
                 .heartbeat(std::time::Duration::from_secs(10))
+                .max_messages_per_batch(100)
+                .expires(std::time::Duration::from_secs(30))
                 .messages()
                 .await?;
 
@@ -333,7 +340,10 @@ impl JsStreamService {
     ) where
         T: EndpointTraits,
     {
+        println!("WAITING TO PROCESS MESSAGE...");
         while let Some(Ok(js_msg)) = messages.next().await {
+            println!("MESSAGES : js_msg={:?}", js_msg);
+            
             log::trace!(
                 "{}Consumer received message: subj='{}.{}', endpoint={}, service={}",
                 log_info.prefix,
diff --git a/scripts/hosting_agent_setup.sh b/scripts/hosting_agent_setup.sh
index 94e36f1..46cc431 100644
--- a/scripts/hosting_agent_setup.sh
+++ b/scripts/hosting_agent_setup.sh
@@ -28,6 +28,7 @@ for cmd in nsc nats; do
 done
 
 # Variables
+NSC_PATH=$1
 OPERATOR_NAME="HOLO"
 SYS_ACCOUNT_NAME="SYS"
 AUTH_ACCOUNT_NAME="AUTH"
@@ -64,7 +65,10 @@ else
             echo "WARNING: AUTH_GUARD user credentials not found. Unable to add the complete Hosting Agent set-up."
         else
             echo "Found the $AUTH_GUARD_USER_NAME credentials file."
-            echo "Set-up complete. Credential files are in the $SHARED_CREDS_DIR/ directory."
+            $AUTH_GUARD_CRED_PATH="{$NSC_PATH}/keys/creds/{$OPERATOR_NAME}/{$AUTH_ACCOUNT_NAME}/"
+            echo "Moving $AUTH_GUARD_USER_NAME creds to the $AUTH_GUARD_CRED_PATH directory."
+            mv $AUTH_GUARD_USER_PATH $AUTH_GUARD_CRED_PATH
+            echo "Set-up complete. Credential files are in the $AUTH_GUARD_CRED_PATH/ directory."
         fi
     fi
 fi

From 6e908d8ebb183b044b1ce2698a0d7608e6237cdd Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 13:46:15 -0600
Subject: [PATCH 66/91] auth service without jetstream

---
 Cargo.lock                                    |   7 +
 rust/clients/host_agent/src/auth/init.rs      |   2 +-
 .../clients/host_agent/src/hostd/workloads.rs |   2 +-
 rust/clients/orchestrator/src/auth.rs         | 427 ++++++++++--------
 rust/clients/orchestrator/src/workloads.rs    |   2 +-
 rust/services/authentication/Cargo.toml       |   5 +-
 rust/services/authentication/src/lib.rs       | 173 ++++---
 rust/services/authentication/src/types.rs     |   8 +
 rust/services/authentication/src/utils.rs     |   6 +-
 rust/util_libs/src/js_stream_service.rs       |   3 +-
 rust/util_libs/src/nats_js_client.rs          |  28 +-
 scripts/orchestrator_setup.sh                 |  16 +-
 12 files changed, 393 insertions(+), 286 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 052aa50..cc67435 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -236,6 +236,7 @@ dependencies = [
  "anyhow",
  "async-nats",
  "async-trait",
+ "base32",
  "bson",
  "bytes",
  "chrono",
@@ -293,6 +294,12 @@ version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
 
+[[package]]
+name = "base32"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076"
+
 [[package]]
 name = "base36"
 version = "0.0.1"
diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index b566d52..b2bcdbc 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -41,7 +41,7 @@ pub async fn run(
     // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host & Hoster =============================================
     let nonce = TextNonce::new().to_string();
     let unique_inbox = &format!(
-        "{}.{}",
+        "{}_{}",
         HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey
     );
     println!(">>> unique_inbox : {}", unique_inbox);
diff --git a/rust/clients/host_agent/src/hostd/workloads.rs b/rust/clients/host_agent/src/hostd/workloads.rs
index a9f1760..56c7843 100644
--- a/rust/clients/host_agent/src/hostd/workloads.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -25,7 +25,7 @@ use workload::{
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
-const HOST_AGENT_INBOX_PREFIX: &str = "_workload_inbox";
+const HOST_AGENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX";
 
 pub async fn run(
     host_pubkey: &str,
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 96e8385..4a26482 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -18,11 +18,13 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
+use async_nats::service::ServiceExt;
 use anyhow::{anyhow, Context, Result};
-use async_nats::Message;
+use futures::StreamExt;
+// use async_nats::Message;
 use authentication::{
     self,
-    types::{self, AuthApiResult},
+    types::{self, AuthErrorPayload},
     AuthServiceApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION,
 };
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
@@ -30,207 +32,266 @@ use nkeys::KeyPair;
 use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
-use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::{sync::Arc, time::Duration};
 use std::str::FromStr;
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{
-        get_event_listeners, get_nats_url, with_event_listeners, get_nats_creds_by_nsc, Credentials,
-        EndpointType, JsClient, NewJsClientParams,
-    },
+    nats_js_client::{get_nats_url, get_nats_creds_by_nsc},
+    js_stream_service::CreateResponse
 };
 
 pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Manager";
-pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_auth_inbox_orchestrator";
+pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX_ORCHESTRATOR";
 
 pub async fn run() -> Result<(), async_nats::Error> {
     println!("inside auth... 0");
 
-    // let admin_account_creds_path = PathBuf::from_str("/home/za/Documents/holo-v2/holo-host/rust/clients/orchestrator/src/tmp/test_admin.creds")?;
-    let admin_account_creds_path = PathBuf::from_str(&get_nats_creds_by_nsc(
-        "HOLO",
-        "AUTH",
-        "auth",
-    ))?;
-    println!(
-        " >>>> admin_account_creds_path: {:#?} ",
-        admin_account_creds_path
-    );
-
-    println!("inside auth... 1");
-
-    // Root Keypair associated with AUTH account
-    let root_account_key_path = std::env::var("ROOT_AUTH_NKEY_PATH")
-        .context("Cannot read ROOT_AUTH_NKEY_PATH from env var")?;
-    let root_account_keypair = Arc::new(
-        try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
-            || {
-                anyhow!(
-                    "Root AUTH Account keypair not found at path {:?}",
-                    root_account_key_path
-                )
-            },
-        )?,
-    );
-
-    println!("inside auth... 2");
-
-    // TODO: REMOVE
-    // let root_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "<>",
-    // )?);
+    // // let admin_account_creds_path = PathBuf::from_str("/home/za/Documents/holo-v2/holo-host/rust/clients/orchestrator/src/tmp/test_admin.creds")?;
+    // let admin_account_creds_path = PathBuf::from_str(&get_nats_creds_by_nsc(
+    //     "HOLO",
+    //     "AUTH",
+    //     "auth",
+    // ))?;
+    // println!(
+    //     " >>>> admin_account_creds_path: {:#?} ",
+    //     admin_account_creds_path
+    // );
+
+    // // Root Keypair associated with AUTH account
+    // let root_account_key_path = std::env::var("ROOT_AUTH_NKEY_PATH")
+    //     .context("Cannot read ROOT_AUTH_NKEY_PATH from env var")?;
+
+    // let root_account_keypair = Arc::new(
+    //     try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
+    //         || {
+    //             anyhow!(
+    //                 "Root AUTH Account keypair not found at path {:?}",
+    //                 root_account_key_path
+    //             )
+    //         },
+    //     )?,
+    // );
+    // let root_account_pubkey = root_account_keypair.public_key().clone();
+    // println!(">>>>>>>>> root account pubkey: {:?}", root_account_pubkey);
+
+    // // AUTH Account Signing Keypair associated with the `auth` user
+    // let signing_account_key_path = std::env::var("SIGNING_AUTH_NKEY_PATH")
+    // .context("Cannot read SIGNING_AUTH_NKEY_PATH from env var")?;
+    // let signing_account_keypair = Arc::new(
+    //     try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
+    //         .ok_or_else(|| {
+    //             anyhow!(
+    //                 "Signing AUTH Account keypair not found at path {:?}",
+    //                 signing_account_key_path
+    //             )
+    //         })?,
+    // );
+    // let signing_account_pubkey = signing_account_keypair.public_key().clone();
+    // println!(">>>>>>>>> signing_account pubkey: {:?}", signing_account_pubkey);
+    let admin_account_creds = "-----BEGIN NATS USER JWT-----
+eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJUTVJDVklaUDRJUlRLWktGSEVTV1BYSzZMM0hDQUdTTFhUREZBTk9IS1ZQSFNNQ0w3UEhBIiwiaWF0IjoxNzM4NTI0MDg5LCJpc3MiOiJBRFQ2TUhSQUgzU0JXWFU1RlRHN0I2WklCU0VXV0UzMkJVNDJKTzRKRE8yV0VSVDZYTVpLRTYzUyIsIm5hbWUiOiJhdXRoIiwic3ViIjoiVUNWQTVZT1haNTZMVzNSRVhEV1VBR1EzVktERE5RVlA0TVNSSFJKRVRTQjJZRlBVQ1FJQTdQT0siLCJuYXRzIjp7InB1YiI6eyJhbGxvdyI6WyJcdTAwM2UiXX0sInN1YiI6eyJhbGxvdyI6WyJcdTAwM2UiXX0sInN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImlzc3Vlcl9hY2NvdW50IjoiQUEzUTdIWVBHTVdSWFcyRkxLNUNCRFZQWVdEUjI3TU5QUE9TT1M3SUdVT0hBWTNMNEdOUkJMSjQiLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.K-cSBzw_wai2BEA7Lzxg2kDThj72lZpTKJbvUff6gSxPjn2KuVJQy5k2mSC9fohBmjcJLSoQ-7JrXA7GN1VMDA
+------END NATS USER JWT------
+
+************************* IMPORTANT *************************
+NKEY Seed printed below can be used to sign and prove identity.
+NKEYs are sensitive and should be treated as secrets.
+
+-----BEGIN USER NKEY SEED-----
+SUALKRFOSR77N6VXQOQRF65RET2GU4D4IP2OEB4546EEX6WLHK2BW6FMZU
+------END USER NKEY SEED------
+
+*************************************************************";    
+    let root_account_keypair = KeyPair::from_seed("SAAINFLMRAAE6GYKTQ4SCXNPPCZQTSSSWB3BU3PDKK7CHZDEDYXHL5IP4E")?;
     let root_account_pubkey = root_account_keypair.public_key().clone();
-    println!("inside auth... 3");
+    println!(">>>>>>>>> root account pubkey: {:?}", root_account_pubkey);
 
-    // AUTH Account Signing Keypair associated with the `auth` user
-    let signing_account_key_path = std::env::var("SIGNING_AUTH_NKEY_PATH")
-        .context("Cannot read SIGNING_AUTH_NKEY_PATH from env var")?;
-    println!("inside auth... 4");
-
-    let signing_account_keypair = Arc::new(
-        try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
-            .ok_or_else(|| {
-                anyhow!(
-                    "Signing AUTH Account keypair not found at path {:?}",
-                    signing_account_key_path
-                )
-            })?,
-    );
-    println!("inside auth... 5");
-
-    // TODO: REMOVE
-    // let signing_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "<>",
-    // )?);
-    let signing_account_pubkey = signing_account_keypair.public_key().clone();
-    println!(
-        ">>>>>>>>> signing_account pubkey: {:?}",
-        signing_account_pubkey
-    );
-    println!("inside auth... 6");
+    let signing_account_keypair = KeyPair::from_seed("SAAL7ULQELTAX5VHVYDDZZ3636AY2AO2O25CRVOPPRFS2KOMVEZV6HTLXI")?;
+    let signing_account_pubkey = signing_account_keypair.public_key();
+    println!(">>>>>>>>> auth_signing_account pubkey: {:?}", signing_account_pubkey);
 
 
     // ==================== Setup NATS ====================
-    // Setup JS Stream Service
-    let auth_stream_service_params = JsServiceParamsPartial {
-        name: AUTH_SRV_NAME.to_string(),
-        description: AUTH_SRV_DESC.to_string(),
-        version: AUTH_SRV_VERSION.to_string(),
-        service_subject: AUTH_SRV_SUBJ.to_string(),
-    };
-    println!("inside auth... 7");
     let nats_url = get_nats_url();
-    let nats_connect_timeout_secs: u64 = 180; 
-
-    let orchestrator_auth_client = JsClient::new(NewJsClientParams {
-        nats_url: get_nats_url(),
-        name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
-        inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
-        service_params: vec![auth_stream_service_params],
-        credentials: Some(Credentials::Path(admin_account_creds_path)),
-        listeners: vec![with_event_listeners(get_event_listeners())],
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(5)),
-    })
-    .await?;
-
-    // let orchestrator_auth_client = tokio::select! {
-    //     client = async {loop {
-    //             let orchestrator_auth_client = JsClient::new(NewJsClientParams {
-    //                 nats_url: nats_url.clone(),
-    //                 name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
-    //                 inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
-    //                 service_params: vec![auth_stream_service_params.clone()],
-    //                 credentials: Some(Credentials::Path(admin_account_creds_path.clone())),
-    //                 listeners: vec![with_event_listeners(get_event_listeners())],
-    //                 ping_interval: Some(Duration::from_secs(10)),
-    //                 request_timeout: Some(Duration::from_secs(5)),
-    //             })
-    //             .await
-    //             .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
-
-    //             match orchestrator_auth_client {
-    //                 Ok(client) => break client,
-    //                 Err(e) => {
-    //                     let duration = tokio::time::Duration::from_millis(100);
-    //                     log::warn!("{}, retrying in {duration:?}", e);
-    //                     tokio::time::sleep(duration).await;
-    //                 }
-    //             }
-    //         }} => client,
-    //     _ = {
-    //         log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
-    //         tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
-    //      } => {
-    //         return Err(format!("timed out waiting for NATS on {nats_url}").into());
-    //     }
-    // };
-
-    println!("inside auth... 8");
+    let nats_connect_timeout_secs: u64 = 180;
+
+    let orchestrator_auth_client = tokio::select! {
+        client = async {loop {
+            let orchestrator_auth_client = async_nats::ConnectOptions::new()
+                .name(ORCHESTRATOR_AUTH_CLIENT_NAME.to_string())
+                .custom_inbox_prefix(ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string())
+                .ping_interval(Duration::from_secs(10))
+                .request_timeout(Some(Duration::from_secs(30)))
+                // .credentials_file(&admin_account_creds_path).await.map_err(|e| anyhow::anyhow!("Error loading credentials file: {e}"))?
+                .credentials(admin_account_creds)?
+                .connect(nats_url.clone())
+                .await
+                .map_err(|e| anyhow::anyhow!("Connecting Orchestrator Auth Client to NATS via {nats_url}: {e}"));
+
+                match orchestrator_auth_client {
+                    Ok(client) => break Ok::<async_nats::Client, async_nats::Error>(client),
+                    Err(e) => {
+                        let duration = tokio::time::Duration::from_millis(100);
+                        log::warn!("{}, retrying in {duration:?}", e);
+                        tokio::time::sleep(duration).await;
+                    }
+                }
+            }} => client?,
+        _ = {
+            log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
+            tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
+         } => {
+            return Err(format!("timed out waiting for NATS on {nats_url}").into());
+        }
+    };
 
     // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-    println!("inside auth... 9");
+    let db_client = MongoDBClient::with_options(client_options)?;
     
     // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
-    let auth_api = AuthServiceApi::new(&client).await?;
-    println!("inside auth... 10");
+    let auth_api = AuthServiceApi::new(&db_client).await?;
+    let auth_api_clone  = auth_api.clone();
 
-    // Register Auth Stream for Orchestrator to consume and process
+    // Register Auth Service for Orchestrator and spawn listener for processing
     let auth_service = orchestrator_auth_client
-        .get_js_service(AUTH_SRV_NAME.to_string())
-        .await
-        .ok_or(anyhow!(
-            "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
-        ))?;
-    println!("inside auth... 11");
-
-    auth_service
-        .add_consumer::<AuthApiResult>(
-            "auth_callout",
-            types::AUTH_CALLOUT_SUBJECT, // consumer stream subj
-            EndpointType::Async(auth_api.call({
-                move |api: AuthServiceApi, msg: Arc<Message>| {
-                    let signing_account_kp = Arc::clone(&signing_account_keypair);
-                    let signing_account_pk = signing_account_pubkey.clone();
-                    let root_account_kp = Arc::clone(&root_account_keypair);
-                    let root_account_pk = root_account_pubkey.clone();
-
-                    async move {
-                        api.handle_auth_callout(
-                            msg,
-                            signing_account_kp,
-                            signing_account_pk,
-                            root_account_kp,
-                            root_account_pk,
-                        )
-                        .await
+        .service_builder()
+        .description(AUTH_SRV_DESC)
+        .start(AUTH_SRV_NAME, AUTH_SRV_VERSION)
+        .await?;
+    
+    // Auth Callout Service
+    let sys_user_group = auth_service.group("$SYS").group("REQ").group("USER");
+    let mut auth_callout = sys_user_group.endpoint("AUTH").await?;
+    let auth_service_info = auth_service.info().await;
+    let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
+
+    tokio::spawn(async move {
+        while let Some(request) = auth_callout.next().await {
+                let signing_account_kp = Arc::clone(&Arc::new(signing_account_keypair.clone()));
+                let signing_account_pk = signing_account_pubkey.clone();
+                let root_account_kp = Arc::clone(&Arc::new(root_account_keypair.clone()));
+                let root_account_pk = root_account_pubkey.clone();
+
+                let maybe_reply = request.message.reply.clone();
+                match auth_api_clone.handle_auth_callout(
+                    Arc::new(request.message),
+                    signing_account_kp,
+                    signing_account_pk,
+                    root_account_kp,
+                    root_account_pk,
+                )
+                .await {
+                    Ok(r) => {
+                        let res_bytes = r.get_response();
+                        if let Some(reply_subject) = maybe_reply {
+                            let _ = orchestrator_auth_client_clone.publish(reply_subject, res_bytes.into()).await.map_err(|e| {
+                            log::error!(
+                                "{}Failed to send success response. Res={:?} Err={:?}",
+                                "NATS-SERVICE-LOG::AUTH::",
+                                r,
+                                e
+                            );
+                        });
+                        }
+                    },
+                    Err(e) => {
+                        let mut err_payload = AuthErrorPayload {
+                            service_info: auth_service_info.clone(),
+                            group: "$SYS.REQ.USER".to_string(),
+                            endpoint: "AUTH".to_string(),
+                            error: format!("{}",e),
+                        };
+    
+                        log::error!(
+                            "{}Failed to handle the endpoint handler. Err={:?}",
+                            "NATS-SERVICE-LOG::AUTH::",
+                            err_payload
+                        );
+    
+                        let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                            err_payload.error = e.to_string();
+                            log::error!(
+                                "{}Failed to deserialize error response. Err={:?}",
+                                "NATS-SERVICE-LOG::AUTH::",
+                                err_payload
+                            );
+                            vec![]
+                        });
+    
+                        let _ = orchestrator_auth_client_clone.publish(format!("{}.ERROR", ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX), err_response.into()).await.map_err(|e| {
+                            log::error!(
+                                "{}Failed to send error response. Err={:?}",
+                                "NATS-SERVICE-LOG::AUTH::",
+                                err_payload
+                            );
+                        });
                     }
                 }
-            })),
-            None,
-        )
-        .await?;
-    println!("inside auth... 12");
-
-    auth_service
-        .add_consumer::<AuthApiResult>(
-            "authorize_host_and_sys",
-            types::AUTHORIZE_SUBJECT, // consumer stream subj
-            EndpointType::Async(auth_api.call(
-                |api: AuthServiceApi, msg: Arc<Message>| async move {
-                    api.handle_handshake_request(msg).await
-                },
-            )),
-            Some(create_callback_subject_to_host("host_pubkey".to_string())),
-        )
-        .await?;
-    println!("inside auth... 13");
+            }
+        });
+
+        // Auth Validation Service
+        let v1_auth_group = auth_service.group(AUTH_SRV_SUBJ); // .group("V1")
+        let mut auth_validation = v1_auth_group.endpoint(types::AUTHORIZE_SUBJECT).await?;
+        let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
+
+        tokio::spawn(async move {
+            while let Some(request) = auth_validation.next().await {
+                let maybe_reply = request.message.reply.clone();
+                match auth_api.handle_handshake_request(
+                    Arc::new(request.message)
+                )
+                .await {
+                    Ok(r) => {
+                        let res_bytes = r.get_response();
+                        if let Some(reply_subject) = maybe_reply {
+                            let _ = orchestrator_auth_client_clone.publish(reply_subject, res_bytes.into()).await.map_err(|e| {
+                            log::error!(
+                                "{}Failed to send success response. Res={:?} Err={:?}",
+                                "NATS-SERVICE-LOG::AUTH::",
+                                r,
+                                e
+                            );
+                        });
+                        }
+                    },
+                    Err(e) => {
+                        let auth_service_info = auth_service.info().await;
+                        let mut err_payload = AuthErrorPayload {
+                            service_info: auth_service_info,
+                            group: "AUTH".to_string(),
+                            endpoint: types::AUTHORIZE_SUBJECT.to_string(),
+                            error: format!("{}",e),
+                        };
+                        log::error!(
+                            "{}Failed to handle the endpoint handler. Err={:?}",
+                            "NATS-SERVICE-LOG::AUTH::",
+                            err_payload
+                        );
+                        let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                            err_payload.error = e.to_string();
+                            log::error!(
+                                "{}Failed to deserialize error response. Err={:?}",
+                                "NATS-SERVICE-LOG::AUTH::",
+                                err_payload
+                            );
+                            vec![]
+                        });
+                        let _ = orchestrator_auth_client_clone.publish("AUTH.ERROR", err_response.into()).await.map_err(|e| {
+                            err_payload.error = e.to_string();
+                            log::error!(
+                                "{}Failed to send error response. Err={:?}",
+                                "NATS-SERVICE-LOG::AUTH::",
+                                err_payload
+                            );
+                        });
+                    }
+                }
+            }
+        });
 
     println!("Orchestrator Auth Service is running. Waiting for requests...");
 
@@ -238,25 +299,15 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    println!("inside auth... 14... closing");
+    println!("closing orchestrator auth service...");
 
     // Close client and drain internal buffer before exiting to make sure all messages are sent
-    orchestrator_auth_client.close().await?;
-    println!("inside auth... 15... closed");
+    orchestrator_auth_client.drain().await?;
+    println!("closed orchestrator auth service");
 
     Ok(())
 }
 
-pub fn create_callback_subject_to_host(tag_name: String) -> ResponseSubjectsGenerator {
-    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
-        if let Some(tag) = tag_map.get(&tag_name) {
-            return vec![format!("AUTH.{}", tag)];
-        }
-        log::error!("Auth Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject 'AUTH.validate'. Fwding response to `AUTH.ERROR.INBOX`.", tag_name);
-        vec!["AUTH.ERROR.INBOX".to_string()]
-    })
-}
-
 fn try_read_keypair_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
     match try_read_from_file(key_file_path)? {
         Some(kps) => Ok(Some(KeyPair::from_seed(&kps)?)),
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 291eadc..6a08ee9 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -36,7 +36,7 @@ use workload::{
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Manager";
-const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_workload_inbox_orchestrator";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX_ORCHESTRATOR";
 
 pub fn create_callback_subject_to_host(
     is_prefix: bool,
diff --git a/rust/services/authentication/Cargo.toml b/rust/services/authentication/Cargo.toml
index 2bdc024..b0a9d0e 100644
--- a/rust/services/authentication/Cargo.toml
+++ b/rust/services/authentication/Cargo.toml
@@ -14,10 +14,11 @@ env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
 thiserror = { workspace = true }
-async-trait = "0.1.83"
-mongodb = "3.1"
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
 url = { version = "2", features = ["serde"] }
+async-trait = "0.1.83"
+mongodb = "3.1"
+base32 = "0.5.1"
 nkeys = "=0.4.4"
 sha2 = "=0.10.8"
 nats-jwt = "0.3.0"
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 86d9b42..71a34fb 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -13,9 +13,9 @@ use anyhow::{Context, Result};
 use async_nats::jetstream::ErrorCode;
 use async_nats::HeaderValue;
 use async_nats::{AuthError, Message};
-use bson::{self, doc, to_document};
+use data_encoding::BASE64URL_NOPAD;
 use core::option::Option::None;
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use mongodb::Client as MongoDBClient;
 use nkeys::KeyPair;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -27,7 +27,7 @@ use util_libs::db::{
     mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
     schemas::{self, Host, Hoster, Role, RoleInfo, User},
 };
-use util_libs::nats_js_client::{AsyncEndpointHandler, JsServiceResponse, ServiceError};
+use util_libs::nats_js_client::{get_nsc_root_path, AsyncEndpointHandler, JsServiceResponse, ServiceError};
 use utils::handle_internal_err;
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
@@ -88,6 +88,7 @@ impl AuthServiceApi {
         .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
         println!("user_data TO VALIDATE  : {:#?}", user_data);
 
+        // TODO:
         // 2. Validate Host signature, returning validation error if not successful
         let host_pubkey = user_data.host_pubkey.as_ref();
         let host_signature = user_data.get_host_signature();
@@ -95,18 +96,19 @@ impl AuthServiceApi {
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
         let raw_payload = serde_json::to_vec(&user_data.clone().without_signature())
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &host_signature) {
-            log::error!(
-                "Error: Failed to validate Signature. Subject='{}'. Err={}",
-                msg.subject,
-                e
-            );
-            return Err(ServiceError::Authentication(AuthError::new(e)));
-        };
+        // if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &host_signature) {
+        //     log::error!(
+        //         "Error: Failed to validate Signature. Subject='{}'. Err={}",
+        //         msg.subject,
+        //         e
+        //     );
+        //     return Err(ServiceError::Authentication(AuthError::new(e)));
+        // };
 
         // 3. If provided, authenticate the Hoster pubkey and email and assign full permissions if successful
         let is_hoster_valid = if user_data.email.is_some() && user_data.hoster_hc_pubkey.is_some() {
             true 
+            // TODO:
             // let hoster_hc_pubkey = user_data.hoster_hc_pubkey.unwrap(); // unwrap is safe here as checked above
             // let hoster_email = user_data.email.unwrap(); // unwrap is safe here as checked above
 
@@ -221,7 +223,7 @@ impl AuthServiceApi {
             let user_unique_auth_subject = &format!("AUTH.{}.>", host_pubkey);
             println!(">>> user_unique_auth_subject : {user_unique_auth_subject}");
 
-            let user_unique_inbox = &format!("_INBOX.{}.>", host_pubkey);
+            let user_unique_inbox = &format!("_AUTH_INBOX_{}.>", host_pubkey);
             println!(">>> user_unique_inbox : {user_unique_inbox}");
 
             let authenticated_user_diagnostics_subject =
@@ -280,12 +282,6 @@ impl AuthServiceApi {
 
         println!("\n\n\n\nencoded_jwt: {:#?}", token);
 
-        // DONE BY JS HANDLER
-        // let res = token.into_bytes();
-        // if let Some(reply) = msg.reply {
-        //     client.publish(reply, res.into()).await?;
-        // }
-
         Ok(types::AuthApiResult {
             result: types::AuthResult::Callout(token),
             maybe_response_tags: None,
@@ -297,13 +293,18 @@ impl AuthServiceApi {
         msg: Arc<Message>,
     ) -> Result<AuthApiResult, ServiceError> {
         log::warn!("INCOMING Message for 'AUTH.validate' : {:?}", msg);
+        println!("msg={:#?}", msg);
 
         // 1. Verify expected data was received
         let signature: &[u8] = match &msg.headers {
-            Some(h) => HeaderValue::as_ref(h.get("X-Signature").ok_or_else(|| {
-                log::error!("Error: Missing x-signature header. Subject='AUTH.authorize'");
-                ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
-            })?),
+            Some(h) => {
+                println!("header={:?}", h);
+                let r = HeaderValue::as_str(h.get("X-Signature").ok_or_else(|| {
+                    log::error!("Error: Missing x-signature header. Subject='AUTH.authorize'");
+                    ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
+                })?);
+                r.as_bytes()
+            },
             None => {
                 log::error!("Error: Missing message headers. Subject='AUTH.authorize'");
                 return Err(ServiceError::Request(format!(
@@ -316,13 +317,18 @@ impl AuthServiceApi {
         let types::AuthJWTPayload {
             host_pubkey,
             maybe_sys_pubkey,
-            nonce: _,
+            ..
         } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
 
+        let decoded_signature = BASE64URL_NOPAD.decode(signature).map_err(|e| {
+            println!("err={}", e);
+            ServiceError::Internal(e.to_string())
+        })?;
+
         // 2. Validate signature
         let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), signature) {
+        if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), &decoded_signature) {
             log::error!(
                 "Error: Failed to validate Signature. Subject='{}'. Err={}",
                 msg.subject,
@@ -331,68 +337,88 @@ impl AuthServiceApi {
             return Err(ServiceError::Request(format!(
                 "{:?}",
                 ErrorCode::BAD_REQUEST
-            )));
+            )));    
         };
 
-        // 4. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
-        if let Some(sys_pubkey) = maybe_sys_pubkey {
-            Command::new("nsc")
-                .arg(format!(
-                    "add user -a SYS -n user_sys_host_{} -k {}",
-                    host_pubkey, sys_pubkey
-                ))
+        println!("'AUTH.validate >>> user_verifying_keypair' : {:?}", user_verifying_keypair);
+        println!("'AUTH.validate >>> maybe_sys_pubkey' : {:?}", maybe_sys_pubkey);
+
+        // 3. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
+        match Command::new("nsc")
+            .args(&[
+                "add", "user",
+                "-a", "WORKLOAD",
+                "-n", &format!("host_user_{}", host_pubkey),
+                "-k", &host_pubkey,
+                "-K", WORKLOAD_SK_ROLE,
+                "--tag", &format!("pubkey:{}", host_pubkey),
+            ])
+            .output()
+            .context("Failed to add host user with provided keys")
+            .map_err(|e| ServiceError::Internal(e.to_string())) {
+                Ok(r) => {
+                    println!("'AUTH.validate >>> add user -a WORKLOAD -n host_user_<pubkey> ...' : {:?}", r);
+                },
+                Err(e) => {
+                    if !e.to_string().contains("already exists") {
+                        return Err(e);
+                    }
+                    println!("'AUTH.validate >>> ERROR: add user -a WORKLOAD -n host_user_<pubkey> ...' : {:?}", e);
+                }
+            };
+        println!("\nadded host user");
+
+        if let Some(sys_pubkey) = maybe_sys_pubkey.clone() {
+            println!("inside handle_handshake_request... 5 sys -- inside");
+
+            match Command::new("nsc")
+                .args(&[
+                    "add", "user",
+                    "-a", "SYS",
+                    "-n", &format!("sys_user_{}", host_pubkey),
+                    "-k", &sys_pubkey,
+                ])
                 .output()
                 .context("Failed to add host sys user with provided keys")
-                .map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-            Command::new("nsc")
-                .arg(format!(
-                    "add user -a WORKLOAD -n user_host_{} -k {} -K {} --tag pubkey:{}",
-                    host_pubkey, host_pubkey, WORKLOAD_SK_ROLE, host_pubkey
-                ))
-                .output()
-                .context("Failed to add host user with provided keys")
-                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+                .map_err(|e| ServiceError::Internal(e.to_string())) {
+                    Ok(r) => {
+                        println!("'AUTH.validate >>> add user -a SYS -n sys_user_<pubkey> ...' : {:?}", r);
+                    },
+                    Err(e) => {
+                        if !e.to_string().contains("already exists") {
+                            return Err(e);
+                        }
+                        println!("'AUTH.validate >>> ERROR: add user -a SYS -n sys_user_<pubkey> ...' : {:?}", e);
+                    }
+                };
         }
+        println!("\nadded sys user for provided host");
 
-        // ..and push auth updates to hub server
-        Command::new("nsc")
-            .arg("push -A")
-            .output()
-            .context("Failed to update resolver config file")
+        // 4. Create User JWT files (automatically signed with respective account key)
+        let host_jwt = std::fs::read_to_string(format!("{}/stores/HOLO/accounts/WORKLOAD/users/host_user_{}.jwt", get_nsc_root_path(), host_pubkey))
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        println!("'AUTH.validate >>> host_jwt' : {:?}", host_jwt);
 
-        // 3. Create User JWT files (automatically signed with respective account key)
-        let sys_jwt_output = Command::new("nsc")
-            .arg(format!(
-                "describe user -n user_sys_host_{} -a SYS --raw",
-                host_pubkey
-            ))
-            .output()
-            .context("Failed to generate host sys user jwt file")
-            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        let sys_jwt = if let Some(_) = maybe_sys_pubkey {
+            std::fs::read_to_string(format!("{}/stores/HOLO/accounts/SYS/users/sys_user_{}.jwt", get_nsc_root_path(), host_pubkey))
+            .map_err(|e| ServiceError::Internal(e.to_string()))?
+        } else { String::new() };
+        println!("'AUTH.validate >>> sys_jwt' : {:?}", sys_jwt);
 
-        let sys_jwt = String::from_utf8(sys_jwt_output.stdout)
-            .context("Command returned invalid UTF-8 output")
-            .map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let host_jwt_output = Command::new("nsc")
-            .arg(format!(
-                "describe user -n user_host_{} -a WORKLOAD --raw",
-                host_pubkey
-            ))
+        // 5. PUSH the auth updates to resolver programmtically by sending jwts to `SYS.REQ.ACCOUNT.PUSH` subject
+        Command::new("nsc")
+            .arg("push -A")
             .output()
-            .context("Failed to generate host user jwt file")
-            .map_err(|e| ServiceError::Internal(e.to_string()))?;
-
-        let host_jwt = String::from_utf8(host_jwt_output.stdout)
-            .context("Command returned invalid UTF-8 output")
+            .context("Failed to update resolver config file")
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        println!("\npushed new jwts to resolver server");
 
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
+        println!("inside handle_handshake_request... 13");
 
-        Ok(AuthApiResult {
+        // 6. Form the result and return
+        let r = AuthApiResult {
             result: types::AuthResult::Authorization(types::AuthJWTResult {
                 host_pubkey: host_pubkey.clone(),
                 status: types::AuthState::Authorized,
@@ -400,7 +426,10 @@ impl AuthServiceApi {
                 sys_jwt,
             }),
             maybe_response_tags: Some(tag_map),
-        })
+        };
+        println!("inside handle_handshake_request... RESULT={:?}", r);
+
+        Ok(r)
     }
 
     // Helper function to initialize mongodb collections
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 2cfaa3a..2fe5be9 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -19,6 +19,14 @@ pub enum AuthState {
     Error(String),   // internal error
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct AuthErrorPayload {
+    pub service_info: async_nats::service::Info,
+    pub group: String,
+    pub endpoint: String,
+    pub error: String,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct AuthJWTPayload {
     pub host_pubkey: String,              // nkey
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index e3001ea..f2f6759 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -1,6 +1,8 @@
 use super::types;
 use anyhow::{anyhow, Result};
 use data_encoding::{BASE32HEX_NOPAD, BASE64URL_NOPAD};
+use base32::Alphabet;
+use base32::decode as base32Decode;
 use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
 use nkeys::KeyPair;
 use serde::Deserialize;
@@ -118,8 +120,8 @@ where
     println!("Public Key (Base32): {}", public_key_b32);
 
     // Decode from Base32 to raw bytes using Rfc4648 (compatible with NATS keys)
-    let public_key_bytes = Some(BASE32HEX_NOPAD.decode(public_key_b32.as_bytes()))
-        .ok_or(anyhow!("Failed to convert public key to bytes"))??;
+    let public_key_bytes = base32Decode(Alphabet::Rfc4648 { padding: false }, &public_key_b32)
+        .expect("failed to convert public key to bytes");
     println!("Decoded Public Key Bytes: {:?}", public_key_bytes);
 
     // Use the decoded key to create a DecodingKey
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index f044dbe..b2d8e67 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -1,5 +1,3 @@
-use super::nats_js_client::EndpointType;
-
 use anyhow::{anyhow, Result};
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
@@ -12,6 +10,7 @@ use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
+use super::nats_js_client::EndpointType;
 
 pub type ResponseSubjectsGenerator =
     Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 0820a77..ee2fe9c 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -61,6 +61,15 @@ where
     }
 }
 
+#[derive(Clone, Debug)]
+pub struct RequestInfo {
+    pub stream_subject: String,
+    pub consumer_name: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+    pub headers: Option<HeaderMap>,
+}
+
 #[derive(Clone, Debug)]
 pub struct PublishInfo {
     pub subject: String,
@@ -224,11 +233,19 @@ impl JsClient {
     }
 
     pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
+        log::debug!(
+            "{}Published message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+        
         let now = Instant::now();
         let result = match payload.headers {
-            Some(h) => {
+            Some(headers) => {
                 self.js
-                    .publish_with_headers(payload.subject.clone(), h, payload.data.clone().into())
+                    .publish_with_headers(payload.subject.clone(), headers, payload.data.clone().into())
                     .await
             }
             None => {
@@ -246,13 +263,6 @@ impl JsClient {
             return Err(Box::new(err));
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index eed4271..9b937b5 100755
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -113,7 +113,7 @@ nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem
 
 ADMIN_SK="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 ADMIN_ROLE_NAME="admin_role"
-nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_workload_inbox_*.>","_auth_inbox_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_workload_inbox_orchestrator.>","_auth_inbox_orchestrator.>" --allow-pub-response
+nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_WORKLOAD_INBOX_*.>","_AUTH_INBOX_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_WORKLOAD_INBOX_ORCHESTRATOR.>","_AUTH_INBOX_ORCHESTRATOR.>" --allow-pub-response
 
 # Step 3: Create AUTH with JetStream with non-scoped signing key
 nsc add account --name $AUTH_ACCOUNT
@@ -121,24 +121,24 @@ nsc edit account --name $AUTH_ACCOUNT --sk generate --js-streams -1 --js-consume
 AUTH_ACCOUNT_PUBKEY=$(nsc describe account $AUTH_ACCOUNT --field sub | jq -r)
 AUTH_SK_ACCOUNT_PUBKEY=$(nsc describe account $AUTH_ACCOUNT --field 'nats.signing_keys[0]' | tr -d '"')
 
-# Step 4: Create "Sentinel" User in AUTH Account
-nsc add user --name $AUTH_GUARD_USER --account $AUTH_ACCOUNT --deny-pubsub ">"
-
-# Step 5: Create WORKLOAD Account with JetStream and scoped signing keys
+# Step 4: Create WORKLOAD Account with JetStream and scoped signing keys
 nsc add account --name $WORKLOAD_ACCOUNT
 nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G --conns -1 --leaf-conns -1
 WORKLOAD_SK="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 WORKLOAD_ROLE_NAME="workload_role"
-nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","_INBOX_{{tag(pubkey)}}.>","_workload_inbox_{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","_INBOX_{{tag(pubkey)}}.>","_workload_inbox_{{tag(pubkey)}}.>" --allow-pub-response
+nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","_WORKLOAD_INBOX_{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","_WORKLOAD_INBOX_{{tag(pubkey)}}.>" --allow-pub-response
 
-# Step 6: Create Operator User in ADMIN Account (for use in Orchestrator)
+# Step 5: Create Orchestrator User in ADMIN Account
 nsc add user --name $ADMIN_USER --account $ADMIN_ACCOUNT -K $ADMIN_ROLE_NAME
 
-# Step 7: Create Operator User in AUTH Account (used in auth service)
+# Step 6: Create Orchestrator User in AUTH Account (used in auth-callout service)
 nsc add user --name $ORCHESTRATOR_AUTH_USER --account $AUTH_ACCOUNT --allow-pubsub ">"
 AUTH_USER_PUBKEY=$(nsc describe user --name $ORCHESTRATOR_AUTH_USER --account $AUTH_ACCOUNT --field sub | jq -r)
 echo "assigned auth user pubkey: $AUTH_USER_PUBKEY"
 
+# Step 7: Create "Sentinel" User in AUTH Account (used by host agents in auth-callout service)
+nsc add user --name $AUTH_GUARD_USER --account $AUTH_ACCOUNT --deny-pubsub ">"
+
 # Step 8: Configure Auth Callout
 echo $AUTH_ACCOUNT_PUBKEY
 echo $AUTH_SK_ACCOUNT_PUBKEY

From c44a448313e613bdc7438f16a5c14681aa2c5fb4 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 13:51:14 -0600
Subject: [PATCH 67/91] nats updates

---
 .env.example                                  |  13 +-
 .gitignore                                    |   1 +
 rust/clients/host_agent/output_dir.host.jwt   |   1 +
 .../host_agent/output_dir.host_sys.jwt        |   1 +
 rust/clients/host_agent/src/auth/config.rs    |  12 +-
 rust/clients/host_agent/src/auth/init.rs      | 164 ++++++----
 .../clients/host_agent/src/hostd/workloads.rs |   4 +-
 rust/clients/host_agent/src/keys.rs           | 129 ++++++--
 rust/clients/host_agent/src/main.rs           |  48 +--
 rust/clients/orchestrator/src/auth.rs         | 301 +++++++++---------
 rust/clients/orchestrator/src/workloads.rs    |   2 +-
 rust/services/authentication/src/lib.rs       |   2 +-
 rust/services/authentication/src/types.rs     |  34 +-
 rust/util_libs/src/nats_js_client.rs          | 122 +++++--
 scripts/orchestrator_setup.sh                 |   4 +-
 15 files changed, 539 insertions(+), 299 deletions(-)
 create mode 100644 rust/clients/host_agent/output_dir.host.jwt
 create mode 100644 rust/clients/host_agent/output_dir.host_sys.jwt

diff --git a/.env.example b/.env.example
index 7ad33bc..ad8c75c 100644
--- a/.env.example
+++ b/.env.example
@@ -1,9 +1,16 @@
+# ALL
 NSC_PATH="<path/to/nsc_dir>"
-HOST_NKEY_PATH="<path>/host.nk"
-SYS_NKEY_PATH="<path>/sys.nk"
 NATS_URL="nats:/<domain>:<port>"
 NATS_LISTEN_PORT="<port>"
-LEAF_SERVER_DEFAULT_LISTEN_PORT="<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
+
+# ORCHESTRATOR
 MONGO_URI="mongodb://<host>:<port>"
+ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH="/home/lisa/.local/share/nats/nsc/local_creds_output/AUTH_ROOT_SK.nk"
+ORCHESTRATOR_ROOT_AUTH_NKEY_PATH="/home/lisa/.local/share/nats/nsc/local_creds_output/AUTH_SK.nk"
+
+# HOSTING AGENT
+HOSTING_AGENT_HOST_NKEY_PATH="<path>/host.nk"
+HOSTING_AGENT_SYS_NKEY_PATH="<path>/sys.nk"
 HPOS_CONFIG_PATH="path/to/file.config";
 DEVICE_SEED_DEFAULT_PASSWORD="device_pw_1234"
diff --git a/.gitignore b/.gitignore
index 19765ef..b1c3a1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,4 @@ rust/*/*/resolver.conf
 leaf_server.conf
 .local
 rust/*/*/tmp/
+rust/*/*/*/tmp/
diff --git a/rust/clients/host_agent/output_dir.host.jwt b/rust/clients/host_agent/output_dir.host.jwt
new file mode 100644
index 0000000..09638f3
--- /dev/null
+++ b/rust/clients/host_agent/output_dir.host.jwt
@@ -0,0 +1 @@
+eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJDS1pMR0pVNUtKWVVVUEpNSDNJTU9OQjVZRFI1V1RUQzdDV1JTR01DWDZUVVhQS1pOWEdRIiwiaWF0IjoxNzM4OTU0OTQ0LCJpc3MiOiJBQ1dZNkZZTFNBRU81QTRZUEZUNDc0REhVNkIyM1VSRk5QV0k0N0lSWUlGQVdDS0xCM1NXRFZJQyIsIm5hbWUiOiJob3N0X3VzZXJfVURTMkE3STRCQ0VDVVJIRTY0QzUyT1JLNklEU09TRTRJTFo3UkpNNElPNEVBWUYzM0I2N0VXRUYiLCJzdWIiOiJVRFMyQTdJNEJDRUNVUkhFNjRDNTJPUks2SURTT1NFNElMWjdSSk00SU80RUFZRjMzQjY3RVdFRiIsIm5hdHMiOnsicHViIjp7fSwic3ViIjp7fSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidGFncyI6WyJwdWJrZXk6dWRzMmE3aTRiY2VjdXJoZTY0YzUyb3JrNmlkc29zZTRpbHo3cmptNGlvNGVheWYzM2I2N2V3ZWYiXSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.GU1VJRaTH42njK_CGcuklnR-8WCNw2zTfZFn83EizwSqS4-xxtxlXJIyGWtOO73D9xMemSc5KgoLi6rGkLR0BA
\ No newline at end of file
diff --git a/rust/clients/host_agent/output_dir.host_sys.jwt b/rust/clients/host_agent/output_dir.host_sys.jwt
new file mode 100644
index 0000000..5a93965
--- /dev/null
+++ b/rust/clients/host_agent/output_dir.host_sys.jwt
@@ -0,0 +1 @@
+eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJQMkJMWkEyM0UyUUFORU0yS1JHNjUzSVpWWExRT0ZCUFg0TTI3S0Y2UVZET1BJNkJTTVNBIiwiaWF0IjoxNzM4OTU2NjAwLCJpc3MiOiJBREY1NkREMkxOVVhCSDRXQUpWSUlKRFFWSjJKQU9SWDVWS1VNU0pFRkFMQ1NHQVZDWEVTUE9GSiIsIm5hbWUiOiJzeXNfdXNlcl9VRFMyQTdJNEJDRUNVUkhFNjRDNTJPUks2SURTT1NFNElMWjdSSk00SU80RUFZRjMzQjY3RVdFRiIsInN1YiI6IlVBQ0paUU9RSzJZMkpGUVZOVjRDSk9SQUVaR1YzR1lUQ0s3VU9TQ0xORVpKUktNT1c0QVRVWlpHIiwibmF0cyI6eyJwdWIiOnt9LCJzdWIiOnt9LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpc3N1ZXJfYWNjb3VudCI6IkFCTFhYU01OSjRXTlpWVU9TUFdXVFJWVTZUNzdHSVpYU0g0S0RRSkRGNjc2VEFFR0JJQU5GSEE2IiwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.1R8jhZJVDgXeUblr8Bi8l5vw-mikO-r54qBOb7D19TY0QrWNtWvCUgqDEopBMScXP3BgCJTjNF3VJBMmLWWcDQ
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/auth/config.rs b/rust/clients/host_agent/src/auth/config.rs
index 15d8a1d..377f725 100644
--- a/rust/clients/host_agent/src/auth/config.rs
+++ b/rust/clients/host_agent/src/auth/config.rs
@@ -17,9 +17,16 @@ pub struct HosterConfig {
 
 impl HosterConfig {
     pub async fn new() -> Result<Self> {
+        println!(">>> inside Hoster Config new fn..");
+
         let (keypair, email) = get_from_config().await?;
+        println!(">>> inside Hoster Config new fn : keypair={:#?}", keypair);
+
         let hc_pubkey = public_key::to_holochain_encoded_agent_key(&keypair.verifying_key());
+        println!(">>> inside Hoster Config new fn : hc_pubkey={}", hc_pubkey);
+
         let holoport_id = public_key::to_base36_id(&keypair.verifying_key());
+        println!(">>> inside Hoster Config new fn : holoport_id={}", holoport_id);
 
         Ok(Self {
             email,
@@ -31,9 +38,11 @@ impl HosterConfig {
 }
 
 async fn get_from_config() -> Result<(SigningKey, String)> {
+    println!("inside config_path...");
+
     let config_path =
         env::var("HPOS_CONFIG_PATH").context("Cannot read HPOS_CONFIG_PATH from env var")?;
-
+    
     let password = env::var("DEVICE_SEED_DEFAULT_PASSWORD")
         .context("Cannot read bundle password from env var")?;
 
@@ -53,6 +62,7 @@ async fn get_from_config() -> Result<(SigningKey, String)> {
                     "unable to unlock the device bundle from {}",
                     &config_path
                 ))?;
+                println!(">>> inside config-path new fn : signing_key={:#?}", signing_key);
             Ok((signing_key, settings.admin.email))
         }
         _ => Err(anyhow!("Unsupported version of hpos config")),
diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index b566d52..b20a6f8 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -22,26 +22,33 @@ use crate::{
     keys::{AuthCredType, Keys},
 };
 use anyhow::Result;
-use async_nats::{HeaderMap, HeaderName, HeaderValue};
-use authentication::types::{AuthApiResult, AuthGuardPayload, AuthJWTPayload, AuthResult}; // , AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION
-use futures::StreamExt;
+use async_nats::{jetstream::context::PublishErrorKind, HeaderMap, HeaderName, HeaderValue, RequestErrorKind};
+use authentication::types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthResult, AuthState};
+use std::time::Duration;
+// use futures::StreamExt;
+use util_libs::nats_js_client::{
+    self, get_event_listeners, get_nats_url, with_event_listeners,
+    Credentials, JsClient, NewJsClientParams,
+};
 use hpos_hal::inventory::HoloInventory;
 use std::str::FromStr;
 use textnonce::TextNonce;
-use util_libs::nats_js_client;
 
-// pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
+pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
 pub const HOST_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX";
 
 pub async fn run(
     mut host_agent_keys: Keys,
 ) -> Result<(Keys, async_nats::Client), async_nats::Error> {
     log::info!("Host Auth Client: Connecting to server...");
-
+    println!("Keys={:#?}", host_agent_keys);
+    
+    println!("inside init auth... 0");
+    
     // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host & Hoster =============================================
     let nonce = TextNonce::new().to_string();
     let unique_inbox = &format!(
-        "{}.{}",
+        "{}_{}",
         HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey
     );
     println!(">>> unique_inbox : {}", unique_inbox);
@@ -50,6 +57,7 @@ pub async fn run(
         ">>> user_unique_auth_subject : {}",
         user_unique_auth_subject
     );
+    println!("inside init auth... 1");
 
     // Fetch Hoster Pubkey and email (from config)
     let mut auth_guard_payload = AuthGuardPayload::default();
@@ -67,111 +75,157 @@ pub async fn run(
         }
     };
     auth_guard_payload = auth_guard_payload.try_add_signature(|p| host_agent_keys.host_sign(p))?;
+    println!("auth_guard_payload={:#?}", auth_guard_payload);
 
     let user_auth_json = serde_json::to_string(&auth_guard_payload)?;
     let user_auth_token = json_to_base64(&user_auth_json)?;
-    let user_creds = if let AuthCredType::Guard(creds) = host_agent_keys.creds.clone() {
+    let user_creds_path = if let AuthCredType::Guard(creds) = host_agent_keys.creds.clone() {
         creds
     } else {
         return Err(async_nats::Error::from(
             "Failed to locate Auth Guard credentials",
         ));
     };
+    println!("user_creds_path={:#?}", user_creds_path);
+
+    let user_creds = "-----BEGIN NATS USER JWT-----
+eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI2MkVCSEhFR0M1RDIyU0lXR1hSU0paNEpWWkdWUk9FVUo3N1BYQ1BPNUU3UDRBTkVHV1RBIiwiaWF0IjoxNzM4NTI3NjgwLCJpc3MiOiJBRFQ2TUhSQUgzU0JXWFU1RlRHN0I2WklCU0VXV0UzMkJVNDJKTzRKRE8yV0VSVDZYTVpLRTYzUyIsIm5hbWUiOiJhdXRoLWd1YXJkIiwic3ViIjoiVUM1N1pETUtOSVhVWE9NNlRISE8zQjVVRUlWQ0JPM0hNRlUzSU5ESVZNTzVCSFZKR1k3R1hIM1UiLCJuYXRzIjp7InB1YiI6eyJkZW55IjpbIlx1MDAzZSJdfSwic3ViIjp7ImRlbnkiOlsiXHUwMDNlIl19LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpc3N1ZXJfYWNjb3VudCI6IkFBM1E3SFlQR01XUlhXMkZMSzVDQkRWUFlXRFIyN01OUFBPU09TN0lHVU9IQVkzTDRHTlJCTEo0IiwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.REQSfDwGzuG0vWDDfHyZdpN-Ens3hhRF1-I-k5akDK9oT8kueW2OWX3lFlgBreNw5JsTgE0fjKDq942QRTygDg
+------END NATS USER JWT------
+
+************************* IMPORTANT *************************
+NKEY Seed printed below can be used to sign and prove identity.
+NKEYs are sensitive and should be treated as secrets.
+
+-----BEGIN USER NKEY SEED-----
+SUABBYL4YAGRRJDOMXP72EUDM4UOFOGJWVPKT6AB7UMNWU2TV4M4PMFXDE
+------END USER NKEY SEED------
+
+*************************************************************";
 
     // Connect to Nats server as auth guard and call NATS AuthCallout
     let nats_url = nats_js_client::get_nats_url();
     let auth_guard_client =
-        async_nats::ConnectOptions::with_credentials(&user_creds.to_string_lossy())?
-            .token(user_auth_token)
+        async_nats::ConnectOptions::new()
+            .name(HOST_AUTH_CLIENT_NAME.to_string())
             .custom_inbox_prefix(unique_inbox.to_string())
+            .ping_interval(Duration::from_secs(10))
+            .request_timeout(Some(Duration::from_secs(30)))
+            .token(user_auth_token)
+            // .credentials_file(&user_creds_path).await?
+            .credentials(user_creds)?
             .connect(nats_url)
             .await?;
 
+    let server_info = auth_guard_client.server_info();
     println!(
         "User connected to server on port {}.  Connection State: {:#?}",
-        auth_guard_client.server_info().port,
+        server_info.port,
         auth_guard_client.connection_state()
     );
 
-    let server_node_id = auth_guard_client.server_info().server_id;
+    let server_node_id = server_info.server_id;
     log::trace!("Host Auth Client: Retrieved Node ID: {}", server_node_id);
 
     // ==================== Handle Host User and SYS Authoriation ============================================================
     let auth_guard_client_clone = auth_guard_client.clone();
-    tokio::spawn({
-        let mut auth_inbox_msgs = auth_guard_client_clone
-            .subscribe(unique_inbox.to_string())
-            .await?;
-        async move {
-            while let Some(msg) = auth_inbox_msgs.next().await {
-                println!("got an AUTH INBOX msg: {:?}", &msg);
-            }
-        }
-    });
+
+    // tokio::spawn({
+    //     let mut auth_inbox_msgs = auth_guard_client_clone
+    //         .subscribe(unique_inbox.to_string())
+    //         .await?;
+    //     async move {
+    //         while let Some(msg) = auth_inbox_msgs.next().await {
+    //             println!("got an AUTH INBOX msg: {:?}", &msg);
+    //         }
+    //     }
+    // });
 
     let payload = AuthJWTPayload {
         host_pubkey: host_agent_keys.host_pubkey.to_string(),
         maybe_sys_pubkey: host_agent_keys.local_sys_pubkey.clone(),
         nonce: TextNonce::new().to_string(),
     };
+    println!("inside init auth... 9");
 
     let payload_bytes = serde_json::to_vec(&payload)?;
+    println!("inside init auth... 10");
+
     let signature = host_agent_keys.host_sign(&payload_bytes)?;
+    println!("inside init auth... 11");
+    println!(" >>> signature >>> {}", signature);
+    println!(" >>> signature.as_bytes() >>> {:?}", signature.as_bytes());
+
     let mut headers = HeaderMap::new();
     headers.insert(
         HeaderName::from_static("X-Signature"),
-        HeaderValue::from_str(&format!("{:?}", signature.as_bytes()))?,
+        HeaderValue::from_str(&signature)?,
     );
-
-    println!("About to send out the {} message", user_unique_auth_subject);
-    let response = auth_guard_client
+    
+    println!("About to send out the {} message", "AUTH.validate");
+    let response_msg = match auth_guard_client
         .request_with_headers(
-            user_unique_auth_subject.to_string(),
+            "AUTH.validate".to_string(),
             headers,
-            payload_bytes.into(),
+            payload_bytes.into()
         )
-        .await?;
+        .await {
+            Ok(msg) => msg,
+            Err(e) => {
+                log::error!("{:#?}", e);
+                if let RequestErrorKind::TimedOut = e.kind() {
+                    println!("inside init auth... 13");
+
+                    // TODO: Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject to ensure has correct permissions
+                    println!("got an AUTH RES ERROR: {:?}", e);
+        
+                    let unauthenticated_user_diagnostics_subject = format!(
+                        "DIAGNOSTICS.unauthenticated.{}",
+                        host_agent_keys.host_pubkey
+                    );
+                    let diganostics = HoloInventory::from_host();
+                    let payload_bytes = serde_json::to_vec(&diganostics)?;                  
+            
+                    if let Ok(_) = auth_guard_client
+                        .publish( unauthenticated_user_diagnostics_subject.to_string(), payload_bytes.into())
+                        .await {
+                            return Ok((host_agent_keys, auth_guard_client));
+                        }
+                }
+                return Err(async_nats::Error::from(e));
+            }
+        };
 
     println!(
         "got an AUTH response: {:?}",
-        std::str::from_utf8(&response.payload).expect("failed to deserialize msg response")
+        std::str::from_utf8(&response_msg.payload).expect("failed to deserialize msg response")
     );
 
-    match serde_json::from_slice::<AuthApiResult>(&response.payload) {
-        Ok(auth_response) => match auth_response.result {
-            AuthResult::Authorization(r) => {
+    println!(
+        "got an AUTH response: {:#?}",
+        serde_json::from_slice::<AuthJWTResult>(&response_msg.payload).expect("failed to serde_json deserialize msg response")
+    );
+
+    if let Ok(auth_response) = serde_json::from_slice::<AuthJWTResult>(&response_msg.payload) {
+        match auth_response.status {
+            AuthState::Authorized => {
+                println!("inside init auth... 13");
+
                 host_agent_keys = host_agent_keys
-                    .save_host_creds(r.host_jwt, r.sys_jwt)
+                    .save_host_creds(auth_response.host_jwt, auth_response.sys_jwt)
                     .await?;
 
-                if let Some(_reply) = response.reply {
+                if let Some(_reply) = response_msg.reply {
                     // Publish the Awk resp to the Orchestrator... (JS)
                 }
             }
             _ => {
-                log::error!("got unexpected AUTH RESPONSE : {:?}", auth_response);
+                println!("inside init auth... 13");
+                log::error!("got unexpected AUTH State : {:?}", auth_response);
             }
-        },
-        Err(e) => {
-            // TODO: Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject to ensure has correct permissions
-            println!("got an AUTH RES ERROR: {:?}", e);
-
-            let unauthenticated_user_diagnostics_subject = format!(
-                "DIAGNOSTICS.unauthenticated.{}",
-                host_agent_keys.host_pubkey
-            );
-            let diganostics = HoloInventory::from_host();
-            let payload_bytes = serde_json::to_vec(&diganostics)?;
-            auth_guard_client
-                .publish(
-                    unauthenticated_user_diagnostics_subject,
-                    payload_bytes.into(),
-                )
-                .await?;
         }
     };
 
-    log::trace!("host_agent_keys: {:#?}", host_agent_keys);
-
+    println!("inside init auth... 14");
+    println!("host_agent_keys: {:#?}", host_agent_keys);
     Ok((host_agent_keys, auth_guard_client))
 }
diff --git a/rust/clients/host_agent/src/hostd/workloads.rs b/rust/clients/host_agent/src/hostd/workloads.rs
index a9f1760..024ec54 100644
--- a/rust/clients/host_agent/src/hostd/workloads.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -54,7 +54,7 @@ pub async fn run(
     // Spin up Nats Client and loaded in the Js Stream Service
     // Nats takes a moment to become responsive, so we try to connect in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let creds = host_creds_path.to_owned().map(Credentials::Path);
+    let creds = host_creds_path.to_owned().map(Credentials::Path).ok_or_else(|| async_nats::Error::from("error"))?;
 
     let host_workload_client = tokio::select! {
         client = async {loop {
@@ -63,7 +63,7 @@ pub async fn run(
                     name: HOST_AGENT_CLIENT_NAME.to_string(),
                     inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
                     service_params: vec![workload_stream_service_params.clone()],
-                    credentials: creds.clone(),
+                    credentials: Some(vec![creds.clone()]),
                     ping_interval: Some(Duration::from_secs(10)),
                     request_timeout: Some(Duration::from_secs(29)),
                     listeners: vec![nats_js_client::with_event_listeners(event_listeners.clone())],
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index b92dc41..2d132a4 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -6,7 +6,7 @@ use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Command;
 use std::str::FromStr;
-use util_libs::nats_js_client::{get_path_buf_from_current_dir, get_nats_creds_by_nsc};
+use util_libs::nats_js_client::{get_nats_creds_by_nsc, get_nsc_root_path, get_path_buf_from_current_dir};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -55,19 +55,42 @@ pub struct Keys {
 
 impl Keys {
     pub fn new() -> Result<Self> {
+        println!("inside Keys new ... 0");
+
         let host_key_path =
-            std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
+            std::env::var("HOSTING_AGENT_HOST_NKEY_PATH").context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
+        println!("inside Keys new ... 1");
+        println!("inside Keys new > host_key_path={}", host_key_path);
+    
         let host_kp = KeyPair::new_user();
+        println!("inside Keys new ... 2");
+        println!("inside Keys new > host_kp={:#?}", host_kp);
+
         write_keypair_to_file(PathBuf::from_str(&host_key_path)?, host_kp.clone())?;
+        println!("inside Keys new ... 3");
+
         let host_pk = host_kp.public_key();
+        println!("inside Keys new > host_pk={}", host_pk);
 
         let sys_key_path =
-            std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
+            std::env::var("HOSTING_AGENT_SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
+        println!("inside Keys new ... 4");
+        println!("inside Keys new > sys_key_path={}", sys_key_path);
+            
         let local_sys_kp = KeyPair::new_user();
+        println!("inside Keys new ... 5");
+        println!("inside Keys new > local_sys_kp={:#?}", local_sys_kp);
+
         write_keypair_to_file(PathBuf::from_str(&sys_key_path)?, local_sys_kp.clone())?;
+        println!("inside Keys new ... 6");
+
         let local_sys_pk = local_sys_kp.public_key();
+        println!("inside Keys new ... 7");
+        println!("inside Keys new > local_sys_pk={}", local_sys_pk);
 
         let auth_guard_creds = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
+        println!("inside Keys new > auth_guard_creds={:#?}", auth_guard_creds);
+        println!("inside Keys new ... 8");
 
         Ok(Self {
             host_keypair: host_kp,
@@ -83,24 +106,37 @@ impl Keys {
         maybe_host_creds_path: &Option<PathBuf>,
         maybe_sys_creds_path: &Option<PathBuf>,
     ) -> Result<Self> {
-        let host_key_path =
-            std::env::var("HOST_NKEY_PATH").context("Cannot read HOST_NKEY_PATH from env var")?;
+        println!("maybe_host_creds_path={:#?}, maybe_sys_creds_path={:#?}", maybe_host_creds_path, maybe_sys_creds_path);
+
+        let host_key_path: String =
+            std::env::var("HOSTING_AGENT_HOST_NKEY_PATH").context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
+        println!("host_key_path={:#?}", host_key_path); 
+
         let host_keypair =
             try_read_keypair_from_file(PathBuf::from_str(&host_key_path.clone())?)?
                 .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
+        println!("host_keypair={:#?}", host_keypair); 
         let host_pk = host_keypair.public_key();
+
         let sys_key_path =
-            std::env::var("SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
+            std::env::var("HOSTING_AGENT_SYS_NKEY_PATH").context("Cannot read HOSTING_AGENT_SYS_NKEY_PATH from env var")?;
+        println!("inside try_from_storage auth... 3");
+        println!("sys_key_path={:#?}", sys_key_path); 
+
         let host_creds_path = maybe_host_creds_path
             .to_owned()
             .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "host")), Ok)?;
+        println!("host_creds_path={:#?}", host_creds_path); 
+
         let sys_creds_path = maybe_sys_creds_path
             .to_owned()
             .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")), Ok)?;
+        println!("sys_creds_path={:#?}", sys_creds_path); 
 
         // Set auth_guard_creds as default:
         let auth_guard_creds =
         PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
+        println!("auth_guard_creds={:#?}", auth_guard_creds); 
 
         let keys = match try_read_keypair_from_file(PathBuf::from_str(&sys_key_path)?)? {
             Some(kp) => {
@@ -122,6 +158,8 @@ impl Keys {
             },
         };
 
+        println!("keys={:#?}", keys); 
+
         Ok(keys.clone().add_creds_paths(host_creds_path, Some(sys_creds_path)).unwrap_or_else(move |e| {
             log::error!("Error: Cannot locate authenticated cred files. Defaulting to auth_guard_creds. Err={}",e);
             keys
@@ -206,39 +244,78 @@ impl Keys {
         host_sys_user_jwt: String,
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
-        let host_path = PathBuf::from_str(&format!("{}.{}", "output_dir", "host.jwt"))?;
-        write_to_file(host_path, host_user_jwt.as_bytes())?;
-        let sys_path = PathBuf::from_str(&format!("{}.{}", "output_dir", "host_sys.jwt"))?;
-        write_to_file(sys_path, host_sys_user_jwt.as_bytes())?;
+        let host_path = PathBuf::from_str(&format!("/{}/{}/{}", get_nsc_root_path(), "local_creds", "host.jwt"))?;
+        println!("host_path ={:?}", host_path);
+        write_to_file(host_path.clone(), host_user_jwt.as_bytes())?;
+        println!("wrote JWT to host file");
+
+        let sys_path = PathBuf::from_str(&format!("/{}/{}/{}", get_nsc_root_path(), "local_creds", "sys.jwt"))?;
+        println!("sys_path ={:?}", sys_path);
+        write_to_file(sys_path.clone(), host_sys_user_jwt.as_bytes())?;
+        println!("wrote JWT to sys file");
+
+        // Import host user jwt to local nsc resolver
+        // TODO: Determine why the following works in cmd line, but doesn't seem to work when run in current program / run
+        Command::new("nsc")
+            .args(&[
+                "import", "user",
+                "--file", &format!("{:?}", host_path)
+            ])
+            .output()
+            .context("Failed to add import new host user on hosting agent.")?;
+        println!("imported host user");
+
+        // Import sys user jwt to local nsc resolver
+        Command::new("nsc")
+            .args(&[
+                "import", "user",
+                "--file", &format!("{:?}", sys_path)
+            ])
+            .output()
+            .context("Failed to add import new sys user on hosting agent.")?;
+        println!("imported sys user");
 
         // Save user creds and sys creds local to hosting agent
-        let host_creds_file_name = "host.creds";
+        let host_user_name = format!("host_user_{}", self.host_pubkey);
+        let host_creds_path =
+            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD",  &host_user_name))?;
         Command::new("nsc")
-            .arg(format!(
-                "generate creds --name user_host_{} --account {} > {}",
-                self.host_pubkey, "WORKLOAD", host_creds_file_name
-            ))
+            .args(&[
+                "generate", "creds",
+                "--name", &host_user_name,
+                "--account", "WORKLOAD",
+                "--output-file", &host_creds_path.to_string_lossy(),
+            ])
             .output()
             .context("Failed to add new operator signing key on hosting agent")?;
+        println!("generated host user creds");
 
         let mut sys_creds_file_name = None;
-        if let Some(sys_pubkey) = self.local_sys_pubkey.as_ref() {
-            let file_name = "host_sys.creds";
-            sys_creds_file_name = Some(get_path_buf_from_current_dir(file_name));
+        if let Some(_) = self.local_sys_pubkey.as_ref() {
+            let sys_user_name = format!("sys_user_{}", self.host_pubkey);
+            let path = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name))?;
             Command::new("nsc")
-                .arg(format!(
-                    "generate creds --name user_host_{} --account {} > {}",
-                    sys_pubkey, "SYS", file_name
-                ))
-                .output()
-                .context("Failed to add new operator signing key on hosting agent")?;
+            .arg(format!(
+                "generate creds --name {} --account {}",
+                sys_user_name, "SYS"
+            ))
+            .args(&[
+                "generate", "creds",
+                "--name", &sys_user_name,
+                "--account", "SYS",
+                "--output-file", &path.to_string_lossy(),
+            ])
+            .output()
+            .context("Failed to add new operator signing key on hosting agent")?;
+            sys_creds_file_name = Some(path);
+            println!("generated sys user creds");
         }
 
         self.to_owned()
-            .add_creds_paths(get_path_buf_from_current_dir(host_creds_file_name), sys_creds_file_name)
+            .add_creds_paths(host_creds_path, sys_creds_file_name)
     }
 
-    pub fn get_host_creds_path(&self) -> Option<PathBuf> {
+    pub fn _get_host_creds_path(&self) -> Option<PathBuf> {
         if let AuthCredType::Authenticated(creds) = self.to_owned().creds {
             return Some(creds.host_creds_path);
         };
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index e642d9d..d7a5eaa 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -22,6 +22,7 @@ use clap::Parser;
 use dotenv::dotenv;
 use hpos_hal::inventory::HoloInventory;
 use thiserror::Error;
+use util_libs::nats_js_client::{JsClient, PublishInfo};
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -50,6 +51,8 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
+    println!("inside host agent main auth... 0");
+
     let mut host_agent_keys = keys::Keys::try_from_storage(
         &args.nats_leafnode_client_creds_path,
         &args.nats_leafnode_client_sys_creds_path,
@@ -60,33 +63,38 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
             async_nats::Error::from(e)
         })
     })?;
+    println!("inside host main auth... 1");
 
     // If user cred file is for the auth_guard user, run loop to authenticate host & hoster...
     if let keys::AuthCredType::Guard(_) = host_agent_keys.creds {
+        println!("inside host main auth... 2a");
         host_agent_keys = run_auth_loop(host_agent_keys).await?;
     }
 
-    // Once authenticated, start leaf server and run workload api calls.
-    let _ = hostd::gen_leaf_server::run(
-        &host_agent_keys.get_host_creds_path(),
-        &args.store_dir,
-        args.hub_url.clone(),
-        args.hub_tls_insecure,
-    )
-    .await;
-
-    let host_workload_client = hostd::workloads::run(
-        &host_agent_keys.host_pubkey,
-        &host_agent_keys.get_host_creds_path(),
-        args.nats_connect_timeout_secs,
-    )
-    .await?;
+    println!("inside host main auth... 2b");
+    println!("Successfully AUTH'D and created new agent keys: {:#?}", host_agent_keys);
+
+    // // Once authenticated, start leaf server and run workload api calls.
+    // let _ = hostd::gen_leaf_server::run(
+    //     &host_agent_keys.get_host_creds_path(),
+    //     &args.store_dir,
+    //     args.hub_url.clone(),
+    //     args.hub_tls_insecure,
+    // )
+    // .await;
+
+    // let host_workload_client = hostd::workloads::run(
+    //     &host_agent_keys.host_pubkey,
+    //     &host_agent_keys.get_host_creds_path(),
+    //     args.nats_connect_timeout_secs,
+    // )
+    // .await?;
 
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    // Close client and drain internal buffer before exiting to make sure all messages are sent
-    host_workload_client.close().await?;
+    // // Close client and drain internal buffer before exiting to make sure all messages are sent
+    // host_workload_client.close().await?;
 
     Ok(())
 }
@@ -115,11 +123,9 @@ async fn run_auth_loop(mut keys: keys::Keys) -> Result<keys::Keys, async_nats::E
                 format!("DIAGNOSTICS.unauthenticated.{}", keys.host_pubkey);
             let diganostics = HoloInventory::from_host();
             let payload_bytes = serde_json::to_vec(&diganostics)?;
+            
             if let Err(e) = auth_guard_client
-                .publish(
-                    unauthenticated_user_diagnostics_subject,
-                    payload_bytes.into(),
-                )
+            .publish(unauthenticated_user_diagnostics_subject, payload_bytes.into())
                 .await
             {
                 log::error!("Encountered error when sending diganostics. Err={:#?}", e);
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index e6249b9..a84f7ae 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -18,11 +18,13 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
+use async_nats::service::ServiceExt;
 use anyhow::{anyhow, Context, Result};
-use async_nats::Message;
+use futures::StreamExt;
+// use async_nats::Message;
 use authentication::{
     self,
-    types::{self, AuthApiResult},
+    types::{self, AuthErrorPayload},
     AuthServiceApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION,
 };
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
@@ -30,19 +32,15 @@ use nkeys::KeyPair;
 use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
-use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::{sync::Arc, time::Duration};
 use std::str::FromStr;
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{
-        get_event_listeners, get_nats_url, with_event_listeners, get_nats_creds_by_nsc, Credentials,
-        EndpointType, JsClient, NewJsClientParams,
-    },
+    nats_js_client::{get_nats_url, get_nats_creds_by_nsc}
 };
 
 pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Manager";
-pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_auth_inbox_orchestrator";
+pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX_ORCHESTRATOR";
 
 pub async fn run() -> Result<(), async_nats::Error> {
     println!("inside auth... 0");
@@ -58,11 +56,10 @@ pub async fn run() -> Result<(), async_nats::Error> {
         admin_account_creds_path
     );
 
-    println!("inside auth... 1");
-
     // Root Keypair associated with AUTH account
-    let root_account_key_path = std::env::var("ROOT_AUTH_NKEY_PATH")
-        .context("Cannot read ROOT_AUTH_NKEY_PATH from env var")?;
+    let root_account_key_path = std::env::var("ORCHESTRATOR_ROOT_AUTH_NKEY_PATH")
+        .context("Cannot read ORCHESTRATOR_ROOT_AUTH_NKEY_PATH from env var")?;
+
     let root_account_keypair = Arc::new(
         try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
             || {
@@ -73,21 +70,11 @@ pub async fn run() -> Result<(), async_nats::Error> {
             },
         )?,
     );
-
-    println!("inside auth... 2");
-
-    // TODO: REMOVE
-    // let root_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "SAAINFLMRAAE6GYKTQ4SCXNPPCZQTSSSWB3BU3PDKK7CHZDEDYXHL5IP4E",
-    // )?);
     let root_account_pubkey = root_account_keypair.public_key().clone();
-    println!("inside auth... 3");
 
     // AUTH Account Signing Keypair associated with the `auth` user
-    let signing_account_key_path = std::env::var("SIGNING_AUTH_NKEY_PATH")
-        .context("Cannot read SIGNING_AUTH_NKEY_PATH from env var")?;
-    println!("inside auth... 4");
-
+    let signing_account_key_path = std::env::var("ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH")
+    .context("Cannot read ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH from env var")?;
     let signing_account_keypair = Arc::new(
         try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
             .ok_or_else(|| {
@@ -97,140 +84,160 @@ pub async fn run() -> Result<(), async_nats::Error> {
                 )
             })?,
     );
-    println!("inside auth... 5");
-
-    // TODO: REMOVE
-    // let signing_account_keypair = Arc::new(KeyPair::from_seed(
-    //     "SAAL7ULQELTAX5VHVYDDZZ3636AY2AO2O25CRVOPPRFS2KOMVEZV6HTLXI",
-    // )?);
     let signing_account_pubkey = signing_account_keypair.public_key().clone();
     println!(
         ">>>>>>>>> signing_account pubkey: {:?}",
         signing_account_pubkey
     );
-    println!("inside auth... 6");
-
 
     // ==================== Setup NATS ====================
-    // Setup JS Stream Service
-    let auth_stream_service_params = JsServiceParamsPartial {
-        name: AUTH_SRV_NAME.to_string(),
-        description: AUTH_SRV_DESC.to_string(),
-        version: AUTH_SRV_VERSION.to_string(),
-        service_subject: AUTH_SRV_SUBJ.to_string(),
-    };
-    println!("inside auth... 7");
     let nats_url = get_nats_url();
-    let nats_connect_timeout_secs: u64 = 180; 
-
-    let orchestrator_auth_client = JsClient::new(NewJsClientParams {
-        nats_url: get_nats_url(),
-        name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
-        inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
-        service_params: vec![auth_stream_service_params],
-        credentials: Some(Credentials::Path(admin_account_creds_path)),
-        listeners: vec![with_event_listeners(get_event_listeners())],
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(5)),
-    })
-    .await?;
-
-    // let orchestrator_auth_client = tokio::select! {
-    //     client = async {loop {
-    //             let orchestrator_auth_client = JsClient::new(NewJsClientParams {
-    //                 nats_url: nats_url.clone(),
-    //                 name: ORCHESTRATOR_AUTH_CLIENT_NAME.to_string(),
-    //                 inbox_prefix: ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string(),
-    //                 service_params: vec![auth_stream_service_params.clone()],
-    //                 credentials: Some(Credentials::Path(admin_account_creds_path.clone())),
-    //                 listeners: vec![with_event_listeners(get_event_listeners())],
-    //                 ping_interval: Some(Duration::from_secs(10)),
-    //                 request_timeout: Some(Duration::from_secs(5)),
-    //             })
-    //             .await
-    //             .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
-
-    //             match orchestrator_auth_client {
-    //                 Ok(client) => break client,
-    //                 Err(e) => {
-    //                     let duration = tokio::time::Duration::from_millis(100);
-    //                     log::warn!("{}, retrying in {duration:?}", e);
-    //                     tokio::time::sleep(duration).await;
-    //                 }
-    //             }
-    //         }} => client,
-    //     _ = {
-    //         log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
-    //         tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
-    //      } => {
-    //         return Err(format!("timed out waiting for NATS on {nats_url}").into());
-    //     }
-    // };
-
-    println!("inside auth... 8");
+    let nats_connect_timeout_secs: u64 = 180;
+
+    let orchestrator_auth_client = tokio::select! {
+        client = async {loop {
+            let orchestrator_auth_client = async_nats::ConnectOptions::new()
+                .name(ORCHESTRATOR_AUTH_CLIENT_NAME.to_string())
+                .custom_inbox_prefix(ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string())
+                .ping_interval(Duration::from_secs(10))
+                .request_timeout(Some(Duration::from_secs(30)))
+                .credentials_file(&admin_account_creds_path).await.map_err(|e| anyhow::anyhow!("Error loading credentials file: {e}"))?
+                .connect(nats_url.clone())
+                .await
+                .map_err(|e| anyhow::anyhow!("Connecting Orchestrator Auth Client to NATS via {nats_url}: {e}"));
+
+                match orchestrator_auth_client {
+                    Ok(client) => break Ok::<async_nats::Client, async_nats::Error>(client),
+                    Err(e) => {
+                        let duration = tokio::time::Duration::from_millis(100);
+                        log::warn!("{}, retrying in {duration:?}", e);
+                        tokio::time::sleep(duration).await;
+                    }
+                }
+            }} => client?,
+        _ = {
+            log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
+            tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
+         } => {
+            return Err(format!("timed out waiting for NATS on {nats_url}").into());
+        }
+    };
 
     // ==================== Setup DB ====================
     // Create a new MongoDB Client and connect it to the cluster
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-    println!("inside auth... 9");
+    let db_client = MongoDBClient::with_options(client_options)?;
     
     // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
-    let auth_api = AuthServiceApi::new(&client).await?;
-    println!("inside auth... 10");
+    let auth_api = AuthServiceApi::new(&db_client).await?;
+    let auth_api_clone  = auth_api.clone();
 
-    // Register Auth Stream for Orchestrator to consume and process
+    // Register Auth Service for Orchestrator and spawn listener for processing
     let auth_service = orchestrator_auth_client
-        .get_js_service(AUTH_SRV_NAME.to_string())
-        .await
-        .ok_or(anyhow!(
-            "Failed to locate Auth Service. Unable to spin up Orchestrator Auth Client."
-        ))?;
-    println!("inside auth... 11");
+        .service_builder()
+        .description(AUTH_SRV_DESC)
+        .start(AUTH_SRV_NAME, AUTH_SRV_VERSION)
+        .await?;
+    
+    // Auth Callout Service
+    let sys_user_group = auth_service.group("$SYS").group("REQ").group("USER");
+    let mut auth_callout = sys_user_group.endpoint("AUTH").await?;
+    let auth_service_info = auth_service.info().await;
+    let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
+
+    tokio::spawn(async move {
+        while let Some(request) = auth_callout.next().await {
+                let signing_account_kp = Arc::clone(&signing_account_keypair);
+                let signing_account_pk = signing_account_pubkey.clone();
+                let root_account_kp = Arc::clone(&root_account_keypair);
+                let root_account_pk = root_account_pubkey.clone();
+
+                if let Err(e) = auth_api_clone.handle_auth_callout(
+                    Arc::new(request.message),
+                    signing_account_kp,
+                    signing_account_pk,
+                    root_account_kp,
+                    root_account_pk,
+                )
+                .await {
+                    let mut err_payload = AuthErrorPayload {
+                        service_info: auth_service_info.clone(),
+                        group: "$SYS.REQ.USER".to_string(),
+                        endpoint: "AUTH".to_string(),
+                        error: format!("{}",e),
+                    };
+
+                    log::error!(
+                        "{}Failed to handle the endpoint handler. Err={:?}",
+                        "NATS-SERVICE-LOG::AUTH::",
+                        err_payload
+                    );
+
+                    let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                        err_payload.error = e.to_string();
+                        log::error!(
+                            "{}Failed to deserialize error response. Err={:?}",
+                            "NATS-SERVICE-LOG::AUTH::",
+                            err_payload
+                        );
+                        vec![]
+                    });
+
+                    let _ = orchestrator_auth_client_clone.publish("_AUTH_INBOX.ERROR", err_response.into()).await.map_err(|e| {
+                        log::error!(
+                            "{}Failed to send error response. Err={:?}",
+                            "NATS-SERVICE-LOG::AUTH::",
+                            err_payload
+                        );
+                    });
+                }
+            }
+        });
 
-    auth_service
-        .add_consumer::<AuthApiResult>(
-            "auth_callout",
-            types::AUTH_CALLOUT_SUBJECT, // consumer stream subj
-            EndpointType::Async(auth_api.call({
-                move |api: AuthServiceApi, msg: Arc<Message>| {
-                    let signing_account_kp = Arc::clone(&signing_account_keypair);
-                    let signing_account_pk = signing_account_pubkey.clone();
-                    let root_account_kp = Arc::clone(&root_account_keypair);
-                    let root_account_pk = root_account_pubkey.clone();
+        // Auth Validation Service
+        let v1_auth_group = auth_service.group(AUTH_SRV_SUBJ).group("V1");
+        let mut auth_validation = v1_auth_group.endpoint(types::AUTHORIZE_SUBJECT).await?;
+        let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
 
-                    async move {
-                        api.handle_auth_callout(
-                            msg,
-                            signing_account_kp,
-                            signing_account_pk,
-                            root_account_kp,
-                            root_account_pk,
-                        )
-                        .await
-                    }
+        tokio::spawn(async move {
+            while let Some(request) = auth_validation.next().await {
+                if let Err(e) = auth_api.handle_handshake_request(
+                    Arc::new(request.message)
+                )
+                .await {
+                    let auth_service_info = auth_service.info().await;
+                    let mut err_payload = AuthErrorPayload {
+                        service_info: auth_service_info,
+                        group: "AUTH.V1".to_string(),
+                        endpoint: types::AUTHORIZE_SUBJECT.to_string(),
+                        error: format!("{}",e),
+                    };
+                    log::warn!(
+                        "{}Failed to handle the endpoint handler. Err={:?}",
+                        "NATS-SERVICE-LOG::AUTH::",
+                        err_payload
+                    );
+                    let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                        err_payload.error = e.to_string();
+                        log::error!(
+                            "{}Failed to deserialize error response. Err={:?}",
+                            "NATS-SERVICE-LOG::AUTH::",
+                            err_payload
+                        );
+                        vec![]
+                    });
+                    let _ = orchestrator_auth_client_clone.publish("_AUTH_INBOX.ERROR", err_response.into()).await.map_err(|e| {
+                        log::error!(
+                            "{}Failed to send error response. Err={:?}",
+                            "NATS-SERVICE-LOG::AUTH::",
+                            err_payload
+                        );
+                    });
                 }
-            })),
-            None,
-        )
-        .await?;
-    println!("inside auth... 12");
-
-    auth_service
-        .add_consumer::<AuthApiResult>(
-            "authorize_host_and_sys",
-            types::AUTHORIZE_SUBJECT, // consumer stream subj
-            EndpointType::Async(auth_api.call(
-                |api: AuthServiceApi, msg: Arc<Message>| async move {
-                    api.handle_handshake_request(msg).await
-                },
-            )),
-            Some(create_callback_subject_to_host("host_pubkey".to_string())),
-        )
-        .await?;
-    println!("inside auth... 13");
+            }
+        });
 
     println!("Orchestrator Auth Service is running. Waiting for requests...");
 
@@ -238,25 +245,15 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    println!("inside auth... 14... closing");
+    println!("closing orchestrator auth service...");
 
     // Close client and drain internal buffer before exiting to make sure all messages are sent
-    orchestrator_auth_client.close().await?;
-    println!("inside auth... 15... closed");
+    orchestrator_auth_client.drain().await?;
+    println!("closed orchestrator auth service");
 
     Ok(())
 }
 
-pub fn create_callback_subject_to_host(tag_name: String) -> ResponseSubjectsGenerator {
-    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
-        if let Some(tag) = tag_map.get(&tag_name) {
-            return vec![format!("AUTH.{}", tag)];
-        }
-        log::error!("Auth Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject 'AUTH.validate'. Fwding response to `AUTH.ERROR.INBOX`.", tag_name);
-        vec!["AUTH.ERROR.INBOX".to_string()]
-    })
-}
-
 fn try_read_keypair_from_file(key_file_path: PathBuf) -> Result<Option<KeyPair>> {
     match try_read_from_file(key_file_path)? {
         Some(kps) => Ok(Some(KeyPair::from_seed(&kps)?)),
@@ -286,4 +283,4 @@ fn try_read_from_file(file_path: PathBuf) -> Result<Option<String>> {
             Ok(None)
         }
     }
-}
+}
\ No newline at end of file
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 291eadc..6a08ee9 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -36,7 +36,7 @@ use workload::{
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Manager";
-const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_workload_inbox_orchestrator";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX_ORCHESTRATOR";
 
 pub fn create_callback_subject_to_host(
     is_prefix: bool,
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 738e4e4..bd69445 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -315,7 +315,7 @@ impl AuthServiceApi {
         let types::AuthJWTPayload {
             host_pubkey,
             maybe_sys_pubkey,
-            nonce: _,
+            ..
         } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
 
         // 2. Validate signature
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 2cfaa3a..64ef095 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -10,6 +10,14 @@ pub const AUTHORIZE_SUBJECT: &str = "validate";
 // NB: This role name *must* match the `ROLE_NAME_WORKLOAD` in the `orchestrator_setup.sh` script file.
 pub const WORKLOAD_SK_ROLE: &str = "workload-role";
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct AuthErrorPayload {
+    pub service_info: async_nats::service::Info,
+    pub group: String,
+    pub endpoint: String,
+    pub error: String,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthState {
     Unauthenticated, // step 0
@@ -22,9 +30,33 @@ pub enum AuthState {
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct AuthJWTPayload {
     pub host_pubkey: String,              // nkey
-    pub maybe_sys_pubkey: Option<String>, // nkey
+    pub maybe_sys_pubkey: Option<String>, // optional nkey
     pub nonce: String,
+    // #[serde(skip_serializing_if = "Vec::is_empty")]
+    // host_signature: Vec<u8>, // used to verify the host keypair
 }
+// // NB: Currently there is no way to pass headers in jetstream requests.
+// // Therefore the host_signature is passed within the b64 encoded `AuthGuardPayload` token
+// impl AuthJWTPayload {
+// pub fn try_add_signature<T>(mut self, sign_handler: T) -> Result<Self>
+//     where
+//         T: Fn(&[u8]) -> Result<String>,
+//     {
+//         let payload_bytes = serde_json::to_vec(&self)?;
+//         let signature = sign_handler(&payload_bytes)?;
+//         self.host_signature = signature.as_bytes().to_vec();
+//         Ok(self)
+//     }
+
+//     pub fn without_signature(mut self) -> Self {
+//         self.host_signature = vec![];
+//         self
+//     }
+
+//     pub fn get_host_signature(&self) -> Vec<u8> {
+//         self.host_signature.clone()
+//     }
+// }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthJWTResult {
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 0820a77..ff81594 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -29,7 +29,7 @@ pub enum ServiceError {
 }
 
 pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
 pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
 pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
 pub type AsyncEndpointHandler<T> = Arc<
@@ -61,6 +61,15 @@ where
     }
 }
 
+// #[derive(Clone, Debug)]
+// pub struct RequestInfo {
+//     pub stream_subject: String,
+//     pub consumer_name: String,
+//     pub msg_id: String,
+//     pub data: Vec<u8>,
+//     pub headers: Option<HeaderMap>,
+// }
+
 #[derive(Clone, Debug)]
 pub struct PublishInfo {
     pub subject: String,
@@ -91,6 +100,7 @@ impl std::fmt::Debug for JsClient {
     }
 }
 
+#[derive(Clone)]
 pub struct JsClient {
     url: String,
     name: String,
@@ -106,6 +116,7 @@ pub struct JsClient {
 pub enum Credentials {
     Path(std::path::PathBuf), // String = pathbuf as string
     Password(String, String),
+    Token(String),
 }
 
 #[derive(Deserialize, Default)]
@@ -115,7 +126,7 @@ pub struct NewJsClientParams {
     pub inbox_prefix: String,
     pub service_params: Vec<JsServiceParamsPartial>,
     #[serde(skip_deserializing)]
-    pub credentials: Option<Credentials>,
+    pub credentials: Option<Vec<Credentials>>,
     pub ping_interval: Option<Duration>,
     pub request_timeout: Option<Duration>, // Defaults to 5s
     #[serde(skip_deserializing)]
@@ -124,32 +135,31 @@ pub struct NewJsClientParams {
 
 impl JsClient {
     pub async fn new(p: NewJsClientParams) -> Result<Self, async_nats::Error> {
-        let connect_options = async_nats::ConnectOptions::new()
+        let mut connect_options: async_nats::ConnectOptions = async_nats::ConnectOptions::new()
             // .require_tls(true)
             .name(&p.name)
             .ping_interval(p.ping_interval.unwrap_or(Duration::from_secs(120)))
             .request_timeout(Some(p.request_timeout.unwrap_or(Duration::from_secs(10))))
             .custom_inbox_prefix(&p.inbox_prefix);
 
-        let client = match p.credentials {
-            Some(c) => match c {
-                Credentials::Password(user, pw) => {
-                    connect_options
-                        .user_and_password(user, pw)
-                        .connect(&p.nats_url)
-                        .await?
-                }
-                Credentials::Path(cp) => {
-                    let path = std::path::Path::new(&cp);
-                    connect_options
-                        .credentials_file(path)
-                        .await?
-                        .connect(&p.nats_url)
-                        .await?
+        if let Some(credentials_list) = p.credentials {
+            for credentials in credentials_list {
+                match credentials {
+                    Credentials::Password(user, pw) => {
+                        connect_options = connect_options.user_and_password(user, pw);
+                    }
+                    Credentials::Path(cp) => {
+                        let path = std::path::Path::new(&cp);
+                        connect_options = connect_options.credentials_file(path).await?;
+                    }
+                    Credentials::Token(t) => {
+                        connect_options = connect_options.token(t);
+                    }
                 }
-            },
-            None => connect_options.connect(&p.nats_url).await?,
-        };
+            }
+        }; 
+
+        let client = connect_options.connect(&p.nats_url).await?;
 
         let jetstream = jetstream::new(client.clone());
         let mut services = vec![];
@@ -223,7 +233,15 @@ impl JsClient {
         }
     }
 
-    pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
+    pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>> {
+        log::debug!(
+            "{}Called Publish message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+        
         let now = Instant::now();
         let result = match payload.headers {
             Some(h) => {
@@ -243,22 +261,58 @@ impl JsClient {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(Box::new(err));
+            return Err(async_nats::error::Error::from(err));
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
         Ok(())
     }
 
+    // pub async fn request(&self, payload: RequestInfo) -> Result<Message, async_nats::error::Error<async_nats::RequestErrorKind>> {
+    //     /// ie: $JS.API.CONSUMER.INFO.AUTH.authorize_host_and_sys, JS.API.CONSUMER.INFO.AUTH.auth_callout
+    //     let js_subject = format!("$JS.API.CONSUMER.INFO.{}.{}", payload.stream_subject, payload.consumer_name);
+    //     log::debug!(
+    //         "{}Published message: subj={}, msg_id={} data={:?}",
+    //         self.service_log_prefix,
+    //         js_subject,
+    //         payload.msg_id,
+    //         payload.data
+    //     );
+
+    //     let now = Instant::now();
+    //     let result = match payload.headers {
+    //         Some(headers) => {
+    //             self.client
+    //                 .request_with_headers(format!("$JS.API.CONSUMER.INFO.{}.{}", payload.stream_subject, payload.consumer_name), headers, payload.data.clone().into())
+    //                 .await
+    //         }
+    //         None => {
+    //             self.client
+    //                 .request(format!("$JS.API.CONSUMER.INFO.{}.{}", payload.stream_subject, payload.consumer_name), payload.data.clone().into())
+    //                 .await
+    //         }
+    //     };
+
+    //     let duration = now.elapsed();
+
+    //     match result {
+    //         Ok(m) => {
+    //             if let Some(ref on_published) = self.on_msg_published_event {
+    //                 on_published(&js_subject, &self.name, duration);
+    //             }
+    //             Ok(m)
+    //         },
+    //         Err(e) => {
+    //             if let Some(ref on_failed) = self.on_msg_failed_event {
+    //                 on_failed(&js_subject, &self.name, duration); // todo: add msg_id
+    //             }
+    //             Err(e)
+    //         }
+    //     }
+    // }
+
     pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
         let mut current_services = self.js_services.unwrap_or_default();
         current_services.extend(js_services);
@@ -296,7 +350,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_published_event = Some(Box::pin(f.clone()));
+        c.on_msg_published_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -305,7 +359,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_failed_event = Some(Box::pin(f.clone()));
+        c.on_msg_failed_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -320,12 +374,12 @@ pub fn get_nats_url() -> String {
 }
 
 pub fn get_nsc_root_path() -> String {
-    std::env::var("NSC_PATH").unwrap_or_else(|_| "/.local/share/nats/nsc".to_string())
+    std::env::var("NSC_PATH").unwrap_or_else(|_| ".local/share/nats/nsc".to_string())
 }
 
 pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {
     format!(
-        "{}/keys/creds/{}/{}/{}.creds",
+        "/{}/keys/creds/{}/{}/{}.creds",
         get_nsc_root_path(),
         operator,
         account,
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index eed4271..fcc3a08 100755
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -113,7 +113,7 @@ nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem
 
 ADMIN_SK="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 ADMIN_ROLE_NAME="admin_role"
-nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_workload_inbox_*.>","_auth_inbox_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_workload_inbox_orchestrator.>","_auth_inbox_orchestrator.>" --allow-pub-response
+nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_WORKLOAD_INBOX_*.>","_AUTH_INBOX_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_WORKLOAD_INBOX_ORCHESTRATOR.>","_AUTH_INBOX_ORCHESTRATOR.>" --allow-pub-response
 
 # Step 3: Create AUTH with JetStream with non-scoped signing key
 nsc add account --name $AUTH_ACCOUNT
@@ -129,7 +129,7 @@ nsc add account --name $WORKLOAD_ACCOUNT
 nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G --conns -1 --leaf-conns -1
 WORKLOAD_SK="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 WORKLOAD_ROLE_NAME="workload_role"
-nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","_INBOX_{{tag(pubkey)}}.>","_workload_inbox_{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","_INBOX_{{tag(pubkey)}}.>","_workload_inbox_{{tag(pubkey)}}.>" --allow-pub-response
+nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","_INBOX_{{tag(pubkey)}}.>","_WORKLOAD_INBOX_{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","_INBOX_{{tag(pubkey)}}.>","_WORKLOAD_INBOX_{{tag(pubkey)}}.>" --allow-pub-response
 
 # Step 6: Create Operator User in ADMIN Account (for use in Orchestrator)
 nsc add user --name $ADMIN_USER --account $ADMIN_ACCOUNT -K $ADMIN_ROLE_NAME

From 622b604d01bd994534dc3271a97f9790fbbd3c2d Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 16:11:30 -0600
Subject: [PATCH 68/91] decode sig

---
 rust/clients/host_agent/src/auth/init.rs  |  9 +++-
 rust/services/authentication/src/lib.rs   | 64 +++++++++++++----------
 rust/services/authentication/src/types.rs |  2 +
 3 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index b20a6f8..421811a 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -24,15 +24,17 @@ use crate::{
 use anyhow::Result;
 use async_nats::{jetstream::context::PublishErrorKind, HeaderMap, HeaderName, HeaderValue, RequestErrorKind};
 use authentication::types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthResult, AuthState};
+use nkeys::KeyPair;
 use std::time::Duration;
 // use futures::StreamExt;
 use util_libs::nats_js_client::{
     self, get_event_listeners, get_nats_url, with_event_listeners,
     Credentials, JsClient, NewJsClientParams,
 };
+use data_encoding::BASE64URL_NOPAD;
+use textnonce::TextNonce;
 use hpos_hal::inventory::HoloInventory;
 use std::str::FromStr;
-use textnonce::TextNonce;
 
 pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
 pub const HOST_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX";
@@ -74,8 +76,11 @@ pub async fn run(
             auth_guard_payload.nonce = nonce;
         }
     };
+    println!("PRIOR TO SIG : auth_guard_payload={:#?}", auth_guard_payload);
+    println!(" SIG OF auth_guard_payload : {:?}", host_agent_keys.host_sign(&serde_json::to_vec(&auth_guard_payload)?));
+
     auth_guard_payload = auth_guard_payload.try_add_signature(|p| host_agent_keys.host_sign(p))?;
-    println!("auth_guard_payload={:#?}", auth_guard_payload);
+    println!("POST SIG : auth_guard_payload={:#?}", auth_guard_payload);
 
     let user_auth_json = serde_json::to_string(&auth_guard_payload)?;
     let user_auth_token = json_to_base64(&user_auth_json)?;
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 71a34fb..397c6ea 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -11,6 +11,7 @@ pub mod types;
 pub mod utils;
 use anyhow::{Context, Result};
 use async_nats::jetstream::ErrorCode;
+use async_nats::service::{NATS_SERVICE_ERROR, NATS_SERVICE_ERROR_CODE};
 use async_nats::HeaderValue;
 use async_nats::{AuthError, Message};
 use data_encoding::BASE64URL_NOPAD;
@@ -92,18 +93,27 @@ impl AuthServiceApi {
         // 2. Validate Host signature, returning validation error if not successful
         let host_pubkey = user_data.host_pubkey.as_ref();
         let host_signature = user_data.get_host_signature();
+        let decoded_sig = BASE64URL_NOPAD.decode(&host_signature)
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
+        println!("host_signature: {:?}", host_signature);
+        println!("decoded_sig: {:?}", decoded_sig);
+
         let user_verifying_keypair = KeyPair::from_public_key(host_pubkey)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        let raw_payload = serde_json::to_vec(&user_data.clone().without_signature())
+        let payload_no_sig = &user_data.clone().without_signature();
+        println!("PAYLOAD WITHOUT SIG: {:#?}", payload_no_sig);
+        let raw_payload = serde_json::to_vec(payload_no_sig)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        // if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &host_signature) {
-        //     log::error!(
-        //         "Error: Failed to validate Signature. Subject='{}'. Err={}",
-        //         msg.subject,
-        //         e
-        //     );
-        //     return Err(ServiceError::Authentication(AuthError::new(e)));
-        // };
+        println!("PAYLOAD WITHOUT SIG AS BYTES: {:?}", raw_payload);
+
+        if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &decoded_sig) {
+            log::error!(
+                "Error: Failed to validate Signature. Subject='{}'. Err={}",
+                msg.subject,
+                e
+            );
+            return Err(ServiceError::Authentication(AuthError::new(e)));
+        };
 
         // 3. If provided, authenticate the Hoster pubkey and email and assign full permissions if successful
         let is_hoster_valid = if user_data.email.is_some() && user_data.hoster_hc_pubkey.is_some() {
@@ -320,29 +330,21 @@ impl AuthServiceApi {
             ..
         } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
 
-        let decoded_signature = BASE64URL_NOPAD.decode(signature).map_err(|e| {
-            println!("err={}", e);
-            ServiceError::Internal(e.to_string())
-        })?;
-
         // 2. Validate signature
+        let decoded_signature = BASE64URL_NOPAD.decode(signature)
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
         let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
         if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), &decoded_signature) {
             log::error!(
                 "Error: Failed to validate Signature. Subject='{}'. Err={}",
-                msg.subject,
+                msg.subject,  
                 e
             );
-            return Err(ServiceError::Request(format!(
-                "{:?}",
-                ErrorCode::BAD_REQUEST
-            )));    
+            return Err(ServiceError::Authentication(AuthError::new(format!("{:?}", e))));    
         };
 
-        println!("'AUTH.validate >>> user_verifying_keypair' : {:?}", user_verifying_keypair);
-        println!("'AUTH.validate >>> maybe_sys_pubkey' : {:?}", maybe_sys_pubkey);
-
         // 3. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
         match Command::new("nsc")
             .args(&[
@@ -358,12 +360,14 @@ impl AuthServiceApi {
             .map_err(|e| ServiceError::Internal(e.to_string())) {
                 Ok(r) => {
                     println!("'AUTH.validate >>> add user -a WORKLOAD -n host_user_<pubkey> ...' : {:?}", r);
+                    let stderr = String::from_utf8_lossy(&r.stderr);
+                    if !r.stderr.is_empty() && !stderr.contains("already exists") {
+                        return Err(ServiceError::Internal(stderr.to_string()));
+                    }
                 },
                 Err(e) => {
-                    if !e.to_string().contains("already exists") {
-                        return Err(e);
-                    }
                     println!("'AUTH.validate >>> ERROR: add user -a WORKLOAD -n host_user_<pubkey> ...' : {:?}", e);
+                    return Err(e);
                 }
             };
         println!("\nadded host user");
@@ -383,12 +387,14 @@ impl AuthServiceApi {
                 .map_err(|e| ServiceError::Internal(e.to_string())) {
                     Ok(r) => {
                         println!("'AUTH.validate >>> add user -a SYS -n sys_user_<pubkey> ...' : {:?}", r);
-                    },
-                    Err(e) => {
-                        if !e.to_string().contains("already exists") {
-                            return Err(e);
+                        let stderr = String::from_utf8_lossy(&r.stderr);
+                        if !r.stderr.is_empty() && !stderr.contains("already exists") {
+                            return Err(ServiceError::Internal(stderr.to_string()));
                         }
+                    },
+                    Err(e) => {   
                         println!("'AUTH.validate >>> ERROR: add user -a SYS -n sys_user_<pubkey> ...' : {:?}", e);
+                        return Err(e);
                     }
                 };
         }
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 8034bb8..a01c561 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -2,6 +2,7 @@ use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
+use data_encoding::BASE64URL_NOPAD;
 
 pub const AUTH_CALLOUT_SUBJECT: &str = "$SYS.REQ.USER.AUTH";
 pub const AUTHORIZE_SUBJECT: &str = "validate";
@@ -120,6 +121,7 @@ impl AuthGuardPayload {
         T: Fn(&[u8]) -> Result<String>,
     {
         let payload_bytes = serde_json::to_vec(&self)?;
+        println!("Going to sign payload_bytes={:?}", payload_bytes);
         let signature = sign_handler(&payload_bytes)?;
         self.host_signature = signature.as_bytes().to_vec();
         Ok(self)

From a49aaaff7dd2a8e9afacdf62882ad87ef5f30574 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 17:07:15 -0600
Subject: [PATCH 69/91] clean key refs

---
 rust/clients/host_agent/src/auth/init.rs  |  13 ---
 rust/clients/host_agent/src/keys.rs       |   2 +-
 rust/clients/orchestrator/src/auth.rs     | 105 +++++++++-------------
 rust/services/authentication/src/lib.rs   |   1 -
 rust/services/authentication/src/types.rs |  24 -----
 rust/util_libs/src/nats_js_client.rs      |   4 +-
 6 files changed, 44 insertions(+), 105 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index 421811a..304e367 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -132,19 +132,6 @@ SUABBYL4YAGRRJDOMXP72EUDM4UOFOGJWVPKT6AB7UMNWU2TV4M4PMFXDE
     log::trace!("Host Auth Client: Retrieved Node ID: {}", server_node_id);
 
     // ==================== Handle Host User and SYS Authoriation ============================================================
-    let auth_guard_client_clone = auth_guard_client.clone();
-
-    // tokio::spawn({
-    //     let mut auth_inbox_msgs = auth_guard_client_clone
-    //         .subscribe(unique_inbox.to_string())
-    //         .await?;
-    //     async move {
-    //         while let Some(msg) = auth_inbox_msgs.next().await {
-    //             println!("got an AUTH INBOX msg: {:?}", &msg);
-    //         }
-    //     }
-    // });
-
     let payload = AuthJWTPayload {
         host_pubkey: host_agent_keys.host_pubkey.to_string(),
         maybe_sys_pubkey: host_agent_keys.local_sys_pubkey.clone(),
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 2d132a4..fb31c76 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -6,7 +6,7 @@ use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Command;
 use std::str::FromStr;
-use util_libs::nats_js_client::{get_nats_creds_by_nsc, get_nsc_root_path, get_path_buf_from_current_dir};
+use util_libs::nats_js_client::{get_nats_creds_by_nsc, get_nsc_root_path};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 4e1e20b..3243929 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -21,7 +21,6 @@ This client is responsible for:
 use async_nats::service::ServiceExt;
 use anyhow::{anyhow, Context, Result};
 use futures::StreamExt;
-// use async_nats::Message;
 use authentication::{
     self,
     types::{self, AuthErrorPayload},
@@ -44,71 +43,50 @@ pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Manager";
 pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX_ORCHESTRATOR";
 
 pub async fn run() -> Result<(), async_nats::Error> {
-    println!("inside auth... 0");
+    let admin_account_creds_path = PathBuf::from_str(&get_nats_creds_by_nsc(
+        "HOLO",
+        "AUTH",
+        "auth",
+    ))?;
+    println!(
+        " >>>> admin_account_creds_path: {:#?} ",
+        admin_account_creds_path
+    );
 
-    // // let admin_account_creds_path = PathBuf::from_str("/home/za/Documents/holo-v2/holo-host/rust/clients/orchestrator/src/tmp/test_admin.creds")?;
-    // let admin_account_creds_path = PathBuf::from_str(&get_nats_creds_by_nsc(
-    //     "HOLO",
-    //     "AUTH",
-    //     "auth",
-    // ))?;
-    // println!(
-    //     " >>>> admin_account_creds_path: {:#?} ",
-    //     admin_account_creds_path
-    // );
+    // Root Keypair associated with AUTH account
+    let root_account_key_path = std::env::var("ORCHESTRATOR_ROOT_AUTH_NKEY_PATH")
+        .context("Cannot read ORCHESTRATOR_ROOT_AUTH_NKEY_PATH from env var")?;
+    println!(">>>>>>>>> root account key_path: {:?}", root_account_key_path);
 
-    // // Root Keypair associated with AUTH account
-    // let root_account_key_path = std::env::var("ROOT_AUTH_NKEY_PATH")
-    //     .context("Cannot read ROOT_AUTH_NKEY_PATH from env var")?;
-
-    // let root_account_keypair = Arc::new(
-    //     try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
-    //         || {
-    //             anyhow!(
-    //                 "Root AUTH Account keypair not found at path {:?}",
-    //                 root_account_key_path
-    //             )
-    //         },
-    //     )?,
-    // );
-    // let root_account_pubkey = root_account_keypair.public_key().clone();
-    // println!(">>>>>>>>> root account pubkey: {:?}", root_account_pubkey);
-
-    // // AUTH Account Signing Keypair associated with the `auth` user
-    // let signing_account_key_path = std::env::var("SIGNING_AUTH_NKEY_PATH")
-    // .context("Cannot read SIGNING_AUTH_NKEY_PATH from env var")?;
-    // let signing_account_keypair = Arc::new(
-    //     try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
-    //         .ok_or_else(|| {
-    //             anyhow!(
-    //                 "Signing AUTH Account keypair not found at path {:?}",
-    //                 signing_account_key_path
-    //             )
-    //         })?,
-    // );
-    // let signing_account_pubkey = signing_account_keypair.public_key().clone();
-    // println!(">>>>>>>>> signing_account pubkey: {:?}", signing_account_pubkey);
-    let admin_account_creds = "-----BEGIN NATS USER JWT-----
-eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJUTVJDVklaUDRJUlRLWktGSEVTV1BYSzZMM0hDQUdTTFhUREZBTk9IS1ZQSFNNQ0w3UEhBIiwiaWF0IjoxNzM4NTI0MDg5LCJpc3MiOiJBRFQ2TUhSQUgzU0JXWFU1RlRHN0I2WklCU0VXV0UzMkJVNDJKTzRKRE8yV0VSVDZYTVpLRTYzUyIsIm5hbWUiOiJhdXRoIiwic3ViIjoiVUNWQTVZT1haNTZMVzNSRVhEV1VBR1EzVktERE5RVlA0TVNSSFJKRVRTQjJZRlBVQ1FJQTdQT0siLCJuYXRzIjp7InB1YiI6eyJhbGxvdyI6WyJcdTAwM2UiXX0sInN1YiI6eyJhbGxvdyI6WyJcdTAwM2UiXX0sInN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImlzc3Vlcl9hY2NvdW50IjoiQUEzUTdIWVBHTVdSWFcyRkxLNUNCRFZQWVdEUjI3TU5QUE9TT1M3SUdVT0hBWTNMNEdOUkJMSjQiLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.K-cSBzw_wai2BEA7Lzxg2kDThj72lZpTKJbvUff6gSxPjn2KuVJQy5k2mSC9fohBmjcJLSoQ-7JrXA7GN1VMDA
-------END NATS USER JWT------
-
-************************* IMPORTANT *************************
-NKEY Seed printed below can be used to sign and prove identity.
-NKEYs are sensitive and should be treated as secrets.
-
------BEGIN USER NKEY SEED-----
-SUALKRFOSR77N6VXQOQRF65RET2GU4D4IP2OEB4546EEX6WLHK2BW6FMZU
-------END USER NKEY SEED------
-
-*************************************************************";    
-    let root_account_keypair = KeyPair::from_seed("SAAINFLMRAAE6GYKTQ4SCXNPPCZQTSSSWB3BU3PDKK7CHZDEDYXHL5IP4E")?;
+    let root_account_keypair = Arc::new(
+        try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
+            || {
+                anyhow!(
+                    "Root AUTH Account keypair not found at path {:?}",
+                    root_account_key_path
+                )
+            },
+        )?,
+    );
     let root_account_pubkey = root_account_keypair.public_key().clone();
     println!(">>>>>>>>> root account pubkey: {:?}", root_account_pubkey);
 
-    let signing_account_keypair = KeyPair::from_seed("SAAL7ULQELTAX5VHVYDDZZ3636AY2AO2O25CRVOPPRFS2KOMVEZV6HTLXI")?;
-    let signing_account_pubkey = signing_account_keypair.public_key();
-    println!(">>>>>>>>> auth_signing_account pubkey: {:?}", signing_account_pubkey);
+    // AUTH Account Signing Keypair associated with the `auth` user
+    let signing_account_key_path = std::env::var("ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH")
+        .context("Cannot read ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH from env var")?;
+    println!(">>>>>>>>> signing account key_path: {:?}", signing_account_key_path);
 
+    let signing_account_keypair = Arc::new(
+        try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
+            .ok_or_else(|| {
+                anyhow!(
+                    "Signing AUTH Account keypair not found at path {:?}",
+                    signing_account_key_path
+                )
+            })?,
+    );
+    let signing_account_pubkey = signing_account_keypair.public_key().clone();
+    println!(">>>>>>>>> signing_account pubkey: {:?}", signing_account_pubkey);
 
     // ==================== Setup NATS ====================
     let nats_url = get_nats_url();
@@ -121,8 +99,7 @@ SUALKRFOSR77N6VXQOQRF65RET2GU4D4IP2OEB4546EEX6WLHK2BW6FMZU
                 .custom_inbox_prefix(ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX.to_string())
                 .ping_interval(Duration::from_secs(10))
                 .request_timeout(Some(Duration::from_secs(30)))
-                // .credentials_file(&admin_account_creds_path).await.map_err(|e| anyhow::anyhow!("Error loading credentials file: {e}"))?
-                .credentials(admin_account_creds)?
+                .credentials_file(&admin_account_creds_path).await.map_err(|e| anyhow::anyhow!("Error loading credentials file: {e}"))?
                 .connect(nats_url.clone())
                 .await
                 .map_err(|e| anyhow::anyhow!("Connecting Orchestrator Auth Client to NATS via {nats_url}: {e}"));
@@ -170,9 +147,9 @@ SUALKRFOSR77N6VXQOQRF65RET2GU4D4IP2OEB4546EEX6WLHK2BW6FMZU
 
     tokio::spawn(async move {
         while let Some(request) = auth_callout.next().await {
-                let signing_account_kp = Arc::clone(&Arc::new(signing_account_keypair.clone()));
+                let signing_account_kp = Arc::clone(&signing_account_keypair.clone());
                 let signing_account_pk = signing_account_pubkey.clone();
-                let root_account_kp = Arc::clone(&Arc::new(root_account_keypair.clone()));
+                let root_account_kp = Arc::clone(&root_account_keypair.clone());
                 let root_account_pk = root_account_pubkey.clone();
 
                 let maybe_reply = request.message.reply.clone();
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 397c6ea..236b9c0 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -11,7 +11,6 @@ pub mod types;
 pub mod utils;
 use anyhow::{Context, Result};
 use async_nats::jetstream::ErrorCode;
-use async_nats::service::{NATS_SERVICE_ERROR, NATS_SERVICE_ERROR_CODE};
 use async_nats::HeaderValue;
 use async_nats::{AuthError, Message};
 use data_encoding::BASE64URL_NOPAD;
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index a01c561..8bcc2fa 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -33,31 +33,7 @@ pub struct AuthJWTPayload {
     pub host_pubkey: String,              // nkey
     pub maybe_sys_pubkey: Option<String>, // optional nkey
     pub nonce: String,
-    // #[serde(skip_serializing_if = "Vec::is_empty")]
-    // host_signature: Vec<u8>, // used to verify the host keypair
 }
-// // NB: Currently there is no way to pass headers in jetstream requests.
-// // Therefore the host_signature is passed within the b64 encoded `AuthGuardPayload` token
-// impl AuthJWTPayload {
-// pub fn try_add_signature<T>(mut self, sign_handler: T) -> Result<Self>
-//     where
-//         T: Fn(&[u8]) -> Result<String>,
-//     {
-//         let payload_bytes = serde_json::to_vec(&self)?;
-//         let signature = sign_handler(&payload_bytes)?;
-//         self.host_signature = signature.as_bytes().to_vec();
-//         Ok(self)
-//     }
-
-//     pub fn without_signature(mut self) -> Self {
-//         self.host_signature = vec![];
-//         self
-//     }
-
-//     pub fn get_host_signature(&self) -> Vec<u8> {
-//         self.host_signature.clone()
-//     }
-// }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AuthJWTResult {
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 3f9df12..c18c05a 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -383,12 +383,12 @@ pub fn get_nats_url() -> String {
 }
 
 pub fn get_nsc_root_path() -> String {
-    std::env::var("NSC_PATH").unwrap_or_else(|_| ".local/share/nats/nsc".to_string())
+    std::env::var("NSC_PATH").unwrap_or_else(|_| "/.local/share/nats/nsc".to_string())
 }
 
 pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {
     format!(
-        "/{}/keys/creds/{}/{}/{}.creds",
+        "{}/keys/creds/{}/{}/{}.creds",
         get_nsc_root_path(),
         operator,
         account,

From e037f2f6e00c3013dc5e54c06d5582b347bebdd3 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 17:50:28 -0600
Subject: [PATCH 70/91] holo agent key clean-up

---
 rust/clients/host_agent/src/auth/init.rs | 30 ++++--------------------
 rust/clients/host_agent/src/keys.rs      | 27 +++++++--------------
 2 files changed, 14 insertions(+), 43 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index 304e367..e614feb 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -22,16 +22,10 @@ use crate::{
     keys::{AuthCredType, Keys},
 };
 use anyhow::Result;
-use async_nats::{jetstream::context::PublishErrorKind, HeaderMap, HeaderName, HeaderValue, RequestErrorKind};
-use authentication::types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthResult, AuthState};
-use nkeys::KeyPair;
+use async_nats::{HeaderMap, HeaderName, HeaderValue, RequestErrorKind};
+use authentication::types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthState};
 use std::time::Duration;
-// use futures::StreamExt;
-use util_libs::nats_js_client::{
-    self, get_event_listeners, get_nats_url, with_event_listeners,
-    Credentials, JsClient, NewJsClientParams,
-};
-use data_encoding::BASE64URL_NOPAD;
+use util_libs::nats_js_client;
 use textnonce::TextNonce;
 use hpos_hal::inventory::HoloInventory;
 use std::str::FromStr;
@@ -93,20 +87,6 @@ pub async fn run(
     };
     println!("user_creds_path={:#?}", user_creds_path);
 
-    let user_creds = "-----BEGIN NATS USER JWT-----
-eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI2MkVCSEhFR0M1RDIyU0lXR1hSU0paNEpWWkdWUk9FVUo3N1BYQ1BPNUU3UDRBTkVHV1RBIiwiaWF0IjoxNzM4NTI3NjgwLCJpc3MiOiJBRFQ2TUhSQUgzU0JXWFU1RlRHN0I2WklCU0VXV0UzMkJVNDJKTzRKRE8yV0VSVDZYTVpLRTYzUyIsIm5hbWUiOiJhdXRoLWd1YXJkIiwic3ViIjoiVUM1N1pETUtOSVhVWE9NNlRISE8zQjVVRUlWQ0JPM0hNRlUzSU5ESVZNTzVCSFZKR1k3R1hIM1UiLCJuYXRzIjp7InB1YiI6eyJkZW55IjpbIlx1MDAzZSJdfSwic3ViIjp7ImRlbnkiOlsiXHUwMDNlIl19LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpc3N1ZXJfYWNjb3VudCI6IkFBM1E3SFlQR01XUlhXMkZMSzVDQkRWUFlXRFIyN01OUFBPU09TN0lHVU9IQVkzTDRHTlJCTEo0IiwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.REQSfDwGzuG0vWDDfHyZdpN-Ens3hhRF1-I-k5akDK9oT8kueW2OWX3lFlgBreNw5JsTgE0fjKDq942QRTygDg
-------END NATS USER JWT------
-
-************************* IMPORTANT *************************
-NKEY Seed printed below can be used to sign and prove identity.
-NKEYs are sensitive and should be treated as secrets.
-
------BEGIN USER NKEY SEED-----
-SUABBYL4YAGRRJDOMXP72EUDM4UOFOGJWVPKT6AB7UMNWU2TV4M4PMFXDE
-------END USER NKEY SEED------
-
-*************************************************************";
-
     // Connect to Nats server as auth guard and call NATS AuthCallout
     let nats_url = nats_js_client::get_nats_url();
     let auth_guard_client =
@@ -116,8 +96,8 @@ SUABBYL4YAGRRJDOMXP72EUDM4UOFOGJWVPKT6AB7UMNWU2TV4M4PMFXDE
             .ping_interval(Duration::from_secs(10))
             .request_timeout(Some(Duration::from_secs(30)))
             .token(user_auth_token)
-            // .credentials_file(&user_creds_path).await?
-            .credentials(user_creds)?
+            .credentials_file(&user_creds_path).await?
+            // .credentials(user_creds)?
             .connect(nats_url)
             .await?;
 
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index fb31c76..f967c06 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -41,7 +41,7 @@ pub struct CredPaths {
 #[derive(Clone)]
 pub enum AuthCredType {
     Guard(PathBuf),           // Default
-    Authenticated(CredPaths), // only assiged after successful hoster authentication
+    Authenticated(CredPaths), // Only assiged after successful hoster authentication
 }
 
 #[derive(Clone)]
@@ -55,42 +55,32 @@ pub struct Keys {
 
 impl Keys {
     pub fn new() -> Result<Self> {
-        println!("inside Keys new ... 0");
-
         let host_key_path =
             std::env::var("HOSTING_AGENT_HOST_NKEY_PATH").context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
-        println!("inside Keys new ... 1");
         println!("inside Keys new > host_key_path={}", host_key_path);
     
         let host_kp = KeyPair::new_user();
-        println!("inside Keys new ... 2");
         println!("inside Keys new > host_kp={:#?}", host_kp);
 
         write_keypair_to_file(PathBuf::from_str(&host_key_path)?, host_kp.clone())?;
-        println!("inside Keys new ... 3");
 
         let host_pk = host_kp.public_key();
         println!("inside Keys new > host_pk={}", host_pk);
 
         let sys_key_path =
             std::env::var("HOSTING_AGENT_SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
-        println!("inside Keys new ... 4");
         println!("inside Keys new > sys_key_path={}", sys_key_path);
             
         let local_sys_kp = KeyPair::new_user();
-        println!("inside Keys new ... 5");
         println!("inside Keys new > local_sys_kp={:#?}", local_sys_kp);
 
         write_keypair_to_file(PathBuf::from_str(&sys_key_path)?, local_sys_kp.clone())?;
-        println!("inside Keys new ... 6");
 
         let local_sys_pk = local_sys_kp.public_key();
-        println!("inside Keys new ... 7");
         println!("inside Keys new > local_sys_pk={}", local_sys_pk);
 
         let auth_guard_creds = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
         println!("inside Keys new > auth_guard_creds={:#?}", auth_guard_creds);
-        println!("inside Keys new ... 8");
 
         Ok(Self {
             host_keypair: host_kp,
@@ -120,17 +110,18 @@ impl Keys {
 
         let sys_key_path =
             std::env::var("HOSTING_AGENT_SYS_NKEY_PATH").context("Cannot read HOSTING_AGENT_SYS_NKEY_PATH from env var")?;
-        println!("inside try_from_storage auth... 3");
         println!("sys_key_path={:#?}", sys_key_path); 
 
+        let host_user_name = format!("host_user_{}", host_pk);
         let host_creds_path = maybe_host_creds_path
             .to_owned()
-            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "host")), Ok)?;
+            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", &host_user_name)), Ok)?;
         println!("host_creds_path={:#?}", host_creds_path); 
 
+        let sys_user_name = format!("sys_user_{}", host_pk);
         let sys_creds_path = maybe_sys_creds_path
             .to_owned()
-            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")), Ok)?;
+            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name)), Ok)?;
         println!("sys_creds_path={:#?}", sys_creds_path); 
 
         // Set auth_guard_creds as default:
@@ -247,12 +238,12 @@ impl Keys {
         let host_path = PathBuf::from_str(&format!("/{}/{}/{}", get_nsc_root_path(), "local_creds", "host.jwt"))?;
         println!("host_path ={:?}", host_path);
         write_to_file(host_path.clone(), host_user_jwt.as_bytes())?;
-        println!("wrote JWT to host file");
+        println!("Wrote JWT to host file");
 
         let sys_path = PathBuf::from_str(&format!("/{}/{}/{}", get_nsc_root_path(), "local_creds", "sys.jwt"))?;
         println!("sys_path ={:?}", sys_path);
         write_to_file(sys_path.clone(), host_sys_user_jwt.as_bytes())?;
-        println!("wrote JWT to sys file");
+        println!("Wrote JWT to sys file");
 
         // Import host user jwt to local nsc resolver
         // TODO: Determine why the following works in cmd line, but doesn't seem to work when run in current program / run
@@ -288,7 +279,7 @@ impl Keys {
             ])
             .output()
             .context("Failed to add new operator signing key on hosting agent")?;
-        println!("generated host user creds");
+        println!("Generated host user creds. creds_path={:?}", host_creds_path);
 
         let mut sys_creds_file_name = None;
         if let Some(_) = self.local_sys_pubkey.as_ref() {
@@ -307,8 +298,8 @@ impl Keys {
             ])
             .output()
             .context("Failed to add new operator signing key on hosting agent")?;
+            println!("Generated sys user creds. creds_path={:?}", path);
             sys_creds_file_name = Some(path);
-            println!("generated sys user creds");
         }
 
         self.to_owned()

From e82b31b3d91747960ba5a9e8cc80875111487596 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 19:08:22 -0600
Subject: [PATCH 71/91] fmt

---
 rust/clients/host_agent/src/auth/config.rs    |  14 +-
 rust/clients/host_agent/src/auth/init.rs      | 141 ++++-------
 .../clients/host_agent/src/hostd/workloads.rs |   5 +-
 rust/clients/host_agent/src/keys.rs           | 157 ++++++------
 rust/clients/host_agent/src/main.rs           |  18 +-
 rust/clients/orchestrator/src/auth.rs         | 238 +++++++++---------
 rust/services/authentication/src/lib.rs       | 198 +++++++--------
 rust/services/authentication/src/types.rs     |   3 +-
 rust/services/authentication/src/utils.rs     |  44 +---
 rust/util_libs/src/js_stream_service.rs       |  15 +-
 rust/util_libs/src/nats_js_client.rs          |  18 +-
 scripts/orchestrator_setup.sh                 |   1 -
 12 files changed, 403 insertions(+), 449 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/config.rs b/rust/clients/host_agent/src/auth/config.rs
index 377f725..2f1c58e 100644
--- a/rust/clients/host_agent/src/auth/config.rs
+++ b/rust/clients/host_agent/src/auth/config.rs
@@ -17,16 +17,9 @@ pub struct HosterConfig {
 
 impl HosterConfig {
     pub async fn new() -> Result<Self> {
-        println!(">>> inside Hoster Config new fn..");
-
         let (keypair, email) = get_from_config().await?;
-        println!(">>> inside Hoster Config new fn : keypair={:#?}", keypair);
-
         let hc_pubkey = public_key::to_holochain_encoded_agent_key(&keypair.verifying_key());
-        println!(">>> inside Hoster Config new fn : hc_pubkey={}", hc_pubkey);
-
         let holoport_id = public_key::to_base36_id(&keypair.verifying_key());
-        println!(">>> inside Hoster Config new fn : holoport_id={}", holoport_id);
 
         Ok(Self {
             email,
@@ -38,14 +31,10 @@ impl HosterConfig {
 }
 
 async fn get_from_config() -> Result<(SigningKey, String)> {
-    println!("inside config_path...");
-
     let config_path =
         env::var("HPOS_CONFIG_PATH").context("Cannot read HPOS_CONFIG_PATH from env var")?;
-    
     let password = env::var("DEVICE_SEED_DEFAULT_PASSWORD")
         .context("Cannot read bundle password from env var")?;
-
     let config_file =
         File::open(&config_path).context(format!("Failed to open config file {}", config_path))?;
 
@@ -55,14 +44,13 @@ async fn get_from_config() -> Result<(SigningKey, String)> {
             settings,
             ..
         } => {
-            // take in password
+            // Take in password
             let signing_key = unlock(&device_bundle, Some(password))
                 .await
                 .context(format!(
                     "unable to unlock the device bundle from {}",
                     &config_path
                 ))?;
-                println!(">>> inside config-path new fn : signing_key={:#?}", signing_key);
             Ok((signing_key, settings.admin.email))
         }
         _ => Err(anyhow!("Unsupported version of hpos config")),
diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index e614feb..6519472 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -24,11 +24,11 @@ use crate::{
 use anyhow::Result;
 use async_nats::{HeaderMap, HeaderName, HeaderValue, RequestErrorKind};
 use authentication::types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthState};
-use std::time::Duration;
-use util_libs::nats_js_client;
-use textnonce::TextNonce;
 use hpos_hal::inventory::HoloInventory;
 use std::str::FromStr;
+use std::time::Duration;
+use textnonce::TextNonce;
+use util_libs::nats_js_client;
 
 pub const HOST_AUTH_CLIENT_NAME: &str = "Host Auth";
 pub const HOST_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX";
@@ -37,23 +37,13 @@ pub async fn run(
     mut host_agent_keys: Keys,
 ) -> Result<(Keys, async_nats::Client), async_nats::Error> {
     log::info!("Host Auth Client: Connecting to server...");
-    println!("Keys={:#?}", host_agent_keys);
-    
-    println!("inside init auth... 0");
-    
+    log::trace!(
+        "Host Agent Keys before authentication request: {:#?}",
+        host_agent_keys
+    );
+
     // ==================== Fetch Config File & Call NATS AuthCallout Service to Authenticate Host & Hoster =============================================
     let nonce = TextNonce::new().to_string();
-    let unique_inbox = &format!(
-        "{}_{}",
-        HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey
-    );
-    println!(">>> unique_inbox : {}", unique_inbox);
-    let user_unique_auth_subject = &format!("AUTH.{}.>", host_agent_keys.host_pubkey);
-    println!(
-        ">>> user_unique_auth_subject : {}",
-        user_unique_auth_subject
-    );
-    println!("inside init auth... 1");
 
     // Fetch Hoster Pubkey and email (from config)
     let mut auth_guard_payload = AuthGuardPayload::default();
@@ -70,11 +60,7 @@ pub async fn run(
             auth_guard_payload.nonce = nonce;
         }
     };
-    println!("PRIOR TO SIG : auth_guard_payload={:#?}", auth_guard_payload);
-    println!(" SIG OF auth_guard_payload : {:?}", host_agent_keys.host_sign(&serde_json::to_vec(&auth_guard_payload)?));
-
     auth_guard_payload = auth_guard_payload.try_add_signature(|p| host_agent_keys.host_sign(p))?;
-    println!("POST SIG : auth_guard_payload={:#?}", auth_guard_payload);
 
     let user_auth_json = serde_json::to_string(&auth_guard_payload)?;
     let user_auth_token = json_to_base64(&user_auth_json)?;
@@ -85,21 +71,23 @@ pub async fn run(
             "Failed to locate Auth Guard credentials",
         ));
     };
-    println!("user_creds_path={:#?}", user_creds_path);
+    let user_unique_inbox = &format!(
+        "{}_{}",
+        HOST_AUTH_CLIENT_INBOX_PREFIX, host_agent_keys.host_pubkey
+    );
 
     // Connect to Nats server as auth guard and call NATS AuthCallout
     let nats_url = nats_js_client::get_nats_url();
-    let auth_guard_client =
-        async_nats::ConnectOptions::new()
-            .name(HOST_AUTH_CLIENT_NAME.to_string())
-            .custom_inbox_prefix(unique_inbox.to_string())
-            .ping_interval(Duration::from_secs(10))
-            .request_timeout(Some(Duration::from_secs(30)))
-            .token(user_auth_token)
-            .credentials_file(&user_creds_path).await?
-            // .credentials(user_creds)?
-            .connect(nats_url)
-            .await?;
+    let auth_guard_client = async_nats::ConnectOptions::new()
+        .name(HOST_AUTH_CLIENT_NAME.to_string())
+        .custom_inbox_prefix(user_unique_inbox.to_string())
+        .ping_interval(Duration::from_secs(10))
+        .request_timeout(Some(Duration::from_secs(30)))
+        .token(user_auth_token)
+        .credentials_file(&user_creds_path)
+        .await?
+        .connect(nats_url)
+        .await?;
 
     let server_info = auth_guard_client.server_info();
     println!(
@@ -117,87 +105,66 @@ pub async fn run(
         maybe_sys_pubkey: host_agent_keys.local_sys_pubkey.clone(),
         nonce: TextNonce::new().to_string(),
     };
-    println!("inside init auth... 9");
-
     let payload_bytes = serde_json::to_vec(&payload)?;
-    println!("inside init auth... 10");
-
     let signature = host_agent_keys.host_sign(&payload_bytes)?;
-    println!("inside init auth... 11");
-    println!(" >>> signature >>> {}", signature);
-    println!(" >>> signature.as_bytes() >>> {:?}", signature.as_bytes());
-
     let mut headers = HeaderMap::new();
     headers.insert(
         HeaderName::from_static("X-Signature"),
         HeaderValue::from_str(&signature)?,
     );
-    
+
     println!("About to send out the {} message", "AUTH.validate");
     let response_msg = match auth_guard_client
-        .request_with_headers(
-            "AUTH.validate".to_string(),
-            headers,
-            payload_bytes.into()
-        )
-        .await {
-            Ok(msg) => msg,
-            Err(e) => {
-                log::error!("{:#?}", e);
-                if let RequestErrorKind::TimedOut = e.kind() {
-                    println!("inside init auth... 13");
-
-                    // TODO: Check to see if error is due to auth error.. if so then try to publish to Diagnostics Subject to ensure has correct permissions
-                    println!("got an AUTH RES ERROR: {:?}", e);
-        
-                    let unauthenticated_user_diagnostics_subject = format!(
-                        "DIAGNOSTICS.unauthenticated.{}",
-                        host_agent_keys.host_pubkey
-                    );
-                    let diganostics = HoloInventory::from_host();
-                    let payload_bytes = serde_json::to_vec(&diganostics)?;                  
-            
-                    if let Ok(_) = auth_guard_client
-                        .publish( unauthenticated_user_diagnostics_subject.to_string(), payload_bytes.into())
-                        .await {
-                            return Ok((host_agent_keys, auth_guard_client));
-                        }
+        .request_with_headers("AUTH.validate".to_string(), headers, payload_bytes.into())
+        .await
+    {
+        Ok(msg) => msg,
+        Err(e) => {
+            log::error!("{:#?}", e);
+            if let RequestErrorKind::TimedOut = e.kind() {
+                let unauthenticated_user_diagnostics_subject = format!(
+                    "DIAGNOSTICS.{}.unauthenticated",
+                    host_agent_keys.host_pubkey
+                );
+                let diganostics = HoloInventory::from_host();
+                let payload_bytes = serde_json::to_vec(&diganostics)?;
+                if (auth_guard_client
+                    .publish(
+                        unauthenticated_user_diagnostics_subject.to_string(),
+                        payload_bytes.into(),
+                    )
+                    .await)
+                    .is_ok()
+                {
+                    return Ok((host_agent_keys, auth_guard_client));
                 }
-                return Err(async_nats::Error::from(e));
             }
-        };
-
-    println!(
-        "got an AUTH response: {:?}",
-        std::str::from_utf8(&response_msg.payload).expect("failed to deserialize msg response")
-    );
+            return Err(async_nats::Error::from(e));
+        }
+    };
 
     println!(
         "got an AUTH response: {:#?}",
-        serde_json::from_slice::<AuthJWTResult>(&response_msg.payload).expect("failed to serde_json deserialize msg response")
+        serde_json::from_slice::<AuthJWTResult>(&response_msg.payload)
+            .expect("failed to serde_json deserialize msg response")
     );
 
     if let Ok(auth_response) = serde_json::from_slice::<AuthJWTResult>(&response_msg.payload) {
         match auth_response.status {
             AuthState::Authorized => {
-                println!("inside init auth... 13");
-
                 host_agent_keys = host_agent_keys
                     .save_host_creds(auth_response.host_jwt, auth_response.sys_jwt)
                     .await?;
-
-                if let Some(_reply) = response_msg.reply {
-                    // Publish the Awk resp to the Orchestrator... (JS)
-                }
             }
             _ => {
-                println!("inside init auth... 13");
                 log::error!("got unexpected AUTH State : {:?}", auth_response);
             }
         }
     };
 
-    println!("inside init auth... 14");
-    println!("host_agent_keys: {:#?}", host_agent_keys);
+    log::trace!(
+        "Host Agent Keys after authentication request: {:#?}",
+        host_agent_keys
+    );
     Ok((host_agent_keys, auth_guard_client))
 }
diff --git a/rust/clients/host_agent/src/hostd/workloads.rs b/rust/clients/host_agent/src/hostd/workloads.rs
index 3a44d5f..95e9b53 100644
--- a/rust/clients/host_agent/src/hostd/workloads.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -54,7 +54,10 @@ pub async fn run(
     // Spin up Nats Client and loaded in the Js Stream Service
     // Nats takes a moment to become responsive, so we try to connect in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let creds = host_creds_path.to_owned().map(Credentials::Path).ok_or_else(|| async_nats::Error::from("error"))?;
+    let creds = host_creds_path
+        .to_owned()
+        .map(Credentials::Path)
+        .ok_or_else(|| async_nats::Error::from("error"))?;
 
     let host_workload_client = tokio::select! {
         client = async {loop {
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index f967c06..193f96e 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -55,32 +55,20 @@ pub struct Keys {
 
 impl Keys {
     pub fn new() -> Result<Self> {
-        let host_key_path =
-            std::env::var("HOSTING_AGENT_HOST_NKEY_PATH").context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
-        println!("inside Keys new > host_key_path={}", host_key_path);
-    
+        let host_key_path = std::env::var("HOSTING_AGENT_HOST_NKEY_PATH")
+            .context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
         let host_kp = KeyPair::new_user();
-        println!("inside Keys new > host_kp={:#?}", host_kp);
-
         write_keypair_to_file(PathBuf::from_str(&host_key_path)?, host_kp.clone())?;
-
         let host_pk = host_kp.public_key();
-        println!("inside Keys new > host_pk={}", host_pk);
 
-        let sys_key_path =
-            std::env::var("HOSTING_AGENT_SYS_NKEY_PATH").context("Cannot read SYS_NKEY_PATH from env var")?;
-        println!("inside Keys new > sys_key_path={}", sys_key_path);
-            
+        let sys_key_path = std::env::var("HOSTING_AGENT_SYS_NKEY_PATH")
+            .context("Cannot read SYS_NKEY_PATH from env var")?;
         let local_sys_kp = KeyPair::new_user();
-        println!("inside Keys new > local_sys_kp={:#?}", local_sys_kp);
-
         write_keypair_to_file(PathBuf::from_str(&sys_key_path)?, local_sys_kp.clone())?;
-
         let local_sys_pk = local_sys_kp.public_key();
-        println!("inside Keys new > local_sys_pk={}", local_sys_pk);
 
-        let auth_guard_creds = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
-        println!("inside Keys new > auth_guard_creds={:#?}", auth_guard_creds);
+        let auth_guard_creds =
+            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
 
         Ok(Self {
             host_keypair: host_kp,
@@ -96,38 +84,42 @@ impl Keys {
         maybe_host_creds_path: &Option<PathBuf>,
         maybe_sys_creds_path: &Option<PathBuf>,
     ) -> Result<Self> {
-        println!("maybe_host_creds_path={:#?}, maybe_sys_creds_path={:#?}", maybe_host_creds_path, maybe_sys_creds_path);
-
-        let host_key_path: String =
-            std::env::var("HOSTING_AGENT_HOST_NKEY_PATH").context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
-        println!("host_key_path={:#?}", host_key_path); 
-
-        let host_keypair =
-            try_read_keypair_from_file(PathBuf::from_str(&host_key_path.clone())?)?
-                .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
-        println!("host_keypair={:#?}", host_keypair); 
+        println!(
+            "maybe_host_creds_path={:#?}, maybe_sys_creds_path={:#?}",
+            maybe_host_creds_path, maybe_sys_creds_path
+        );
+
+        let host_key_path: String = std::env::var("HOSTING_AGENT_HOST_NKEY_PATH")
+            .context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
+        println!("host_key_path={:#?}", host_key_path);
+
+        let host_keypair = try_read_keypair_from_file(PathBuf::from_str(&host_key_path.clone())?)?
+            .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
+        println!("host_keypair={:#?}", host_keypair);
         let host_pk = host_keypair.public_key();
 
-        let sys_key_path =
-            std::env::var("HOSTING_AGENT_SYS_NKEY_PATH").context("Cannot read HOSTING_AGENT_SYS_NKEY_PATH from env var")?;
-        println!("sys_key_path={:#?}", sys_key_path); 
+        let sys_key_path = std::env::var("HOSTING_AGENT_SYS_NKEY_PATH")
+            .context("Cannot read HOSTING_AGENT_SYS_NKEY_PATH from env var")?;
+        println!("sys_key_path={:#?}", sys_key_path);
 
         let host_user_name = format!("host_user_{}", host_pk);
-        let host_creds_path = maybe_host_creds_path
-            .to_owned()
-            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", &host_user_name)), Ok)?;
-        println!("host_creds_path={:#?}", host_creds_path); 
+        let host_creds_path = maybe_host_creds_path.to_owned().map_or_else(
+            || PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", &host_user_name)),
+            Ok,
+        )?;
+        println!("host_creds_path={:#?}", host_creds_path);
 
         let sys_user_name = format!("sys_user_{}", host_pk);
-        let sys_creds_path = maybe_sys_creds_path
-            .to_owned()
-            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name)), Ok)?;
-        println!("sys_creds_path={:#?}", sys_creds_path); 
+        let sys_creds_path = maybe_sys_creds_path.to_owned().map_or_else(
+            || PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name)),
+            Ok,
+        )?;
+        println!("sys_creds_path={:#?}", sys_creds_path);
 
         // Set auth_guard_creds as default:
         let auth_guard_creds =
-        PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
-        println!("auth_guard_creds={:#?}", auth_guard_creds); 
+            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
+        println!("auth_guard_creds={:#?}", auth_guard_creds);
 
         let keys = match try_read_keypair_from_file(PathBuf::from_str(&sys_key_path)?)? {
             Some(kp) => {
@@ -149,7 +141,7 @@ impl Keys {
             },
         };
 
-        println!("keys={:#?}", keys); 
+        println!("keys={:#?}", keys);
 
         Ok(keys.clone().add_creds_paths(host_creds_path, Some(sys_creds_path)).unwrap_or_else(move |e| {
             log::error!("Error: Cannot locate authenticated cred files. Defaulting to auth_guard_creds. Err={}",e);
@@ -158,8 +150,10 @@ impl Keys {
     }
 
     pub fn _add_local_sys(mut self, sys_key_path: Option<PathBuf>) -> Result<Self> {
-        let sys_key_path = sys_key_path
-            .map_or_else(|| PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")), Ok)?;
+        let sys_key_path = sys_key_path.map_or_else(
+            || PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "HPOS", "sys")),
+            Ok,
+        )?;
 
         let mut is_new_key = false;
 
@@ -235,12 +229,22 @@ impl Keys {
         host_sys_user_jwt: String,
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
-        let host_path = PathBuf::from_str(&format!("/{}/{}/{}", get_nsc_root_path(), "local_creds", "host.jwt"))?;
+        let host_path = PathBuf::from_str(&format!(
+            "/{}/{}/{}",
+            get_nsc_root_path(),
+            "local_creds",
+            "host.jwt"
+        ))?;
         println!("host_path ={:?}", host_path);
         write_to_file(host_path.clone(), host_user_jwt.as_bytes())?;
         println!("Wrote JWT to host file");
 
-        let sys_path = PathBuf::from_str(&format!("/{}/{}/{}", get_nsc_root_path(), "local_creds", "sys.jwt"))?;
+        let sys_path = PathBuf::from_str(&format!(
+            "/{}/{}/{}",
+            get_nsc_root_path(),
+            "local_creds",
+            "sys.jwt"
+        ))?;
         println!("sys_path ={:?}", sys_path);
         write_to_file(sys_path.clone(), host_sys_user_jwt.as_bytes())?;
         println!("Wrote JWT to sys file");
@@ -248,20 +252,14 @@ impl Keys {
         // Import host user jwt to local nsc resolver
         // TODO: Determine why the following works in cmd line, but doesn't seem to work when run in current program / run
         Command::new("nsc")
-            .args(&[
-                "import", "user",
-                "--file", &format!("{:?}", host_path)
-            ])
+            .args(["import", "user", "--file", &format!("{:?}", host_path)])
             .output()
             .context("Failed to add import new host user on hosting agent.")?;
         println!("imported host user");
 
         // Import sys user jwt to local nsc resolver
         Command::new("nsc")
-            .args(&[
-                "import", "user",
-                "--file", &format!("{:?}", sys_path)
-            ])
+            .args(["import", "user", "--file", &format!("{:?}", sys_path)])
             .output()
             .context("Failed to add import new sys user on hosting agent.")?;
         println!("imported sys user");
@@ -269,35 +267,46 @@ impl Keys {
         // Save user creds and sys creds local to hosting agent
         let host_user_name = format!("host_user_{}", self.host_pubkey);
         let host_creds_path =
-            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD",  &host_user_name))?;
+            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", &host_user_name))?;
         Command::new("nsc")
-            .args(&[
-                "generate", "creds",
-                "--name", &host_user_name,
-                "--account", "WORKLOAD",
-                "--output-file", &host_creds_path.to_string_lossy(),
+            .args([
+                "generate",
+                "creds",
+                "--name",
+                &host_user_name,
+                "--account",
+                "WORKLOAD",
+                "--output-file",
+                &host_creds_path.to_string_lossy(),
             ])
             .output()
             .context("Failed to add new operator signing key on hosting agent")?;
-        println!("Generated host user creds. creds_path={:?}", host_creds_path);
+        println!(
+            "Generated host user creds. creds_path={:?}",
+            host_creds_path
+        );
 
         let mut sys_creds_file_name = None;
-        if let Some(_) = self.local_sys_pubkey.as_ref() {
+        if self.local_sys_pubkey.as_ref().is_some() {
             let sys_user_name = format!("sys_user_{}", self.host_pubkey);
             let path = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name))?;
             Command::new("nsc")
-            .arg(format!(
-                "generate creds --name {} --account {}",
-                sys_user_name, "SYS"
-            ))
-            .args(&[
-                "generate", "creds",
-                "--name", &sys_user_name,
-                "--account", "SYS",
-                "--output-file", &path.to_string_lossy(),
-            ])
-            .output()
-            .context("Failed to add new operator signing key on hosting agent")?;
+                .arg(format!(
+                    "generate creds --name {} --account {}",
+                    sys_user_name, "SYS"
+                ))
+                .args([
+                    "generate",
+                    "creds",
+                    "--name",
+                    &sys_user_name,
+                    "--account",
+                    "SYS",
+                    "--output-file",
+                    &path.to_string_lossy(),
+                ])
+                .output()
+                .context("Failed to add new operator signing key on hosting agent")?;
             println!("Generated sys user creds. creds_path={:?}", path);
             sys_creds_file_name = Some(path);
         }
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index d7a5eaa..3a0d614 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -63,16 +63,17 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
             async_nats::Error::from(e)
         })
     })?;
-    println!("inside host main auth... 1");
+    println!("Host Agent Keys={:#?}", host_agent_keys);
 
     // If user cred file is for the auth_guard user, run loop to authenticate host & hoster...
     if let keys::AuthCredType::Guard(_) = host_agent_keys.creds {
-        println!("inside host main auth... 2a");
         host_agent_keys = run_auth_loop(host_agent_keys).await?;
     }
 
-    println!("inside host main auth... 2b");
-    println!("Successfully AUTH'D and created new agent keys: {:#?}", host_agent_keys);
+    println!(
+        "Successfully AUTH'D and created new agent keys: {:#?}",
+        host_agent_keys
+    );
 
     // // Once authenticated, start leaf server and run workload api calls.
     // let _ = hostd::gen_leaf_server::run(
@@ -120,12 +121,15 @@ async fn run_auth_loop(mut keys: keys::Keys) -> Result<keys::Keys, async_nats::E
 
         while max_time_interval > now.signed_duration_since(start) {
             let unauthenticated_user_diagnostics_subject =
-                format!("DIAGNOSTICS.unauthenticated.{}", keys.host_pubkey);
+                format!("DIAGNOSTICS.{}.unauthenticated", keys.host_pubkey);
             let diganostics = HoloInventory::from_host();
             let payload_bytes = serde_json::to_vec(&diganostics)?;
-            
+
             if let Err(e) = auth_guard_client
-            .publish(unauthenticated_user_diagnostics_subject, payload_bytes.into())
+                .publish(
+                    unauthenticated_user_diagnostics_subject,
+                    payload_bytes.into(),
+                )
                 .await
             {
                 log::error!("Encountered error when sending diganostics. Err={:#?}", e);
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 3243929..3e8b39e 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -18,36 +18,33 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
-use async_nats::service::ServiceExt;
 use anyhow::{anyhow, Context, Result};
-use futures::StreamExt;
+use async_nats::service::ServiceExt;
 use authentication::{
     self,
     types::{self, AuthErrorPayload},
     AuthServiceApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION,
 };
+use futures::StreamExt;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use nkeys::KeyPair;
 use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
-use std::{sync::Arc, time::Duration};
 use std::str::FromStr;
+use std::{sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    nats_js_client::{get_nats_url, get_nats_creds_by_nsc},
-    js_stream_service::CreateResponse
+    js_stream_service::CreateResponse,
+    nats_js_client::{get_nats_creds_by_nsc, get_nats_url},
 };
 
 pub const ORCHESTRATOR_AUTH_CLIENT_NAME: &str = "Orchestrator Auth Manager";
 pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX_ORCHESTRATOR";
 
 pub async fn run() -> Result<(), async_nats::Error> {
-    let admin_account_creds_path = PathBuf::from_str(&get_nats_creds_by_nsc(
-        "HOLO",
-        "AUTH",
-        "auth",
-    ))?;
+    let admin_account_creds_path =
+        PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth"))?;
     println!(
         " >>>> admin_account_creds_path: {:#?} ",
         admin_account_creds_path
@@ -56,26 +53,20 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Root Keypair associated with AUTH account
     let root_account_key_path = std::env::var("ORCHESTRATOR_ROOT_AUTH_NKEY_PATH")
         .context("Cannot read ORCHESTRATOR_ROOT_AUTH_NKEY_PATH from env var")?;
-    println!(">>>>>>>>> root account key_path: {:?}", root_account_key_path);
-
     let root_account_keypair = Arc::new(
-        try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?.ok_or_else(
-            || {
+        try_read_keypair_from_file(PathBuf::from_str(&root_account_key_path.clone())?)?
+            .ok_or_else(|| {
                 anyhow!(
                     "Root AUTH Account keypair not found at path {:?}",
                     root_account_key_path
                 )
-            },
-        )?,
+            })?,
     );
     let root_account_pubkey = root_account_keypair.public_key().clone();
-    println!(">>>>>>>>> root account pubkey: {:?}", root_account_pubkey);
 
     // AUTH Account Signing Keypair associated with the `auth` user
     let signing_account_key_path = std::env::var("ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH")
         .context("Cannot read ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH from env var")?;
-    println!(">>>>>>>>> signing account key_path: {:?}", signing_account_key_path);
-
     let signing_account_keypair = Arc::new(
         try_read_keypair_from_file(PathBuf::from_str(&signing_account_key_path.clone())?)?
             .ok_or_else(|| {
@@ -86,7 +77,6 @@ pub async fn run() -> Result<(), async_nats::Error> {
             })?,
     );
     let signing_account_pubkey = signing_account_keypair.public_key().clone();
-    println!(">>>>>>>>> signing_account pubkey: {:?}", signing_account_pubkey);
 
     // ==================== Setup NATS ====================
     let nats_url = get_nats_url();
@@ -114,10 +104,10 @@ pub async fn run() -> Result<(), async_nats::Error> {
                 }
             }} => client?,
         _ = {
-            log::debug!("will time out waiting for NATS after {nats_connect_timeout_secs:?}");
+            log::debug!("Will time out waiting for NATS after {nats_connect_timeout_secs:?}...");
             tokio::time::sleep(tokio::time::Duration::from_secs(nats_connect_timeout_secs))
          } => {
-            return Err(format!("timed out waiting for NATS on {nats_url}").into());
+            return Err(format!("Timed out waiting for NATS on {nats_url}").into());
         }
     };
 
@@ -126,11 +116,11 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let mongo_uri = get_mongodb_url();
     let client_options = ClientOptions::parse(mongo_uri).await?;
     let db_client = MongoDBClient::with_options(client_options)?;
-    
+
     // ==================== Setup API & Register Endpoints ====================
     // Generate the Auth API with access to db
     let auth_api = AuthServiceApi::new(&db_client).await?;
-    let auth_api_clone  = auth_api.clone();
+    let auth_api_clone = auth_api.clone();
 
     // Register Auth Service for Orchestrator and spawn listener for processing
     let auth_service = orchestrator_auth_client
@@ -138,7 +128,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         .description(AUTH_SRV_DESC)
         .start(AUTH_SRV_NAME, AUTH_SRV_VERSION)
         .await?;
-    
+
     // Auth Callout Service
     let sys_user_group = auth_service.group("$SYS").group("REQ").group("USER");
     let mut auth_callout = sys_user_group.endpoint("AUTH").await?;
@@ -147,117 +137,135 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     tokio::spawn(async move {
         while let Some(request) = auth_callout.next().await {
-                let signing_account_kp = Arc::clone(&signing_account_keypair.clone());
-                let signing_account_pk = signing_account_pubkey.clone();
-                let root_account_kp = Arc::clone(&root_account_keypair.clone());
-                let root_account_pk = root_account_pubkey.clone();
+            let signing_account_kp = Arc::clone(&signing_account_keypair.clone());
+            let signing_account_pk = signing_account_pubkey.clone();
+            let root_account_kp = Arc::clone(&root_account_keypair.clone());
+            let root_account_pk = root_account_pubkey.clone();
 
-                let maybe_reply = request.message.reply.clone();
-                match auth_api_clone.handle_auth_callout(
+            let maybe_reply = request.message.reply.clone();
+            match auth_api_clone
+                .handle_auth_callout(
                     Arc::new(request.message),
                     signing_account_kp,
                     signing_account_pk,
                     root_account_kp,
                     root_account_pk,
                 )
-                .await {
-                    Ok(r) => {
-                        let res_bytes = r.get_response();
-                        if let Some(reply_subject) = maybe_reply {
-                            let _ = orchestrator_auth_client_clone.publish(reply_subject, res_bytes.into()).await.map_err(|e| {
-                            log::error!(
-                                "{}Failed to send success response. Res={:?} Err={:?}",
-                                "NATS-SERVICE-LOG::AUTH::",
-                                r,
-                                e
-                            );
-                        });
-                        }
-                    },
-                    Err(e) => {
-                        let mut err_payload = AuthErrorPayload {
-                            service_info: auth_service_info.clone(),
-                            group: "$SYS.REQ.USER".to_string(),
-                            endpoint: "AUTH".to_string(),
-                            error: format!("{}",e),
-                        };
-    
+                .await
+            {
+                Ok(r) => {
+                    let res_bytes = r.get_response();
+                    if let Some(reply_subject) = maybe_reply {
+                        let _ = orchestrator_auth_client_clone
+                            .publish(reply_subject, res_bytes)
+                            .await
+                            .map_err(|e| {
+                                log::error!(
+                                    "{}Failed to send success response. Res={:?} Err={:?}",
+                                    "NATS-SERVICE-LOG::AUTH::",
+                                    r,
+                                    e
+                                );
+                            });
+                    }
+                }
+                Err(e) => {
+                    let mut err_payload = AuthErrorPayload {
+                        service_info: auth_service_info.clone(),
+                        group: "$SYS.REQ.USER".to_string(),
+                        endpoint: "AUTH".to_string(),
+                        error: format!("{}", e),
+                    };
+
+                    log::error!(
+                        "{}Failed to handle the endpoint handler. Err={:?}",
+                        "NATS-SERVICE-LOG::AUTH::",
+                        err_payload
+                    );
+
+                    let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                        err_payload.error = e.to_string();
                         log::error!(
-                            "{}Failed to handle the endpoint handler. Err={:?}",
+                            "{}Failed to deserialize error response. Err={:?}",
                             "NATS-SERVICE-LOG::AUTH::",
                             err_payload
                         );
-    
-                        let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                        vec![]
+                    });
+
+                    let _ = orchestrator_auth_client_clone
+                        .publish(
+                            format!("{}.ERROR", ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX),
+                            err_response.into(),
+                        )
+                        .await
+                        .map_err(|e| {
                             err_payload.error = e.to_string();
-                            log::error!(
-                                "{}Failed to deserialize error response. Err={:?}",
-                                "NATS-SERVICE-LOG::AUTH::",
-                                err_payload
-                            );
-                            vec![]
-                        });
-    
-                        let _ = orchestrator_auth_client_clone.publish(format!("{}.ERROR", ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX), err_response.into()).await.map_err(|e| {
                             log::error!(
                                 "{}Failed to send error response. Err={:?}",
                                 "NATS-SERVICE-LOG::AUTH::",
                                 err_payload
                             );
                         });
-                    }
                 }
             }
-        });
+        }
+    });
 
-        // Auth Validation Service
-        let v1_auth_group = auth_service.group(AUTH_SRV_SUBJ); // .group("V1")
-        let mut auth_validation = v1_auth_group.endpoint(types::AUTHORIZE_SUBJECT).await?;
-        let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
+    // Auth Validation Service
+    let v1_auth_group = auth_service.group(AUTH_SRV_SUBJ); // .group("V1")
+    let mut auth_validation = v1_auth_group.endpoint(types::AUTHORIZE_SUBJECT).await?;
+    let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
 
-        tokio::spawn(async move {
-            while let Some(request) = auth_validation.next().await {
-                let maybe_reply = request.message.reply.clone();
-                match auth_api.handle_handshake_request(
-                    Arc::new(request.message)
-                )
-                .await {
-                    Ok(r) => {
-                        let res_bytes = r.get_response();
-                        if let Some(reply_subject) = maybe_reply {
-                            let _ = orchestrator_auth_client_clone.publish(reply_subject, res_bytes.into()).await.map_err(|e| {
-                            log::error!(
-                                "{}Failed to send success response. Res={:?} Err={:?}",
-                                "NATS-SERVICE-LOG::AUTH::",
-                                r,
-                                e
-                            );
-                        });
-                        }
-                    },
-                    Err(e) => {
-                        let auth_service_info = auth_service.info().await;
-                        let mut err_payload = AuthErrorPayload {
-                            service_info: auth_service_info,
-                            group: "AUTH".to_string(),
-                            endpoint: types::AUTHORIZE_SUBJECT.to_string(),
-                            error: format!("{}",e),
-                        };
+    tokio::spawn(async move {
+        while let Some(request) = auth_validation.next().await {
+            let maybe_reply = request.message.reply.clone();
+            match auth_api
+                .handle_handshake_request(Arc::new(request.message))
+                .await
+            {
+                Ok(r) => {
+                    let res_bytes = r.get_response();
+                    if let Some(reply_subject) = maybe_reply {
+                        let _ = orchestrator_auth_client_clone
+                            .publish(reply_subject, res_bytes)
+                            .await
+                            .map_err(|e| {
+                                log::error!(
+                                    "{}Failed to send success response. Res={:?} Err={:?}",
+                                    "NATS-SERVICE-LOG::AUTH::",
+                                    r,
+                                    e
+                                );
+                            });
+                    }
+                }
+                Err(e) => {
+                    let auth_service_info = auth_service.info().await;
+                    let mut err_payload = AuthErrorPayload {
+                        service_info: auth_service_info,
+                        group: "AUTH".to_string(),
+                        endpoint: types::AUTHORIZE_SUBJECT.to_string(),
+                        error: format!("{}", e),
+                    };
+                    log::error!(
+                        "{}Failed to handle the endpoint handler. Err={:?}",
+                        "NATS-SERVICE-LOG::AUTH::",
+                        err_payload
+                    );
+                    let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
+                        err_payload.error = e.to_string();
                         log::error!(
-                            "{}Failed to handle the endpoint handler. Err={:?}",
+                            "{}Failed to deserialize error response. Err={:?}",
                             "NATS-SERVICE-LOG::AUTH::",
                             err_payload
                         );
-                        let err_response = serde_json::to_vec(&err_payload).unwrap_or_else(|e| {
-                            err_payload.error = e.to_string();
-                            log::error!(
-                                "{}Failed to deserialize error response. Err={:?}",
-                                "NATS-SERVICE-LOG::AUTH::",
-                                err_payload
-                            );
-                            vec![]
-                        });
-                        let _ = orchestrator_auth_client_clone.publish("AUTH.ERROR", err_response.into()).await.map_err(|e| {
+                        vec![]
+                    });
+                    let _ = orchestrator_auth_client_clone
+                        .publish("AUTH.ERROR", err_response.into())
+                        .await
+                        .map_err(|e| {
                             err_payload.error = e.to_string();
                             log::error!(
                                 "{}Failed to send error response. Err={:?}",
@@ -265,10 +273,10 @@ pub async fn run() -> Result<(), async_nats::Error> {
                                 err_payload
                             );
                         });
-                    }
                 }
             }
-        });
+        }
+    });
 
     println!("Orchestrator Auth Service is running. Waiting for requests...");
 
@@ -276,11 +284,11 @@ pub async fn run() -> Result<(), async_nats::Error> {
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    println!("closing orchestrator auth service...");
+    println!("Closing orchestrator auth service...");
 
     // Close client and drain internal buffer before exiting to make sure all messages are sent
     orchestrator_auth_client.drain().await?;
-    println!("closed orchestrator auth service");
+    println!("Closed orchestrator auth service");
 
     Ok(())
 }
@@ -314,4 +322,4 @@ fn try_read_from_file(file_path: PathBuf) -> Result<Option<String>> {
             Ok(None)
         }
     }
-}
\ No newline at end of file
+}
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 236b9c0..b9f9194 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -13,8 +13,8 @@ use anyhow::{Context, Result};
 use async_nats::jetstream::ErrorCode;
 use async_nats::HeaderValue;
 use async_nats::{AuthError, Message};
-use data_encoding::BASE64URL_NOPAD;
 use core::option::Option::None;
+use data_encoding::BASE64URL_NOPAD;
 use mongodb::Client as MongoDBClient;
 use nkeys::KeyPair;
 use serde::{Deserialize, Serialize};
@@ -27,7 +27,9 @@ use util_libs::db::{
     mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
     schemas::{self, Host, Hoster, Role, RoleInfo, User},
 };
-use util_libs::nats_js_client::{get_nsc_root_path, AsyncEndpointHandler, JsServiceResponse, ServiceError};
+use util_libs::nats_js_client::{
+    get_nsc_root_path, AsyncEndpointHandler, JsServiceResponse, ServiceError,
+};
 use utils::handle_internal_err;
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
@@ -61,49 +63,43 @@ impl AuthServiceApi {
         auth_root_account_keypair: Arc<KeyPair>,
         auth_root_account_pubkey: String,
     ) -> Result<AuthApiResult, ServiceError> {
+        log::info!("Incoming message for '$SYS.REQ.USER.AUTH' : {:#?}", msg);
+
         // 1. Verify expected data was received
         let auth_request_token = String::from_utf8_lossy(&msg.payload).to_string();
-        println!("auth_request_token  : {:?}", auth_request_token);
-
-        let auth_request_claim =
-            utils::decode_jwt::<types::NatsAuthorizationRequestClaim>(&auth_request_token)
-                .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
-        println!(
-            "\nauth REQUEST - main claim : {}",
-            serde_json::to_string_pretty(&auth_request_claim).unwrap()
-        );
+        let auth_request_claim = utils::decode_jwt::<types::NatsAuthorizationRequestClaim>(
+            &auth_request_token,
+            &auth_signing_account_pubkey,
+        )
+        .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
 
         let auth_request_user_claim = utils::decode_jwt::<types::UserClaim>(
             &auth_request_claim.auth_request.connect_opts.user_jwt,
+            &auth_signing_account_pubkey,
         )
         .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
-        println!(
-            "\nauth REQUEST - user claim : {}",
-            serde_json::to_string_pretty(&auth_request_user_claim).unwrap()
-        );
 
-        let user_data: types::AuthGuardPayload = utils::base64_to_data::<types::AuthGuardPayload>(
+        if auth_request_user_claim.generic_claim_data.issuer != auth_signing_account_pubkey {
+            let e = "Error: Failed to validate issuer for auth user.";
+            log::error!("{} Subject='{}'.", e, msg.subject);
+            return Err(ServiceError::Authentication(AuthError::new(e)));
+        };
+
+        // 2. Validate Host signature, returning validation error if not successful
+        let user_data = utils::base64_to_data::<types::AuthGuardPayload>(
             &auth_request_claim.auth_request.connect_opts.user_auth_token,
         )
         .map_err(|e| ServiceError::Authentication(AuthError::new(e)))?;
-        println!("user_data TO VALIDATE  : {:#?}", user_data);
-
-        // TODO:
-        // 2. Validate Host signature, returning validation error if not successful
         let host_pubkey = user_data.host_pubkey.as_ref();
         let host_signature = user_data.get_host_signature();
-        let decoded_sig = BASE64URL_NOPAD.decode(&host_signature)
+        let decoded_sig = BASE64URL_NOPAD
+            .decode(&host_signature)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        println!("host_signature: {:?}", host_signature);
-        println!("decoded_sig: {:?}", decoded_sig);
-
         let user_verifying_keypair = KeyPair::from_public_key(host_pubkey)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
         let payload_no_sig = &user_data.clone().without_signature();
-        println!("PAYLOAD WITHOUT SIG: {:#?}", payload_no_sig);
         let raw_payload = serde_json::to_vec(payload_no_sig)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        println!("PAYLOAD WITHOUT SIG AS BYTES: {:?}", raw_payload);
 
         if let Err(e) = user_verifying_keypair.verify(raw_payload.as_ref(), &decoded_sig) {
             log::error!(
@@ -116,7 +112,7 @@ impl AuthServiceApi {
 
         // 3. If provided, authenticate the Hoster pubkey and email and assign full permissions if successful
         let is_hoster_valid = if user_data.email.is_some() && user_data.hoster_hc_pubkey.is_some() {
-            true 
+            true
             // TODO:
             // let hoster_hc_pubkey = user_data.hoster_hc_pubkey.unwrap(); // unwrap is safe here as checked above
             // let hoster_email = user_data.email.unwrap(); // unwrap is safe here as checked above
@@ -230,14 +226,8 @@ impl AuthServiceApi {
         let permissions = if is_hoster_valid {
             // If successful, assign personalized inbox and auth permissions
             let user_unique_auth_subject = &format!("AUTH.{}.>", host_pubkey);
-            println!(">>> user_unique_auth_subject : {user_unique_auth_subject}");
-
             let user_unique_inbox = &format!("_AUTH_INBOX_{}.>", host_pubkey);
-            println!(">>> user_unique_inbox : {user_unique_inbox}");
-
-            let authenticated_user_diagnostics_subject =
-                &format!("DIAGNOSTICS.{}.>", host_pubkey);
-            println!(">>> authenticated_user_diagnostics_subject : {authenticated_user_diagnostics_subject}");
+            let authenticated_user_diagnostics_subject = &format!("DIAGNOSTICS.{}.>", host_pubkey);
 
             types::Permissions {
                 publish: types::PermissionLimits {
@@ -289,8 +279,6 @@ impl AuthServiceApi {
         let token = utils::encode_jwt(&claim_str, &auth_root_account_keypair)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        println!("\n\n\n\nencoded_jwt: {:#?}", token);
-
         Ok(types::AuthApiResult {
             result: types::AuthResult::Callout(token),
             maybe_response_tags: None,
@@ -301,19 +289,18 @@ impl AuthServiceApi {
         &self,
         msg: Arc<Message>,
     ) -> Result<AuthApiResult, ServiceError> {
-        log::warn!("INCOMING Message for 'AUTH.validate' : {:?}", msg);
-        println!("msg={:#?}", msg);
+        log::info!("Incoming message for 'AUTH.validate' : {:#?}", msg);
 
         // 1. Verify expected data was received
         let signature: &[u8] = match &msg.headers {
             Some(h) => {
                 println!("header={:?}", h);
                 let r = HeaderValue::as_str(h.get("X-Signature").ok_or_else(|| {
-                    log::error!("Error: Missing x-signature header. Subject='AUTH.authorize'");
+                    log::error!("Error: Missing X-Signature header. Subject='AUTH.authorize'");
                     ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
                 })?);
                 r.as_bytes()
-            },
+            }
             None => {
                 log::error!("Error: Missing message headers. Subject='AUTH.authorize'");
                 return Err(ServiceError::Request(format!(
@@ -330,7 +317,8 @@ impl AuthServiceApi {
         } = Self::convert_msg_to_type::<types::AuthJWTPayload>(msg.clone())?;
 
         // 2. Validate signature
-        let decoded_signature = BASE64URL_NOPAD.decode(signature)
+        let decoded_signature = BASE64URL_NOPAD
+            .decode(signature)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
         let user_verifying_keypair = KeyPair::from_public_key(&host_pubkey)
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
@@ -338,77 +326,94 @@ impl AuthServiceApi {
         if let Err(e) = user_verifying_keypair.verify(msg.payload.as_ref(), &decoded_signature) {
             log::error!(
                 "Error: Failed to validate Signature. Subject='{}'. Err={}",
-                msg.subject,  
+                msg.subject,
                 e
             );
-            return Err(ServiceError::Authentication(AuthError::new(format!("{:?}", e))));    
+            return Err(ServiceError::Authentication(AuthError::new(format!(
+                "{:?}",
+                e
+            ))));
         };
 
         // 3. Add User keys to nsc resolver (and automatically create account-signed refernce to user key)
         match Command::new("nsc")
-            .args(&[
-                "add", "user",
-                "-a", "WORKLOAD",
-                "-n", &format!("host_user_{}", host_pubkey),
-                "-k", &host_pubkey,
-                "-K", WORKLOAD_SK_ROLE,
-                "--tag", &format!("pubkey:{}", host_pubkey),
+            .args([
+                "add",
+                "user",
+                "-a",
+                "WORKLOAD",
+                "-n",
+                &format!("host_user_{}", host_pubkey),
+                "-k",
+                &host_pubkey,
+                "-K",
+                WORKLOAD_SK_ROLE,
+                "--tag",
+                &format!("pubkey:{}", host_pubkey),
             ])
             .output()
             .context("Failed to add host user with provided keys")
-            .map_err(|e| ServiceError::Internal(e.to_string())) {
-                Ok(r) => {
-                    println!("'AUTH.validate >>> add user -a WORKLOAD -n host_user_<pubkey> ...' : {:?}", r);
-                    let stderr = String::from_utf8_lossy(&r.stderr);
-                    if !r.stderr.is_empty() && !stderr.contains("already exists") {
-                        return Err(ServiceError::Internal(stderr.to_string()));
-                    }
-                },
-                Err(e) => {
-                    println!("'AUTH.validate >>> ERROR: add user -a WORKLOAD -n host_user_<pubkey> ...' : {:?}", e);
-                    return Err(e);
+            .map_err(|e| ServiceError::Internal(e.to_string()))
+        {
+            Ok(r) => {
+                let stderr = String::from_utf8_lossy(&r.stderr);
+                if !r.stderr.is_empty() && !stderr.contains("already exists") {
+                    return Err(ServiceError::Internal(stderr.to_string()));
                 }
-            };
-        println!("\nadded host user");
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        };
 
         if let Some(sys_pubkey) = maybe_sys_pubkey.clone() {
             println!("inside handle_handshake_request... 5 sys -- inside");
 
             match Command::new("nsc")
-                .args(&[
-                    "add", "user",
-                    "-a", "SYS",
-                    "-n", &format!("sys_user_{}", host_pubkey),
-                    "-k", &sys_pubkey,
+                .args([
+                    "add",
+                    "user",
+                    "-a",
+                    "SYS",
+                    "-n",
+                    &format!("sys_user_{}", host_pubkey),
+                    "-k",
+                    &sys_pubkey,
                 ])
                 .output()
                 .context("Failed to add host sys user with provided keys")
-                .map_err(|e| ServiceError::Internal(e.to_string())) {
-                    Ok(r) => {
-                        println!("'AUTH.validate >>> add user -a SYS -n sys_user_<pubkey> ...' : {:?}", r);
-                        let stderr = String::from_utf8_lossy(&r.stderr);
-                        if !r.stderr.is_empty() && !stderr.contains("already exists") {
-                            return Err(ServiceError::Internal(stderr.to_string()));
-                        }
-                    },
-                    Err(e) => {   
-                        println!("'AUTH.validate >>> ERROR: add user -a SYS -n sys_user_<pubkey> ...' : {:?}", e);
-                        return Err(e);
+                .map_err(|e| ServiceError::Internal(e.to_string()))
+            {
+                Ok(r) => {
+                    let stderr = String::from_utf8_lossy(&r.stderr);
+                    if !r.stderr.is_empty() && !stderr.contains("already exists") {
+                        return Err(ServiceError::Internal(stderr.to_string()));
                     }
-                };
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            };
         }
-        println!("\nadded sys user for provided host");
 
         // 4. Create User JWT files (automatically signed with respective account key)
-        let host_jwt = std::fs::read_to_string(format!("{}/stores/HOLO/accounts/WORKLOAD/users/host_user_{}.jwt", get_nsc_root_path(), host_pubkey))
-            .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        println!("'AUTH.validate >>> host_jwt' : {:?}", host_jwt);
+        let host_jwt = std::fs::read_to_string(format!(
+            "{}/stores/HOLO/accounts/WORKLOAD/users/host_user_{}.jwt",
+            get_nsc_root_path(),
+            host_pubkey
+        ))
+        .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
-        let sys_jwt = if let Some(_) = maybe_sys_pubkey {
-            std::fs::read_to_string(format!("{}/stores/HOLO/accounts/SYS/users/sys_user_{}.jwt", get_nsc_root_path(), host_pubkey))
+        let sys_jwt = if maybe_sys_pubkey.is_some() {
+            std::fs::read_to_string(format!(
+                "{}/stores/HOLO/accounts/SYS/users/sys_user_{}.jwt",
+                get_nsc_root_path(),
+                host_pubkey
+            ))
             .map_err(|e| ServiceError::Internal(e.to_string()))?
-        } else { String::new() };
-        println!("'AUTH.validate >>> sys_jwt' : {:?}", sys_jwt);
+        } else {
+            String::new()
+        };
 
         // 5. PUSH the auth updates to resolver programmtically by sending jwts to `SYS.REQ.ACCOUNT.PUSH` subject
         Command::new("nsc")
@@ -420,10 +425,9 @@ impl AuthServiceApi {
 
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
-        println!("inside handle_handshake_request... 13");
 
         // 6. Form the result and return
-        let r = AuthApiResult {
+        Ok(AuthApiResult {
             result: types::AuthResult::Authorization(types::AuthJWTResult {
                 host_pubkey: host_pubkey.clone(),
                 status: types::AuthState::Authorized,
@@ -431,10 +435,7 @@ impl AuthServiceApi {
                 sys_jwt,
             }),
             maybe_response_tags: Some(tag_map),
-        };
-        println!("inside handle_handshake_request... RESULT={:?}", r);
-
-        Ok(r)
+        })
     }
 
     // Helper function to initialize mongodb collections
@@ -479,12 +480,3 @@ impl AuthServiceApi {
         })
     }
 }
-
-
-// example:
-// [1] Subject: AUTH.UDS2A7I4BCECURHE64C52ORK6IDSOSE4ILZ7RJM4IO4EAYF33B67EWEF.> Received: 2025-02-05T21:19:52-06:00
-//   X-Signature: [80, 71, 109, 80, 76, 99, 48, 122, 56, 113, 112, 48, 101, 95, 57, 107, 45, 105, 78, 75, 72, 67, 66, 97, 120, 117, 102, 110, 100, 72, 110, 53, 101, 74, 82, 77, 52, 121, 65, 66, 85, 53, 48, 109, 101, 51, 107, 54, 50, 65, 89, 81, 85, 51, 52, 50, 80, 81, 74, 49, 119, 90, 118, 104, 112, 100, 68, 109, 99, 105, 49, 69, 101, 85, 116, 67, 48, 118, 68, 89, 74, 86, 56, 86, 65, 103]
-// {"host_pubkey":"UDS2A7I4BCECURHE64C52ORK6IDSOSE4ILZ7RJM4IO4EAYF33B67EWEF","maybe_sys_pubkey":"UACJZQOQK2Y2JFQVNV4CJORAEZGV3GYTCK7UOSCLNEZJRKMOW4ATUZZG","nonce":"zq7bDlgqpGcAAAAA3ItWHoUsldKNZg7/"}
-
-// - subject: _INBOX.Uwce1Uabie65ojhlucmyhB.vy24bmby
-// - subject: _INBOX.5RgE68PiQieODvqbf4Yn1s.6f14XeRJ
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 8bcc2fa..3f6df5e 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -2,7 +2,6 @@ use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
-use data_encoding::BASE64URL_NOPAD;
 
 pub const AUTH_CALLOUT_SUBJECT: &str = "$SYS.REQ.USER.AUTH";
 pub const AUTHORIZE_SUBJECT: &str = "validate";
@@ -45,7 +44,7 @@ pub struct AuthJWTResult {
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthResult {
-    Callout(String), // stringifiedAuthResponseClaim
+    Callout(String), // stringified `AuthResponseClaim`
     Authorization(AuthJWTResult),
 }
 
diff --git a/rust/services/authentication/src/utils.rs b/rust/services/authentication/src/utils.rs
index f2f6759..66f22ba 100644
--- a/rust/services/authentication/src/utils.rs
+++ b/rust/services/authentication/src/utils.rs
@@ -1,8 +1,8 @@
 use super::types;
 use anyhow::{anyhow, Result};
-use data_encoding::{BASE32HEX_NOPAD, BASE64URL_NOPAD};
-use base32::Alphabet;
 use base32::decode as base32Decode;
+use base32::Alphabet;
+use data_encoding::{BASE32HEX_NOPAD, BASE64URL_NOPAD};
 use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
 use nkeys::KeyPair;
 use serde::Deserialize;
@@ -52,28 +52,20 @@ pub fn hash_claim(claims_str: &str) -> Vec<u8> {
 pub fn encode_jwt(claims_str: &str, signing_kp: &Arc<KeyPair>) -> Result<String> {
     const JWT_HEADER: &str = r#"{"typ":"JWT","alg":"ed25519-nkey"}"#;
     let b64_header: String = BASE64URL_NOPAD.encode(JWT_HEADER.as_bytes());
-    println!("encoded b64 header: {:?}", b64_header);
     let b64_body = BASE64URL_NOPAD.encode(claims_str.as_bytes());
-    println!("encoded header: {:?}", b64_body);
-
     let jwt_half = format!("{b64_header}.{b64_body}");
     let sig = signing_kp.sign(jwt_half.as_bytes())?;
     let b64_sig = BASE64URL_NOPAD.encode(&sig);
-
-    let token = format!("{jwt_half}.{b64_sig}");
-    Ok(token)
+    Ok(format!("{jwt_half}.{b64_sig}"))
 }
 
 /// Convert token into the
-pub fn decode_jwt<T>(token: &str) -> Result<T>
+pub fn decode_jwt<T>(token: &str, auth_signing_account_pubkey: &str) -> Result<T>
 where
     T: for<'de> Deserialize<'de> + std::fmt::Debug,
 {
     // Decode and replace custom `ed25519-nkey` to `EdDSA`
     let parts: Vec<&str> = token.split('.').collect();
-    println!("parts: {:?}", parts);
-    println!("parts.len() : {:?}", parts.len());
-
     if parts.len() != 3 {
         return Err(anyhow!("Invalid JWT format"));
     }
@@ -81,7 +73,6 @@ where
     // Decode base64 JWT header and fix the algorithm field
     let header_json = BASE64URL_NOPAD.decode(parts[0].as_bytes())?;
     let mut header: Value = serde_json::from_slice(&header_json).expect("failed to create header");
-    println!("header: {:?}", header);
 
     // Manually fix the algorithm name
     if let Some(alg) = header.get_mut("alg") {
@@ -89,11 +80,7 @@ where
             *alg = serde_json::Value::String("EdDSA".to_string());
         }
     }
-    println!("after header: {:?}", header);
-
     let modified_header = BASE64URL_NOPAD.encode(&serde_json::to_vec(&header)?);
-    println!("modified_header: {:?}", modified_header);
-
     let part_1_json = BASE64URL_NOPAD.decode(parts[1].as_bytes())?;
     let mut part_1: Value = serde_json::from_slice(&part_1_json)?;
     if part_1.get("exp").is_none() {
@@ -109,34 +96,24 @@ where
         part_1 = serde_json::to_value(b)?;
     }
     let modified_part_1 = BASE64URL_NOPAD.encode(&serde_json::to_vec(&part_1)?);
-
     let modified_token = format!("{}.{}.{}", modified_header, modified_part_1, parts[2]);
-    println!("modified_token: {:?}", modified_token);
-
-    let account_kp =
-        KeyPair::from_public_key("ABYGJO6B2OJTXL7DLL7EGR45RQ4I2CKM4D5XYYUSUBZJ7HJJF67E54VC")?;
-
-    let public_key_b32 = account_kp.public_key();
-    println!("Public Key (Base32): {}", public_key_b32);
 
     // Decode from Base32 to raw bytes using Rfc4648 (compatible with NATS keys)
-    let public_key_bytes = base32Decode(Alphabet::Rfc4648 { padding: false }, &public_key_b32)
-        .expect("failed to convert public key to bytes");
-    println!("Decoded Public Key Bytes: {:?}", public_key_bytes);
+    let public_key_bytes = base32Decode(
+        Alphabet::Rfc4648 { padding: false },
+        auth_signing_account_pubkey,
+    )
+    .expect("Failed to convert public key to bytes");
 
     // Use the decoded key to create a DecodingKey
     let decoding_key = DecodingKey::from_ed_der(&public_key_bytes);
-    println!(">>>>>>> decoded key");
 
     // Validate the token with the correct algorithm
     let mut validation = Validation::new(Algorithm::EdDSA);
     validation.insecure_disable_signature_validation();
     validation.validate_aud = false; // Disable audience validation
-    println!("passed validation");
 
     let token_data = decode::<T>(&modified_token, &decoding_key, &validation)?;
-    // println!("token_data: {:#?}", token_data);
-
     Ok(token_data.claims)
 }
 
@@ -187,9 +164,8 @@ pub fn generate_auth_response_claim(
     let hashed_user_claim_bytes = hash_claim(&user_claim_str);
     user_claim.generic_claim_data.jwt_id = Some(BASE32HEX_NOPAD.encode(&hashed_user_claim_bytes));
     user_claim_str = serde_json::to_string(&user_claim)?;
-    let user_token = encode_jwt(&user_claim_str, &auth_signing_account_keypair)?;
-    println!("user_token: {:#?}", user_token);
 
+    let user_token = encode_jwt(&user_claim_str, &auth_signing_account_keypair)?;
     let outer_nats_claim = types::ClaimData {
         issuer: auth_root_account_pubkey.clone(), // Must be the pubkey of the keypair that signs the claim
         subcriber: auth_request_claim.auth_request.user_nkey.clone(),
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index b2d8e67..fb159dd 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -1,3 +1,4 @@
+use super::nats_js_client::EndpointType;
 use anyhow::{anyhow, Result};
 use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
 use async_nats::jetstream::stream::{self, Info, Stream};
@@ -10,7 +11,6 @@ use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
-use super::nats_js_client::EndpointType;
 
 pub type ResponseSubjectsGenerator =
     Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
@@ -225,11 +225,12 @@ impl JsStreamService {
         T: EndpointTraits,
     {
         // Avoid adding the Service Subject prefix if the Endpoint Subject name starts with global keywords $SYS or $JS
-        let consumer_subject = if endpoint_subject.starts_with("$SYS") || endpoint_subject.starts_with("$JS") {
-            endpoint_subject.to_string()
-        } else {
-            format!("{}.{}", self.service_subject, endpoint_subject)
-        };
+        let consumer_subject =
+            if endpoint_subject.starts_with("$SYS") || endpoint_subject.starts_with("$JS") {
+                endpoint_subject.to_string()
+            } else {
+                format!("{}.{}", self.service_subject, endpoint_subject)
+            };
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
@@ -342,7 +343,7 @@ impl JsStreamService {
         println!("WAITING TO PROCESS MESSAGE...");
         while let Some(Ok(js_msg)) = messages.next().await {
             println!("MESSAGES : js_msg={:?}", js_msg);
-            
+
             log::trace!(
                 "{}Consumer received message: subj='{}.{}', endpoint={}, service={}",
                 log_info.prefix,
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index c18c05a..d32c41f 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -166,7 +166,7 @@ impl JsClient {
                     }
                 }
             }
-        }; 
+        };
 
         let client = connect_options.connect(&p.nats_url).await?;
 
@@ -242,7 +242,11 @@ impl JsClient {
         }
     }
 
-    pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>> {
+    pub async fn publish(
+        &self,
+        payload: PublishInfo,
+    ) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>>
+    {
         log::debug!(
             "{}Called Publish message: subj={}, msg_id={} data={:?}",
             self.service_log_prefix,
@@ -250,12 +254,16 @@ impl JsClient {
             payload.msg_id,
             payload.data
         );
-        
+
         let now = Instant::now();
         let result = match payload.headers {
             Some(headers) => {
                 self.js
-                    .publish_with_headers(payload.subject.clone(), headers, payload.data.clone().into())
+                    .publish_with_headers(
+                        payload.subject.clone(),
+                        headers,
+                        payload.data.clone().into(),
+                    )
                     .await
             }
             None => {
@@ -270,7 +278,7 @@ impl JsClient {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(async_nats::error::Error::from(err));
+            return Err(err);
         }
 
         if let Some(ref on_published) = self.on_msg_published_event {
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index 9b937b5..40cd11a 100755
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -150,7 +150,6 @@ nsc describe operator --raw --output-file $SHARED_CREDS_DIR/$OPERATOR.jwt
 nsc describe account --name SYS --raw --output-file $SHARED_CREDS_DIR/$SYS_ACCOUNT.jwt
 nsc generate creds --name $AUTH_GUARD_USER --account $AUTH_ACCOUNT --output-file $SHARED_CREDS_DIR/$AUTH_GUARD_USER.creds
 
-# ADMIN_SK=$(nsc describe account ADMIN --field 'nats.signing_keys[0].key' | tr -d '"')
 extract_signing_key ADMIN $ADMIN_SK
 echo "extracted ADMIN signing key"
 

From 5fecd3e75b96708617d0344335464ca9c5c11d75 Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 20:21:57 -0600
Subject: [PATCH 72/91] clean up comments and logs

---
 rust/clients/host_agent/src/auth/init.rs |  2 +-
 rust/clients/host_agent/src/keys.rs      | 88 ++++++++++--------------
 rust/clients/host_agent/src/main.rs      |  5 +-
 3 files changed, 38 insertions(+), 57 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index 6519472..af90f5e 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -144,7 +144,7 @@ pub async fn run(
     };
 
     println!(
-        "got an AUTH response: {:#?}",
+        "Received AUTH response: {:#?}",
         serde_json::from_slice::<AuthJWTResult>(&response_msg.payload)
             .expect("failed to serde_json deserialize msg response")
     );
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 193f96e..67150e2 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -41,7 +41,7 @@ pub struct CredPaths {
 #[derive(Clone)]
 pub enum AuthCredType {
     Guard(PathBuf),           // Default
-    Authenticated(CredPaths), // Only assiged after successful hoster authentication
+    Authenticated(CredPaths), // Only assigned after successful hoster authentication
 }
 
 #[derive(Clone)]
@@ -84,65 +84,47 @@ impl Keys {
         maybe_host_creds_path: &Option<PathBuf>,
         maybe_sys_creds_path: &Option<PathBuf>,
     ) -> Result<Self> {
-        println!(
-            "maybe_host_creds_path={:#?}, maybe_sys_creds_path={:#?}",
-            maybe_host_creds_path, maybe_sys_creds_path
-        );
-
         let host_key_path: String = std::env::var("HOSTING_AGENT_HOST_NKEY_PATH")
             .context("Cannot read HOSTING_AGENT_HOST_NKEY_PATH from env var")?;
-        println!("host_key_path={:#?}", host_key_path);
-
         let host_keypair = try_read_keypair_from_file(PathBuf::from_str(&host_key_path.clone())?)?
             .ok_or_else(|| anyhow!("Host keypair not found at path {:?}", host_key_path))?;
-        println!("host_keypair={:#?}", host_keypair);
         let host_pk = host_keypair.public_key();
 
-        let sys_key_path = std::env::var("HOSTING_AGENT_SYS_NKEY_PATH")
-            .context("Cannot read HOSTING_AGENT_SYS_NKEY_PATH from env var")?;
-        println!("sys_key_path={:#?}", sys_key_path);
+        let auth_guard_creds =
+            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
 
         let host_user_name = format!("host_user_{}", host_pk);
         let host_creds_path = maybe_host_creds_path.to_owned().map_or_else(
             || PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "WORKLOAD", &host_user_name)),
             Ok,
         )?;
-        println!("host_creds_path={:#?}", host_creds_path);
 
         let sys_user_name = format!("sys_user_{}", host_pk);
         let sys_creds_path = maybe_sys_creds_path.to_owned().map_or_else(
             || PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name)),
             Ok,
         )?;
-        println!("sys_creds_path={:#?}", sys_creds_path);
 
-        // Set auth_guard_creds as default:
-        let auth_guard_creds =
-            PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth_guard"))?;
-        println!("auth_guard_creds={:#?}", auth_guard_creds);
+        let mut default_keys = Self {
+            host_keypair,
+            host_pubkey: host_pk,
+            local_sys_keypair: None,
+            local_sys_pubkey: None,
+            creds: AuthCredType::Guard(auth_guard_creds),  // Set auth_guard_creds as default user cred
+        };
 
+        let sys_key_path = std::env::var("HOSTING_AGENT_SYS_NKEY_PATH")
+            .context("Cannot read HOSTING_AGENT_SYS_NKEY_PATH from env var")?;
         let keys = match try_read_keypair_from_file(PathBuf::from_str(&sys_key_path)?)? {
             Some(kp) => {
                 let local_sys_pk = kp.public_key();
-                Self {
-                    host_keypair,
-                    host_pubkey: host_pk,
-                    local_sys_keypair: Some(kp),
-                    local_sys_pubkey: Some(local_sys_pk),
-                    creds: AuthCredType::Guard(auth_guard_creds),
-                }
+                default_keys.local_sys_keypair = Some(kp);
+                default_keys.local_sys_pubkey = Some(local_sys_pk);
+                default_keys
             }
-            None => Self {
-                host_keypair,
-                host_pubkey: host_pk,
-                local_sys_keypair: None,
-                local_sys_pubkey: None,
-                creds: AuthCredType::Guard(auth_guard_creds),
-            },
+            None => default_keys,
         };
 
-        println!("keys={:#?}", keys);
-
         Ok(keys.clone().add_creds_paths(host_creds_path, Some(sys_creds_path)).unwrap_or_else(move |e| {
             log::error!("Error: Cannot locate authenticated cred files. Defaulting to auth_guard_creds. Err={}",e);
             keys
@@ -230,39 +212,45 @@ impl Keys {
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
         let host_path = PathBuf::from_str(&format!(
-            "/{}/{}/{}",
+            "{}/{}/{}",
             get_nsc_root_path(),
             "local_creds",
             "host.jwt"
         ))?;
-        println!("host_path ={:?}", host_path);
+        log::trace!("host_path={:?}", host_path);
         write_to_file(host_path.clone(), host_user_jwt.as_bytes())?;
-        println!("Wrote JWT to host file");
+        log::trace!("Wrote JWT to host file");
 
         let sys_path = PathBuf::from_str(&format!(
-            "/{}/{}/{}",
+            "{}/{}/{}",
             get_nsc_root_path(),
             "local_creds",
             "sys.jwt"
         ))?;
-        println!("sys_path ={:?}", sys_path);
+        log::trace!("sys_path={:?}", sys_path);
         write_to_file(sys_path.clone(), host_sys_user_jwt.as_bytes())?;
-        println!("Wrote JWT to sys file");
+        log::trace!("Wrote JWT to sys file");
 
         // Import host user jwt to local nsc resolver
         // TODO: Determine why the following works in cmd line, but doesn't seem to work when run in current program / run
         Command::new("nsc")
-            .args(["import", "user", "--file", &format!("{:?}", host_path)])
+            .arg("import")
+            .arg("user")
+            .arg("--file")
+            .arg(&format!("{:?}", host_path))
             .output()
             .context("Failed to add import new host user on hosting agent.")?;
-        println!("imported host user");
+        log::trace!("Imported host user successfully");
 
         // Import sys user jwt to local nsc resolver
         Command::new("nsc")
-            .args(["import", "user", "--file", &format!("{:?}", sys_path)])
+            .arg("import")
+            .arg("user")
+            .arg("--file")
+            .arg(&format!("{:?}", sys_path))
             .output()
             .context("Failed to add import new sys user on hosting agent.")?;
-        println!("imported sys user");
+        log::trace!("Imported sys user successfully");
 
         // Save user creds and sys creds local to hosting agent
         let host_user_name = format!("host_user_{}", self.host_pubkey);
@@ -280,8 +268,8 @@ impl Keys {
                 &host_creds_path.to_string_lossy(),
             ])
             .output()
-            .context("Failed to add new operator signing key on hosting agent")?;
-        println!(
+            .context("Failed to add host user key to hosting agent")?;
+        log::trace!(
             "Generated host user creds. creds_path={:?}",
             host_creds_path
         );
@@ -291,10 +279,6 @@ impl Keys {
             let sys_user_name = format!("sys_user_{}", self.host_pubkey);
             let path = PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "SYS", &sys_user_name))?;
             Command::new("nsc")
-                .arg(format!(
-                    "generate creds --name {} --account {}",
-                    sys_user_name, "SYS"
-                ))
                 .args([
                     "generate",
                     "creds",
@@ -306,8 +290,8 @@ impl Keys {
                     &path.to_string_lossy(),
                 ])
                 .output()
-                .context("Failed to add new operator signing key on hosting agent")?;
-            println!("Generated sys user creds. creds_path={:?}", path);
+                .context("Failed to add sys user key to hosting agent")?;
+            log::trace!("Generated sys user creds. creds_path={:?}", path);
             sys_creds_file_name = Some(path);
         }
 
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 3a0d614..05c29da 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -51,19 +51,16 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
-    println!("inside host agent main auth... 0");
-
     let mut host_agent_keys = keys::Keys::try_from_storage(
         &args.nats_leafnode_client_creds_path,
         &args.nats_leafnode_client_sys_creds_path,
     )
     .or_else(|_| {
         keys::Keys::new().map_err(|e| {
-            eprintln!("Failed to create new keys: {:?}", e);
+            log::error!("Failed to create new keys: {:?}", e);
             async_nats::Error::from(e)
         })
     })?;
-    println!("Host Agent Keys={:#?}", host_agent_keys);
 
     // If user cred file is for the auth_guard user, run loop to authenticate host & hoster...
     if let keys::AuthCredType::Guard(_) = host_agent_keys.creds {

From c54a11701de13bb2aeb662ed82aab1fff6980355 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 21:20:52 -0600
Subject: [PATCH 73/91]  update docs

---
 rust/clients/host_agent/src/auth/init.rs  | 35 ++++++++++--------
 rust/clients/host_agent/src/keys.rs       | 19 +++++-----
 rust/clients/host_agent/src/main.rs       | 39 ++++++++++----------
 rust/clients/orchestrator/src/auth.rs     | 40 ++++++++++-----------
 rust/services/authentication/src/lib.rs   | 27 ++++++++------
 rust/services/authentication/src/types.rs |  3 --
 rust/util_libs/src/nats_js_client.rs      | 43 -----------------------
 7 files changed, 85 insertions(+), 121 deletions(-)

diff --git a/rust/clients/host_agent/src/auth/init.rs b/rust/clients/host_agent/src/auth/init.rs
index af90f5e..ae31cb6 100644
--- a/rust/clients/host_agent/src/auth/init.rs
+++ b/rust/clients/host_agent/src/auth/init.rs
@@ -1,18 +1,17 @@
 /*
 This client is associated with the:
-    - ADMIN account
+    - AUTH account
     - auth guard user
 
 Nb: Once the host and hoster are validated, and the host creds file is created,
-...this client should close and the hostd workload manager should spin up.
+...this client should safely close and then the `hostd.workload` manager should spin up.
 
 This client is responsible for:
-    - generating new key for host / and accessing hoster key from provided config file
-    - registering with the host auth service to:
-        - get hub operator jwt and hub sys account jwt
-        - send "nkey" version of host pubkey as file to hub
-        - get user jwt from hub and create user creds file with provided file path
-    - publishing to `auth.start` to initilize the auth handshake and validate the host/hoster
+    - generating new key for host and accessing hoster key from provided config file
+    - calling the host auth service to:
+        - validate hoster hc pubkey and email
+        - send the host pubkey to the orchestrator to register with the orchestrator key resovler
+        - get user jwt from orchestrator and create user creds file with provided file path
     - returning the host pubkey and closing client cleanly
 */
 
@@ -23,7 +22,10 @@ use crate::{
 };
 use anyhow::Result;
 use async_nats::{HeaderMap, HeaderName, HeaderValue, RequestErrorKind};
-use authentication::types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthState};
+use authentication::{
+    types::{AuthGuardPayload, AuthJWTPayload, AuthJWTResult, AuthState},
+    AUTH_SRV_SUBJ, VALIDATE_AUTH_SUBJECT,
+};
 use hpos_hal::inventory::HoloInventory;
 use std::str::FromStr;
 use std::time::Duration;
@@ -113,9 +115,16 @@ pub async fn run(
         HeaderValue::from_str(&signature)?,
     );
 
-    println!("About to send out the {} message", "AUTH.validate");
+    println!(
+        "About to send out the {}.{} message",
+        AUTH_SRV_SUBJ, VALIDATE_AUTH_SUBJECT
+    );
     let response_msg = match auth_guard_client
-        .request_with_headers("AUTH.validate".to_string(), headers, payload_bytes.into())
+        .request_with_headers(
+            format!("{}.{}", AUTH_SRV_SUBJ, VALIDATE_AUTH_SUBJECT),
+            headers,
+            payload_bytes.into(),
+        )
         .await
     {
         Ok(msg) => msg,
@@ -162,9 +171,5 @@ pub async fn run(
         }
     };
 
-    log::trace!(
-        "Host Agent Keys after authentication request: {:#?}",
-        host_agent_keys
-    );
     Ok((host_agent_keys, auth_guard_client))
 }
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 67150e2..0bcb68c 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -79,7 +79,7 @@ impl Keys {
         })
     }
 
-    // NB: Only call when trying to load an already authenticated Host and Sys User
+    // NB: Only call when trying to load an already authenticated host user (with or without a sys user)
     pub fn try_from_storage(
         maybe_host_creds_path: &Option<PathBuf>,
         maybe_sys_creds_path: &Option<PathBuf>,
@@ -110,7 +110,7 @@ impl Keys {
             host_pubkey: host_pk,
             local_sys_keypair: None,
             local_sys_pubkey: None,
-            creds: AuthCredType::Guard(auth_guard_creds),  // Set auth_guard_creds as default user cred
+            creds: AuthCredType::Guard(auth_guard_creds), // Set auth_guard_creds as default user cred
         };
 
         let sys_key_path = std::env::var("HOSTING_AGENT_SYS_NKEY_PATH")
@@ -125,7 +125,10 @@ impl Keys {
             None => default_keys,
         };
 
-        Ok(keys.clone().add_creds_paths(host_creds_path, Some(sys_creds_path)).unwrap_or_else(move |e| {
+        Ok(keys.clone().add_creds_paths(
+            host_creds_path,
+            Some(sys_creds_path)
+        ).unwrap_or_else(move |e| {
             log::error!("Error: Cannot locate authenticated cred files. Defaulting to auth_guard_creds. Err={}",e);
             keys
         }))
@@ -159,7 +162,7 @@ impl Keys {
     pub fn add_creds_paths(
         mut self,
         host_creds_file_path: PathBuf,
-        sys_creds_file_path: Option<PathBuf>,
+        maybe_sys_creds_file_path: Option<PathBuf>,
     ) -> Result<Self> {
         match host_creds_file_path.try_exists() {
             Ok(is_ok) => {
@@ -170,7 +173,7 @@ impl Keys {
                     ));
                 }
 
-                let creds = match sys_creds_file_path {
+                let creds = match maybe_sys_creds_file_path {
                     Some(sys_path) => match sys_path.try_exists() {
                         Ok(is_ok) => {
                             if !is_ok {
@@ -237,7 +240,7 @@ impl Keys {
             .arg("import")
             .arg("user")
             .arg("--file")
-            .arg(&format!("{:?}", host_path))
+            .arg(format!("{:?}", host_path))
             .output()
             .context("Failed to add import new host user on hosting agent.")?;
         log::trace!("Imported host user successfully");
@@ -247,7 +250,7 @@ impl Keys {
             .arg("import")
             .arg("user")
             .arg("--file")
-            .arg(&format!("{:?}", sys_path))
+            .arg(format!("{:?}", sys_path))
             .output()
             .context("Failed to add import new sys user on hosting agent.")?;
         log::trace!("Imported sys user successfully");
@@ -299,7 +302,7 @@ impl Keys {
             .add_creds_paths(host_creds_path, sys_creds_file_name)
     }
 
-    pub fn _get_host_creds_path(&self) -> Option<PathBuf> {
+    pub fn get_host_creds_path(&self) -> Option<PathBuf> {
         if let AuthCredType::Authenticated(creds) = self.to_owned().creds {
             return Some(creds.host_creds_path);
         };
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 05c29da..d1938dc 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -22,7 +22,6 @@ use clap::Parser;
 use dotenv::dotenv;
 use hpos_hal::inventory::HoloInventory;
 use thiserror::Error;
-use util_libs::nats_js_client::{JsClient, PublishInfo};
 
 #[derive(Error, Debug)]
 pub enum AgentCliError {
@@ -67,32 +66,32 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
         host_agent_keys = run_auth_loop(host_agent_keys).await?;
     }
 
-    println!(
-        "Successfully AUTH'D and created new agent keys: {:#?}",
+    log::trace!(
+        "Host Agent Keys after successful authentication: {:#?}",
         host_agent_keys
     );
 
-    // // Once authenticated, start leaf server and run workload api calls.
-    // let _ = hostd::gen_leaf_server::run(
-    //     &host_agent_keys.get_host_creds_path(),
-    //     &args.store_dir,
-    //     args.hub_url.clone(),
-    //     args.hub_tls_insecure,
-    // )
-    // .await;
-
-    // let host_workload_client = hostd::workloads::run(
-    //     &host_agent_keys.host_pubkey,
-    //     &host_agent_keys.get_host_creds_path(),
-    //     args.nats_connect_timeout_secs,
-    // )
-    // .await?;
+    // Once authenticated, start leaf server and run workload api calls.
+    let _ = hostd::gen_leaf_server::run(
+        &host_agent_keys.get_host_creds_path(),
+        &args.store_dir,
+        args.hub_url.clone(),
+        args.hub_tls_insecure,
+    )
+    .await;
+
+    let host_workload_client = hostd::workloads::run(
+        &host_agent_keys.host_pubkey,
+        &host_agent_keys.get_host_creds_path(),
+        args.nats_connect_timeout_secs,
+    )
+    .await?;
 
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    // // Close client and drain internal buffer before exiting to make sure all messages are sent
-    // host_workload_client.close().await?;
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    host_workload_client.close().await?;
 
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 3e8b39e..fb9d1a6 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -1,29 +1,27 @@
 /*
 This client is associated with the:
-    - ADMIN account
-    - orchestrator user
+    - AUTH account
+    - (the orchestrator's) auth user
 
 This client is responsible for:
     - initalizing connection and handling interface with db
-    - registering with the host auth service to:
-        - handling auth requests by:
-            - validating user signature
-            - validating hoster pubkey
-            - validating hoster email
-            - bidirectionally pairing hoster and host
-            - interfacing with hub nsc resolver and hub credential files
-            - adding user to hub
-            - creating signed jwt for user
-            - adding user jwt file to user collection (with ttl)
+    - registering the `handle_auth_callout` and  `handle_auth_validation` fns as core nats service group endpoints:
+        - NB: These endpoints will consider authentiction successful if:
+            - user signature is valid
+            - hoster pubkey is valid
+            - hoster email is valid
+            - succesfully paired hoster and host in mongodb
+            - succesfully added user to resolver on hub (orchestrator side)
+            - succesfully created signed jwt for user
+            - succesfully added user jwt file to user collection in mongodb (with ttl)
     - keeping service running until explicitly cancelled out
 */
 
 use anyhow::{anyhow, Context, Result};
 use async_nats::service::ServiceExt;
 use authentication::{
-    self,
-    types::{self, AuthErrorPayload},
-    AuthServiceApi, AUTH_SRV_DESC, AUTH_SRV_NAME, AUTH_SRV_SUBJ, AUTH_SRV_VERSION,
+    types::AuthErrorPayload, AuthServiceApi, AUTH_CALLOUT_SUBJECT, AUTH_SRV_DESC, AUTH_SRV_NAME,
+    AUTH_SRV_SUBJ, AUTH_SRV_VERSION, VALIDATE_AUTH_SUBJECT,
 };
 use futures::StreamExt;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
@@ -131,7 +129,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Auth Callout Service
     let sys_user_group = auth_service.group("$SYS").group("REQ").group("USER");
-    let mut auth_callout = sys_user_group.endpoint("AUTH").await?;
+    let mut auth_callout = sys_user_group.endpoint(AUTH_CALLOUT_SUBJECT).await?;
     let auth_service_info = auth_service.info().await;
     let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
 
@@ -173,7 +171,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
                     let mut err_payload = AuthErrorPayload {
                         service_info: auth_service_info.clone(),
                         group: "$SYS.REQ.USER".to_string(),
-                        endpoint: "AUTH".to_string(),
+                        endpoint: AUTH_CALLOUT_SUBJECT.to_string(),
                         error: format!("{}", e),
                     };
 
@@ -214,14 +212,14 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Auth Validation Service
     let v1_auth_group = auth_service.group(AUTH_SRV_SUBJ); // .group("V1")
-    let mut auth_validation = v1_auth_group.endpoint(types::AUTHORIZE_SUBJECT).await?;
+    let mut auth_validation = v1_auth_group.endpoint(VALIDATE_AUTH_SUBJECT).await?;
     let orchestrator_auth_client_clone = orchestrator_auth_client.clone();
 
     tokio::spawn(async move {
         while let Some(request) = auth_validation.next().await {
             let maybe_reply = request.message.reply.clone();
             match auth_api
-                .handle_handshake_request(Arc::new(request.message))
+                .handle_auth_validation(Arc::new(request.message))
                 .await
             {
                 Ok(r) => {
@@ -244,8 +242,8 @@ pub async fn run() -> Result<(), async_nats::Error> {
                     let auth_service_info = auth_service.info().await;
                     let mut err_payload = AuthErrorPayload {
                         service_info: auth_service_info,
-                        group: "AUTH".to_string(),
-                        endpoint: types::AUTHORIZE_SUBJECT.to_string(),
+                        group: AUTH_SRV_SUBJ.to_string(),
+                        endpoint: VALIDATE_AUTH_SUBJECT.to_string(),
                         error: format!("{}", e),
                     };
                     log::error!(
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index b9f9194..dafc3ca 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -1,10 +1,11 @@
 /*
 Service Name: AUTH
 Subject: "AUTH.>"
-Provisioning Account: ADMIN Account (ie: This service is exclusively permissioned to the ADMIN account.)
+Provisioning Account: AUTH Account (ie: This service is exclusively permissioned to the AUTH account.)
 Users: orchestrator & noauth
 Endpoints & Managed Subjects:
-    - handle_handshake_request: AUTH.validate
+    - handle_auth_callout: $SYS.REQ.USER.AUTH
+    - handle_auth_validation: AUTH.validate
 */
 
 pub mod types;
@@ -23,12 +24,12 @@ use std::future::Future;
 use std::process::Command;
 use std::sync::Arc;
 use types::{AuthApiResult, WORKLOAD_SK_ROLE};
-use util_libs::db::{
-    mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-    schemas::{self, Host, Hoster, Role, RoleInfo, User},
-};
-use util_libs::nats_js_client::{
-    get_nsc_root_path, AsyncEndpointHandler, JsServiceResponse, ServiceError,
+use util_libs::{
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Hoster, Role, RoleInfo, User},
+    },
+    nats_js_client::{get_nsc_root_path, AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 use utils::handle_internal_err;
 
@@ -38,6 +39,12 @@ pub const AUTH_SRV_VERSION: &str = "0.0.1";
 pub const AUTH_SRV_DESC: &str =
     "This service handles the Authentication flow the Host and the Orchestrator.";
 
+// Service Endpoint Names:
+pub const VALIDATE_AUTH_SUBJECT: &str = "validate";
+// NB: Do not change this subject name unless NATS.io has changed the naming of their auth permissions subject.
+// NB: `AUTH_CALLOUT_SUBJECT` attached to the global subject `$SYS.REQ.USER`
+pub const AUTH_CALLOUT_SUBJECT: &str = "AUTH";
+
 #[derive(Clone, Debug)]
 pub struct AuthServiceApi {
     pub user_collection: MongoCollection<User>,
@@ -285,7 +292,7 @@ impl AuthServiceApi {
         })
     }
 
-    pub async fn handle_handshake_request(
+    pub async fn handle_auth_validation(
         &self,
         msg: Arc<Message>,
     ) -> Result<AuthApiResult, ServiceError> {
@@ -367,8 +374,6 @@ impl AuthServiceApi {
         };
 
         if let Some(sys_pubkey) = maybe_sys_pubkey.clone() {
-            println!("inside handle_handshake_request... 5 sys -- inside");
-
             match Command::new("nsc")
                 .args([
                     "add",
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 3f6df5e..fe37261 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -3,9 +3,6 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 
-pub const AUTH_CALLOUT_SUBJECT: &str = "$SYS.REQ.USER.AUTH";
-pub const AUTHORIZE_SUBJECT: &str = "validate";
-
 // The workload_sk_role is assigned when the host agent is created during the auth flow.
 // NB: This role name *must* match the `ROLE_NAME_WORKLOAD` in the `orchestrator_setup.sh` script file.
 pub const WORKLOAD_SK_ROLE: &str = "workload-role";
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index d32c41f..56d4151 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -287,49 +287,6 @@ impl JsClient {
         Ok(())
     }
 
-    // pub async fn request(&self, payload: RequestInfo) -> Result<Message, async_nats::error::Error<async_nats::RequestErrorKind>> {
-    //     /// ie: $JS.API.CONSUMER.INFO.AUTH.authorize_host_and_sys, JS.API.CONSUMER.INFO.AUTH.auth_callout
-    //     let js_subject = format!("$JS.API.CONSUMER.INFO.{}.{}", payload.stream_subject, payload.consumer_name);
-    //     log::debug!(
-    //         "{}Published message: subj={}, msg_id={} data={:?}",
-    //         self.service_log_prefix,
-    //         js_subject,
-    //         payload.msg_id,
-    //         payload.data
-    //     );
-
-    //     let now = Instant::now();
-    //     let result = match payload.headers {
-    //         Some(headers) => {
-    //             self.client
-    //                 .request_with_headers(format!("$JS.API.CONSUMER.INFO.{}.{}", payload.stream_subject, payload.consumer_name), headers, payload.data.clone().into())
-    //                 .await
-    //         }
-    //         None => {
-    //             self.client
-    //                 .request(format!("$JS.API.CONSUMER.INFO.{}.{}", payload.stream_subject, payload.consumer_name), payload.data.clone().into())
-    //                 .await
-    //         }
-    //     };
-
-    //     let duration = now.elapsed();
-
-    //     match result {
-    //         Ok(m) => {
-    //             if let Some(ref on_published) = self.on_msg_published_event {
-    //                 on_published(&js_subject, &self.name, duration);
-    //             }
-    //             Ok(m)
-    //         },
-    //         Err(e) => {
-    //             if let Some(ref on_failed) = self.on_msg_failed_event {
-    //                 on_failed(&js_subject, &self.name, duration); // todo: add msg_id
-    //             }
-    //             Err(e)
-    //         }
-    //     }
-    // }
-
     pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
         let mut current_services = self.js_services.unwrap_or_default();
         current_services.extend(js_services);

From a1e9c09ed2a968143ac737861430d365fe0de26e Mon Sep 17 00:00:00 2001
From: Jetttech <ldjetton21@gmail.com>
Date: Mon, 10 Feb 2025 21:22:25 -0600
Subject: [PATCH 74/91] remove print logs

---
 rust/clients/orchestrator/src/auth.rs     | 10 +++-------
 rust/services/authentication/src/lib.rs   |  5 +----
 rust/services/authentication/src/types.rs |  1 -
 rust/util_libs/src/js_stream_service.rs   |  3 ---
 4 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/rust/clients/orchestrator/src/auth.rs b/rust/clients/orchestrator/src/auth.rs
index 3e8b39e..6ae9974 100644
--- a/rust/clients/orchestrator/src/auth.rs
+++ b/rust/clients/orchestrator/src/auth.rs
@@ -45,10 +45,6 @@ pub const ORCHESTRATOR_AUTH_CLIENT_INBOX_PREFIX: &str = "_AUTH_INBOX_ORCHESTRATO
 pub async fn run() -> Result<(), async_nats::Error> {
     let admin_account_creds_path =
         PathBuf::from_str(&get_nats_creds_by_nsc("HOLO", "AUTH", "auth"))?;
-    println!(
-        " >>>> admin_account_creds_path: {:#?} ",
-        admin_account_creds_path
-    );
 
     // Root Keypair associated with AUTH account
     let root_account_key_path = std::env::var("ORCHESTRATOR_ROOT_AUTH_NKEY_PATH")
@@ -278,17 +274,17 @@ pub async fn run() -> Result<(), async_nats::Error> {
         }
     });
 
-    println!("Orchestrator Auth Service is running. Waiting for requests...");
+    log::debug!("Orchestrator Auth Service is running. Waiting for requests...");
 
     // ==================== Close and Clean Client ====================
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    println!("Closing orchestrator auth service...");
+    log::debug!("Closing orchestrator auth service...");
 
     // Close client and drain internal buffer before exiting to make sure all messages are sent
     orchestrator_auth_client.drain().await?;
-    println!("Closed orchestrator auth service");
+    log::debug!("Closed orchestrator auth service");
 
     Ok(())
 }
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index b9f9194..a078e74 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -294,7 +294,6 @@ impl AuthServiceApi {
         // 1. Verify expected data was received
         let signature: &[u8] = match &msg.headers {
             Some(h) => {
-                println!("header={:?}", h);
                 let r = HeaderValue::as_str(h.get("X-Signature").ok_or_else(|| {
                     log::error!("Error: Missing X-Signature header. Subject='AUTH.authorize'");
                     ServiceError::Request(format!("{:?}", ErrorCode::BAD_REQUEST))
@@ -367,8 +366,6 @@ impl AuthServiceApi {
         };
 
         if let Some(sys_pubkey) = maybe_sys_pubkey.clone() {
-            println!("inside handle_handshake_request... 5 sys -- inside");
-
             match Command::new("nsc")
                 .args([
                     "add",
@@ -421,7 +418,7 @@ impl AuthServiceApi {
             .output()
             .context("Failed to update resolver config file")
             .map_err(|e| ServiceError::Internal(e.to_string()))?;
-        println!("\npushed new jwts to resolver server");
+        log::trace!("\nPushed new jwts to resolver server");
 
         let mut tag_map: HashMap<String, String> = HashMap::new();
         tag_map.insert("host_pubkey".to_string(), host_pubkey.clone());
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index 3f6df5e..2649400 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -96,7 +96,6 @@ impl AuthGuardPayload {
         T: Fn(&[u8]) -> Result<String>,
     {
         let payload_bytes = serde_json::to_vec(&self)?;
-        println!("Going to sign payload_bytes={:?}", payload_bytes);
         let signature = sign_handler(&payload_bytes)?;
         self.host_signature = signature.as_bytes().to_vec();
         Ok(self)
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index fb159dd..e90354f 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -340,10 +340,7 @@ impl JsStreamService {
     ) where
         T: EndpointTraits,
     {
-        println!("WAITING TO PROCESS MESSAGE...");
         while let Some(Ok(js_msg)) = messages.next().await {
-            println!("MESSAGES : js_msg={:?}", js_msg);
-
             log::trace!(
                 "{}Consumer received message: subj='{}.{}', endpoint={}, service={}",
                 log_info.prefix,

From 56ca9226a95263d1b67f4347651b9c23d7e49b65 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 11 Feb 2025 08:39:06 -0600
Subject: [PATCH 75/91]  update inbox naming

---
 rust/clients/host_agent/src/hostd/workloads.rs | 4 +++-
 rust/clients/orchestrator/src/workloads.rs     | 2 +-
 scripts/orchestrator_setup.sh                  | 4 ++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workloads.rs b/rust/clients/host_agent/src/hostd/workloads.rs
index 95e9b53..ec30ba8 100644
--- a/rust/clients/host_agent/src/hostd/workloads.rs
+++ b/rust/clients/host_agent/src/hostd/workloads.rs
@@ -59,12 +59,14 @@ pub async fn run(
         .map(Credentials::Path)
         .ok_or_else(|| async_nats::Error::from("error"))?;
 
+    let pubkey_lowercase = host_pubkey.to_string().to_lowercase();
+
     let host_workload_client = tokio::select! {
         client = async {loop {
                 let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
                     nats_url: nats_url.clone(),
                     name: HOST_AGENT_CLIENT_NAME.to_string(),
-                    inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
+                    inbox_prefix: format!("{}.{}", pubkey_lowercase, HOST_AGENT_INBOX_PREFIX),
                     service_params: vec![workload_stream_service_params.clone()],
                     credentials: Some(vec![creds.clone()]),
                     ping_interval: Some(Duration::from_secs(10)),
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 6a08ee9..1f6e9aa 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -36,7 +36,7 @@ use workload::{
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Manager";
-const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX_ORCHESTRATOR";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "ORCHESTRATOR._WORKLOAD_INBOX";
 
 pub fn create_callback_subject_to_host(
     is_prefix: bool,
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index 40cd11a..b9484d0 100755
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -113,7 +113,7 @@ nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem
 
 ADMIN_SK="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 ADMIN_ROLE_NAME="admin_role"
-nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_WORKLOAD_INBOX_*.>","_AUTH_INBOX_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","_WORKLOAD_INBOX_ORCHESTRATOR.>","_AUTH_INBOX_ORCHESTRATOR.>" --allow-pub-response
+nsc edit signing-key --sk $ADMIN_SK --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","*._WORKLOAD_INBOX.>","_AUTH_INBOX_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","ORCHESTRATOR._WORKLOAD_INBOX.>","_AUTH_INBOX_ORCHESTRATOR.>" --allow-pub-response
 
 # Step 3: Create AUTH with JetStream with non-scoped signing key
 nsc add account --name $AUTH_ACCOUNT
@@ -126,7 +126,7 @@ nsc add account --name $WORKLOAD_ACCOUNT
 nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G --conns -1 --leaf-conns -1
 WORKLOAD_SK="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 WORKLOAD_ROLE_NAME="workload_role"
-nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","_WORKLOAD_INBOX_{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","_WORKLOAD_INBOX_{{tag(pubkey)}}.>" --allow-pub-response
+nsc edit signing-key --sk $WORKLOAD_SK --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","{{tag(pubkey)}}._WORKLOAD_INBOX.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","{{tag(pubkey)}}._WORKLOAD_INBOX.>" --allow-pub-response
 
 # Step 5: Create Orchestrator User in ADMIN Account
 nsc add user --name $ADMIN_USER --account $ADMIN_ACCOUNT -K $ADMIN_ROLE_NAME

From da58996b0cfd789f6399fe5b552e30c4ba71c54d Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 11 Feb 2025 16:11:05 -0600
Subject: [PATCH 76/91] auth clean up

---
 rust/clients/host_agent/output_dir.host.jwt     |  1 -
 rust/clients/host_agent/output_dir.host_sys.jwt |  1 -
 rust/clients/host_agent/src/keys.rs             | 12 +++++-------
 rust/services/authentication/src/lib.rs         | 14 +++-----------
 rust/services/authentication/src/types.rs       |  4 ++--
 rust/util_libs/src/nats_js_client.rs            | 17 +++++++++++++----
 6 files changed, 23 insertions(+), 26 deletions(-)
 delete mode 100644 rust/clients/host_agent/output_dir.host.jwt
 delete mode 100644 rust/clients/host_agent/output_dir.host_sys.jwt

diff --git a/rust/clients/host_agent/output_dir.host.jwt b/rust/clients/host_agent/output_dir.host.jwt
deleted file mode 100644
index 09638f3..0000000
--- a/rust/clients/host_agent/output_dir.host.jwt
+++ /dev/null
@@ -1 +0,0 @@
-eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJDS1pMR0pVNUtKWVVVUEpNSDNJTU9OQjVZRFI1V1RUQzdDV1JTR01DWDZUVVhQS1pOWEdRIiwiaWF0IjoxNzM4OTU0OTQ0LCJpc3MiOiJBQ1dZNkZZTFNBRU81QTRZUEZUNDc0REhVNkIyM1VSRk5QV0k0N0lSWUlGQVdDS0xCM1NXRFZJQyIsIm5hbWUiOiJob3N0X3VzZXJfVURTMkE3STRCQ0VDVVJIRTY0QzUyT1JLNklEU09TRTRJTFo3UkpNNElPNEVBWUYzM0I2N0VXRUYiLCJzdWIiOiJVRFMyQTdJNEJDRUNVUkhFNjRDNTJPUks2SURTT1NFNElMWjdSSk00SU80RUFZRjMzQjY3RVdFRiIsIm5hdHMiOnsicHViIjp7fSwic3ViIjp7fSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidGFncyI6WyJwdWJrZXk6dWRzMmE3aTRiY2VjdXJoZTY0YzUyb3JrNmlkc29zZTRpbHo3cmptNGlvNGVheWYzM2I2N2V3ZWYiXSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.GU1VJRaTH42njK_CGcuklnR-8WCNw2zTfZFn83EizwSqS4-xxtxlXJIyGWtOO73D9xMemSc5KgoLi6rGkLR0BA
\ No newline at end of file
diff --git a/rust/clients/host_agent/output_dir.host_sys.jwt b/rust/clients/host_agent/output_dir.host_sys.jwt
deleted file mode 100644
index 5a93965..0000000
--- a/rust/clients/host_agent/output_dir.host_sys.jwt
+++ /dev/null
@@ -1 +0,0 @@
-eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJQMkJMWkEyM0UyUUFORU0yS1JHNjUzSVpWWExRT0ZCUFg0TTI3S0Y2UVZET1BJNkJTTVNBIiwiaWF0IjoxNzM4OTU2NjAwLCJpc3MiOiJBREY1NkREMkxOVVhCSDRXQUpWSUlKRFFWSjJKQU9SWDVWS1VNU0pFRkFMQ1NHQVZDWEVTUE9GSiIsIm5hbWUiOiJzeXNfdXNlcl9VRFMyQTdJNEJDRUNVUkhFNjRDNTJPUks2SURTT1NFNElMWjdSSk00SU80RUFZRjMzQjY3RVdFRiIsInN1YiI6IlVBQ0paUU9RSzJZMkpGUVZOVjRDSk9SQUVaR1YzR1lUQ0s3VU9TQ0xORVpKUktNT1c0QVRVWlpHIiwibmF0cyI6eyJwdWIiOnt9LCJzdWIiOnt9LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpc3N1ZXJfYWNjb3VudCI6IkFCTFhYU01OSjRXTlpWVU9TUFdXVFJWVTZUNzdHSVpYU0g0S0RRSkRGNjc2VEFFR0JJQU5GSEE2IiwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.1R8jhZJVDgXeUblr8Bi8l5vw-mikO-r54qBOb7D19TY0QrWNtWvCUgqDEopBMScXP3BgCJTjNF3VJBMmLWWcDQ
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index 0bcb68c..b6ba329 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -6,7 +6,7 @@ use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Command;
 use std::str::FromStr;
-use util_libs::nats_js_client::{get_nats_creds_by_nsc, get_nsc_root_path};
+use util_libs::nats_js_client::{get_nats_creds_by_nsc, get_local_creds_path};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -215,9 +215,8 @@ impl Keys {
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
         let host_path = PathBuf::from_str(&format!(
-            "{}/{}/{}",
-            get_nsc_root_path(),
-            "local_creds",
+            "{}/{}",
+            get_local_creds_path(),
             "host.jwt"
         ))?;
         log::trace!("host_path={:?}", host_path);
@@ -225,9 +224,8 @@ impl Keys {
         log::trace!("Wrote JWT to host file");
 
         let sys_path = PathBuf::from_str(&format!(
-            "{}/{}/{}",
-            get_nsc_root_path(),
-            "local_creds",
+            "{}/{}",
+            get_local_creds_path(),
             "sys.jwt"
         ))?;
         log::trace!("sys_path={:?}", sys_path);
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index a591dd5..2d26942 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -29,7 +29,7 @@ use util_libs::{
         mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
         schemas::{self, Host, Hoster, Role, RoleInfo, User},
     },
-    nats_js_client::{get_nsc_root_path, AsyncEndpointHandler, JsServiceResponse, ServiceError},
+    nats_js_client::{get_nats_jwt_by_nsc, AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 use utils::handle_internal_err;
 
@@ -401,19 +401,11 @@ impl AuthServiceApi {
         }
 
         // 4. Create User JWT files (automatically signed with respective account key)
-        let host_jwt = std::fs::read_to_string(format!(
-            "{}/stores/HOLO/accounts/WORKLOAD/users/host_user_{}.jwt",
-            get_nsc_root_path(),
-            host_pubkey
-        ))
+        let host_jwt = std::fs::read_to_string(get_nats_jwt_by_nsc("HOLO","WORKLOAD", &format!("host_user_{}.jwt", host_pubkey)))
         .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         let sys_jwt = if maybe_sys_pubkey.is_some() {
-            std::fs::read_to_string(format!(
-                "{}/stores/HOLO/accounts/SYS/users/sys_user_{}.jwt",
-                get_nsc_root_path(),
-                host_pubkey
-            ))
+            std::fs::read_to_string(get_nats_jwt_by_nsc("HOLO","SYS", &format!("sys_user_{}.jwt", host_pubkey)))
             .map_err(|e| ServiceError::Internal(e.to_string()))?
         } else {
             String::new()
diff --git a/rust/services/authentication/src/types.rs b/rust/services/authentication/src/types.rs
index eca2e38..ff32c29 100644
--- a/rust/services/authentication/src/types.rs
+++ b/rust/services/authentication/src/types.rs
@@ -4,8 +4,8 @@ use std::collections::HashMap;
 use util_libs::js_stream_service::{CreateResponse, CreateTag, EndpointTraits};
 
 // The workload_sk_role is assigned when the host agent is created during the auth flow.
-// NB: This role name *must* match the `ROLE_NAME_WORKLOAD` in the `orchestrator_setup.sh` script file.
-pub const WORKLOAD_SK_ROLE: &str = "workload-role";
+// NB: This role name *must* match the `ROLE_NAME_WORKLOAD` in the `hub_auth_setup.sh` script file.
+pub const WORKLOAD_SK_ROLE: &str = "workload_role";
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum AuthState {
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 56d4151..28caeb3 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -347,10 +347,14 @@ pub fn get_nats_url() -> String {
     })
 }
 
-pub fn get_nsc_root_path() -> String {
+fn get_nsc_root_path() -> String {
     std::env::var("NSC_PATH").unwrap_or_else(|_| "/.local/share/nats/nsc".to_string())
 }
 
+pub fn get_local_creds_path() -> String {
+    std::env::var("LOCAL_CREDS_PATH").unwrap_or_else(|_| format!("{}/local_creds", get_nsc_root_path()))
+}
+
 pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {
     format!(
         "{}/keys/creds/{}/{}/{}.creds",
@@ -361,9 +365,14 @@ pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> Strin
     )
 }
 
-pub fn get_path_buf_from_current_dir(file_name: &str) -> PathBuf {
-    let current_dir_path = std::env::current_dir().expect("Failed to locate current directory.");
-    current_dir_path.join(file_name)
+pub fn get_nats_jwt_by_nsc(operator: &str, account: &str, user: &str) -> String {
+    format!(
+        "{}/stores/{}/accounts/{}/users/{}.jwt",
+        get_nsc_root_path(),
+        operator,
+        account,
+        user
+    )
 }
 
 pub fn get_event_listeners() -> Vec<EventListener> {

From 03386b2b4abb331399ce226c7c5bfd269a8d1bc4 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 11 Feb 2025 16:45:26 -0600
Subject: [PATCH 77/91] fmt

---
 .env.example                            |  7 ++++---
 rust/clients/host_agent/src/keys.rs     | 14 +++-----------
 rust/services/authentication/src/lib.rs | 12 ++++++++++--
 rust/util_libs/src/nats_js_client.rs    | 13 ++-----------
 4 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/.env.example b/.env.example
index ad8c75c..4a7a1ab 100644
--- a/.env.example
+++ b/.env.example
@@ -3,14 +3,15 @@ NSC_PATH="<path/to/nsc_dir>"
 NATS_URL="nats:/<domain>:<port>"
 NATS_LISTEN_PORT="<port>"
 LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
+LOCAL_CREDS_PATH="<path/to/dir>"
 
 # ORCHESTRATOR
 MONGO_URI="mongodb://<host>:<port>"
-ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH="/home/lisa/.local/share/nats/nsc/local_creds_output/AUTH_ROOT_SK.nk"
-ORCHESTRATOR_ROOT_AUTH_NKEY_PATH="/home/lisa/.local/share/nats/nsc/local_creds_output/AUTH_SK.nk"
+ORCHESTRATOR_SIGNING_AUTH_NKEY_PATH="/.local/share/nats/nsc/local_creds/AUTH_ROOT_SK.nk"
+ORCHESTRATOR_ROOT_AUTH_NKEY_PATH="/.local/share/nats/nsc/local_creds/AUTH_SK.nk"
 
 # HOSTING AGENT
 HOSTING_AGENT_HOST_NKEY_PATH="<path>/host.nk"
 HOSTING_AGENT_SYS_NKEY_PATH="<path>/sys.nk"
 HPOS_CONFIG_PATH="path/to/file.config";
-DEVICE_SEED_DEFAULT_PASSWORD="device_pw_1234"
+DEVICE_SEED_DEFAULT_PASSWORD="device_pw_1234"
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/keys.rs b/rust/clients/host_agent/src/keys.rs
index b6ba329..2747c0c 100644
--- a/rust/clients/host_agent/src/keys.rs
+++ b/rust/clients/host_agent/src/keys.rs
@@ -6,7 +6,7 @@ use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Command;
 use std::str::FromStr;
-use util_libs::nats_js_client::{get_nats_creds_by_nsc, get_local_creds_path};
+use util_libs::nats_js_client::{get_local_creds_path, get_nats_creds_by_nsc};
 
 impl std::fmt::Debug for Keys {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -214,20 +214,12 @@ impl Keys {
         host_sys_user_jwt: String,
     ) -> Result<Self> {
         //  Save user jwt and sys jwt local to hosting agent
-        let host_path = PathBuf::from_str(&format!(
-            "{}/{}",
-            get_local_creds_path(),
-            "host.jwt"
-        ))?;
+        let host_path = PathBuf::from_str(&format!("{}/{}", get_local_creds_path(), "host.jwt"))?;
         log::trace!("host_path={:?}", host_path);
         write_to_file(host_path.clone(), host_user_jwt.as_bytes())?;
         log::trace!("Wrote JWT to host file");
 
-        let sys_path = PathBuf::from_str(&format!(
-            "{}/{}",
-            get_local_creds_path(),
-            "sys.jwt"
-        ))?;
+        let sys_path = PathBuf::from_str(&format!("{}/{}", get_local_creds_path(), "sys.jwt"))?;
         log::trace!("sys_path={:?}", sys_path);
         write_to_file(sys_path.clone(), host_sys_user_jwt.as_bytes())?;
         log::trace!("Wrote JWT to sys file");
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index 2d26942..eb90ad1 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -401,11 +401,19 @@ impl AuthServiceApi {
         }
 
         // 4. Create User JWT files (automatically signed with respective account key)
-        let host_jwt = std::fs::read_to_string(get_nats_jwt_by_nsc("HOLO","WORKLOAD", &format!("host_user_{}.jwt", host_pubkey)))
+        let host_jwt = std::fs::read_to_string(get_nats_jwt_by_nsc(
+            "HOLO",
+            "WORKLOAD",
+            &format!("host_user_{}.jwt", host_pubkey),
+        ))
         .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         let sys_jwt = if maybe_sys_pubkey.is_some() {
-            std::fs::read_to_string(get_nats_jwt_by_nsc("HOLO","SYS", &format!("sys_user_{}.jwt", host_pubkey)))
+            std::fs::read_to_string(get_nats_jwt_by_nsc(
+                "HOLO",
+                "SYS",
+                &format!("sys_user_{}.jwt", host_pubkey),
+            ))
             .map_err(|e| ServiceError::Internal(e.to_string()))?
         } else {
             String::new()
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index 28caeb3..5d66474 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -9,7 +9,6 @@ use std::error::Error;
 use std::fmt;
 use std::fmt::Debug;
 use std::future::Future;
-use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -61,15 +60,6 @@ where
     }
 }
 
-// #[derive(Clone, Debug)]
-// pub struct RequestInfo {
-//     pub stream_subject: String,
-//     pub consumer_name: String,
-//     pub msg_id: String,
-//     pub data: Vec<u8>,
-//     pub headers: Option<HeaderMap>,
-// }
-
 #[derive(Clone, Debug)]
 pub struct RequestInfo {
     pub stream_subject: String,
@@ -352,7 +342,8 @@ fn get_nsc_root_path() -> String {
 }
 
 pub fn get_local_creds_path() -> String {
-    std::env::var("LOCAL_CREDS_PATH").unwrap_or_else(|_| format!("{}/local_creds", get_nsc_root_path()))
+    std::env::var("LOCAL_CREDS_PATH")
+        .unwrap_or_else(|_| format!("{}/local_creds", get_nsc_root_path()))
 }
 
 pub fn get_nats_creds_by_nsc(operator: &str, account: &str, user: &str) -> String {

From 0f714ed8f5e7a5930e98a0a5980485caf995822f Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Mon, 17 Feb 2025 19:09:28 -0600
Subject: [PATCH 78/91] clean-up

---
 nix/modules/nixos/holo-host-agent.nix     | 49 +++++++++++++++++++++++
 rust/clients/host_agent/src/auth/utils.rs | 49 +++++++++++++++++++++++
 rust/clients/host_agent/src/main.rs       | 48 +---------------------
 rust/services/authentication/src/lib.rs   |  8 ++--
 4 files changed, 103 insertions(+), 51 deletions(-)

diff --git a/nix/modules/nixos/holo-host-agent.nix b/nix/modules/nixos/holo-host-agent.nix
index 39833c5..ad10830 100644
--- a/nix/modules/nixos/holo-host-agent.nix
+++ b/nix/modules/nixos/holo-host-agent.nix
@@ -64,6 +64,41 @@ in
         default = "${cfg.nats.listenHost}:${builtins.toString cfg.nats.listenPort}";
       };
 
+      nscPath = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/.local/share/nats/nsc";
+      };
+
+      sharedCredsPath = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.nats.nscPath}/shared_creds";
+      };
+
+      localCredsPath = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.nats.nscPath}/local_creds";
+      };
+
+      hostNkeyPath = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.nats.localCredsPath}/host.nk";
+      };
+
+      sysNkeyPath = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.nats.localCredsPath}/sys.nk";
+      };
+
+      hposCredsPath = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/holo-host-agent/server-key-config.json";
+      };
+
+      hposCredsPw = lib.mkOption {
+        type = lib.types.str;
+        default = "pass";
+      };
+
       hub = {
         url = lib.mkOption {
           type = lib.types.str;
@@ -96,6 +131,12 @@ in
         {
           RUST_LOG = cfg.rust.log;
           RUST_BACKTRACE = cfg.rust.backtrace;
+          NSC_PATH = cfg.nats.nscPath;
+          LOCAL_CREDS_PATH = cfg.nats.localCredsPath;
+          HOSTING_AGENT_HOST_NKEY_PATH = cfg.nats.hostNkeyPath;
+          HOSTING_AGENT_SYS_NKEY_PATH = cfg.nats.sysNkeyPath;
+          HPOS_CONFIG_PATH = cfg.nats.hposCredsPath;
+          DEVICE_SEED_DEFAULT_PASSWORD = builtins.toString cfg.nats.hposCredsPw;
           NATS_LISTEN_PORT = builtins.toString cfg.nats.listenPort;
         }
         // lib.attrsets.optionalAttrs (cfg.nats.url != null) {
@@ -106,6 +147,14 @@ in
         pkgs.nats-server
       ];
 
+      preStart = ''
+        echo "Start Host Auth Setup"
+        mkdir -p ${cfg.nats.hostNkeyPath}
+        mkdir -p ${cfg.nats.sysNkeyPath}
+        mkdir -p ${cfg.nats.hposCredsPath}
+        echo "Finshed Host Auth Setup"
+      '';
+
       script =
         let
           extraDaemonizeArgsList = lib.attrsets.mapAttrsToList (
diff --git a/rust/clients/host_agent/src/auth/utils.rs b/rust/clients/host_agent/src/auth/utils.rs
index cef4ffe..89e0178 100644
--- a/rust/clients/host_agent/src/auth/utils.rs
+++ b/rust/clients/host_agent/src/auth/utils.rs
@@ -1,4 +1,7 @@
+use crate::{auth, keys};
+use anyhow::Result;
 use data_encoding::BASE64URL_NOPAD;
+use hpos_hal::inventory::HoloInventory;
 
 /// Encode a json string into a b64 string
 pub fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
@@ -7,3 +10,49 @@ pub fn json_to_base64(json_data: &str) -> Result<String, serde_json::Error> {
     let encoded = BASE64URL_NOPAD.encode(json_string.as_bytes());
     Ok(encoded)
 }
+
+pub async fn run_auth_loop(mut keys: keys::Keys) -> Result<keys::Keys, async_nats::Error> {
+    let mut start = chrono::Utc::now();
+    loop {
+        log::debug!("About to run the Hosting Agent Authentication Service");
+        let auth_guard_client: async_nats::Client;
+        (keys, auth_guard_client) = auth::init::run(keys).await?;
+
+        // If authenicated creds exist, then auth call was successful.
+        // Close buffer, exit loop, and return.
+        if let keys::AuthCredType::Authenticated(_) = keys.creds {
+            auth_guard_client.drain().await?;
+            break;
+        }
+
+        // Otherwise, send diagonostics and wait 24hrs, then exit while loop and retry auth.
+        // TODO: Discuss interval for sending diagnostic reports and wait duration before retrying auth with team.
+        let now = chrono::Utc::now();
+        let max_time_interval = chrono::TimeDelta::hours(24);
+
+        while max_time_interval > now.signed_duration_since(start) {
+            let pubkey_lowercase = keys.host_pubkey.to_string().to_lowercase();
+            let unauthenticated_user_inventory_subject =
+                format!("INVENTORY.update.{}.unauthenticated", pubkey_lowercase);
+            let inventory = HoloInventory::from_host();
+            let payload_bytes = serde_json::to_vec(&inventory)?;
+
+            if let Err(e) = auth_guard_client
+                .publish(unauthenticated_user_inventory_subject, payload_bytes.into())
+                .await
+            {
+                log::error!(
+                    "Encountered error when sending inventory as unauthenticated user. Err={:#?}",
+                    e
+                );
+            };
+            tokio::time::sleep(chrono::TimeDelta::hours(24).to_std()?).await;
+        }
+
+        // Close and drain internal buffer before exiting to make sure all messages are sent.
+        auth_guard_client.drain().await?;
+        start = chrono::Utc::now();
+    }
+
+    Ok(keys)
+}
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index d1938dc..293fbcd 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -20,7 +20,6 @@ use agent_cli::DaemonzeArgs;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
-use hpos_hal::inventory::HoloInventory;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -63,7 +62,7 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
 
     // If user cred file is for the auth_guard user, run loop to authenticate host & hoster...
     if let keys::AuthCredType::Guard(_) = host_agent_keys.creds {
-        host_agent_keys = run_auth_loop(host_agent_keys).await?;
+        host_agent_keys = auth::utils::run_auth_loop(host_agent_keys).await?;
     }
 
     log::trace!(
@@ -95,48 +94,3 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
 
     Ok(())
 }
-
-async fn run_auth_loop(mut keys: keys::Keys) -> Result<keys::Keys, async_nats::Error> {
-    let mut start = chrono::Utc::now();
-    loop {
-        log::debug!("About to run the Hosting Agent Authentication Service");
-        let auth_guard_client: async_nats::Client;
-        (keys, auth_guard_client) = auth::init::run(keys).await?;
-
-        // If authenicated creds exist, then auth call was successful.
-        // Close buffer, exit loop, and return.
-        if let keys::AuthCredType::Authenticated(_) = keys.creds {
-            auth_guard_client.drain().await?;
-            break;
-        }
-
-        // Otherwise, send diagonostics every 1hr for the next 24hrs, then exit while loop and retry auth.
-        // TODO: Discuss interval for sending diagnostic reports and wait duration before retrying auth with team.
-        let now = chrono::Utc::now();
-        let max_time_interval = chrono::TimeDelta::days(1);
-
-        while max_time_interval > now.signed_duration_since(start) {
-            let unauthenticated_user_diagnostics_subject =
-                format!("DIAGNOSTICS.{}.unauthenticated", keys.host_pubkey);
-            let diganostics = HoloInventory::from_host();
-            let payload_bytes = serde_json::to_vec(&diganostics)?;
-
-            if let Err(e) = auth_guard_client
-                .publish(
-                    unauthenticated_user_diagnostics_subject,
-                    payload_bytes.into(),
-                )
-                .await
-            {
-                log::error!("Encountered error when sending diganostics. Err={:#?}", e);
-            };
-            tokio::time::sleep(chrono::TimeDelta::hours(1).to_std()?).await;
-        }
-
-        // Close and drain internal buffer before exiting to make sure all messages are sent.
-        auth_guard_client.drain().await?;
-        start = chrono::Utc::now();
-    }
-
-    Ok(keys)
-}
diff --git a/rust/services/authentication/src/lib.rs b/rust/services/authentication/src/lib.rs
index eb90ad1..10869db 100644
--- a/rust/services/authentication/src/lib.rs
+++ b/rust/services/authentication/src/lib.rs
@@ -2,7 +2,7 @@
 Service Name: AUTH
 Subject: "AUTH.>"
 Provisioning Account: AUTH Account (ie: This service is exclusively permissioned to the AUTH account.)
-Users: orchestrator & noauth
+Users: orchestrator auth user & auth guard user
 Endpoints & Managed Subjects:
     - handle_auth_callout: $SYS.REQ.USER.AUTH
     - handle_auth_validation: AUTH.validate
@@ -26,12 +26,12 @@ use std::sync::Arc;
 use types::{AuthApiResult, WORKLOAD_SK_ROLE};
 use util_libs::{
     db::{
-        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{self, Host, Hoster, Role, RoleInfo, User},
+        mongodb::{IntoIndexes, MongoCollection}, // , MongoDbAPI
+        schemas::{self, Host, Hoster, User}, //  Role, RoleInfo,
     },
     nats_js_client::{get_nats_jwt_by_nsc, AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
-use utils::handle_internal_err;
+// use utils::handle_internal_err;
 
 pub const AUTH_SRV_NAME: &str = "AUTH";
 pub const AUTH_SRV_SUBJ: &str = "AUTH";

From fd4df0654bb269e2c7b2f740f5877cce0f97a649 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 18 Feb 2025 16:09:39 -0600
Subject: [PATCH 79/91] tidy

---
 .env.example                                  |  16 +-
 .gitignore                                    |   3 +-
 .vscode/settings.json                         |   9 +-
 rust/clients/host_agent/Cargo.toml            |  16 +-
 .../host_agent/src/hostd/gen_leaf_server.rs   |   4 +-
 .../host_agent/src/hostd/workload_manager.rs  |  18 +-
 rust/clients/orchestrator/src/main.rs         |   5 +-
 rust/clients/orchestrator/src/workloads.rs    |  18 +-
 rust/services/workload/src/host_api.rs        |   4 +-
 .../services/workload/src/orchestrator_api.rs | 278 +++++++++++++-----
 rust/services/workload/src/types.rs           |   2 +-
 rust/util_libs/src/js_stream_service.rs       |  10 +-
 rust/util_libs/src/nats_js_client.rs          | 150 +++++-----
 rust/util_libs/src/nats_server.rs             |  10 +-
 rust/util_libs/src/nats_types.rs              | 145 ---------
 rust/util_libs/test_configs/hub_server.conf   |  22 --
 .../test_configs/hub_server_pw_auth.conf      |  22 --
 17 files changed, 360 insertions(+), 372 deletions(-)
 delete mode 100644 rust/util_libs/src/nats_types.rs
 delete mode 100644 rust/util_libs/test_configs/hub_server.conf
 delete mode 100644 rust/util_libs/test_configs/hub_server_pw_auth.conf

diff --git a/.env.example b/.env.example
index bfe8157..3ca30fd 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,14 @@
-NSC_PATH = "<path/to/nsc_dir>"
+# ALL
+NSC_PATH="<path/to/nsc_dir>"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LOCAL_CREDS_PATH="<path/to/dir>"
+
+# ORCHESTRATOR
+MONGO_URI="mongodb://<host>:<port>"
+
+# HOSTING AGENT
 HOST_CREDS_FILE_PATH = "ops/admin.creds"
-MONGO_URI = "mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 41cbad9..1812acf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,4 +22,5 @@ rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf
 .local
-
+rust/*/*/*/tmp/
+rust/*/*/*/*/tmp/
diff --git a/.vscode/settings.json b/.vscode/settings.json
index f78dcd9..4b8209b 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -14,5 +14,12 @@
         "command": ["treefmt-nix", "--stdin", "neverexists.nix"]
       }
     }
-  }
+  },
+  "[rust]": {
+    "editor.defaultFormatter": "rust-lang.rust-analyzer",
+  },
+  "files.readonlyInclude": {
+    "**/.cargo/registry/src/**/*.rs": true,
+    "**/lib/rustlib/src/rust/library/**/*.rs": true,
+  },
 }
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 5ed82ab..9d7462b 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -14,15 +14,21 @@ log = { workspace = true }
 dotenv = { workspace = true }
 clap = { workspace = true }
 thiserror = { workspace = true }
+env_logger = { workspace = true }
 url = { version = "2", features = ["serde"] }
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
-env_logger = { workspace = true }
-mongodb = "3.1"
+ed25519-dalek = { version = "2.1.1" }
+nkeys = "=0.4.4"
+sha2 = "=0.10.8"
+nats-jwt = "0.3.0"
+data-encoding = "2.7.0"
+jsonwebtoken = "9.3.0"
+textnonce = "1.0.0"
 chrono = "0.4.0"
 bytes = "1.8.0"
-nkeys = "=0.4.4"
 rand = "0.8.5"
+tempfile = "3.15.0"
+hpos-hal = { path = "../../hpos-hal" }
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
-hpos-hal = { path = "../../hpos-hal" }
-tempfile = "3.15.0"
+
diff --git a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 906946c..f70d1c3 100644
--- a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -4,7 +4,7 @@ use anyhow::Context;
 use tempfile::tempdir;
 use util_libs::nats_server::{
     JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
-    LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT, LEAF_SERVE_NAME,
+    LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
 };
 
 pub async fn run(
@@ -58,7 +58,7 @@ pub async fn run(
 
     // Create a new Leaf Server instance
     let leaf_server = LeafServer::new(
-        LEAF_SERVE_NAME,
+        None,
         LEAF_SERVER_CONFIG_PATH,
         leaf_client_conn_domain,
         leaf_client_conn_port,
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index 0d9fe39..6257afb 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -23,7 +23,7 @@ use workload::{
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
-const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
+const HOST_AGENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX";
 
 // TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
 pub async fn run(
@@ -35,6 +35,8 @@ pub async fn run(
     log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
+    let pubkey_lowercase = host_pubkey.to_string().to_lowercase();
+
     // ==================== Setup NATS ====================
     // Connect to Nats server
     let nats_url = nats_js_client::get_nats_url();
@@ -93,7 +95,7 @@ pub async fn run(
     
     // Register Workload Streams for Host Agent to consume and process
     // NB: Subjects are published by orchestrator
-    let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
+    let workload_install_subject = serde_json::to_string(&WorkloadServiceSubjects::Install)?;
     let workload_send_status_subject = serde_json::to_string(&WorkloadServiceSubjects::SendStatus)?;
     let workload_uninstall_subject = serde_json::to_string(&WorkloadServiceSubjects::Uninstall)?;
     let workload_update_installed_subject = serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
@@ -107,11 +109,11 @@ pub async fn run(
 
     workload_service
         .add_consumer::<WorkloadApiResult>(
-            "start_workload", // consumer name
-            &format!("{}.{}", host_pubkey, workload_start_subject), // consumer stream subj
+            "install_workload", // consumer name
+            &format!("{}.{}", pubkey_lowercase, workload_install_subject), // consumer stream subj
             EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
-                    api.start_workload(msg).await
+                    api.install_workload(msg).await
                 })
             ),
             None,
@@ -121,7 +123,7 @@ pub async fn run(
     workload_service
         .add_consumer::<WorkloadApiResult>(
             "update_installed_workload", // consumer name
-            &format!("{}.{}", host_pubkey, workload_update_installed_subject), // consumer stream subj
+            &format!("{}.{}", pubkey_lowercase, workload_update_installed_subject), // consumer stream subj
             EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.update_workload(msg).await
@@ -134,7 +136,7 @@ pub async fn run(
     workload_service
         .add_consumer::<WorkloadApiResult>(
             "uninstall_workload", // consumer name
-            &format!("{}.{}", host_pubkey, workload_uninstall_subject), // consumer stream subj
+            &format!("{}.{}", pubkey_lowercase, workload_uninstall_subject), // consumer stream subj
             EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
@@ -147,7 +149,7 @@ pub async fn run(
     workload_service
         .add_consumer::<WorkloadApiResult>(
             "send_workload_status", // consumer name
-            &format!("{}.{}", host_pubkey, workload_send_status_subject), // consumer stream subj
+            &format!("{}.{}", pubkey_lowercase, workload_send_status_subject), // consumer stream subj
             EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.send_workload_status(msg).await
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index 8e5aa3e..bbcb245 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -10,8 +10,7 @@ async fn main() -> Result<(), async_nats::Error> {
     // TODO: invoke auth service (once ready)
 
     // Run workload service
-    if let Err(e) = workloads::run().await {
-        log::error!("{}", e)
-    }
+    workloads::run().await?
+
     Ok(())
 }
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index c5bd37e..28b5f33 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -1,7 +1,7 @@
 /*
 This client is associated with the:
-    - WORKLOAD account
-    - orchestrator user
+    - ADMIN account
+    - admin user
 
 This client is responsible for:
     - initalizing connection and handling interface with db
@@ -10,7 +10,7 @@ This client is responsible for:
         - handling requests to update workloads
         - handling requests to remove workloads
         - handling workload status updates
-        - interfacing with mongodb DB
+    - interfacing with mongodb DB
     - keeping service running until explicitly cancelled out
 */
 
@@ -86,14 +86,14 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let workload_api = OrchestratorWorkloadApi::new(&client).await?;
 
     // Register Workload Streams for Orchestrator to consume and proceess
-    // NB: These subjects below are published by external Developer, the Nats-DB-Connector, or the Host Agent
+    // NB: These subjects are published by external Developer (via external api), the Nats-DB-Connector, or the Hosting Agent
     let workload_add_subject = serde_json::to_string(&WorkloadServiceSubjects::Add)?;
     let workload_update_subject = serde_json::to_string(&WorkloadServiceSubjects::Update)?;
     let workload_remove_subject = serde_json::to_string(&WorkloadServiceSubjects::Remove)?;
     let workload_db_insert_subject = serde_json::to_string(&WorkloadServiceSubjects::Insert)?;
     let workload_db_modification_subject = serde_json::to_string(&WorkloadServiceSubjects::Modify)?;
     let workload_handle_status_subject = serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
-    let workload_start_subject = serde_json::to_string(&WorkloadServiceSubjects::Start)?;
+    let workload_install_subject = serde_json::to_string(&WorkloadServiceSubjects::Install)?;
     let workload_update_installed_subject = serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
 
     let workload_service = orchestrator_workload_client
@@ -107,7 +107,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     workload_service
         .add_consumer::<WorkloadApiResult>(
             "add_workload", // consumer name
-             &workload_add_subject, // consumer stream subj
+            &workload_add_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.add_workload(msg).await
@@ -120,7 +120,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         workload_service
         .add_consumer::<WorkloadApiResult>(
             "update_workload", // consumer name
-             &workload_update_subject, // consumer stream subj
+            &workload_update_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.update_workload(msg).await
@@ -134,7 +134,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
     workload_service
         .add_consumer::<WorkloadApiResult>(
             "remove_workload", // consumer name
-             &workload_remove_subject, // consumer stream subj
+            &workload_remove_subject, // consumer stream subj
             EndpointType::Async(workload_api.call(|api: OrchestratorWorkloadApi, msg: Arc<Message>| {
                 async move {
                     api.remove_workload(msg).await
@@ -154,7 +154,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
                     api.handle_db_insertion(msg).await
                 }
             })),
-            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_start_subject)),
+            Some(create_callback_subject_to_host(true, "assigned_hosts".to_string(), workload_install_subject)),
         )
         .await?;
 
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index 028b581..5621f3f 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -1,6 +1,6 @@
 /*
 Endpoints & Managed Subjects:
-    - `start_workload`: handles the "WORKLOAD.<host_pukey>.start." subject
+    - `install_workload`: handles the "WORKLOAD.<host_pukey>.install." subject
     - `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
     - `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
     - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
@@ -24,7 +24,7 @@ pub struct HostWorkloadApi {}
 impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
-    pub async fn start_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn install_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index a49db80..50110de 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -32,6 +32,7 @@ pub struct OrchestratorWorkloadApi {
     pub workload_collection: MongoCollection<schemas::Workload>,
     pub host_collection: MongoCollection<schemas::Host>,
     pub user_collection: MongoCollection<schemas::User>,
+    pub developer_collection: MongoCollection<schemas::Developer>,
 }
 
 impl WorkloadServiceApi for OrchestratorWorkloadApi {}
@@ -42,6 +43,7 @@ impl OrchestratorWorkloadApi {
             workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME).await?,
             host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
             user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME).await?,
         })
     }
 
@@ -81,9 +83,20 @@ impl OrchestratorWorkloadApi {
             WorkloadState::Running,
             |workload: schemas::Workload| async move {
                 let workload_query = doc! { "_id":  workload._id.clone() };
-                let updated_workload_doc = to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
-                log::info!("Successfully updated workload. MongodDB Workload ID={:?}", workload._id);
+
+                let updated_workload_doc =
+                    to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection
+                    .update_one_within(
+                        workload_query,
+                        UpdateModifications::Document(doc! { "$set": updated_workload_doc }),
+                    )
+                    .await?;
+                log::info!(
+                    "Successfully updated workload. MongodDB Workload ID={:?}",
+                    workload._id
+                );
                 Ok(WorkloadApiResult {
                     result: WorkloadResult {
                         status: WorkloadStatus {
@@ -146,15 +159,9 @@ impl OrchestratorWorkloadApi {
                     return Err(ServiceError::Internal(err_msg));
                 };
 
-                // 1. Perform sanity check to ensure workload is not already assigned to a host
-                // ...and if so, exit fn
-                // todo: check for to ensure assigned host *still* has enough capacity for updated workload
+                // 1. Perform sanity check to ensure workload is not already assigned to a host and if so, exit fn
                 if !workload.assigned_hosts.is_empty() {
                     log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    let mut tag_map: HashMap<String, String> = HashMap::new();
-                    for (index, host_pubkey) in workload.assigned_hosts.into_iter().enumerate() {
-                        tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
-                    }
 
                     return Ok(WorkloadApiResult {
                         result: WorkloadResult {
@@ -165,61 +172,94 @@ impl OrchestratorWorkloadApi {
                             },
                             workload: None
                         },
-                        maybe_response_tags: Some(tag_map)
+                        maybe_response_tags: None
                     });
                 }
 
                 // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
-                let host_filter = doc! {
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },      
-                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
-                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk }
-                };
-                let eligible_hosts = self.host_collection.get_many_from(host_filter).await? ;
-                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_hosts);
-
-                // 3. Randomly choose host/node
-                let host = match eligible_hosts.choose(&mut rand::thread_rng()) {
-                    Some(h) => h,
-                    None => {
-                        // todo: Try to get another host up to 5 times, if fails thereafter, return error
-                        let err_msg = format!("Failed to locate an eligible host to support the required workload capacity. Workload={:?}", workload);
-                        return Err(ServiceError::Internal(err_msg));
+                // & randomly choose host(s)
+                let eligible_host_ids = self.find_hosts_meeting_workload_criteria(workload.clone()).await?;
+                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_host_ids);
+
+                // 3. Update the selected host records with the assigned Workload ID
+                // NB: This will attempt to assign the hosts up to 5 times.. then exit loop with warning message
+                let assigned_host_ids: Vec<schemas::MongoDbId>;
+                let mut unassigned_host_ids: Vec<schemas::MongoDbId> = eligible_host_ids.clone();
+                let mut exit_flag = 0;
+                loop {
+                    let updated_host_result = self.host_collection
+                        .update_many_within(
+                        doc! {
+                            "_id": { "$in": unassigned_host_ids.clone() },
+                            // Currently we only allow a single workload per host
+                            "assigned_workloads": { "$size": 0 }
+                        },
+                        UpdateModifications::Document(doc! {
+                            "$set": {
+                                // Currently we only allow a single workload per host
+                                "assigned_workloads": vec![workload_id]
+                            }
+                        }),
+                    )
+                    .await?;
+
+                    if updated_host_result.matched_count == unassigned_host_ids.len() as u64 {
+                        log::debug!(
+                            "Successfully updated Host records with the new workload id {}. Host_IDs={:?} Update_Result={:?}",
+                            workload_id,
+                            eligible_host_ids,
+                            updated_host_result
+                        );
+                        assigned_host_ids = eligible_host_ids;
+                        break;
+                    } else if exit_flag == 5 {
+                        let unassigned_host_hashset: HashSet<schemas::MongoDbId> = unassigned_host_ids.into_iter().collect();
+                        assigned_host_ids =  eligible_host_ids.into_iter().filter(|id| !unassigned_host_hashset.contains(id)).collect();
+                        log::warn!("Exiting loop after 5 attempts to assign the workload to the min number of hosts. Only able to assign {} hosts. Workload_ID={}, Assigned_Host_IDs={:?}",
+                            workload.min_hosts,
+                            workload_id,
+                            assigned_host_ids
+                        );
+                        break;
                     }
-                };
 
-                // Note: The `_id` is an option because it is only generated upon the intial insertion of a record in
-                // a mongodb collection. This also means that whenever a record is fetched from mongodb, it must have the `_id` feild.
-                // Using `unwrap` is therefore safe.
-                let host_id = host._id.to_owned().unwrap();
-                
+                    log::warn!("Failed to update all selected host records with workload_id.");
+                    log::debug!("Fetching paired host records to see which one(s) still remain unassigned to workload...");
+                    let unassigned_hosts= self.host_collection.get_many_from(doc! {
+                        "_id": { "$in": eligible_host_ids.clone() },
+                        "assigned_workloads": { "$size": 0 }
+                    }).await?;
+
+                    unassigned_host_ids = unassigned_hosts.into_iter().map(|h| h._id.unwrap_or_default()).collect();
+                    exit_flag += 1;
+                }
+
                 // 4. Update the Workload Collection with the assigned Host ID
-                let workload_query = doc! { "_id":  workload_id.clone() };
-                let updated_workload = &Workload {
-                    assigned_hosts: vec![host_id],
-                    ..workload.clone()
-                };
-                let updated_workload_doc = to_document(updated_workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
+                let updated_workload_result = self.workload_collection
+                    .update_one_within(
+                        doc! {
+                            "_id": workload_id
+                        },
+                        UpdateModifications::Document(doc! {
+                            "$set": [{
+                                "state": bson::to_bson(&WorkloadState::Assigned)
+                                    .map_err(|e| ServiceError::Internal(e.to_string()))?
+                                }, {
+                                "assigned_hosts": assigned_host_ids.clone()
+                            }]
+                        }),
+                    )
+                    .await?;
+
                 log::trace!(
                     "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
                     updated_workload_result
                 );
-                
-                // 5. Update the Host Collection with the assigned Workload ID
-                let host_query = doc! { "_id":  host.clone()._id };
-                let updated_host_doc =  to_document(&Host {
-                    assigned_workloads: vec![workload_id.clone()],
-                    ..host.to_owned()
-                }).map_err(|e| ServiceError::Internal(e.to_string()))?;
-                let updated_host_result = self.host_collection.update_one_within(host_query, UpdateModifications::Document(updated_host_doc)).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
-                    updated_host_result
-                );
+
+                // 5. Create tag map with host ids to inform nats to publish message to these hosts with workload install status                
                 let mut tag_map: HashMap<String, String> = HashMap::new();
-                for (index, host_pubkey) in updated_workload.assigned_hosts.iter().cloned().enumerate() {
-                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey);
+                for (index, host_pubkey) in assigned_host_ids.iter().cloned().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
                 }
                 Ok(WorkloadApiResult {
                     result: WorkloadResult {
@@ -238,51 +278,149 @@ impl OrchestratorWorkloadApi {
         .await
     }
 
-    // Zeeshan to take a look:
     // NB: Automatically published by the nats-db-connector
-    pub async fn handle_db_modification(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    // triggers on mongodb [workload] collection (update)
+    pub async fn handle_db_modification(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.modify'");
-        
+
         let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
         log::trace!("New workload to assign. Workload={:#?}", workload);
-        
-        // TODO: ...handle the use case for the update entry change stream 
 
-        // let workload_request_bytes = serde_json::to_vec(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+        // 1. remove workloads from existing hosts
+        self.host_collection
+            .inner
+            .update_many(
+                doc! {},
+                doc! { "$pull": { "assigned_workloads": workload._id } },
+            )
+            .await
+            .map_err(ServiceError::Database)?;
 
-        let success_status = WorkloadStatus {
-            id: workload._id.clone(),
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Running,
+        log::info!(
+            "Remove workload from previous hosts. Workload={:#?}",
+            workload._id
+        );
+
+        if !workload.metadata.is_deleted {
+            // 3. add workload to specific hosts
+            self.host_collection
+                .inner
+                .update_one(
+                    doc! { "_id": { "$in": workload.clone().assigned_hosts } },
+                    doc! { "$push": { "assigned_workloads": workload._id } },
+                )
+                .await
+                .map_err(ServiceError::Database)?;
+
+            log::info!("Added workload to new hosts. Workload={:#?}", workload._id);
+        } else {
+            log::info!(
+                "Skipping (reason: deleted) - Added workload to new hosts. Workload={:#?}",
+                workload._id
+            );
+        }
+
+        let status = WorkloadStatus {
+            id: workload._id,
+            desired: WorkloadState::Updating,
+            actual: WorkloadState::Updating,
         };
+        log::info!("Workload update successful. Workload={:#?}", workload._id);
 
         Ok(WorkloadApiResult {
             result: WorkloadResult {
-                status: success_status,
-                workload: Some(workload)
+                status,
+                workload: Some(workload),
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
     // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
 
         let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
         log::trace!("Workload status to update. Status={:?}", workload_status);
 
-        // TODO: ...handle the use case for the workload status update within the orchestrator
+        let workload_status_id = workload_status
+            .id
+            .ok_or_else(|| ServiceError::Internal("Failed to read ._id from record".to_string()))?;
+
+        self.workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_status_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": {
+                        "state": bson::to_bson(&workload_status.actual)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                    }
+                }),
+            )
+            .await?;
 
         Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status: workload_status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
+    // Looks through existing hosts to find possible hosts for a given workload
+    // returns the minimum number of hosts required for workload
+    async fn find_hosts_meeting_workload_criteria(
+        &self,
+        workload: Workload,
+    ) -> Result<Vec<schemas::MongoDbId>, ServiceError> {
+        let pipeline = vec![
+            doc! {
+                "$match": {
+                    // verify there are enough system resources
+                    "$expr": { "$gte": [{ "$sum": "$inventory.drive" }, Bson::Int64(workload.system_specs.capacity.drive as i64)]},
+                    "$expr": { "$gte": [{ "$size": "$inventory.cpus" }, Bson::Int64(workload.system_specs.capacity.cores)]},
+
+                    // limit how many workloads a single host can have
+                    "assigned_workloads": { "$lt": 1 }
+                }
+            },
+            doc! {
+                // the maximum number of hosts returned should be the minimum hosts required by workload
+                // sample randomized results and always return back at least 1 result
+                "$sample": std::cmp::min(workload.min_hosts as i32, 1),
+
+                // only return the `host._id` feilds
+                "$project": { "_id": 1 }
+            },
+        ];
+        let host_ids = self
+            .host_collection
+            .aggregate::<schemas::MongoDbId>(pipeline)
+            .await?;
+        if host_ids.is_empty() {
+            let err_msg = format!(
+                "Failed to locate a compatible host for workload. Workload_Id={:?}",
+                workload._id
+            );
+            return Err(ServiceError::Internal(err_msg));
+        } else if workload.min_hosts > host_ids.len() as u16 {
+            log::warn!(
+                "Failed to locate the the min required number of hosts for workload. Workload_Id={:?}",
+                workload._id
+            );
+        }
+        Ok(host_ids)
+    }
+
     // Helper function to initialize mongodb collections
     async fn init_collection<T>(
         client: &MongoDBClient,
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index f68d56e..b0ced72 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -12,7 +12,7 @@ pub enum WorkloadServiceSubjects {
     Modify, // db change stream trigger
     HandleStatusUpdate,
     SendStatus,
-    Start,
+    Install,
     Uninstall,
     UpdateInstalled
 }
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/js_stream_service.rs
index 94383ab..a544111 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/js_stream_service.rs
@@ -215,13 +215,19 @@ impl JsStreamService {
     where
         T: EndpointTraits,
     {
-        let full_subject = format!("{}.{}", self.service_subject, endpoint_subject);
+        // Avoid adding the Service Subject prefix if the Endpoint Subject name starts with global keywords $SYS or $JS
+        let consumer_subject =
+            if endpoint_subject.starts_with("$SYS") || endpoint_subject.starts_with("$JS") {
+                endpoint_subject.to_string()
+            } else {
+                format!("{}.{}", self.service_subject, endpoint_subject)
+            };
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
             durable_name: Some(consumer_name.to_string()),
             ack_policy: AckPolicy::Explicit,
-            filter_subject: full_subject,
+            filter_subject: consumer_subject,
             ..Default::default()
         };
 
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats_js_client.rs
index ce2a0ee..89215a0 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats_js_client.rs
@@ -26,7 +26,7 @@ pub enum ServiceError {
 }
 
 pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
 pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
 pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
 pub type AsyncEndpointHandler<T> = Arc<
@@ -81,7 +81,7 @@ impl std::fmt::Debug for JsClient {
             .field("url", &self.url)
             .field("name", &self.name)
             .field("client", &self.client)
-            .field("js", &self.js)
+            .field("js_context", &self.js)
             .field("js_services", &self.js_services)
             .field("service_log_prefix", &self.service_log_prefix)
             .finish()
@@ -94,7 +94,7 @@ pub struct JsClient {
     on_msg_published_event: Option<EventHandler>,
     on_msg_failed_event: Option<EventHandler>,
     client: async_nats::Client, // inner_client
-    pub js: jetstream::Context,
+    pub js_context: jetstream::Context,
     pub js_services: Option<Vec<JsStreamService>>,
     service_log_prefix: String,
 }
@@ -106,14 +106,14 @@ pub struct NewJsClientParams {
     pub inbox_prefix: String,
     #[serde(default)]
     pub service_params: Vec<JsServiceParamsPartial>,
-    #[serde(skip_deserializing)]
-    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
     #[serde(default)]
     pub credentials_path: Option<String>,
     #[serde(default)]
     pub ping_interval: Option<Duration>,
     #[serde(default)]
     pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
 }
 
 impl JsClient {
@@ -150,64 +150,34 @@ impl JsClient {
             .await?;
             services.push(service);
         }
+        
+        let log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
+        log::info!("{}Connected to NATS server at {}", log_prefix, p.nats_url);
 
-        let js_services = if services.is_empty() {
-            None
-        } else {
-            Some(services)
-        };
-
-        let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
-
-        let mut default_client = JsClient {
+        let mut js_client = JsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
             on_msg_failed_event: None,
+            js_services: None,
+            js_context: jetstream::new(client.clone()),
+            service_log_prefix: log_prefix,
             client,
-            js: jetstream,
-            js_services,
-            service_log_prefix: service_log_prefix.clone(),
         };
 
-        for opt in p.opts {
-            opt(&mut default_client);
+        for listener in p.listeners {
+            listener(&mut js_client);
         }
 
-        log::info!(
-            "{}Connected to NATS server at {}",
-            service_log_prefix,
-            default_client.url
-        );
         Ok(default_client)
     }
 
-    pub fn name(&self) -> &str {
-        &self.name
-    }
-
     pub fn get_server_info(&self) -> ServerInfo {
         self.client.server_info()
     }
 
-    pub async fn monitor(&self) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            Err(Box::new(ErrClientDisconnected))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub async fn close(&self) -> Result<(), async_nats::Error> {
-        self.client.drain().await?;
-        Ok(())
-    }
-
-    pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            return Err(Box::new(ErrClientDisconnected));
-        }
-        let stream = &self.js.get_stream(stream_name).await?;
+    pub async fn get_stream_info(&self, stream_name: &str) -> Result<(), async_nats::Error> {
+        let stream = &self.js_context.get_stream(stream_name).await?;
         let info = stream.get_info().await?;
         log::debug!(
             "{}JetStream info: stream:{}, info:{:?}",
@@ -218,17 +188,46 @@ impl JsClient {
         Ok(())
     }
 
-    pub async fn publish(&self, payload: PublishInfo) -> Result<(), async_nats::Error> {
+    pub async fn check_connection(
+        &self,
+    ) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state = self.client.connection_state();
+        if let async_nats::connection::State::Disconnected = conn_state {
+            Err(Box::new(ErrClientDisconnected))
+        } else {
+            Ok(conn_state)
+        }
+    }
+
+    pub async fn publish(
+        &self,
+        payload: PublishInfo,
+    ) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>>
+    {
+        log::debug!(
+            "{}Called Publish message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+
         let now = Instant::now();
         let result = match payload.headers {
-            Some(h) => self
-                .js
-                .publish_with_headers(payload.subject.clone(), h, payload.data.clone().into())
-                .await,
-            None => self
-                .js
-                .publish(payload.subject.clone(), payload.data.clone().into())
-                .await
+            Some(headers) => {
+                self.js_context
+                    .publish_with_headers(
+                        payload.subject.clone(),
+                        headers,
+                        payload.data.clone().into(),
+                    )
+                    .await
+            }
+            None => {
+                self.js_context
+                    .publish(payload.subject.clone(), payload.data.clone().into())
+                    .await
+            }
         };
 
         let duration = now.elapsed();
@@ -236,27 +235,33 @@ impl JsClient {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(Box::new(err));
+            return Err(err);
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
         Ok(())
     }
 
-    pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
-        let mut current_services = self.js_services.unwrap_or_default();
-        current_services.extend(js_services);
+    pub async fn add_js_service(
+        &mut self,
+        params: JsServiceParamsPartial,
+    ) -> Result<(), async_nats::Error> {
+        let new_service = JsStreamService::new(
+            self.js_context.to_owned(),
+            &params.name,
+            &params.description,
+            &params.version,
+            &params.service_subject,
+        )
+        .await?;
+
+        let mut current_services = self.js_services.to_owned().unwrap_or_default();
+        current_services.push(new_service);
         self.js_services = Some(current_services);
-        self
+
+        Ok(())
     }
 
     pub async fn get_js_service(&self, js_service_name: String) -> Option<&JsStreamService> {
@@ -267,6 +272,11 @@ impl JsClient {
         }
         None
     }
+
+    pub async fn close(&self) -> Result<(), async_nats::Error> {
+        self.client.drain().await?;
+        Ok(())
+    }
 }
 
 // Client Options:
@@ -284,7 +294,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_published_event = Some(Box::pin(f.clone()));
+        c.on_msg_published_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -293,7 +303,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_failed_event = Some(Box::pin(f.clone()));
+        c.on_msg_failed_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -330,7 +340,7 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     };
 
     let event_listeners = vec![
-        on_msg_published_event(published_msg_handler),
+        on_msg_published_event(published_msg_handler), // Shouldn't this be the 'NATS_LISTEN_PORT'?
         on_msg_failed_event(failure_handler),
     ];
 
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats_server.rs
index 602d04f..eda6428 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats_server.rs
@@ -56,7 +56,7 @@ impl Default for LeafNodeRemoteTlsConfig {
 
 #[derive(Debug, Clone)]
 pub struct LeafServer {
-    pub name: String,
+    pub name: Option<String>,
     pub config_path: String,
     host: String,
     pub port: u16,
@@ -69,7 +69,7 @@ pub struct LeafServer {
 // TODO: consider merging this with the `LeafServer` struct
 #[derive(Serialize)]
 struct NatsConfig {
-    server_name: String,
+    server_name: Option<String>,
     host: String,
     port: u16,
     jetstream: JetStreamConfig,
@@ -88,7 +88,7 @@ impl LeafServer {
     // Instantiate a new leaf server
     #[allow(clippy::too_many_arguments)]
     pub fn new(
-        server_name: &str,
+        server_name: Option<&str>,
         new_config_path: &str,
         host: &str,
         port: u16,
@@ -98,7 +98,7 @@ impl LeafServer {
     ) -> Self {
         Self {
             name: server_name.to_string(),
-            config_path: new_config_path.to_string(),
+            config_path: new_config_path.map(ToString::to_string),
             host: host.to_string(),
             port,
             jetstream_config,
@@ -142,7 +142,7 @@ impl LeafServer {
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .spawn()
-            .expect("Failed to start NATS server");
+            .context("Failed to start NATS server");
 
         // TODO: wait for a readiness indicator
         std::thread::sleep(std::time::Duration::from_millis(100));
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
deleted file mode 100644
index ece916b..0000000
--- a/rust/util_libs/src/nats_types.rs
+++ /dev/null
@@ -1,145 +0,0 @@
-/* --------
-NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
-IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
-TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */
-
-use serde::{Deserialize, Serialize};
-
-/// JWT claims for NATS compatible jwts
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Claims {
-    /// Time when the token was issued in seconds since the unix epoch
-    #[serde(rename = "iat")]
-    pub issued_at: i64,
-
-    /// Public key of the issuer signing nkey
-    #[serde(rename = "iss")]
-    pub issuer: String,
-
-    /// Base32 hash of the claims where this is empty
-    #[serde(rename = "jti")]
-    pub jwt_id: String,
-
-    /// Public key of the account or user the JWT is being issued to
-    pub sub: String,
-
-    /// Friendly name
-    pub name: String,
-
-    /// NATS claims
-    pub nats: NatsClaims,
-
-    /// Time when the token expires (in seconds since the unix epoch)
-    #[serde(rename = "exp", skip_serializing_if = "Option::is_none")]
-    pub expires: Option<i64>,
-}
-
-/// NATS claims describing settings for the user or account
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "lowercase")]
-pub enum NatsClaims {
-    /// Claims for NATS users
-    User {
-        /// Publish and subscribe permissions for the user
-        #[serde(flatten)]
-        permissions: NatsPermissionsMap,
-
-        /// Public key/id of the account that issued the JWT
-        issuer_account: String,
-
-        /// Maximum nuber of subscriptions the user can have
-        subs: i64,
-
-        /// Maximum size of the message data the user can send in bytes
-        data: i64,
-
-        /// Maximum size of the entire message payload the user can send in bytes
-        payload: i64,
-
-        /// If true, the user isn't challenged on connection. Typically used for websocket
-        /// connections as the browser won't have/want to have the user's private key.
-        bearer_token: bool,
-
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-    /// Claims for NATS accounts
-    Account {
-        /// Configuration for the limits for this account
-        limits: NatsAccountLimits,
-
-        /// List of signing keys (public key) this account uses
-        #[serde(skip_serializing_if = "Vec::is_empty")]
-        signing_keys: Vec<String>,
-
-        /// Default publish and subscribe permissions users under this account will have if not
-        /// specified otherwise
-        /// default_permissions: NatsPermissionsMap,
-        ///
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-}
-
-/// List of subjects that are allowed and/or denied
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissions {
-    /// List of subject patterns that are allowed
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub allow: Vec<String>,
-
-    /// List of subject patterns that are denied
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub deny: Vec<String>,
-}
-
-impl NatsPermissions {
-    /// Returns `true` if the allow and deny list are both empty
-    #[must_use]
-    pub fn is_empty(&self) -> bool {
-        self.allow.is_empty() && self.deny.is_empty()
-    }
-}
-
-/// Publish and subcribe permissons
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissionsMap {
-    /// Permissions for which subjects can be published to
-    #[serde(rename = "pub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub publish: NatsPermissions,
-
-    /// Permissions for which subjects can be subscribed to
-    #[serde(rename = "sub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub subscribe: NatsPermissions,
-}
-
-/// Limits on what an account or users in the account can do
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NatsAccountLimits {
-    /// Maximum nuber of subscriptions the account
-    pub subs: i64,
-
-    /// Maximum size of the message data a user can send in bytes
-    pub data: i64,
-
-    /// Maximum size of the entire message payload a user can send in bytes
-    pub payload: i64,
-
-    /// Maxiumum number of imports for the account
-    pub imports: i64,
-
-    /// Maxiumum number of exports for the account
-    pub exports: i64,
-
-    /// If true, exports can contain wildcards
-    pub wildcards: bool,
-
-    /// Maximum number of active connections
-    pub conn: i64,
-
-    /// Maximum number of leaf node connections
-    pub leaf: i64,
-}
diff --git a/rust/util_libs/test_configs/hub_server.conf b/rust/util_libs/test_configs/hub_server.conf
deleted file mode 100644
index 0184098..0000000
--- a/rust/util_libs/test_configs/hub_server.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-operator: "./test-auth/test-operator/test-operator.jwt"
-system_account: SYS
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-include ./resolver.conf
-
-# logging options
-debug:   true
-trace:   true
-logtime: false
diff --git a/rust/util_libs/test_configs/hub_server_pw_auth.conf b/rust/util_libs/test_configs/hub_server_pw_auth.conf
deleted file mode 100644
index 51eeb3f..0000000
--- a/rust/util_libs/test_configs/hub_server_pw_auth.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-authorization {
-    user: "test-user"
-    password: "pw-12345"
-}
-
-# logging options
-debug:   true
-trace:   true
-logtime: false

From 68b8cd66f2b1d632a281f0bddf4d3a8ae14f4219 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Tue, 18 Feb 2025 23:42:38 -0600
Subject: [PATCH 80/91] clean-up

---
 Cargo.lock                                    |  33 +++-
 .../host_agent/src/hostd/gen_leaf_server.rs   |  13 +-
 .../host_agent/src/hostd/workload_manager.rs  | 123 ++++++------
 rust/clients/orchestrator/Cargo.toml          |   2 +-
 rust/clients/orchestrator/src/main.rs         |   1 +
 rust/clients/orchestrator/src/utils.rs        |  24 +++
 rust/clients/orchestrator/src/workloads.rs    | 138 ++++++-------
 rust/services/workload/Cargo.toml             |   2 +
 rust/services/workload/src/host_api.rs        |  55 +++---
 rust/services/workload/src/lib.rs             |  47 ++---
 .../services/workload/src/orchestrator_api.rs |  42 ++--
 rust/services/workload/src/types.rs           |  16 +-
 rust/util_libs/src/db/mongodb.rs              |   2 +-
 rust/util_libs/src/lib.rs                     |   4 +-
 .../jetstream_client.rs}                      |  81 +-------
 .../jetstream_service.rs}                     | 166 ++++------------
 .../{nats_server.rs => nats/leaf_server.rs}   |   0
 rust/util_libs/src/nats/mod.rs                |   4 +
 rust/util_libs/src/nats/types.rs              | 185 ++++++++++++++++++
 scripts/orchestrator_setup.sh                 |   5 +-
 20 files changed, 518 insertions(+), 425 deletions(-)
 create mode 100644 rust/clients/orchestrator/src/utils.rs
 rename rust/util_libs/src/{nats_js_client.rs => nats/jetstream_client.rs} (81%)
 rename rust/util_libs/src/{js_stream_service.rs => nats/jetstream_service.rs} (79%)
 rename rust/util_libs/src/{nats_server.rs => nats/leaf_server.rs} (100%)
 create mode 100644 rust/util_libs/src/nats/mod.rs
 create mode 100644 rust/util_libs/src/nats/types.rs

diff --git a/Cargo.lock b/Cargo.lock
index 34e96e0..3b65716 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1407,6 +1407,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -2950,7 +2956,7 @@ dependencies = [
  "serde",
  "serde_json",
  "strum 0.23.0",
- "strum_macros",
+ "strum_macros 0.23.1",
  "thiserror 1.0.69",
  "url",
 ]
@@ -3472,6 +3478,12 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
 
+[[package]]
+name = "strum"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
+
 [[package]]
 name = "strum_macros"
 version = "0.23.1"
@@ -3485,6 +3497,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "strum_macros"
+version = "0.25.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.98",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -3910,9 +3935,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.16"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
+checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
 
 [[package]]
 name = "unicode-normalization"
@@ -4554,6 +4579,8 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
+ "strum 0.25.0",
+ "strum_macros 0.25.3",
  "thiserror 2.0.11",
  "tokio",
  "url",
diff --git a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 590db89..37fa4f2 100644
--- a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -2,9 +2,9 @@ use std::{path::PathBuf, time::Duration};
 
 use anyhow::Context;
 use tempfile::tempdir;
-use util_libs::{
-    nats_js_client,
-    nats_server::{
+use util_libs::nats::{
+    jetstream_client,
+    leaf_server::{
         JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
         LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
     },
@@ -17,7 +17,7 @@ pub async fn run(
     hub_url: String,
     hub_tls_insecure: bool,
     nats_connect_timeout_secs: u64,
-) -> anyhow::Result<nats_js_client::JsClient> {
+) -> anyhow::Result<jetstream_client::JsClient> {
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
         .map(|var| var.parse().expect("can't parse into number"))
@@ -95,19 +95,18 @@ pub async fn run(
     // Spin up Nats Client
     // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
     const HOST_AGENT_CLIENT_NAME: &str = "Host Agent Bare";
 
     let nats_client = tokio::select! {
         client = async {loop {
-                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                let host_workload_client = jetstream_client::JsClient::new(jetstream_client::NewJsClientParams {
                     nats_url:nats_url.clone(),
                     name:HOST_AGENT_CLIENT_NAME.to_string(),
                     ping_interval:Some(Duration::from_secs(10)),
                     request_timeout:Some(Duration::from_secs(29)),
-
                     inbox_prefix: Default::default(),
                     service_params:Default::default(),
                     credentials_path: Default::default(),
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload_manager.rs
index c23bd20..30850f5 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload_manager.rs
@@ -13,15 +13,13 @@ This client is responsible for subscribing to workload streams that handle:
 use anyhow::{anyhow, Result};
 use async_nats::Message;
 use std::{path::PathBuf, sync::Arc, time::Duration};
-use util_libs::{
-    js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType},
+use util_libs::nats::{
+    jetstream_client,
+    types::{ConsumerBuilder, EndpointType, JsServiceParamsPartial},
 };
 use workload::{
-    host_api::HostWorkloadApi,
-    types::{WorkloadApiResult, WorkloadServiceSubjects},
-    WorkloadServiceApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ,
-    WORKLOAD_SRV_VERSION,
+    host_api::HostWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
+    WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
@@ -31,7 +29,7 @@ const HOST_AGENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX";
 pub async fn run(
     host_pubkey: &str,
     host_creds_path: &Option<PathBuf>,
-) -> Result<nats_js_client::JsClient, async_nats::Error> {
+) -> Result<jetstream_client::JsClient, async_nats::Error> {
     log::info!("Host Agent Client: Connecting to server...");
     log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
@@ -40,10 +38,10 @@ pub async fn run(
 
     // ==================== Setup NATS ====================
     // Connect to Nats server
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
-    let event_listeners = nats_js_client::get_event_listeners();
+    let event_listeners = jetstream_client::get_event_listeners();
 
     // Setup JS Stream Service
     let workload_stream_service_params = JsServiceParamsPartial {
@@ -54,22 +52,23 @@ pub async fn run(
     };
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-        nats_url: nats_url.clone(),
-        name: HOST_AGENT_CLIENT_NAME.to_string(),
-        inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
-        service_params: vec![workload_stream_service_params.clone()],
-        credentials_path: host_creds_path
-            .as_ref()
-            .map(|path| path.to_string_lossy().to_string()),
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(29)),
-        listeners: vec![nats_js_client::with_event_listeners(
-            event_listeners.clone(),
-        )],
-    })
-    .await
-    .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
+    let host_workload_client =
+        jetstream_client::JsClient::new(jetstream_client::NewJsClientParams {
+            nats_url: nats_url.clone(),
+            name: HOST_AGENT_CLIENT_NAME.to_string(),
+            inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
+            service_params: vec![workload_stream_service_params.clone()],
+            credentials_path: host_creds_path
+                .as_ref()
+                .map(|path| path.to_string_lossy().to_string()),
+            ping_interval: Some(Duration::from_secs(10)),
+            request_timeout: Some(Duration::from_secs(29)),
+            listeners: vec![jetstream_client::with_event_listeners(
+                event_listeners.clone(),
+            )],
+        })
+        .await
+        .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
 
     // ==================== Setup API & Register Endpoints ====================
     // Instantiate the Workload API
@@ -77,12 +76,6 @@ pub async fn run(
 
     // Register Workload Streams for Host Agent to consume and process
     // NB: Subjects are published by orchestrator
-    let workload_install_subject = serde_json::to_string(&WorkloadServiceSubjects::Install)?;
-    let workload_send_status_subject = serde_json::to_string(&WorkloadServiceSubjects::SendStatus)?;
-    let workload_uninstall_subject = serde_json::to_string(&WorkloadServiceSubjects::Uninstall)?;
-    let workload_update_installed_subject =
-        serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
-
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -91,55 +84,73 @@ pub async fn run(
         ))?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "install_workload",                                            // consumer name
-            &format!("{}.{}", pubkey_lowercase, workload_install_subject), // consumer stream subj
-            EndpointType::Async(
+        .add_consumer(ConsumerBuilder {
+            name: "install_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Install.as_ref().to_string()
+            ), // consumer stream subj
+            handler: EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.install_workload(msg).await
                 }),
             ),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "update_installed_workload", // consumer name
-            &format!("{}.{}", pubkey_lowercase, workload_update_installed_subject), // consumer stream subj
-            EndpointType::Async(
+        .add_consumer(ConsumerBuilder {
+            name: "update_installed_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::UpdateInstalled
+                    .as_ref()
+                    .to_string()
+            ), // consumer stream subj
+            handler: EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.update_workload(msg).await
                 }),
             ),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "uninstall_workload",                                            // consumer name
-            &format!("{}.{}", pubkey_lowercase, workload_uninstall_subject), // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "uninstall_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Uninstall.as_ref().to_string()
+            ), // consumer stream subj
+            handler: EndpointType::Async(workload_api.call(
                 |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
                 },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "send_workload_status", // consumer name
-            &format!("{}.{}", pubkey_lowercase, workload_send_status_subject), // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "send_workload_status".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::SendStatus.as_ref().to_string()
+            ), // consumer stream subj
+            handler: EndpointType::Async(workload_api.call(
                 |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.send_workload_status(msg).await
                 },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     Ok(host_workload_client)
diff --git a/rust/clients/orchestrator/Cargo.toml b/rust/clients/orchestrator/Cargo.toml
index 5ad6260..c961b0c 100644
--- a/rust/clients/orchestrator/Cargo.toml
+++ b/rust/clients/orchestrator/Cargo.toml
@@ -12,7 +12,7 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
-thiserror = "2.0"
+thiserror = { workspace = true }
 url = { version = "2", features = ["serde"] }
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
 env_logger = { workspace = true }
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
index 9858343..19aa705 100644
--- a/rust/clients/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/main.rs
@@ -1,4 +1,5 @@
 mod extern_api;
+mod utils;
 mod workloads;
 use anyhow::Result;
 use dotenv::dotenv;
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
new file mode 100644
index 0000000..5694213
--- /dev/null
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -0,0 +1,24 @@
+use std::{collections::HashMap, sync::Arc};
+use util_libs::nats::types::ResponseSubjectsGenerator;
+
+pub fn create_callback_subject_to_host(
+    is_prefix: bool,
+    tag_name: String,
+    sub_subject_name: String,
+) -> ResponseSubjectsGenerator {
+    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
+        if is_prefix {
+            let matching_tags = tag_map.into_iter().fold(vec![], |mut acc, (k, v)| {
+                if k.starts_with(&tag_name) {
+                    acc.push(v)
+                }
+                acc
+            });
+            return matching_tags;
+        } else if let Some(tag) = tag_map.get(&tag_name) {
+            return vec![format!("{}.{}", tag, sub_subject_name)];
+        }
+        log::error!("WORKLOAD Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `WORKLOAD.ERROR.INBOX`.", tag_name, sub_subject_name);
+        vec!["WORKLOAD.ERROR.INBOX".to_string()]
+    })
+}
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index 59e3708..c88c7e4 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -14,52 +14,31 @@ This client is responsible for:
     - keeping service running until explicitly cancelled out
 */
 
+use super::utils;
 use anyhow::{anyhow, Result};
 use async_nats::Message;
 use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::{sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::{JsServiceParamsPartial, ResponseSubjectsGenerator},
-    nats_js_client::{self, EndpointType, JsClient, NewJsClientParams},
+    nats::{
+        jetstream_client::{self, JsClient, NewJsClientParams},
+        types::{ConsumerBuilder, EndpointType, JsServiceParamsPartial},
+    },
 };
 use workload::{
-    orchestrator_api::OrchestratorWorkloadApi,
-    types::{WorkloadApiResult, WorkloadServiceSubjects},
-    WorkloadServiceApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ,
-    WORKLOAD_SRV_VERSION,
+    orchestrator_api::OrchestratorWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
+    WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
 
 const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
-const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "_orchestrator_workload_inbox";
-
-pub fn create_callback_subject_to_host(
-    is_prefix: bool,
-    tag_name: String,
-    sub_subject_name: String,
-) -> ResponseSubjectsGenerator {
-    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
-        if is_prefix {
-            let matching_tags = tag_map.into_iter().fold(vec![], |mut acc, (k, v)| {
-                if k.starts_with(&tag_name) {
-                    acc.push(v)
-                }
-                acc
-            });
-            return matching_tags;
-        } else if let Some(tag) = tag_map.get(&tag_name) {
-            return vec![format!("{}.{}", tag, sub_subject_name)];
-        }
-        log::error!("WORKLOAD Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `WORKLOAD.ERROR.INBOX`.", tag_name, sub_subject_name);
-        vec!["WORKLOAD.ERROR.INBOX".to_string()]
-    })
-}
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "ORCHESTRATOR._WORKLOAD_INBOX";
 
 pub async fn run() -> Result<(), async_nats::Error> {
     // ==================== Setup NATS ====================
-    let nats_url = nats_js_client::get_nats_url();
-    let creds_path = nats_js_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
-    let event_listeners = nats_js_client::get_event_listeners();
+    let nats_url = jetstream_client::get_nats_url();
+    let creds_path = jetstream_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
+    let event_listeners = jetstream_client::get_event_listeners();
 
     // Setup JS Stream Service
     let workload_stream_service_params = JsServiceParamsPartial {
@@ -77,7 +56,7 @@ pub async fn run() -> Result<(), async_nats::Error> {
         credentials_path: Some(creds_path),
         ping_interval: Some(Duration::from_secs(10)),
         request_timeout: Some(Duration::from_secs(5)),
-        listeners: vec![nats_js_client::with_event_listeners(event_listeners)],
+        listeners: vec![jetstream_client::with_event_listeners(event_listeners)],
     })
     .await?;
 
@@ -93,17 +72,6 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Register Workload Streams for Orchestrator to consume and proceess
     // NB: These subjects are published by external Developer (via external api), the Nats-DB-Connector, or the Hosting Agent
-    let workload_add_subject = serde_json::to_string(&WorkloadServiceSubjects::Add)?;
-    let workload_update_subject = serde_json::to_string(&WorkloadServiceSubjects::Update)?;
-    let workload_remove_subject = serde_json::to_string(&WorkloadServiceSubjects::Remove)?;
-    let workload_db_insert_subject = serde_json::to_string(&WorkloadServiceSubjects::Insert)?;
-    let workload_db_modification_subject = serde_json::to_string(&WorkloadServiceSubjects::Modify)?;
-    let workload_handle_status_subject =
-        serde_json::to_string(&WorkloadServiceSubjects::HandleStatusUpdate)?;
-    let workload_install_subject = serde_json::to_string(&WorkloadServiceSubjects::Install)?;
-    let workload_update_installed_subject =
-        serde_json::to_string(&WorkloadServiceSubjects::UpdateInstalled)?;
-
     let workload_service = orchestrator_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -113,91 +81,95 @@ pub async fn run() -> Result<(), async_nats::Error> {
 
     // Published by Developer
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "add_workload",        // consumer name
-            &workload_add_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "add_workload".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Add.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
                 |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.add_workload(msg).await
                 },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "update_workload",        // consumer name
-            &workload_update_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "update_workload".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Update.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
                 |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.update_workload(msg).await
                 },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "remove_workload",        // consumer name
-            &workload_remove_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "remove_workload".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Remove.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
                 |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.remove_workload(msg).await
                 },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     // Automatically published by the Nats-DB-Connector
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "handle_db_insertion",       // consumer name
-            &workload_db_insert_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "handle_db_insertion".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Insert.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
                 |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.handle_db_insertion(msg).await
                 },
             )),
-            Some(create_callback_subject_to_host(
+            response_subject_fn: Some(utils::create_callback_subject_to_host(
                 true,
                 "assigned_hosts".to_string(),
-                workload_install_subject,
+                WorkloadServiceSubjects::Install.as_ref().to_string(),
             )),
-        )
+        })
         .await?;
 
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "handle_db_modification",          // consumer name
-            &workload_db_modification_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "handle_db_modification".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Modify.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
                 |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.handle_db_modification(msg).await
                 },
             )),
-            Some(create_callback_subject_to_host(
+            response_subject_fn: Some(utils::create_callback_subject_to_host(
                 true,
                 "assigned_hosts".to_string(),
-                workload_update_installed_subject,
+                WorkloadServiceSubjects::UpdateInstalled
+                    .as_ref()
+                    .to_string(),
             )),
-        )
+        })
         .await?;
 
     // Published by the Host Agent
     workload_service
-        .add_consumer::<WorkloadApiResult>(
-            "handle_status_update",          // consumer name
-            &workload_handle_status_subject, // consumer stream subj
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "handle_status_update".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::HandleStatusUpdate
+                .as_ref()
+                .to_string(),
+            handler: EndpointType::Async(workload_api.call(
                 |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
                     api.handle_status_update(msg).await
                 },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     // ==================== Close and Clean Client ====================
diff --git a/rust/services/workload/Cargo.toml b/rust/services/workload/Cargo.toml
index fb929d3..60dfe59 100644
--- a/rust/services/workload/Cargo.toml
+++ b/rust/services/workload/Cargo.toml
@@ -14,6 +14,8 @@ env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
 thiserror = { workspace = true }
+strum = "0.25"
+strum_macros = "0.25"
 async-trait = "0.1.83"
 semver = "1.0.24"
 rand = "0.8.5"
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
index 5621f3f..00354ea 100644
--- a/rust/services/workload/src/host_api.rs
+++ b/rust/services/workload/src/host_api.rs
@@ -10,12 +10,12 @@ use crate::types::WorkloadResult;
 
 use super::{types::WorkloadApiResult, WorkloadServiceApi};
 use anyhow::Result;
+use async_nats::Message;
 use core::option::Option::None;
 use std::{fmt::Debug, sync::Arc};
-use async_nats::Message;
 use util_libs::{
-    nats_js_client::ServiceError,
-    db::schemas::{WorkloadState, WorkloadStatus}
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::ServiceError,
 };
 
 #[derive(Debug, Clone, Default)]
@@ -24,7 +24,10 @@ pub struct HostWorkloadApi {}
 impl WorkloadServiceApi for HostWorkloadApi {}
 
 impl HostWorkloadApi {
-    pub async fn install_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn install_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -52,16 +55,19 @@ impl HostWorkloadApi {
             }
         };
 
-        Ok(WorkloadApiResult  {
+        Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
-    pub async fn update_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -69,7 +75,6 @@ impl HostWorkloadApi {
         log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
 
         let status = if let Some(workload) = message_payload.workload {
-
             // TODO: Talk through with Stefan
             // 1. Connect to interface for Nix and instruct systemd to install workload...
             // eg: nix_install_with(workload)
@@ -89,17 +94,20 @@ impl HostWorkloadApi {
                 actual: WorkloadState::Error(err_msg),
             }
         };
-            
-        Ok(WorkloadApiResult  {
+
+        Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
-    pub async fn uninstall_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -110,7 +118,7 @@ impl HostWorkloadApi {
             // TODO: Talk through with Stefan
             // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
             // nix_uninstall_with(workload_id)
-    
+
             // 2. Respond to endpoint request
             WorkloadStatus {
                 id: workload._id,
@@ -127,18 +135,21 @@ impl HostWorkloadApi {
             }
         };
 
-        Ok(WorkloadApiResult  {
+        Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
 
     // For host agent ? or elsewhere ?
     // TODO: Talk through with Stefan
-    pub async fn send_workload_status(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
         let msg_subject = msg.subject.clone().into_string();
         log::trace!("Incoming message for '{}'", msg_subject);
 
@@ -150,9 +161,9 @@ impl HostWorkloadApi {
         Ok(WorkloadApiResult {
             result: WorkloadResult {
                 status: workload_status,
-                workload: None
+                workload: None,
             },
-            maybe_response_tags: None
+            maybe_response_tags: None,
         })
     }
-}
\ No newline at end of file
+}
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index d0c3fba..33d094f 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -5,22 +5,22 @@ Provisioning Account: WORKLOAD
 Users: orchestrator & host
 */
 
-pub mod orchestrator_api;
 pub mod host_api;
+pub mod orchestrator_api;
 pub mod types;
 
 use anyhow::Result;
-use core::option::Option::None;
 use async_nats::jetstream::ErrorCode;
-use async_trait::async_trait;
-use std::{fmt::Debug, sync::Arc};
 use async_nats::Message;
-use std::future::Future;
+use async_trait::async_trait;
+use core::option::Option::None;
 use serde::Deserialize;
+use std::future::Future;
+use std::{fmt::Debug, sync::Arc};
 use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
-    nats_js_client::{ServiceError, AsyncEndpointHandler, JsServiceResponse},
-    db::schemas::{WorkloadState, WorkloadStatus}
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::{AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
@@ -28,26 +28,24 @@ pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
 pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
 
-
 #[async_trait]
 pub trait WorkloadServiceApi
 where
     Self: std::fmt::Debug + Clone + 'static,
 {
-    fn call<F, Fut>(
-        &self,
-        handler: F,
-    ) -> AsyncEndpointHandler<WorkloadApiResult>
+    fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<WorkloadApiResult>
     where
         F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
-        Self: Send + Sync
+        Self: Send + Sync,
     {
-        let api = self.to_owned(); 
-        Arc::new(move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
-            let api_clone = api.clone();
-            Box::pin(handler(api_clone, msg))
-        })
+        let api = self.to_owned();
+        Arc::new(
+            move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
+                let api_clone = api.clone();
+                Box::pin(handler(api_clone, msg))
+            },
+        )
     }
 
     fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
@@ -56,11 +54,14 @@ where
     {
         let payload_buf = msg.payload.to_vec();
         serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
-            let err_msg = format!("Error: Failed to deserialize payload. Subject='{}' Err={}", msg.subject.clone().into_string(), e);
+            let err_msg = format!(
+                "Error: Failed to deserialize payload. Subject='{}' Err={}",
+                msg.subject.clone().into_string(),
+                e
+            );
             log::error!("{}", err_msg);
             ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
         })
-        
     }
 
     // Helper function to streamline the processing of incoming workload messages
@@ -95,11 +96,11 @@ where
                 WorkloadApiResult {
                     result: WorkloadResult {
                         status,
-                        workload: None
+                        workload: None,
                     },
-                    maybe_response_tags: None
+                    maybe_response_tags: None,
                 }
             }
         })
     }
-}
\ No newline at end of file
+}
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
index 95d8b1c..9db39fa 100644
--- a/rust/services/workload/src/orchestrator_api.rs
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -27,7 +27,7 @@ use util_libs::{
         mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
         schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
     },
-    nats_js_client::ServiceError,
+    nats::types::ServiceError,
 };
 
 #[derive(Debug, Clone)]
@@ -112,7 +112,7 @@ impl OrchestratorWorkloadApi {
 
                 self.workload_collection
                     .update_one_within(
-                        doc! { "_id":  workload._id.clone() },
+                        doc! { "_id":  workload._id },
                         UpdateModifications::Document(doc! { "$set": updated_workload_doc }),
                     )
                     .await?;
@@ -153,7 +153,7 @@ impl OrchestratorWorkloadApi {
                     .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
                 self.workload_collection.update_one_within(
-                    doc! { "_id":  workload_id.clone() },
+                    doc! { "_id":  workload_id },
                     UpdateModifications::Document(doc! {
                         "$set": {
                             "metadata.is_deleted": true,
@@ -436,7 +436,11 @@ impl OrchestratorWorkloadApi {
     }
 
     // Verifies that a host meets the workload criteria
-    fn verify_host_meets_workload_criteria(&self, assigned_host: Host, workload: Workload) -> bool {
+    fn verify_host_meets_workload_criteria(
+        &self,
+        assigned_host: &Host,
+        workload: &Workload,
+    ) -> bool {
         if assigned_host.remaining_capacity.disk < workload.system_specs.capacity.disk {
             return false;
         }
@@ -454,8 +458,7 @@ impl OrchestratorWorkloadApi {
         Ok(self
             .host_collection
             .get_many_from(doc! { "assigned_workloads": workload_id })
-            .await
-            .map_err(|e| e)?)
+            .await?)
     }
 
     async fn remove_workload_from_hosts(&self, workload_id: ObjectId) -> Result<()> {
@@ -484,22 +487,21 @@ impl OrchestratorWorkloadApi {
         let mut needed_host_count = workload.min_hosts;
         let mut still_eligible_host_ids: Vec<ObjectId> = vec![];
 
-        if maybe_existing_hosts.is_some() {
-            let hosts = maybe_existing_hosts.unwrap();
+        if let Some(hosts) = maybe_existing_hosts {
             still_eligible_host_ids = hosts.into_iter()
-                .filter(|h| {
-                    move |workload| {
-                        return self.verify_host_meets_workload_criteria(h.to_owned(), workload);
-                    };
-                    false
+                .filter_map(|h| {
+                    if self.verify_host_meets_workload_criteria(&h, &workload) {
+                        h._id.ok_or_else(|| {
+                            ServiceError::Internal(format!(
+                                "No `_id` found for workload. Unable to proceed verifying host eligibility. Workload={:?}",
+                                workload
+                            ))
+                        }).ok()
+                    } else {
+                        None
+                    }
                 })
-                .map(|h| h
-                    ._id
-                    .ok_or_else(|| 
-                        ServiceError::Internal(format!("No `_id` found for workload.  Unable to proceed verifying host elibility. Workload={:?}", workload))
-                    )
-                )
-                .collect::<Result<Vec<ObjectId>, ServiceError>>()?;
+                .collect();
             needed_host_count -= still_eligible_host_ids.len() as u16;
         }
 
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index b0ced72..76e0f82 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,8 +1,12 @@
-use std::collections::HashMap;
-use util_libs::{db::schemas::{self, WorkloadStatus}, js_stream_service::{CreateResponse, CreateTag, EndpointTraits}};
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use strum_macros::AsRefStr;
+use util_libs::{
+    db::schemas::{self, WorkloadStatus},
+    nats::types::{CreateResponse, CreateTag, EndpointTraits},
+};
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, AsRefStr)]
 #[serde(rename_all = "snake_case")]
 pub enum WorkloadServiceSubjects {
     Add,
@@ -14,7 +18,7 @@ pub enum WorkloadServiceSubjects {
     SendStatus,
     Install,
     Uninstall,
-    UpdateInstalled
+    UpdateInstalled,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -26,7 +30,7 @@ pub struct WorkloadResult {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadApiResult {
     pub result: WorkloadResult,
-    pub maybe_response_tags: Option<HashMap<String, String>>
+    pub maybe_response_tags: Option<HashMap<String, String>>,
 }
 impl EndpointTraits for WorkloadApiResult {}
 impl CreateTag for WorkloadApiResult {
@@ -42,4 +46,4 @@ impl CreateResponse for WorkloadApiResult {
             Err(e) => e.to_string().into(),
         }
     }
-}
\ No newline at end of file
+}
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index 819aa60..332c741 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,4 +1,4 @@
-use crate::nats_js_client::ServiceError;
+use crate::nats::types::ServiceError;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use bson::oid::ObjectId;
diff --git a/rust/util_libs/src/lib.rs b/rust/util_libs/src/lib.rs
index 662a7cd..f1b0265 100644
--- a/rust/util_libs/src/lib.rs
+++ b/rust/util_libs/src/lib.rs
@@ -1,4 +1,2 @@
 pub mod db;
-pub mod js_stream_service;
-pub mod nats_js_client;
-pub mod nats_server;
+pub mod nats;
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats/jetstream_client.rs
similarity index 81%
rename from rust/util_libs/src/nats_js_client.rs
rename to rust/util_libs/src/nats/jetstream_client.rs
index fbfc8ca..e75c2d3 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats/jetstream_client.rs
@@ -1,80 +1,17 @@
-use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
-use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
-
+use super::{
+    jetstream_service::JsStreamService,
+    leaf_server::LEAF_SERVER_DEFAULT_LISTEN_PORT,
+    types::{
+        ErrClientDisconnected, EventHandler, EventListener, JsServiceParamsPartial, PublishInfo,
+    },
+};
 use anyhow::Result;
-use async_nats::{jetstream, HeaderMap, Message, ServerInfo};
+use async_nats::{jetstream, ServerInfo};
 use core::option::Option::None;
-use serde::{Deserialize, Serialize};
-use std::error::Error;
-use std::fmt;
-use std::fmt::Debug;
-use std::future::Future;
-use std::pin::Pin;
+use serde::Deserialize;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Request Error: {0}")]
-    Request(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-    #[error("Nats Error: {0}")]
-    NATS(String),
-    #[error("Internal Error: {0}")]
-    Internal(String),
-}
-
-pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
-pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
-        + Send
-        + Sync,
->;
-
-#[derive(Clone)]
-pub enum EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
-{
-    Sync(EndpointHandler<T>),
-    Async(AsyncEndpointHandler<T>),
-}
-
-impl<T> std::fmt::Debug for EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let t = match &self {
-            EndpointType::Async(_) => "EndpointType::Async(<function>)",
-            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
-        };
-
-        write!(f, "{}", t)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct PublishInfo {
-    pub subject: String,
-    pub msg_id: String,
-    pub data: Vec<u8>,
-    pub headers: Option<HeaderMap>,
-}
-
-#[derive(Debug)]
-pub struct ErrClientDisconnected;
-impl fmt::Display for ErrClientDisconnected {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Could not reach nats: connection closed")
-    }
-}
-impl Error for ErrClientDisconnected {}
-
 impl std::fmt::Debug for JsClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("JsClient")
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/nats/jetstream_service.rs
similarity index 79%
rename from rust/util_libs/src/js_stream_service.rs
rename to rust/util_libs/src/nats/jetstream_service.rs
index a544111..d32be89 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/nats/jetstream_service.rs
@@ -1,110 +1,17 @@
-use super::nats_js_client::EndpointType;
-
+use super::types::{
+    ConsumerBuilder, ConsumerExt, ConsumerExtTrait, EndpointTraits, EndpointType,
+    JsStreamServiceInfo, LogInfo, ResponseSubjectsGenerator,
+};
 use anyhow::{anyhow, Result};
-use std::any::Any;
-use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
+use async_nats::jetstream::consumer::{self, AckPolicy};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
-use async_trait::async_trait;
 use futures::StreamExt;
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-pub type ResponseSubjectsGenerator = Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
-
-pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> HashMap<String, String>;
-}
-
-pub trait CreateResponse: Send + Sync {
-    fn get_response(&self) -> bytes::Bytes;
-}
-
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + CreateResponse + 'static
-{}
-
-#[async_trait]
-pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
-    fn get_name(&self) -> &str;
-    fn get_consumer(&self) -> PullConsumer;
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
-}
-
-impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
-where
-    T: EndpointTraits,
-{
-    type Error = anyhow::Error;
-
-    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
-        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
-            Ok(*endpoint)
-        } else {
-            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
-        }
-    }
-}
-
-#[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    name: String,
-    consumer: PullConsumer,
-    handler: EndpointType<T>,
-    #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>,
-}
-
-#[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    fn get_name(&self) -> &str {
-        &self.name
-    }
-    fn get_consumer(&self) -> PullConsumer {
-        self.consumer.clone()
-    }
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
-        Box::new(self.handler.clone())
-    }
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
-        self.response_subject_fn.clone()
-    }
-}
-
-#[allow(dead_code)]
-#[derive(Clone, Debug)]
-pub struct JsStreamServiceInfo<'a> {
-    pub name: &'a str,
-    pub version: &'a str,
-    pub service_subject: &'a str,
-}
-
-struct LogInfo {
-    prefix: String,
-    service_name: String,
-    service_subject: String,
-    endpoint_name: String,
-    endpoint_subject: String,
-}
-
-#[derive(Clone, Deserialize, Default)]
-pub struct JsServiceParamsPartial {
-    pub name: String,
-    pub description: String,
-    pub version: String,
-    pub service_subject: String,
-}
-
 /// Microservice for Jetstream Streams
 // This setup creates only one subject for the stream (eg: "WORKLOAD.>") and sets up
 // all consumers of the stream to listen to stream subjects beginning with that subject (eg: "WORKLOAD.start")
@@ -198,7 +105,6 @@ impl JsStreamService {
         let handler: EndpointType<T> = EndpointType::try_from(endpoint_trait_obj)?;
 
         Ok(ConsumerExt {
-            name: consumer_ext.get_name().to_string(),
             consumer: consumer_ext.get_consumer(),
             handler,
             response_subject_fn: consumer_ext.get_response(),
@@ -207,25 +113,26 @@ impl JsStreamService {
 
     pub async fn add_consumer<T>(
         &self,
-        consumer_name: &str,
-        endpoint_subject: &str,
-        endpoint_type: EndpointType<T>,
-        response_subject_fn: Option<ResponseSubjectsGenerator>,
+        builder_params: ConsumerBuilder<T>,
     ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
         // Avoid adding the Service Subject prefix if the Endpoint Subject name starts with global keywords $SYS or $JS
-        let consumer_subject =
-            if endpoint_subject.starts_with("$SYS") || endpoint_subject.starts_with("$JS") {
-                endpoint_subject.to_string()
-            } else {
-                format!("{}.{}", self.service_subject, endpoint_subject)
-            };
+        let consumer_subject = if builder_params.endpoint_subject.starts_with("$SYS")
+            || builder_params.endpoint_subject.starts_with("$JS")
+        {
+            builder_params.endpoint_subject.to_string()
+        } else {
+            format!(
+                "{}.{}",
+                self.service_subject, builder_params.endpoint_subject
+            )
+        };
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
-            durable_name: Some(consumer_name.to_string()),
+            durable_name: Some(builder_params.name.to_string()),
             ack_policy: AckPolicy::Explicit,
             filter_subject: consumer_subject,
             ..Default::default()
@@ -235,28 +142,28 @@ impl JsStreamService {
             .stream
             .write()
             .await
-            .get_or_create_consumer(consumer_name, consumer_config)
+            .get_or_create_consumer(&builder_params.name, consumer_config)
             .await?;
 
         let consumer_with_handler = ConsumerExt {
-            name: consumer_name.to_string(),
             consumer,
-            handler: endpoint_type,
-            response_subject_fn,
+            handler: builder_params.handler,
+            response_subject_fn: builder_params.response_subject_fn,
         };
 
-        self.local_consumers
-            .write()
-            .await
-            .insert(consumer_name.to_string(), Arc::new(consumer_with_handler));
+        self.local_consumers.write().await.insert(
+            builder_params.name.to_string(),
+            Arc::new(consumer_with_handler),
+        );
 
-        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(consumer_name).await?;
-        self.spawn_consumer_handler::<T>(consumer_name).await?;
+        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(&builder_params.name).await?;
+        self.spawn_consumer_handler::<T>(&builder_params.name)
+            .await?;
 
         log::debug!(
             "{}Added the {} local consumer",
             self.service_log_prefix,
-            endpoint_consumer.name,
+            builder_params.name,
         );
 
         Ok(endpoint_consumer)
@@ -287,12 +194,19 @@ impl JsStreamService {
                 .messages()
                 .await?;
 
+            let consumer_info = consumer.info().await?;
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
-                endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
+                endpoint_name: consumer_info
+                    .config
+                    .durable_name
+                    .clone()
+                    .unwrap_or("Consumer Name Not Found".to_string())
+                    .clone(),
+                endpoint_subject: consumer_info.config.filter_subject.clone(),
             };
 
             let service_context = self.js_context.clone();
@@ -349,8 +263,8 @@ impl JsStreamService {
                     let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
-                },
-                Err(err) => (err.to_string().into(), HashMap::new())
+                }
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
@@ -550,4 +464,4 @@ mod tests {
         let result = service.spawn_consumer_handler(consumer_name).await;
         assert!(result.is_ok(), "Failed to spawn consumer handler");
     }
-}
\ No newline at end of file
+}
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats/leaf_server.rs
similarity index 100%
rename from rust/util_libs/src/nats_server.rs
rename to rust/util_libs/src/nats/leaf_server.rs
diff --git a/rust/util_libs/src/nats/mod.rs b/rust/util_libs/src/nats/mod.rs
new file mode 100644
index 0000000..a320ee4
--- /dev/null
+++ b/rust/util_libs/src/nats/mod.rs
@@ -0,0 +1,4 @@
+pub mod jetstream_client;
+pub mod jetstream_service;
+pub mod leaf_server;
+pub mod types;
diff --git a/rust/util_libs/src/nats/types.rs b/rust/util_libs/src/nats/types.rs
new file mode 100644
index 0000000..2bbfe08
--- /dev/null
+++ b/rust/util_libs/src/nats/types.rs
@@ -0,0 +1,185 @@
+use super::jetstream_client::JsClient;
+use anyhow::Result;
+use async_nats::jetstream::consumer::PullConsumer;
+use async_nats::{HeaderMap, Message};
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt;
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
+pub type AsyncEndpointHandler<T> = Arc<
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
+        + Send
+        + Sync,
+>;
+pub type ResponseSubjectsGenerator =
+    Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
+
+pub trait EndpointTraits:
+    Serialize
+    + for<'de> Deserialize<'de>
+    + Send
+    + Sync
+    + Clone
+    + Debug
+    + CreateTag
+    + CreateResponse
+    + 'static
+{
+}
+
+pub trait CreateTag: Send + Sync {
+    fn get_tags(&self) -> HashMap<String, String>;
+}
+
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
+}
+
+#[async_trait]
+pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
+    fn get_consumer(&self) -> PullConsumer;
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
+}
+
+#[async_trait]
+impl<T> ConsumerExtTrait for ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    fn get_consumer(&self) -> PullConsumer {
+        self.consumer.clone()
+    }
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
+        Box::new(self.handler.clone())
+    }
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
+        self.response_subject_fn.clone()
+    }
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    pub consumer: PullConsumer,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerBuilder<T>
+where
+    T: EndpointTraits,
+{
+    pub name: String,
+    pub endpoint_subject: String,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[allow(dead_code)]
+#[derive(Clone, Debug)]
+pub struct JsStreamServiceInfo<'a> {
+    pub name: &'a str,
+    pub version: &'a str,
+    pub service_subject: &'a str,
+}
+
+#[derive(Clone, Debug)]
+pub struct LogInfo {
+    pub prefix: String,
+    pub service_name: String,
+    pub service_subject: String,
+    pub endpoint_name: String,
+    pub endpoint_subject: String,
+}
+
+#[derive(Clone)]
+pub enum EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
+{
+    Sync(EndpointHandler<T>),
+    Async(AsyncEndpointHandler<T>),
+}
+
+impl<T> std::fmt::Debug for EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let t = match &self {
+            EndpointType::Async(_) => "EndpointType::Async(<function>)",
+            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
+        };
+
+        write!(f, "{}", t)
+    }
+}
+impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
+where
+    T: EndpointTraits,
+{
+    type Error = anyhow::Error;
+
+    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
+        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
+            Ok(*endpoint)
+        } else {
+            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
+        }
+    }
+}
+
+#[derive(Clone, Deserialize, Default)]
+pub struct JsServiceParamsPartial {
+    pub name: String,
+    pub description: String,
+    pub version: String,
+    pub service_subject: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct PublishInfo {
+    pub subject: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+    pub headers: Option<HeaderMap>,
+}
+
+#[derive(Debug)]
+pub struct ErrClientDisconnected;
+impl fmt::Display for ErrClientDisconnected {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Could not reach nats: connection closed")
+    }
+}
+impl Error for ErrClientDisconnected {}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index 347ea6c..a3cc658 100644
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -72,14 +72,14 @@ nsc add account --name $ADMIN_ACCOUNT
 nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
 SIGNING_KEY_ADMIN="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 ROLE_NAME_ADMIN="admin_role"
-nsc edit signing-key --sk $SIGNING_KEY_ADMIN --role $ROLE_NAME_ADMIN --allow-pub "ADMIN_>" --allow-sub "ADMIN_>" --allow-pub-response
+nsc edit signing-key --sk $SIGNING_KEY_ADMIN --role $ROLE_NAME_ADMIN --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","INVENTORY.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","*._WORKLOAD_INBOX.>","_AUTH_INBOX_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","INVENTORY.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","ORCHESTRATOR._WORKLOAD_INBOX.>","_AUTH_INBOX_ORCHESTRATOR.>" --allow-pub-response
 
 # Step 3: Create WORKLOAD Account with JetStream and scoped signing key
 nsc add account --name $WORKLOAD_ACCOUNT
 nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
 SIGNING_KEY_WORKLOAD="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
 ROLE_NAME_WORKLOAD="workload-role"
-nsc edit signing-key --sk $SIGNING_KEY_WORKLOAD --role $ROLE_NAME_WORKLOAD --allow-pub "WORKLOAD.>" --allow-sub "WORKLOAD.>" --allow-pub-response
+nsc edit signing-key --sk $SIGNING_KEY_WORKLOAD --role $ROLE_NAME_WORKLOAD --allow-pub "WORKLOAD.>","{{tag(pubkey)}}._WORKLOAD_INBOX.>","INVENTORY.update.{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","{{tag(pubkey)}}._WORKLOAD_INBOX.>","INVENTORY.update.{{tag(pubkey)}}.>" --allow-pub-response
 
 # Step 4: Create User "orchestrator" in ADMIN Account // noauth
 nsc add user --name admin --account $ADMIN_ACCOUNT
@@ -100,3 +100,4 @@ nsc generate config --nats-resolver --sys-account $SYS_ACCOUNT --force --config-
 nsc push -A
 
 echo "Setup complete. JWTs and resolver file are in the $JWT_OUTPUT_DIR/ directory."
+echo "!! Don't forget to start the NATS server and push the credentials to the server with 'nsc push -A' !!"

From 88d976dc3834848c004348d872353d1793022233 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Wed, 19 Feb 2025 00:29:13 -0600
Subject: [PATCH 81/91] rename workload account, clean structure

---
 .../host_agent/src/hostd/gen_leaf_server.rs   |  3 +-
 rust/clients/host_agent/src/hostd/mod.rs      |  2 +-
 .../{workload_manager.rs => workload.rs}      | 57 +++++++++---------
 rust/clients/host_agent/src/main.rs           |  4 +-
 rust/clients/orchestrator/src/workloads.rs    |  8 +--
 rust/services/workload/src/lib.rs             |  5 +-
 rust/util_libs/src/nats/jetstream_client.rs   | 29 ++--------
 rust/util_libs/src/nats/types.rs              | 19 +++++-
 scripts/orchestrator_setup.sh                 | 58 ++++++++++---------
 9 files changed, 94 insertions(+), 91 deletions(-)
 rename rust/clients/host_agent/src/hostd/{workload_manager.rs => workload.rs} (74%)

diff --git a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 37fa4f2..1b0efc3 100644
--- a/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -8,6 +8,7 @@ use util_libs::nats::{
         JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
         LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
     },
+    types::JsClientBuilder,
 };
 
 pub async fn run(
@@ -102,7 +103,7 @@ pub async fn run(
 
     let nats_client = tokio::select! {
         client = async {loop {
-                let host_workload_client = jetstream_client::JsClient::new(jetstream_client::NewJsClientParams {
+                let host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
                     nats_url:nats_url.clone(),
                     name:HOST_AGENT_CLIENT_NAME.to_string(),
                     ping_interval:Some(Duration::from_secs(10)),
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
index 6a971dc..3e7c23b 100644
--- a/rust/clients/host_agent/src/hostd/mod.rs
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -1,2 +1,2 @@
-pub mod workload_manager;
 pub mod gen_leaf_server;
+pub mod workload;
diff --git a/rust/clients/host_agent/src/hostd/workload_manager.rs b/rust/clients/host_agent/src/hostd/workload.rs
similarity index 74%
rename from rust/clients/host_agent/src/hostd/workload_manager.rs
rename to rust/clients/host_agent/src/hostd/workload.rs
index 30850f5..b5a425d 100644
--- a/rust/clients/host_agent/src/hostd/workload_manager.rs
+++ b/rust/clients/host_agent/src/hostd/workload.rs
@@ -1,6 +1,6 @@
 /*
  This client is associated with the:
-    - WORKLOAD account
+    - HPOS account
     - host user
 
 This client is responsible for subscribing to workload streams that handle:
@@ -15,7 +15,7 @@ use async_nats::Message;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use util_libs::nats::{
     jetstream_client,
-    types::{ConsumerBuilder, EndpointType, JsServiceParamsPartial},
+    types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
 };
 use workload::{
     host_api::HostWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
@@ -44,7 +44,7 @@ pub async fn run(
     let event_listeners = jetstream_client::get_event_listeners();
 
     // Setup JS Stream Service
-    let workload_stream_service_params = JsServiceParamsPartial {
+    let workload_stream_service = JsServiceBuilder {
         name: WORKLOAD_SRV_NAME.to_string(),
         description: WORKLOAD_SRV_DESC.to_string(),
         version: WORKLOAD_SRV_VERSION.to_string(),
@@ -52,23 +52,22 @@ pub async fn run(
     };
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client =
-        jetstream_client::JsClient::new(jetstream_client::NewJsClientParams {
-            nats_url: nats_url.clone(),
-            name: HOST_AGENT_CLIENT_NAME.to_string(),
-            inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
-            service_params: vec![workload_stream_service_params.clone()],
-            credentials_path: host_creds_path
-                .as_ref()
-                .map(|path| path.to_string_lossy().to_string()),
-            ping_interval: Some(Duration::from_secs(10)),
-            request_timeout: Some(Duration::from_secs(29)),
-            listeners: vec![jetstream_client::with_event_listeners(
-                event_listeners.clone(),
-            )],
-        })
-        .await
-        .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
+    let host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
+        nats_url: nats_url.clone(),
+        name: HOST_AGENT_CLIENT_NAME.to_string(),
+        inbox_prefix: format!("{}.{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
+        service_params: vec![workload_stream_service.clone()],
+        credentials_path: host_creds_path
+            .as_ref()
+            .map(|path| path.to_string_lossy().to_string()),
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(29)),
+        listeners: vec![jetstream_client::with_event_listeners(
+            event_listeners.clone(),
+        )],
+    })
+    .await
+    .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
 
     // ==================== Setup API & Register Endpoints ====================
     // Instantiate the Workload API
@@ -89,8 +88,8 @@ pub async fn run(
             endpoint_subject: format!(
                 "{}.{}",
                 pubkey_lowercase,
-                WorkloadServiceSubjects::Install.as_ref().to_string()
-            ), // consumer stream subj
+                WorkloadServiceSubjects::Install.as_ref()
+            ),
             handler: EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.install_workload(msg).await
@@ -106,10 +105,8 @@ pub async fn run(
             endpoint_subject: format!(
                 "{}.{}",
                 pubkey_lowercase,
-                WorkloadServiceSubjects::UpdateInstalled
-                    .as_ref()
-                    .to_string()
-            ), // consumer stream subj
+                WorkloadServiceSubjects::UpdateInstalled.as_ref()
+            ),
             handler: EndpointType::Async(
                 workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.update_workload(msg).await
@@ -125,8 +122,8 @@ pub async fn run(
             endpoint_subject: format!(
                 "{}.{}",
                 pubkey_lowercase,
-                WorkloadServiceSubjects::Uninstall.as_ref().to_string()
-            ), // consumer stream subj
+                WorkloadServiceSubjects::Uninstall.as_ref()
+            ),
             handler: EndpointType::Async(workload_api.call(
                 |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
@@ -142,8 +139,8 @@ pub async fn run(
             endpoint_subject: format!(
                 "{}.{}",
                 pubkey_lowercase,
-                WorkloadServiceSubjects::SendStatus.as_ref().to_string()
-            ), // consumer stream subj
+                WorkloadServiceSubjects::SendStatus.as_ref()
+            ),
             handler: EndpointType::Async(workload_api.call(
                 |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.send_workload_status(msg).await
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 4dbf4d8..6a67832 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -1,6 +1,6 @@
 /*
 This client is associated with the:
-  - WORKLOAD account
+  - HPOS account
   - host user
 
 This client is responsible for subscribing the host agent to workload stream endpoints:
@@ -60,7 +60,7 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // TODO: would it be a good idea to reuse this client in the workload_manager and elsewhere later on?
     bare_client.close().await?;
 
-    let host_workload_client = hostd::workload_manager::run(
+    let host_workload_client = hostd::workload::run(
         "host_id_placeholder>",
         &args.nats_leafnode_client_creds_path,
     )
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
index c88c7e4..f456fea 100644
--- a/rust/clients/orchestrator/src/workloads.rs
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -22,8 +22,8 @@ use std::{sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
     nats::{
-        jetstream_client::{self, JsClient, NewJsClientParams},
-        types::{ConsumerBuilder, EndpointType, JsServiceParamsPartial},
+        jetstream_client::{self, JsClient},
+        types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
     },
 };
 use workload::{
@@ -41,14 +41,14 @@ pub async fn run() -> Result<(), async_nats::Error> {
     let event_listeners = jetstream_client::get_event_listeners();
 
     // Setup JS Stream Service
-    let workload_stream_service_params = JsServiceParamsPartial {
+    let workload_stream_service_params = JsServiceBuilder {
         name: WORKLOAD_SRV_NAME.to_string(),
         description: WORKLOAD_SRV_DESC.to_string(),
         version: WORKLOAD_SRV_VERSION.to_string(),
         service_subject: WORKLOAD_SRV_SUBJ.to_string(),
     };
 
-    let orchestrator_workload_client = JsClient::new(NewJsClientParams {
+    let orchestrator_workload_client = JsClient::new(JsClientBuilder {
         nats_url,
         name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
         inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 33d094f..e28d4d6 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -1,7 +1,8 @@
 /*
 Service Name: WORKLOAD
 Subject: "WORKLOAD.>"
-Provisioning Account: WORKLOAD
+Provisioning Account: ADMIN
+Importing Account: HPOS
 Users: orchestrator & host
 */
 
@@ -23,7 +24,7 @@ use util_libs::{
     nats::types::{AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 
-pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
+pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD_SERVICE";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
 pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
diff --git a/rust/util_libs/src/nats/jetstream_client.rs b/rust/util_libs/src/nats/jetstream_client.rs
index e75c2d3..965755e 100644
--- a/rust/util_libs/src/nats/jetstream_client.rs
+++ b/rust/util_libs/src/nats/jetstream_client.rs
@@ -2,13 +2,13 @@ use super::{
     jetstream_service::JsStreamService,
     leaf_server::LEAF_SERVER_DEFAULT_LISTEN_PORT,
     types::{
-        ErrClientDisconnected, EventHandler, EventListener, JsServiceParamsPartial, PublishInfo,
+        ErrClientDisconnected, EventHandler, EventListener, JsClientBuilder, JsServiceBuilder,
+        PublishInfo,
     },
 };
 use anyhow::Result;
 use async_nats::{jetstream, ServerInfo};
 use core::option::Option::None;
-use serde::Deserialize;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -36,25 +36,8 @@ pub struct JsClient {
     service_log_prefix: String,
 }
 
-#[derive(Deserialize, Default)]
-pub struct NewJsClientParams {
-    pub nats_url: String,
-    pub name: String,
-    pub inbox_prefix: String,
-    #[serde(default)]
-    pub service_params: Vec<JsServiceParamsPartial>,
-    #[serde(default)]
-    pub credentials_path: Option<String>,
-    #[serde(default)]
-    pub ping_interval: Option<Duration>,
-    #[serde(default)]
-    pub request_timeout: Option<Duration>, // Defaults to 5s
-    #[serde(skip_deserializing)]
-    pub listeners: Vec<EventListener>,
-}
-
 impl JsClient {
-    pub async fn new(p: NewJsClientParams) -> Result<Self, async_nats::Error> {
+    pub async fn new(p: JsClientBuilder) -> Result<Self, async_nats::Error> {
         let connect_options = async_nats::ConnectOptions::new()
             // .require_tls(true)
             .name(&p.name)
@@ -183,7 +166,7 @@ impl JsClient {
 
     pub async fn add_js_service(
         &mut self,
-        params: JsServiceParamsPartial,
+        params: JsServiceBuilder,
     ) -> Result<(), async_nats::Error> {
         let new_service = JsStreamService::new(
             self.js_context.to_owned(),
@@ -289,8 +272,8 @@ pub fn get_event_listeners() -> Vec<EventListener> {
 mod tests {
     use super::*;
 
-    pub fn get_default_params() -> NewJsClientParams {
-        NewJsClientParams {
+    pub fn get_default_params() -> JsClientBuilder {
+        JsClientBuilder {
             nats_url: "localhost:4222".to_string(),
             name: "test_client".to_string(),
             inbox_prefix: "_UNIQUE_INBOX".to_string(),
diff --git a/rust/util_libs/src/nats/types.rs b/rust/util_libs/src/nats/types.rs
index 2bbfe08..863fdec 100644
--- a/rust/util_libs/src/nats/types.rs
+++ b/rust/util_libs/src/nats/types.rs
@@ -147,8 +147,25 @@ where
     }
 }
 
+#[derive(Deserialize, Default)]
+pub struct JsClientBuilder {
+    pub nats_url: String,
+    pub name: String,
+    pub inbox_prefix: String,
+    #[serde(default)]
+    pub service_params: Vec<JsServiceBuilder>,
+    #[serde(default)]
+    pub credentials_path: Option<String>,
+    #[serde(default)]
+    pub ping_interval: Option<Duration>,
+    #[serde(default)]
+    pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
+}
+
 #[derive(Clone, Deserialize, Default)]
-pub struct JsServiceParamsPartial {
+pub struct JsServiceBuilder {
     pub name: String,
     pub description: String,
     pub version: String,
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index a3cc658..1e7ef8c 100644
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -11,12 +11,12 @@
 # The operator is generated and named "HOLO" and a system account is generated and named SYS.  Both are assigned two signing keys and are associated with the JWT server.  The JWT server URL set to nats://0.0.0.0:4222.
 
 # Account Creation:
-# Two accounts, named "ADMIN" and "WORKLOAD", are created with JetStream enabled.  Both are associated with the HOLO Operator.
+# Two accounts, named "ADMIN" and "HPOS", are created with JetStream enabled.  Both are associated with the HOLO Operator.
 # Each account has a signing key with a randomly generated role name, which is assigned scoped permissions to allow only users assigned to the signing key to publish and subscribe to their respective streams.
 
 # User Creation:
 # One user named "admin" is created under the "ADMIN" account.
-# One user named "orchestrator" is created under the "WORKLOAD" account.
+# One user named "orchestrator" is created under the "HPOS" account.
 
 # JWT Generation:
 # JWT files are generated for the operator and both accounts, saved in the jwt_output/ directory.
@@ -25,16 +25,16 @@
 #   - OPERATOR_NAME
 #   - SYS_ACCOUNT
 #   - ACCOUNT_JWT_SERVER
-#   - WORKLOAD_ACCOUNT
+#   - HPOS_ACCOUNT
 #   - ADMIN_ACCOUNT
 #   - JWT_OUTPUT_DIR
 #   - RESOLVER_FILE
 
 # Output:
 # One Operator: HOLO
-# Two accounts: HOLO/ADMIN & `HOLO/WORKLOAD`
-# Two Users: /HOLO/ADMIN/admin & HOLO/WORKLOAD/orchestrator
-# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and workload_account.jwt in the `jwt_output` directory.
+# Two accounts: HOLO/ADMIN & `HOLO/HPOS`
+# One Users: /HOLO/ADMIN/admin
+# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and hpos_account.jwt in the `jwt_output` directory.
 # --------
 
 set -e # Exit on any error
@@ -50,11 +50,13 @@ for cmd in nsc nats; do
 done
 
 # Variables
+NATS_SERVER_HOST=$1
+NATS_PORT="4222"
+OPERATOR_SERVICE_URL="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
+ACCOUNT_JWT_SERVER="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
 OPERATOR_NAME="HOLO"
 SYS_ACCOUNT="SYS"
-ACCOUNT_JWT_SERVER="nats://143.244.144.52:4222"
-OPERATOR_SERVICE_URL="nats://143.244.144.52:4222"
-WORKLOAD_ACCOUNT="WORKLOAD"
+HPOS_ACCOUNT="HPOS"
 ADMIN_ACCOUNT="ADMIN"
 JWT_OUTPUT_DIR="jwt_output"
 RESOLVER_FILE="main-resolver.conf"
@@ -70,34 +72,36 @@ nsc edit operator --sk generate
 # Step 2: Create ADMIN_Account with JetStream and scoped signing key
 nsc add account --name $ADMIN_ACCOUNT
 nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_ADMIN="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_ADMIN="admin_role"
-nsc edit signing-key --sk $SIGNING_KEY_ADMIN --role $ROLE_NAME_ADMIN --allow-pub "ADMIN.>","AUTH.>","WORKLOAD.>","INVENTORY.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","*._WORKLOAD_INBOX.>","_AUTH_INBOX_*.>" --allow-sub "ADMIN.>","AUTH.>","WORKLOAD.>","INVENTORY.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","ORCHESTRATOR._WORKLOAD_INBOX.>","_AUTH_INBOX_ORCHESTRATOR.>" --allow-pub-response
-
-# Step 3: Create WORKLOAD Account with JetStream and scoped signing key
-nsc add account --name $WORKLOAD_ACCOUNT
-nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_WORKLOAD="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_WORKLOAD="workload-role"
-nsc edit signing-key --sk $SIGNING_KEY_WORKLOAD --role $ROLE_NAME_WORKLOAD --allow-pub "WORKLOAD.>","{{tag(pubkey)}}._WORKLOAD_INBOX.>","INVENTORY.update.{{tag(pubkey)}}.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","{{tag(pubkey)}}._WORKLOAD_INBOX.>","INVENTORY.update.{{tag(pubkey)}}.>" --allow-pub-response
-
-# Step 4: Create User "orchestrator" in ADMIN Account // noauth
+ADMIN_SIGNING_KEY="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+ADMIN_ROLE_NAME="admin_role"
+nsc edit signing-key --sk $ADMIN_SIGNING_KEY --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","*._WORKLOAD_INBOX.>" --allow-sub "ADMIN.>""WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","ORCHESTRATOR._WORKLOAD_INBOX.>" --allow-pub-response
+
+# Step 3: Create HPOS Account with JetStream and scoped signing key
+nsc add account --name $HPOS_ACCOUNT
+nsc edit account --name $HPOS_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
+HPOS_SIGNING_KEY="$(echo "$(nsc edit account -n $HPOS_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+WORKLOAD_ROLE_NAME="workload-role"
+nsc edit signing-key --sk $HPOS_SIGNING_KEY --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","{{tag(pubkey)}}._WORKLOAD_INBOX.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","{{tag(pubkey)}}._WORKLOAD_INBOX.>" --allow-pub-response
+
+# Step 4: Create User "admin" in ADMIN Account
 nsc add user --name admin --account $ADMIN_ACCOUNT
 
-# Step 5: Create User "orchestrator" in WORKLOAD Account
-nsc add user --name orchestrator --account $WORKLOAD_ACCOUNT
+# Step 5: Export/Import WORKLOAD Service Stream between ADMIN and HPOS accounts
+# Share orchestrator (as admin user) workload streams with host
+nsc add export --name "WORKLOAD_SERVICE" --subject "WORKLOAD.>" --account ADMIN
+nsc add import --src-account ADMIN --name "WORKLOAD_SERVICE" --local-subject "WORKLOAD.>" --account HPOS
+# Share host workload streams with orchestrator (as admin user)
+nsc add export --name "WORKLOAD_SERVICE" --subject "WORKLOAD.>" --account HPOS
+nsc add import --src-account HPOS --name "WORKLOAD_SERVICE" --local-subject "WORKLOAD.>" --account ADMIN
 
 # Step 6: Generate JWT files
 nsc describe operator --raw --output-file $JWT_OUTPUT_DIR/holo_operator.jwt
 nsc describe account --name SYS --raw --output-file $JWT_OUTPUT_DIR/sys_account.jwt
-nsc describe account --name $WORKLOAD_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/workload_account.jwt
+nsc describe account --name $HPOS_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/hpos_account.jwt
 nsc describe account --name $ADMIN_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/admin_account.jwt
 
 # Step 7: Generate Resolver Config
 nsc generate config --nats-resolver --sys-account $SYS_ACCOUNT --force --config-file $RESOLVER_FILE
 
-# Step 8: Push credentials to NATS server
-nsc push -A
-
 echo "Setup complete. JWTs and resolver file are in the $JWT_OUTPUT_DIR/ directory."
 echo "!! Don't forget to start the NATS server and push the credentials to the server with 'nsc push -A' !!"

From d4b9edf9a50bbd6d72328318656868fe5561a4cd Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 16:55:43 -0600
Subject: [PATCH 82/91] update README

---
 README.md | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 8914767..be84d2e 100644
--- a/README.md
+++ b/README.md
@@ -22,14 +22,11 @@ The code is grouped by language or framework name.
 /Cargo.toml
 /Cargo.lock
 /rust/ # all rust code lives here
-/rust/common/Cargo.toml
-/rust/common/src/lib.rs
-/rust/holo-agentctl/Cargo.toml
-/rust/holo-agentctl/src/main.rs
-/rust/holo-agentd/Cargo.toml
-/rust/holo-agentd/src/main.rs
-/rust/holo-hqd/Cargo.toml
-/rust/holo-hqd/src/main.rs
+/rust/clients/
+/rust/services/
+/rust/hpos-hal/
+/rust/netdiag/
+/rust/util_libs/
 ```
 
 ### Pulumi for Infrastructure-as-Code

From 273208322d89e8f0cac5dff8d60edea6bb943803 Mon Sep 17 00:00:00 2001
From: Lisa Jetton <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 16:57:08 -0600
Subject: [PATCH 83/91] refactor/util-libs (#73)

* refactor util_libs

* adjust codebase to new types

* update .env.example
---
 .env.example                                  |  13 +-
 .gitignore                                    |   2 +-
 nix/lib/default.nix                           |   6 +-
 nix/modules/nixos/holo-nats-server.nix        |   2 +-
 .../clients/host_agent/src/gen_leaf_server.rs |  17 +-
 .../host_agent/src/workload_manager.rs        |  79 +++---
 rust/services/workload/src/lib.rs             |  69 +++--
 rust/services/workload/src/types.rs           |   2 +-
 rust/util_libs/src/db/mongodb.rs              | 162 +++++-------
 rust/util_libs/src/db/schemas.rs              |  89 ++++---
 rust/util_libs/src/lib.rs                     |   5 +-
 .../jetstream_client.rs}                      | 250 +++++++-----------
 .../jetstream_service.rs}                     | 158 +++--------
 .../{nats_server.rs => nats/leaf_server.rs}   |  10 +-
 rust/util_libs/src/nats/mod.rs                |   4 +
 rust/util_libs/src/nats/types.rs              | 190 +++++++++++++
 rust/util_libs/src/nats_types.rs              | 145 ----------
 rust/util_libs/test_configs/hub_server.conf   |  22 --
 .../test_configs/hub_server_pw_auth.conf      |  22 --
 19 files changed, 536 insertions(+), 711 deletions(-)
 rename rust/util_libs/src/{nats_js_client.rs => nats/jetstream_client.rs} (55%)
 rename rust/util_libs/src/{js_stream_service.rs => nats/jetstream_service.rs} (79%)
 rename rust/util_libs/src/{nats_server.rs => nats/leaf_server.rs} (98%)
 create mode 100644 rust/util_libs/src/nats/mod.rs
 create mode 100644 rust/util_libs/src/nats/types.rs
 delete mode 100644 rust/util_libs/src/nats_types.rs
 delete mode 100644 rust/util_libs/test_configs/hub_server.conf
 delete mode 100644 rust/util_libs/test_configs/hub_server_pw_auth.conf

diff --git a/.env.example b/.env.example
index bfe8157..9473a3e 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,11 @@
-NSC_PATH = "<path/to/nsc_dir>"
+# ALL
+NSC_PATH="<path/to/nsc_dir>"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LOCAL_CREDS_PATH="<path/to/dir>"
+
+# HOSTING AGENT
 HOST_CREDS_FILE_PATH = "ops/admin.creds"
-MONGO_URI = "mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 1812acf..dbb5350 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,7 +17,7 @@ target/
 rust/*/tmp
 rust/*/jwt
 rust/*/*/test_leaf_server/*
-rust/*/*/test_leaf_server.conf
+rust/*/*/*.conf
 rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf
diff --git a/nix/lib/default.nix b/nix/lib/default.nix
index 3c1aa40..f2cd411 100644
--- a/nix/lib/default.nix
+++ b/nix/lib/default.nix
@@ -1,4 +1,8 @@
-{ inputs, flake, ... }:
+{
+  inputs,
+  flake,
+  ...
+}:
 
 {
   mkCraneLib =
diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index 5db4af6..9744ed4 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -125,4 +125,4 @@ in
       }
     );
   };
-}
+}
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/gen_leaf_server.rs
index 61dbd4d..ef019a9 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/gen_leaf_server.rs
@@ -2,12 +2,13 @@ use std::{path::PathBuf, time::Duration};
 
 use anyhow::Context;
 use tempfile::tempdir;
-use util_libs::{
-    nats_js_client,
-    nats_server::{
+use util_libs::nats::{
+    jetstream_client,
+    leaf_server::{
         JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
         LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
     },
+    types::JsClientBuilder,
 };
 
 pub async fn run(
@@ -17,7 +18,7 @@ pub async fn run(
     hub_url: String,
     hub_tls_insecure: bool,
     nats_connect_timeout_secs: u64,
-) -> anyhow::Result<nats_js_client::JsClient> {
+) -> anyhow::Result<jetstream_client::JsClient> {
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
         .map(|var| var.parse().expect("can't parse into number"))
@@ -95,22 +96,20 @@ pub async fn run(
     // Spin up Nats Client
     // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
     const HOST_AGENT_CLIENT_NAME: &str = "Host Agent Bare";
 
     let nats_client = tokio::select! {
         client = async {loop {
-                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                let host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
                     nats_url:nats_url.clone(),
                     name:HOST_AGENT_CLIENT_NAME.to_string(),
                     ping_interval:Some(Duration::from_secs(10)),
                     request_timeout:Some(Duration::from_secs(29)),
-
                     inbox_prefix: Default::default(),
-                    service_params:Default::default(),
-                    opts: Default::default(),
+                    listeners: Default::default(),
                     credentials_path: Default::default()
                 })
                 .await
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
index dd741d6..58a89df 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/workload_manager.rs
@@ -17,8 +17,10 @@ use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use util_libs::{
     db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType},
+    nats::{
+        jetstream_client,
+        types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
+    },
 };
 use workload::{
     WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
@@ -31,36 +33,29 @@ const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
 pub async fn run(
     host_pubkey: &str,
     host_creds_path: &Option<PathBuf>,
-) -> Result<nats_js_client::JsClient, async_nats::Error> {
+) -> Result<jetstream_client::JsClient, async_nats::Error> {
     log::info!("HPOS Agent Client: Connecting to server...");
     log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
+    let pubkey_lowercase = host_pubkey.to_string().to_lowercase();
+
     // ==================== NATS Setup ====================
     // Connect to Nats server
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
-    let event_listeners = nats_js_client::get_event_listeners();
-
-    // Setup JS Stream Service
-    let workload_stream_service_params = JsServiceParamsPartial {
-        name: WORKLOAD_SRV_NAME.to_string(),
-        description: WORKLOAD_SRV_DESC.to_string(),
-        version: WORKLOAD_SRV_VERSION.to_string(),
-        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
-    };
+    let event_listeners = jetstream_client::get_event_listeners();
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+    let mut host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
         nats_url: nats_url.clone(),
         name: HOST_AGENT_CLIENT_NAME.to_string(),
         inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
-        service_params: vec![workload_stream_service_params.clone()],
         credentials_path: host_creds_path
             .as_ref()
             .map(|path| path.to_string_lossy().to_string()),
-        opts: vec![nats_js_client::with_event_listeners(
+        listeners: vec![jetstream_client::with_event_listeners(
             event_listeners.clone(),
         )],
         ping_interval: Some(Duration::from_secs(10)),
@@ -78,9 +73,19 @@ pub async fn run(
     // Generate the Workload API with access to db
     let workload_api = WorkloadApi::new(&client).await?;
 
-    // ==================== API ENDPOINTS ====================
+    // ==================== Setup JS Stream Service ====================
     // Register Workload Streams for Host Agent to consume
     // NB: Subjects are published by orchestrator or nats-db-connector
+    let workload_stream_service = JsServiceBuilder {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+    host_workload_client
+        .add_js_service(workload_stream_service)
+        .await?;
+
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -89,40 +94,40 @@ pub async fn run(
         ))?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "start_workload",
-            "start",
-            EndpointType::Async(workload_api.call(
+        .add_consumer(ConsumerBuilder {
+            name: "install_workload".to_string(),
+            endpoint_subject: format!("{}.{}", pubkey_lowercase, "start_workload",),
+            handler: EndpointType::Async(workload_api.call(
                 |api: WorkloadApi, msg: Arc<Message>| async move { api.start_workload(msg).await },
             )),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "send_workload_status",
-            "send_status",
-            EndpointType::Async(
+        .add_consumer(ConsumerBuilder {
+            name: "uninstall_workload".to_string(),
+            endpoint_subject: format!("{}.{}", pubkey_lowercase, "uninstall",),
+            handler: EndpointType::Async(
                 workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.send_workload_status(msg).await
+                    api.uninstall_workload(msg).await
                 }),
             ),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "uninstall_workload",
-            "uninstall",
-            EndpointType::Async(
+        .add_consumer(ConsumerBuilder {
+            name: "send_workload_status".to_string(),
+            endpoint_subject: format!("{}.{}", pubkey_lowercase, "send_status",),
+            handler: EndpointType::Async(
                 workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.uninstall_workload(msg).await
+                    api.send_workload_status(msg).await
                 }),
             ),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     Ok(host_workload_client)
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 75dad8c..07859cc 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -21,13 +21,13 @@ use bson::{doc, to_document, DateTime};
 use mongodb::{options::UpdateModifications, Client as MongoDBClient};
 use serde::{Deserialize, Serialize};
 use std::future::Future;
-use std::{fmt::Debug, str::FromStr, sync::Arc};
+use std::{fmt::Debug, sync::Arc};
 use util_libs::{
     db::{
         mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
         schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
     },
-    nats_js_client,
+    nats::types::{AsyncEndpointHandler, JsServiceResponse},
 };
 
 pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
@@ -55,14 +55,14 @@ impl WorkloadApi {
         })
     }
 
-    pub fn call<F, Fut>(&self, handler: F) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+    pub fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<types::ApiResult>
     where
         F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
         Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
     {
         let api = self.to_owned();
         Arc::new(
-            move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+            move |msg: Arc<Message>| -> JsServiceResponse<types::ApiResult> {
                 let api_clone = api.clone();
                 Box::pin(handler(api_clone, msg))
             },
@@ -86,12 +86,12 @@ impl WorkloadApi {
                         workload_id
                     );
                     let updated_workload = schemas::Workload {
-                        _id: Some(ObjectId::from_str(&workload_id)?),
+                        _id: Some(workload_id),
                         ..workload
                     };
                     Ok(types::ApiResult(
                         WorkloadStatus {
-                            id: updated_workload._id.map(|oid| oid.to_hex()),
+                            id: updated_workload._id,
                             desired: WorkloadState::Reported,
                             actual: WorkloadState::Reported,
                         },
@@ -134,7 +134,7 @@ impl WorkloadApi {
                     );
                     Ok(types::ApiResult(
                         WorkloadStatus {
-                            id: workload._id.map(|oid| oid.to_hex()),
+                            id: workload._id,
                             desired: WorkloadState::Reported,
                             actual: WorkloadState::Reported,
                         },
@@ -171,7 +171,7 @@ impl WorkloadApi {
                 );
                 Ok(types::ApiResult(
                     WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
+                        id: Some(workload_id),
                         desired: WorkloadState::Removed,
                         actual: WorkloadState::Removed,
                     },
@@ -210,7 +210,7 @@ impl WorkloadApi {
                 "$project": {
                     "_id": 1
                 }
-            }
+            },
         ];
         let results = self.host_collection.aggregate(pipeline).await?;
         if results.is_empty() {
@@ -248,7 +248,7 @@ impl WorkloadApi {
                     log::warn!("Attempted to assign host for new workload, but host already exists.");
                     return Ok(types::ApiResult(
                         WorkloadStatus {
-                            id: Some(workload_id.to_hex()),
+                            id: Some(workload_id),
                             desired: WorkloadState::Assigned,
                             actual: WorkloadState::Assigned,
                         },
@@ -286,7 +286,7 @@ impl WorkloadApi {
                     }
                 };
                 let updated_host_result = self.host_collection.update_many_within(
-                    host_query, 
+                    host_query,
                     UpdateModifications::Document(updated_host_doc)
                 ).await?;
                 log::trace!(
@@ -296,7 +296,7 @@ impl WorkloadApi {
 
                 Ok(types::ApiResult(
                     WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
+                        id: Some(workload_id),
                         desired: WorkloadState::Assigned,
                         actual: WorkloadState::Assigned,
                     },
@@ -325,15 +325,13 @@ impl WorkloadApi {
         log::trace!("Workload to update. Workload={:#?}", workload.clone());
 
         // 1. remove workloads from existing hosts
-        self.host_collection.mongo_error_handler(
-            self.host_collection
-                .collection
-                .update_many(
-                    doc! {},
-                    doc! { "$pull": { "assigned_workloads": workload._id } },
-                )
-                .await,
-        )?;
+        self.host_collection
+            .inner
+            .update_many(
+                doc! {},
+                doc! { "$pull": { "assigned_workloads": workload._id } },
+            )
+            .await?;
         log::info!(
             "Remove workload from previous hosts. Workload={:#?}",
             workload._id
@@ -341,15 +339,13 @@ impl WorkloadApi {
 
         if !workload.metadata.is_deleted {
             // 3. add workload to specific hosts
-            self.host_collection.mongo_error_handler(
-                self.host_collection
-                    .collection
-                    .update_one(
-                        doc! { "_id": { "$in": workload.clone().assigned_hosts } },
-                        doc! { "$push": { "assigned_workloads": workload._id } },
-                    )
-                    .await,
-            )?;
+            self.host_collection
+                .inner
+                .update_one(
+                    doc! { "_id": { "$in": workload.clone().assigned_hosts } },
+                    doc! { "$push": { "assigned_workloads": workload._id } },
+                )
+                .await?;
             log::info!("Added workload to new hosts. Workload={:#?}", workload._id);
         } else {
             log::info!(
@@ -359,7 +355,7 @@ impl WorkloadApi {
         }
 
         let success_status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
+            id: workload._id,
             desired: WorkloadState::Updating,
             actual: WorkloadState::Updating,
         };
@@ -381,15 +377,12 @@ impl WorkloadApi {
         if workload_status.id.is_none() {
             return Err(anyhow!("Got a status update for workload without an id!"));
         }
-        let workload_status_id = workload_status
-            .id
-            .clone()
-            .expect("workload is not provided");
+        let workload_status_id = workload_status.id.expect("workload is not provided");
 
         self.workload_collection
             .update_one_within(
                 doc! {
-                    "_id": ObjectId::parse_str(workload_status_id)?
+                    "_id": workload_status_id
                 },
                 UpdateModifications::Document(doc! {
                     "$set": {
@@ -418,7 +411,7 @@ impl WorkloadApi {
 
         // 2. Respond to endpoint request
         let status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
+            id: workload._id,
             desired: WorkloadState::Running,
             actual: WorkloadState::Unknown("..".to_string()),
         };
@@ -440,7 +433,7 @@ impl WorkloadApi {
 
         // 2. Respond to endpoint request
         let status = WorkloadStatus {
-            id: Some(workload_id),
+            id: Some(ObjectId::parse_str(workload_id)?),
             desired: WorkloadState::Uninstalled,
             actual: WorkloadState::Unknown("..".to_string()),
         };
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 912ffb6..9832fc7 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,7 +1,7 @@
 use serde::{Deserialize, Serialize};
 use util_libs::{
     db::schemas::WorkloadStatus,
-    js_stream_service::{CreateTag, EndpointTraits},
+    nats::types::{CreateTag, EndpointTraits},
 };
 
 pub use String as WorkloadId;
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index b2974bb..332c741 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,47 +1,39 @@
-use anyhow::{anyhow, Context, Result};
+use crate::nats::types::ServiceError;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
-use bson::{self, doc, Document};
+use bson::oid::ObjectId;
+use bson::{self, Document};
 use futures::stream::TryStreamExt;
 use mongodb::options::UpdateModifications;
-use mongodb::results::{DeleteResult, UpdateResult};
+use mongodb::results::UpdateResult;
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
 
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Internal Error: {0}")]
-    Internal(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-}
-
 #[async_trait]
 pub trait MongoDbAPI<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync,
 {
-    fn mongo_error_handler<ReturnType>(
+    type Error;
+
+    async fn aggregate<R: for<'de> Deserialize<'de>>(
         &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType>;
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>>;
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>>;
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
-    async fn insert_one_into(&self, item: T) -> Result<String>;
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(
+        pipeline: Vec<Document>,
+    ) -> Result<Vec<R>, Self::Error>;
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error>;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error>;
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error>;
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn update_many_within(
+    ) -> Result<UpdateResult, Self::Error>;
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
-    async fn delete_all_from(&self) -> Result<DeleteResult>;
+    ) -> Result<UpdateResult, Self::Error>;
 }
 
 pub trait IntoIndexes {
@@ -53,7 +45,7 @@ pub struct MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
 {
-    pub collection: Collection<T>,
+    pub inner: Collection<T>,
     indices: Vec<IndexModel>,
 }
 
@@ -72,7 +64,7 @@ where
         let indices = vec![];
 
         Ok(MongoCollection {
-            collection,
+            inner: collection,
             indices,
         })
     }
@@ -94,7 +86,7 @@ where
         self.indices = indices.clone();
 
         // Apply the indices to the mongodb collection schema
-        self.collection.create_indexes(indices).await?;
+        self.inner.create_indexes(indices).await?;
         Ok(self)
     }
 }
@@ -104,109 +96,86 @@ impl<T> MongoDbAPI<T> for MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes + Debug,
 {
-    fn mongo_error_handler<ReturnType>(
-        &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType> {
-        let rtn = result.map_err(ServiceError::Database)?;
-        Ok(rtn)
-    }
+    type Error = ServiceError;
 
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>> {
-        log::info!("aggregate pipeline {:?}", pipeline);
-        let cursor = self.collection.aggregate(pipeline).await?;
+    async fn aggregate<R>(&self, pipeline: Vec<Document>) -> Result<Vec<R>, Self::Error>
+    where
+        R: for<'de> Deserialize<'de>,
+    {
+        log::trace!("Aggregate pipeline {:?}", pipeline);
+        let cursor = self.inner.aggregate(pipeline).await?;
 
         let results_doc: Vec<bson::Document> =
             cursor.try_collect().await.map_err(ServiceError::Database)?;
 
-        let results: Vec<T> = results_doc
+        let results: Vec<R> = results_doc
             .into_iter()
             .map(|doc| {
-                bson::from_document::<T>(doc).with_context(|| "failed to deserialize document")
+                bson::from_document::<R>(doc).with_context(|| "Failed to deserialize document")
             })
-            .collect::<Result<Vec<T>>>()?;
+            .collect::<Result<Vec<R>>>()
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         Ok(results)
     }
 
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>> {
-        log::info!("get_one_from filter {:?}", filter);
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error> {
+        log::trace!("Get_one_from filter {:?}", filter);
 
         let item = self
-            .collection
+            .inner
             .find_one(filter)
             .await
             .map_err(ServiceError::Database)?;
 
-        log::info!("item {:?}", item);
+        log::debug!("get_one_from item {:?}", item);
         Ok(item)
     }
 
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>> {
-        let cursor = self.collection.find(filter).await?;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error> {
+        let cursor = self.inner.find(filter).await?;
         let results: Vec<T> = cursor.try_collect().await.map_err(ServiceError::Database)?;
         Ok(results)
     }
 
-    async fn insert_one_into(&self, item: T) -> Result<String> {
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error> {
         let result = self
-            .collection
+            .inner
             .insert_one(item)
             .await
             .map_err(ServiceError::Database)?;
 
-        Ok(result.inserted_id.to_string())
-    }
+        let mongo_id = result
+            .inserted_id
+            .as_object_id()
+            .ok_or(ServiceError::Internal(format!(
+                "Failed to read the insert id after inserting item. insert_result={:?}.",
+                result
+            )))?;
 
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>> {
-        let result = self
-            .collection
-            .insert_many(items)
-            .await
-            .map_err(ServiceError::Database)?;
-
-        let ids = result
-            .inserted_ids
-            .values()
-            .map(|id| id.to_string())
-            .collect();
-        Ok(ids)
+        Ok(mongo_id)
     }
 
-    async fn update_one_within(
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_one(query, updated_doc)
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_many(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn update_many_within(
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_many(query, updated_doc)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult> {
-        self.collection
-            .delete_one(query)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_all_from(&self) -> Result<DeleteResult> {
-        self.collection
-            .delete_many(doc! {})
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_one(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 }
 
@@ -328,7 +297,7 @@ mod tests {
                     updated_at: Some(DateTime::now()),
                     deleted_at: None,
                 },
-                device_id: "Vf3IceiD".to_string(),
+                device_id: "placeholder_pubkey_host".to_string(),
                 ip_address: "127.0.0.1".to_string(),
                 remaining_capacity: Capacity {
                     memory: 16,
@@ -359,9 +328,9 @@ mod tests {
         let host_1 = get_mock_host();
         let host_2 = get_mock_host();
         let host_3 = get_mock_host();
-        host_api
-            .insert_many_into(vec![host_1.clone(), host_2.clone(), host_3.clone()])
-            .await?;
+        host_api.insert_one_into(host_1.clone()).await?;
+        host_api.insert_one_into(host_2.clone()).await?;
+        host_api.insert_one_into(host_3.clone()).await?;
 
         // get many docs
         let ids = vec![
@@ -383,13 +352,8 @@ mod tests {
         assert!(updated_ids.contains(&ids[1]));
         assert!(updated_ids.contains(&ids[2]));
 
-        // delete all documents
-        let DeleteResult { deleted_count, .. } = host_api.delete_all_from().await?;
-        assert_eq!(deleted_count, 4);
-        let fetched_host = host_api.get_one_from(filter_one).await?;
-        let fetched_hosts = host_api.get_many_from(filter_many).await?;
-        assert!(fetched_host.is_none());
-        assert!(fetched_hosts.is_empty());
+        // Delete collection and all documents therein.
+        let _ = host_api.inner.drop();
 
         Ok(())
     }
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 5a7945b..ed8cd92 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -14,18 +14,18 @@ pub const HOST_COLLECTION_NAME: &str = "host";
 pub const WORKLOAD_COLLECTION_NAME: &str = "workload";
 
 // Provide type Alias for HosterPubKey
-pub use String as HosterPubKey;
-
-// Provide type Alias for DeveloperPubkey
-pub use String as DeveloperPubkey;
-
-// Provide type Alias for DeveloperJWT
-pub use String as DeveloperJWT;
+pub use String as PubKey;
 
 // Provide type Alias for SemVer (semantic versioning)
 pub use String as SemVer;
 
 // ==================== User Schema ====================
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct RoleInfo {
+    pub collection_id: ObjectId, // Hoster/Developer colleciton Mongodb ID ref
+    pub pubkey: PubKey,          //  Hoster/Developer Pubkey *INDEXED*
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum UserPermission {
     Admin,
@@ -46,24 +46,24 @@ pub struct User {
     pub metadata: Metadata,
     pub jurisdiction: String,
     pub permissions: Vec<UserPermission>,
-    pub user_info_id: Option<ObjectId>,
-    pub developer: Option<ObjectId>,
-    pub hoster: Option<ObjectId>,
+    pub user_info_id: Option<ObjectId>, // *INDEXED*
+    pub developer: Option<RoleInfo>,    // *INDEXED*
+    pub hoster: Option<RoleInfo>,       // *INDEXED*
 }
 
-// No Additional Indexing for Developer
+// Indexing for User
 impl IntoIndexes for User {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
 
-        // add user_info index
-        let user_info_index_doc = doc! { "user_info": 1 };
-        let user_info_index_opts = Some(
+        // add user_info_id index
+        let user_info_id_index_doc = doc! { "user_info_id": 1 };
+        let user_info_id_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("user_info_index".to_string()))
+                .name(Some("user_info_id_index".to_string()))
                 .build(),
         );
-        indices.push((user_info_index_doc, user_info_index_opts));
+        indices.push((user_info_id_index_doc, user_info_id_index_opts));
 
         // add developer index
         let developer_index_doc = doc! { "developer": 1 };
@@ -75,10 +75,10 @@ impl IntoIndexes for User {
         indices.push((developer_index_doc, developer_index_opts));
 
         // add host index
-        let host_index_doc = doc! { "host": 1 };
+        let host_index_doc = doc! { "hoster": 1 };
         let host_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("host_index".to_string()))
+                .name(Some("hoster_index".to_string()))
                 .build(),
         );
         indices.push((host_index_doc, host_index_opts));
@@ -93,7 +93,7 @@ pub struct UserInfo {
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
     pub user_id: ObjectId,
-    pub email: String,
+    pub email: String, // *INDEXED*
     pub given_names: String,
     pub family_name: String,
 }
@@ -101,7 +101,6 @@ pub struct UserInfo {
 impl IntoIndexes for UserInfo {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         // add email index
         let email_index_doc = doc! { "email": 1 };
         let email_index_opts = Some(
@@ -110,7 +109,6 @@ impl IntoIndexes for UserInfo {
                 .build(),
         );
         indices.push((email_index_doc, email_index_opts));
-
         Ok(indices)
     }
 }
@@ -121,8 +119,8 @@ pub struct Developer {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user._id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub active_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub user_id: ObjectId,
+    pub active_workloads: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Developer
@@ -138,8 +136,8 @@ pub struct Hoster {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user.id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub assigned_hosts: Vec<ObjectId>, // MongoDB ID refs to `host._id`
+    pub user_id: ObjectId,
+    pub assigned_hosts: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Hoster
@@ -162,29 +160,27 @@ pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub assigned_hoster: ObjectId,
-    pub device_id: String, // *INDEXED*, Auto-generated Nats server ID
+    pub device_id: PubKey, // *INDEXED*
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
     pub avg_network_speed: i64,
     pub avg_latency: i64,
-    pub assigned_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub assigned_hoster: ObjectId,
+    pub assigned_workloads: Vec<ObjectId>,
 }
 
 impl IntoIndexes for Host {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         //  Add Device ID Index
-        let device_id_index_doc = doc! { "device_id": 1 };
-        let device_id_index_opts = Some(
+        let pubkey_index_doc = doc! { "device_id": 1 };
+        let pubkey_index_opts = Some(
             IndexOptions::builder()
                 .name(Some("device_id_index".to_string()))
                 .build(),
         );
-        indices.push((device_id_index_doc, device_id_index_opts));
-
+        indices.push((pubkey_index_doc, pubkey_index_opts));
         Ok(indices)
     }
 }
@@ -197,24 +193,27 @@ pub enum WorkloadState {
     Pending,
     Installed,
     Running,
+    Updating,
+    Updated,
     Removed,
     Uninstalled,
-    Updating,
     Error(String),   // String = error message
     Unknown(String), // String = context message
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
-    pub id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<ObjectId>,
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SystemSpecs {
-    pub capacity: Capacity, // network_speed: i64
-                            // uptime: i64
+    pub capacity: Capacity,
+    pub avg_network_speed: i64, // Mbps
+    pub avg_uptime: f64, //  decimal value between 0-1 representing avg uptime over past month
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -222,14 +221,14 @@ pub struct Workload {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub state: WorkloadState,
-    pub assigned_developer: ObjectId, // *INDEXED*, Developer Mongodb ID
+    pub assigned_developer: ObjectId, // *INDEXED*
     pub version: SemVer,
     pub nix_pkg: String, // (Includes everthing needed to deploy workload - ie: binary & env pkg & deps, etc)
     pub min_hosts: u16,
     pub system_specs: SystemSpecs,
-    pub assigned_hosts: Vec<ObjectId>, // Host Device IDs (eg: assigned nats server id)
-                                        // pub status: WorkloadStatus,
+    pub assigned_hosts: Vec<ObjectId>,
+    // pub state: WorkloadState,
+    pub status: WorkloadStatus,
 }
 
 impl Default for Workload {
@@ -252,7 +251,6 @@ impl Default for Workload {
                 updated_at: Some(DateTime::now()),
                 deleted_at: None,
             },
-            state: WorkloadState::Reported,
             version: semver,
             nix_pkg: String::new(),
             assigned_developer: ObjectId::new(),
@@ -263,8 +261,15 @@ impl Default for Workload {
                     disk: 400,
                     cores: 20,
                 },
+                avg_network_speed: 200,
+                avg_uptime: 0.8,
             },
             assigned_hosts: Vec::new(),
+            status: WorkloadStatus {
+                id: None, // skips serialization when `None`
+                desired: WorkloadState::Unknown("default state".to_string()),
+                actual: WorkloadState::Unknown("default state".to_string()),
+            },
         }
     }
 }
diff --git a/rust/util_libs/src/lib.rs b/rust/util_libs/src/lib.rs
index 861376d..f1b0265 100644
--- a/rust/util_libs/src/lib.rs
+++ b/rust/util_libs/src/lib.rs
@@ -1,5 +1,2 @@
 pub mod db;
-pub mod js_stream_service;
-pub mod nats_js_client;
-pub mod nats_server;
-pub mod nats_types;
+pub mod nats;
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats/jetstream_client.rs
similarity index 55%
rename from rust/util_libs/src/nats_js_client.rs
rename to rust/util_libs/src/nats/jetstream_client.rs
index cbd1fed..f8a2643 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats/jetstream_client.rs
@@ -1,73 +1,24 @@
-use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
-use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
-
+use super::{
+    jetstream_service::JsStreamService,
+    leaf_server::LEAF_SERVER_DEFAULT_LISTEN_PORT,
+    types::{
+        ErrClientDisconnected, EventHandler, EventListener, JsClientBuilder, JsServiceBuilder,
+        PublishInfo,
+    },
+};
 use anyhow::Result;
-use async_nats::{jetstream, Message, ServerInfo};
-use serde::{Deserialize, Serialize};
-use std::error::Error;
-use std::fmt;
-use std::fmt::Debug;
-use std::future::Future;
-use std::pin::Pin;
+use async_nats::{jetstream, ServerInfo};
+use core::option::Option::None;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
-pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
-        + Send
-        + Sync,
->;
-
-#[derive(Clone)]
-pub enum EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
-{
-    Sync(EndpointHandler<T>),
-    Async(AsyncEndpointHandler<T>),
-}
-
-impl<T> std::fmt::Debug for EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let t = match &self {
-            EndpointType::Async(_) => "EndpointType::Async(<function>)",
-            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
-        };
-
-        write!(f, "{}", t)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct SendRequest {
-    pub subject: String,
-    pub msg_id: String,
-    pub data: Vec<u8>,
-}
-
-#[derive(Debug)]
-pub struct ErrClientDisconnected;
-impl fmt::Display for ErrClientDisconnected {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Could not reach nats: connection closed")
-    }
-}
-impl Error for ErrClientDisconnected {}
-
 impl std::fmt::Debug for JsClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("JsClient")
             .field("url", &self.url)
             .field("name", &self.name)
             .field("client", &self.client)
-            .field("js", &self.js)
+            .field("js_context", &self.js_context)
             .field("js_services", &self.js_services)
             .field("service_log_prefix", &self.service_log_prefix)
             .finish()
@@ -80,30 +31,13 @@ pub struct JsClient {
     on_msg_published_event: Option<EventHandler>,
     on_msg_failed_event: Option<EventHandler>,
     client: async_nats::Client, // inner_client
-    pub js: jetstream::Context,
+    pub js_context: jetstream::Context,
     pub js_services: Option<Vec<JsStreamService>>,
     service_log_prefix: String,
 }
 
-#[derive(Deserialize, Default)]
-pub struct NewJsClientParams {
-    pub nats_url: String,
-    pub name: String,
-    pub inbox_prefix: String,
-    #[serde(default)]
-    pub service_params: Vec<JsServiceParamsPartial>,
-    #[serde(skip_deserializing)]
-    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
-    #[serde(default)]
-    pub credentials_path: Option<String>,
-    #[serde(default)]
-    pub ping_interval: Option<Duration>,
-    #[serde(default)]
-    pub request_timeout: Option<Duration>, // Defaults to 5s
-}
-
 impl JsClient {
-    pub async fn new(p: NewJsClientParams) -> Result<Self, async_nats::Error> {
+    pub async fn new(p: JsClientBuilder) -> Result<Self, async_nats::Error> {
         let connect_options = async_nats::ConnectOptions::new()
             // .require_tls(true)
             .name(&p.name)
@@ -123,77 +57,33 @@ impl JsClient {
             None => connect_options.connect(&p.nats_url).await?,
         };
 
-        let jetstream = jetstream::new(client.clone());
-        let mut services = vec![];
-        for params in p.service_params {
-            let service = JsStreamService::new(
-                jetstream.clone(),
-                &params.name,
-                &params.description,
-                &params.version,
-                &params.service_subject,
-            )
-            .await?;
-            services.push(service);
-        }
+        let log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
+        log::info!("{}Connected to NATS server at {}", log_prefix, p.nats_url);
 
-        let js_services = if services.is_empty() {
-            None
-        } else {
-            Some(services)
-        };
-
-        let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
-
-        let mut default_client = JsClient {
+        let mut js_client = JsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
             on_msg_failed_event: None,
+            js_services: None,
+            js_context: jetstream::new(client.clone()),
+            service_log_prefix: log_prefix,
             client,
-            js: jetstream,
-            js_services,
-            service_log_prefix: service_log_prefix.clone(),
         };
 
-        for opt in p.opts {
-            opt(&mut default_client);
+        for listener in p.listeners {
+            listener(&mut js_client);
         }
 
-        log::info!(
-            "{}Connected to NATS server at {}",
-            service_log_prefix,
-            default_client.url
-        );
-        Ok(default_client)
-    }
-
-    pub fn name(&self) -> &str {
-        &self.name
+        Ok(js_client)
     }
 
     pub fn get_server_info(&self) -> ServerInfo {
         self.client.server_info()
     }
 
-    pub async fn monitor(&self) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            Err(Box::new(ErrClientDisconnected))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub async fn close(&self) -> Result<(), async_nats::Error> {
-        self.client.drain().await?;
-        Ok(())
-    }
-
-    pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            return Err(Box::new(ErrClientDisconnected));
-        }
-        let stream = &self.js.get_stream(stream_name).await?;
+    pub async fn get_stream_info(&self, stream_name: &str) -> Result<(), async_nats::Error> {
+        let stream = &self.js_context.get_stream(stream_name).await?;
         let info = stream.get_info().await?;
         log::debug!(
             "{}JetStream info: stream:{}, info:{:?}",
@@ -204,43 +94,80 @@ impl JsClient {
         Ok(())
     }
 
-    pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
-        Ok(())
+    pub async fn check_connection(
+        &self,
+    ) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state = self.client.connection_state();
+        if let async_nats::connection::State::Disconnected = conn_state {
+            Err(Box::new(ErrClientDisconnected))
+        } else {
+            Ok(conn_state)
+        }
     }
 
-    pub async fn publish(&self, payload: &SendRequest) -> Result<(), async_nats::Error> {
+    pub async fn publish(
+        &self,
+        payload: PublishInfo,
+    ) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>>
+    {
+        log::debug!(
+            "{}Called Publish message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+
         let now = Instant::now();
-        let result = self
-            .js
-            .publish(payload.subject.clone(), payload.data.clone().into())
-            .await;
+        let result = match payload.headers {
+            Some(headers) => {
+                self.js_context
+                    .publish_with_headers(
+                        payload.subject.clone(),
+                        headers,
+                        payload.data.clone().into(),
+                    )
+                    .await
+            }
+            None => {
+                self.js_context
+                    .publish(payload.subject.clone(), payload.data.clone().into())
+                    .await
+            }
+        };
 
         let duration = now.elapsed();
         if let Err(err) = result {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(Box::new(err));
+            return Err(err);
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
         Ok(())
     }
 
-    pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
-        let mut current_services = self.js_services.unwrap_or_default();
-        current_services.extend(js_services);
+    pub async fn add_js_service(
+        &mut self,
+        params: JsServiceBuilder,
+    ) -> Result<(), async_nats::Error> {
+        let new_service = JsStreamService::new(
+            self.js_context.to_owned(),
+            &params.name,
+            &params.description,
+            &params.version,
+            &params.service_subject,
+        )
+        .await?;
+
+        let mut current_services = self.js_services.to_owned().unwrap_or_default();
+        current_services.push(new_service);
         self.js_services = Some(current_services);
-        self
+
+        Ok(())
     }
 
     pub async fn get_js_service(&self, js_service_name: String) -> Option<&JsStreamService> {
@@ -251,6 +178,11 @@ impl JsClient {
         }
         None
     }
+
+    pub async fn close(&self) -> Result<(), async_nats::Error> {
+        self.client.drain().await?;
+        Ok(())
+    }
 }
 
 // Client Options:
@@ -268,7 +200,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_published_event = Some(Box::pin(f.clone()));
+        c.on_msg_published_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -277,7 +209,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_failed_event = Some(Box::pin(f.clone()));
+        c.on_msg_failed_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -314,7 +246,7 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     };
 
     let event_listeners = vec![
-        on_msg_published_event(published_msg_handler),
+        on_msg_published_event(published_msg_handler), // Shouldn't this be the 'NATS_LISTEN_PORT'?
         on_msg_failed_event(failure_handler),
     ];
 
@@ -326,8 +258,8 @@ pub fn get_event_listeners() -> Vec<EventListener> {
 mod tests {
     use super::*;
 
-    pub fn get_default_params() -> NewJsClientParams {
-        NewJsClientParams {
+    pub fn get_default_params() -> JsClientBuilder {
+        JsClientBuilder {
             nats_url: "localhost:4222".to_string(),
             name: "test_client".to_string(),
             inbox_prefix: "_UNIQUE_INBOX".to_string(),
@@ -353,7 +285,7 @@ mod tests {
     async fn test_nats_js_client_publish() {
         let params = get_default_params();
         let client = JsClient::new(params).await.unwrap();
-        let payload = SendRequest {
+        let payload = PublishInfo {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
             data: b"Hello, NATS!".to_vec(),
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/nats/jetstream_service.rs
similarity index 79%
rename from rust/util_libs/src/js_stream_service.rs
rename to rust/util_libs/src/nats/jetstream_service.rs
index 0860c3b..a01f476 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/nats/jetstream_service.rs
@@ -1,108 +1,17 @@
-use super::nats_js_client::EndpointType;
-
+use super::types::{
+    ConsumerBuilder, ConsumerExt, ConsumerExtTrait, EndpointTraits, EndpointType,
+    JsStreamServiceInfo, LogInfo, ResponseSubjectsGenerator,
+};
 use anyhow::{anyhow, Result};
-use std::any::Any;
-// use async_nats::jetstream::message::Message;
-use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
+use async_nats::jetstream::consumer::{self, AckPolicy};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
-use async_trait::async_trait;
 use futures::StreamExt;
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
-
-pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> Option<Vec<String>>;
-}
-
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
-{
-}
-
-#[async_trait]
-pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
-    fn get_name(&self) -> &str;
-    fn get_consumer(&self) -> PullConsumer;
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
-}
-
-impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
-where
-    T: EndpointTraits,
-{
-    type Error = anyhow::Error;
-
-    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
-        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
-            Ok(*endpoint)
-        } else {
-            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
-        }
-    }
-}
-
-#[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    name: String,
-    consumer: PullConsumer,
-    handler: EndpointType<T>,
-    #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>,
-}
-
-#[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    fn get_name(&self) -> &str {
-        &self.name
-    }
-    fn get_consumer(&self) -> PullConsumer {
-        self.consumer.clone()
-    }
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
-        Box::new(self.handler.clone())
-    }
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
-        self.response_subject_fn.clone()
-    }
-}
-
-#[allow(dead_code)]
-#[derive(Clone, Debug)]
-pub struct JsStreamServiceInfo<'a> {
-    pub name: &'a str,
-    pub version: &'a str,
-    pub service_subject: &'a str,
-}
-
-struct LogInfo {
-    prefix: String,
-    service_name: String,
-    service_subject: String,
-    endpoint_name: String,
-    endpoint_subject: String,
-}
-
-#[derive(Clone, Deserialize, Default)]
-pub struct JsServiceParamsPartial {
-    pub name: String,
-    pub description: String,
-    pub version: String,
-    pub service_subject: String,
-}
-
 /// Microservice for Jetstream Streams
 // This setup creates only one subject for the stream (eg: "WORKLOAD.>") and sets up
 // all consumers of the stream to listen to stream subjects beginning with that subject (eg: "WORKLOAD.start")
@@ -196,30 +105,30 @@ impl JsStreamService {
         let handler: EndpointType<T> = EndpointType::try_from(endpoint_trait_obj)?;
 
         Ok(ConsumerExt {
-            name: consumer_ext.get_name().to_string(),
             consumer: consumer_ext.get_consumer(),
             handler,
             response_subject_fn: consumer_ext.get_response(),
         })
     }
 
-    pub async fn add_local_consumer<T>(
+    pub async fn add_consumer<T>(
         &self,
-        consumer_name: &str,
-        endpoint_subject: &str,
-        endpoint_type: EndpointType<T>,
-        response_subject_fn: Option<ResponseSubjectsGenerator>,
+        builder_params: ConsumerBuilder<T>,
     ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
-        let full_subject = format!("{}.{}", self.service_subject, endpoint_subject);
+        // Add the Service Subject prefix
+        let consumer_subject = format!(
+            "{}.{}",
+            self.service_subject, builder_params.endpoint_subject
+        );
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
-            durable_name: Some(consumer_name.to_string()),
+            durable_name: Some(builder_params.name.to_string()),
             ack_policy: AckPolicy::Explicit,
-            filter_subject: full_subject,
+            filter_subject: consumer_subject,
             ..Default::default()
         };
 
@@ -227,28 +136,28 @@ impl JsStreamService {
             .stream
             .write()
             .await
-            .get_or_create_consumer(consumer_name, consumer_config)
+            .get_or_create_consumer(&builder_params.name, consumer_config)
             .await?;
 
         let consumer_with_handler = ConsumerExt {
-            name: consumer_name.to_string(),
             consumer,
-            handler: endpoint_type,
-            response_subject_fn,
+            handler: builder_params.handler,
+            response_subject_fn: builder_params.response_subject_fn,
         };
 
-        self.local_consumers
-            .write()
-            .await
-            .insert(consumer_name.to_string(), Arc::new(consumer_with_handler));
+        self.local_consumers.write().await.insert(
+            builder_params.name.to_string(),
+            Arc::new(consumer_with_handler),
+        );
 
-        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(consumer_name).await?;
-        self.spawn_consumer_handler::<T>(consumer_name).await?;
+        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(&builder_params.name).await?;
+        self.spawn_consumer_handler::<T>(&builder_params.name)
+            .await?;
 
         log::debug!(
             "{}Added the {} local consumer",
             self.service_log_prefix,
-            endpoint_consumer.name,
+            builder_params.name,
         );
 
         Ok(endpoint_consumer)
@@ -279,12 +188,19 @@ impl JsStreamService {
                 .messages()
                 .await?;
 
+            let consumer_info = consumer.info().await?;
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
-                endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
+                endpoint_name: consumer_info
+                    .config
+                    .durable_name
+                    .clone()
+                    .unwrap_or("Consumer Name Not Found".to_string())
+                    .clone(),
+                endpoint_subject: consumer_info.config.filter_subject.clone(),
             };
 
             let service_context = self.js_context.clone();
@@ -498,7 +414,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_js_service_add_local_consumer() {
+    async fn test_js_service_add_consumer() {
         let context = setup_jetstream().await;
         let service = get_default_js_service(context).await;
 
@@ -508,7 +424,7 @@ mod tests {
         let response_subject = Some("response.subject".to_string());
 
         let consumer = service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
@@ -533,7 +449,7 @@ mod tests {
         let response_subject = None;
 
         service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats/leaf_server.rs
similarity index 98%
rename from rust/util_libs/src/nats_server.rs
rename to rust/util_libs/src/nats/leaf_server.rs
index 4e9030b..dfc40f1 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats/leaf_server.rs
@@ -143,7 +143,7 @@ impl LeafServer {
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .spawn()
-            .expect("Failed to start NATS server");
+            .context("Failed to start NATS server")?;
 
         // TODO: wait for a readiness indicator
         std::thread::sleep(std::time::Duration::from_millis(100));
@@ -192,8 +192,8 @@ mod tests {
     const TMP_JS_DIR: &str = "./tmp";
     const TEST_AUTH_DIR: &str = "./tmp/test-auth";
     const OPERATOR_NAME: &str = "test-operator";
-    const USER_ACCOUNT_NAME: &str = "hpos-account";
-    const USER_NAME: &str = "hpos-user";
+    const USER_ACCOUNT_NAME: &str = "host-account";
+    const USER_NAME: &str = "host-user";
     const NEW_LEAF_CONFIG_PATH: &str = "./test_configs/leaf_server.conf";
     // NB: if changed, the resolver file path must also be changed in the `hub-server.conf` iteself as well.
     const RESOLVER_FILE_PATH: &str = "./test_configs/resolver.conf";
@@ -227,7 +227,7 @@ mod tests {
             .output()
             .expect("Failed to create edit operator");
 
-        // Create hpos account (with js enabled)
+        // Create host account (with js enabled)
         Command::new("nsc")
             .args(["add", "account", USER_ACCOUNT_NAME])
             .output()
@@ -245,7 +245,7 @@ mod tests {
             .output()
             .expect("Failed to create edit account");
 
-        // Create user for hpos account
+        // Create user for host account
         Command::new("nsc")
             .args(["add", "user", USER_NAME])
             .args(["--account", USER_ACCOUNT_NAME])
diff --git a/rust/util_libs/src/nats/mod.rs b/rust/util_libs/src/nats/mod.rs
new file mode 100644
index 0000000..a320ee4
--- /dev/null
+++ b/rust/util_libs/src/nats/mod.rs
@@ -0,0 +1,4 @@
+pub mod jetstream_client;
+pub mod jetstream_service;
+pub mod leaf_server;
+pub mod types;
diff --git a/rust/util_libs/src/nats/types.rs b/rust/util_libs/src/nats/types.rs
new file mode 100644
index 0000000..23f959b
--- /dev/null
+++ b/rust/util_libs/src/nats/types.rs
@@ -0,0 +1,190 @@
+use super::jetstream_client::JsClient;
+use anyhow::Result;
+use async_nats::jetstream::consumer::PullConsumer;
+use async_nats::{HeaderMap, Message};
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::error::Error;
+use std::fmt;
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
+pub type AsyncEndpointHandler<T> = Arc<
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
+        + Send
+        + Sync,
+>;
+pub type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
+
+pub trait EndpointTraits:
+    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
+{
+}
+
+pub trait CreateTag: Send + Sync {
+    fn get_tags(&self) -> Option<Vec<String>>;
+}
+
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
+}
+
+#[async_trait]
+pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
+    fn get_consumer(&self) -> PullConsumer;
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
+}
+
+#[async_trait]
+impl<T> ConsumerExtTrait for ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    fn get_consumer(&self) -> PullConsumer {
+        self.consumer.clone()
+    }
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
+        Box::new(self.handler.clone())
+    }
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
+        self.response_subject_fn.clone()
+    }
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    pub consumer: PullConsumer,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerBuilder<T>
+where
+    T: EndpointTraits,
+{
+    pub name: String,
+    pub endpoint_subject: String,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[allow(dead_code)]
+#[derive(Clone, Debug)]
+pub struct JsStreamServiceInfo<'a> {
+    pub name: &'a str,
+    pub version: &'a str,
+    pub service_subject: &'a str,
+}
+
+#[derive(Clone, Debug)]
+pub struct LogInfo {
+    pub prefix: String,
+    pub service_name: String,
+    pub service_subject: String,
+    pub endpoint_name: String,
+    pub endpoint_subject: String,
+}
+
+#[derive(Clone)]
+pub enum EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
+{
+    Sync(EndpointHandler<T>),
+    Async(AsyncEndpointHandler<T>),
+}
+
+impl<T> std::fmt::Debug for EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let t = match &self {
+            EndpointType::Async(_) => "EndpointType::Async(<function>)",
+            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
+        };
+
+        write!(f, "{}", t)
+    }
+}
+impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
+where
+    T: EndpointTraits,
+{
+    type Error = anyhow::Error;
+
+    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
+        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
+            Ok(*endpoint)
+        } else {
+            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
+        }
+    }
+}
+
+#[derive(Deserialize, Default)]
+pub struct JsClientBuilder {
+    pub nats_url: String,
+    pub name: String,
+    pub inbox_prefix: String,
+    #[serde(default)]
+    pub credentials_path: Option<String>,
+    #[serde(default)]
+    pub ping_interval: Option<Duration>,
+    #[serde(default)]
+    pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
+}
+
+#[derive(Clone, Deserialize, Default)]
+pub struct JsServiceBuilder {
+    pub name: String,
+    pub description: String,
+    pub version: String,
+    pub service_subject: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct PublishInfo {
+    pub subject: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+    pub headers: Option<HeaderMap>,
+}
+
+#[derive(Debug)]
+pub struct ErrClientDisconnected;
+impl fmt::Display for ErrClientDisconnected {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Could not reach nats: connection closed")
+    }
+}
+impl Error for ErrClientDisconnected {}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
deleted file mode 100644
index ece916b..0000000
--- a/rust/util_libs/src/nats_types.rs
+++ /dev/null
@@ -1,145 +0,0 @@
-/* --------
-NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
-IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
-TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */
-
-use serde::{Deserialize, Serialize};
-
-/// JWT claims for NATS compatible jwts
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Claims {
-    /// Time when the token was issued in seconds since the unix epoch
-    #[serde(rename = "iat")]
-    pub issued_at: i64,
-
-    /// Public key of the issuer signing nkey
-    #[serde(rename = "iss")]
-    pub issuer: String,
-
-    /// Base32 hash of the claims where this is empty
-    #[serde(rename = "jti")]
-    pub jwt_id: String,
-
-    /// Public key of the account or user the JWT is being issued to
-    pub sub: String,
-
-    /// Friendly name
-    pub name: String,
-
-    /// NATS claims
-    pub nats: NatsClaims,
-
-    /// Time when the token expires (in seconds since the unix epoch)
-    #[serde(rename = "exp", skip_serializing_if = "Option::is_none")]
-    pub expires: Option<i64>,
-}
-
-/// NATS claims describing settings for the user or account
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "lowercase")]
-pub enum NatsClaims {
-    /// Claims for NATS users
-    User {
-        /// Publish and subscribe permissions for the user
-        #[serde(flatten)]
-        permissions: NatsPermissionsMap,
-
-        /// Public key/id of the account that issued the JWT
-        issuer_account: String,
-
-        /// Maximum nuber of subscriptions the user can have
-        subs: i64,
-
-        /// Maximum size of the message data the user can send in bytes
-        data: i64,
-
-        /// Maximum size of the entire message payload the user can send in bytes
-        payload: i64,
-
-        /// If true, the user isn't challenged on connection. Typically used for websocket
-        /// connections as the browser won't have/want to have the user's private key.
-        bearer_token: bool,
-
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-    /// Claims for NATS accounts
-    Account {
-        /// Configuration for the limits for this account
-        limits: NatsAccountLimits,
-
-        /// List of signing keys (public key) this account uses
-        #[serde(skip_serializing_if = "Vec::is_empty")]
-        signing_keys: Vec<String>,
-
-        /// Default publish and subscribe permissions users under this account will have if not
-        /// specified otherwise
-        /// default_permissions: NatsPermissionsMap,
-        ///
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-}
-
-/// List of subjects that are allowed and/or denied
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissions {
-    /// List of subject patterns that are allowed
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub allow: Vec<String>,
-
-    /// List of subject patterns that are denied
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub deny: Vec<String>,
-}
-
-impl NatsPermissions {
-    /// Returns `true` if the allow and deny list are both empty
-    #[must_use]
-    pub fn is_empty(&self) -> bool {
-        self.allow.is_empty() && self.deny.is_empty()
-    }
-}
-
-/// Publish and subcribe permissons
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissionsMap {
-    /// Permissions for which subjects can be published to
-    #[serde(rename = "pub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub publish: NatsPermissions,
-
-    /// Permissions for which subjects can be subscribed to
-    #[serde(rename = "sub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub subscribe: NatsPermissions,
-}
-
-/// Limits on what an account or users in the account can do
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NatsAccountLimits {
-    /// Maximum nuber of subscriptions the account
-    pub subs: i64,
-
-    /// Maximum size of the message data a user can send in bytes
-    pub data: i64,
-
-    /// Maximum size of the entire message payload a user can send in bytes
-    pub payload: i64,
-
-    /// Maxiumum number of imports for the account
-    pub imports: i64,
-
-    /// Maxiumum number of exports for the account
-    pub exports: i64,
-
-    /// If true, exports can contain wildcards
-    pub wildcards: bool,
-
-    /// Maximum number of active connections
-    pub conn: i64,
-
-    /// Maximum number of leaf node connections
-    pub leaf: i64,
-}
diff --git a/rust/util_libs/test_configs/hub_server.conf b/rust/util_libs/test_configs/hub_server.conf
deleted file mode 100644
index 0184098..0000000
--- a/rust/util_libs/test_configs/hub_server.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-operator: "./test-auth/test-operator/test-operator.jwt"
-system_account: SYS
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-include ./resolver.conf
-
-# logging options
-debug:   true
-trace:   true
-logtime: false
diff --git a/rust/util_libs/test_configs/hub_server_pw_auth.conf b/rust/util_libs/test_configs/hub_server_pw_auth.conf
deleted file mode 100644
index 51eeb3f..0000000
--- a/rust/util_libs/test_configs/hub_server_pw_auth.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-authorization {
-    user: "test-user"
-    password: "pw-12345"
-}
-
-# logging options
-debug:   true
-trace:   true
-logtime: false

From 8800afbe0069eec4892939ca6074dcc9f74a1f28 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:00:19 -0600
Subject: [PATCH 84/91] update .gitignore

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 1812acf..dbb5350 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,7 +17,7 @@ target/
 rust/*/tmp
 rust/*/jwt
 rust/*/*/test_leaf_server/*
-rust/*/*/test_leaf_server.conf
+rust/*/*/*.conf
 rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf

From 573a2570962f589e0244f321aa9050b185e5fef5 Mon Sep 17 00:00:00 2001
From: Lisa Jetton <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:01:29 -0600
Subject: [PATCH 85/91] refactor workload service (#71)

* refactor service
---
 .env.example                                  |  13 +-
 Cargo.lock                                    |  30 +-
 nix/lib/default.nix                           |   6 +-
 nix/modules/nixos/holo-nats-server.nix        |   2 +-
 .../clients/host_agent/src/gen_leaf_server.rs |  17 +-
 .../host_agent/src/workload_manager.rs        | 159 +++--
 rust/services/workload/Cargo.toml             |   3 +
 rust/services/workload/src/host_api.rs        | 169 +++++
 rust/services/workload/src/lib.rs             | 526 ++------------
 .../services/workload/src/orchestrator_api.rs | 662 ++++++++++++++++++
 rust/services/workload/src/types.rs           |  49 +-
 rust/util_libs/src/db/mongodb.rs              | 162 ++---
 rust/util_libs/src/db/schemas.rs              |  89 +--
 rust/util_libs/src/lib.rs                     |   5 +-
 .../jetstream_client.rs}                      | 250 +++----
 .../jetstream_service.rs}                     | 165 ++---
 .../{nats_server.rs => nats/leaf_server.rs}   |  10 +-
 rust/util_libs/src/nats/mod.rs                |   4 +
 rust/util_libs/src/nats/types.rs              | 200 ++++++
 rust/util_libs/src/nats_types.rs              | 145 ----
 rust/util_libs/test_configs/hub_server.conf   |  22 -
 .../test_configs/hub_server_pw_auth.conf      |  22 -
 22 files changed, 1521 insertions(+), 1189 deletions(-)
 create mode 100644 rust/services/workload/src/host_api.rs
 create mode 100644 rust/services/workload/src/orchestrator_api.rs
 rename rust/util_libs/src/{nats_js_client.rs => nats/jetstream_client.rs} (55%)
 rename rust/util_libs/src/{js_stream_service.rs => nats/jetstream_service.rs} (78%)
 rename rust/util_libs/src/{nats_server.rs => nats/leaf_server.rs} (98%)
 create mode 100644 rust/util_libs/src/nats/mod.rs
 create mode 100644 rust/util_libs/src/nats/types.rs
 delete mode 100644 rust/util_libs/src/nats_types.rs
 delete mode 100644 rust/util_libs/test_configs/hub_server.conf
 delete mode 100644 rust/util_libs/test_configs/hub_server_pw_auth.conf

diff --git a/.env.example b/.env.example
index bfe8157..9473a3e 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,11 @@
-NSC_PATH = "<path/to/nsc_dir>"
+# ALL
+NSC_PATH="<path/to/nsc_dir>"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LOCAL_CREDS_PATH="<path/to/dir>"
+
+# HOSTING AGENT
 HOST_CREDS_FILE_PATH = "ops/admin.creds"
-MONGO_URI = "mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index aada039..58ba09a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1348,6 +1348,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -2724,7 +2730,7 @@ dependencies = [
  "serde",
  "serde_json",
  "strum 0.23.0",
- "strum_macros",
+ "strum_macros 0.23.1",
  "thiserror 1.0.69",
  "url",
 ]
@@ -3240,6 +3246,12 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
 
+[[package]]
+name = "strum"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
+
 [[package]]
 name = "strum_macros"
 version = "0.23.1"
@@ -3253,6 +3265,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "strum_macros"
+version = "0.25.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.90",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -4261,6 +4286,7 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
+ "async-trait",
  "bson",
  "bytes",
  "chrono",
@@ -4274,6 +4300,8 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
+ "strum 0.25.0",
+ "strum_macros 0.25.3",
  "thiserror 2.0.9",
  "tokio",
  "url",
diff --git a/nix/lib/default.nix b/nix/lib/default.nix
index 3c1aa40..f2cd411 100644
--- a/nix/lib/default.nix
+++ b/nix/lib/default.nix
@@ -1,4 +1,8 @@
-{ inputs, flake, ... }:
+{
+  inputs,
+  flake,
+  ...
+}:
 
 {
   mkCraneLib =
diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index 5db4af6..9744ed4 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -125,4 +125,4 @@ in
       }
     );
   };
-}
+}
\ No newline at end of file
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/gen_leaf_server.rs
index 61dbd4d..ef019a9 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/gen_leaf_server.rs
@@ -2,12 +2,13 @@ use std::{path::PathBuf, time::Duration};
 
 use anyhow::Context;
 use tempfile::tempdir;
-use util_libs::{
-    nats_js_client,
-    nats_server::{
+use util_libs::nats::{
+    jetstream_client,
+    leaf_server::{
         JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
         LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
     },
+    types::JsClientBuilder,
 };
 
 pub async fn run(
@@ -17,7 +18,7 @@ pub async fn run(
     hub_url: String,
     hub_tls_insecure: bool,
     nats_connect_timeout_secs: u64,
-) -> anyhow::Result<nats_js_client::JsClient> {
+) -> anyhow::Result<jetstream_client::JsClient> {
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
         .map(|var| var.parse().expect("can't parse into number"))
@@ -95,22 +96,20 @@ pub async fn run(
     // Spin up Nats Client
     // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
     const HOST_AGENT_CLIENT_NAME: &str = "Host Agent Bare";
 
     let nats_client = tokio::select! {
         client = async {loop {
-                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                let host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
                     nats_url:nats_url.clone(),
                     name:HOST_AGENT_CLIENT_NAME.to_string(),
                     ping_interval:Some(Duration::from_secs(10)),
                     request_timeout:Some(Duration::from_secs(29)),
-
                     inbox_prefix: Default::default(),
-                    service_params:Default::default(),
-                    opts: Default::default(),
+                    listeners: Default::default(),
                     credentials_path: Default::default()
                 })
                 .await
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
index dd741d6..3f76cda 100644
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ b/rust/clients/host_agent/src/workload_manager.rs
@@ -1,86 +1,80 @@
 /*
  This client is associated with the:
-- WORKLOAD account
-- hpos user
+    - HPOS account
+    - host user
 
-// This client is responsible for:
-  - subscribing to workload streams
-    - installing new workloads
-    - removing workloads
+This client is responsible for subscribing to workload streams that handle:
+    - installing new workloads onto the hosting device
+    - removing workloads from the hosting device
     - sending workload status upon request
-    - sending active periodic workload reports
+    - sending out active periodic workload reports
 */
 
 use anyhow::{anyhow, Result};
 use async_nats::Message;
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
 use std::{path::PathBuf, sync::Arc, time::Duration};
-use util_libs::{
-    db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType},
+use util_libs::nats::{
+    jetstream_client,
+    types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
 };
 use workload::{
-    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+    host_api::HostWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
+    WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
 };
 
 const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
-const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
+const HOST_AGENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX";
 
 // TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
 pub async fn run(
     host_pubkey: &str,
     host_creds_path: &Option<PathBuf>,
-) -> Result<nats_js_client::JsClient, async_nats::Error> {
-    log::info!("HPOS Agent Client: Connecting to server...");
+) -> Result<jetstream_client::JsClient, async_nats::Error> {
+    log::info!("Host Agent Client: Connecting to server...");
     log::info!("host_creds_path : {:?}", host_creds_path);
     log::info!("host_pubkey : {}", host_pubkey);
 
-    // ==================== NATS Setup ====================
+    let pubkey_lowercase = host_pubkey.to_string().to_lowercase();
+
+    // ==================== Setup NATS ====================
     // Connect to Nats server
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
-    let event_listeners = nats_js_client::get_event_listeners();
-
-    // Setup JS Stream Service
-    let workload_stream_service_params = JsServiceParamsPartial {
-        name: WORKLOAD_SRV_NAME.to_string(),
-        description: WORKLOAD_SRV_DESC.to_string(),
-        version: WORKLOAD_SRV_VERSION.to_string(),
-        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
-    };
+    let event_listeners = jetstream_client::get_event_listeners();
 
     // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+    let mut host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
         nats_url: nats_url.clone(),
         name: HOST_AGENT_CLIENT_NAME.to_string(),
-        inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
-        service_params: vec![workload_stream_service_params.clone()],
+        inbox_prefix: format!("{}.{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
         credentials_path: host_creds_path
             .as_ref()
             .map(|path| path.to_string_lossy().to_string()),
-        opts: vec![nats_js_client::with_event_listeners(
-            event_listeners.clone(),
-        )],
         ping_interval: Some(Duration::from_secs(10)),
         request_timeout: Some(Duration::from_secs(29)),
+        listeners: vec![jetstream_client::with_event_listeners(
+            event_listeners.clone(),
+        )],
     })
     .await
     .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
+    // ==================== Setup JS Stream Service ====================
+    // Instantiate the Workload API
+    let workload_api = HostWorkloadApi::default();
 
-    // ==================== DB Setup ====================
-    // Create a new MongoDB Client and connect it to the cluster
-    let mongo_uri = get_mongodb_url();
-    let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-
-    // Generate the Workload API with access to db
-    let workload_api = WorkloadApi::new(&client).await?;
-
-    // ==================== API ENDPOINTS ====================
     // Register Workload Streams for Host Agent to consume
     // NB: Subjects are published by orchestrator or nats-db-connector
+    let workload_stream_service = JsServiceBuilder {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+    host_workload_client
+        .add_js_service(workload_stream_service)
+        .await?;
+
     let workload_service = host_workload_client
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
@@ -89,40 +83,71 @@ pub async fn run(
         ))?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "start_workload",
-            "start",
-            EndpointType::Async(workload_api.call(
-                |api: WorkloadApi, msg: Arc<Message>| async move { api.start_workload(msg).await },
-            )),
-            None,
-        )
+        .add_consumer(ConsumerBuilder {
+            name: "install_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Install.as_ref()
+            ),
+            handler: EndpointType::Async(
+                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.install_workload(msg).await
+                }),
+            ),
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "send_workload_status",
-            "send_status",
-            EndpointType::Async(
-                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.send_workload_status(msg).await
+        .add_consumer(ConsumerBuilder {
+            name: "update_installed_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::UpdateInstalled.as_ref()
+            ),
+            handler: EndpointType::Async(
+                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.update_workload(msg).await
                 }),
             ),
-            None,
-        )
+            response_subject_fn: None,
+        })
         .await?;
 
     workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "uninstall_workload",
-            "uninstall",
-            EndpointType::Async(
-                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
+        .add_consumer(ConsumerBuilder {
+            name: "uninstall_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Uninstall.as_ref()
+            ),
+            handler: EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
                     api.uninstall_workload(msg).await
-                }),
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "send_workload_status".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::SendStatus.as_ref()
             ),
-            None,
-        )
+            handler: EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.send_workload_status(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
         .await?;
 
     Ok(host_workload_client)
diff --git a/rust/services/workload/Cargo.toml b/rust/services/workload/Cargo.toml
index abc7708..60dfe59 100644
--- a/rust/services/workload/Cargo.toml
+++ b/rust/services/workload/Cargo.toml
@@ -14,6 +14,9 @@ env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
 thiserror = { workspace = true }
+strum = "0.25"
+strum_macros = "0.25"
+async-trait = "0.1.83"
 semver = "1.0.24"
 rand = "0.8.5"
 mongodb = "3.1"
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
new file mode 100644
index 0000000..00354ea
--- /dev/null
+++ b/rust/services/workload/src/host_api.rs
@@ -0,0 +1,169 @@
+/*
+Endpoints & Managed Subjects:
+    - `install_workload`: handles the "WORKLOAD.<host_pukey>.install." subject
+    - `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
+    - `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
+    - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
+*/
+
+use crate::types::WorkloadResult;
+
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
+use anyhow::Result;
+use async_nats::Message;
+use core::option::Option::None;
+use std::{fmt::Debug, sync::Arc};
+use util_libs::{
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::ServiceError,
+};
+
+#[derive(Debug, Clone, Default)]
+pub struct HostWorkloadApi {}
+
+impl WorkloadServiceApi for HostWorkloadApi {}
+
+impl HostWorkloadApi {
+    pub async fn install_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Running,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
+            // nix_uninstall_with(workload_id)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    // For host agent ? or elsewhere ?
+    // TODO: Talk through with Stefan
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
+
+        // Send updated status:
+        // NB: This will send the update to both the requester (if one exists)
+        // and will broadcast the update to for any `response_subject` address registred for the endpoint
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+}
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 75dad8c..e28d4d6 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -1,482 +1,68 @@
 /*
 Service Name: WORKLOAD
 Subject: "WORKLOAD.>"
-Provisioning Account: WORKLOAD
-Users: orchestrator & hpos
-Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
-- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
-- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
-- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+Provisioning Account: ADMIN
+Importing Account: HPOS
+Users: orchestrator & host
 */
 
+pub mod host_api;
+pub mod orchestrator_api;
 pub mod types;
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
+use async_nats::jetstream::ErrorCode;
 use async_nats::Message;
-use bson::oid::ObjectId;
-use bson::{doc, to_document, DateTime};
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use serde::{Deserialize, Serialize};
+use async_trait::async_trait;
+use core::option::Option::None;
+use serde::Deserialize;
 use std::future::Future;
-use std::{fmt::Debug, str::FromStr, sync::Arc};
+use std::{fmt::Debug, sync::Arc};
+use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
-    db::{
-        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
-    },
-    nats_js_client,
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::{AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 
-pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
+pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD_SERVICE";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
-pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
-
-#[derive(Debug, Clone)]
-pub struct WorkloadApi {
-    pub workload_collection: MongoCollection<schemas::Workload>,
-    pub host_collection: MongoCollection<schemas::Host>,
-    pub user_collection: MongoCollection<schemas::User>,
-    pub developer_collection: MongoCollection<schemas::Developer>,
-}
-
-impl WorkloadApi {
-    pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
-                .await?,
-            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
-            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME)
-                .await?,
-        })
-    }
-
-    pub fn call<F, Fut>(&self, handler: F) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
+
+#[async_trait]
+pub trait WorkloadServiceApi
+where
+    Self: std::fmt::Debug + Clone + 'static,
+{
+    fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<WorkloadApiResult>
     where
-        F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+        F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
+        Self: Send + Sync,
     {
         let api = self.to_owned();
         Arc::new(
-            move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+            move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
                 let api_clone = api.clone();
                 Box::pin(handler(api_clone, msg))
             },
         )
     }
 
-    /*******************************  For Orchestrator   *********************************/
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self
-            .process_request(
-                msg,
-                WorkloadState::Reported,
-                |workload: schemas::Workload| async move {
-                    let workload_id = self
-                        .workload_collection
-                        .insert_one_into(workload.clone())
-                        .await?;
-                    log::info!(
-                        "Successfully added workload. MongodDB Workload ID={:?}",
-                        workload_id
-                    );
-                    let updated_workload = schemas::Workload {
-                        _id: Some(ObjectId::from_str(&workload_id)?),
-                        ..workload
-                    };
-                    Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: updated_workload._id.map(|oid| oid.to_hex()),
-                            desired: WorkloadState::Reported,
-                            actual: WorkloadState::Reported,
-                        },
-                        None,
-                    ))
-                },
-                WorkloadState::Error,
-            )
-            .await)
-    }
-
-    pub async fn update_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self
-            .process_request(
-                msg,
-                WorkloadState::Running,
-                |workload: schemas::Workload| async move {
-                    let workload_query = doc! { "_id":  workload._id };
-
-                    // update workload updated_at
-                    let mut workload_doc = workload.clone();
-                    workload_doc.metadata.updated_at = Some(DateTime::now());
-
-                    // convert workload to document and submit to mongodb
-                    let updated_workload = to_document(&workload_doc)?;
-                    self.workload_collection
-                        .update_one_within(
-                            workload_query,
-                            UpdateModifications::Document(doc! { "$set": updated_workload }),
-                        )
-                        .await?;
-
-                    log::info!(
-                        "Successfully updated workload. MongodDB Workload ID={:?}",
-                        workload._id
-                    );
-                    Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: workload._id.map(|oid| oid.to_hex()),
-                            desired: WorkloadState::Reported,
-                            actual: WorkloadState::Reported,
-                        },
-                        None,
-                    ))
-                },
-                WorkloadState::Error,
-            )
-            .await)
-    }
-
-    pub async fn remove_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.remove'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Removed,
-            |workload_id: bson::oid::ObjectId| async move {
-                let workload_query = doc! { "_id":  workload_id };
-                self.workload_collection.update_one_within(
-                    workload_query,
-                    UpdateModifications::Document(doc! {
-                        "$set": {
-                            "metadata.is_deleted": true,
-                            "metadata.deleted_at": DateTime::now()
-                        }
-                    })
-                ).await?;
-                log::info!(
-                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
-                    workload_id
-                );
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
-                        desired: WorkloadState::Removed,
-                        actual: WorkloadState::Removed,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // looks through existing hosts to find possible hosts for a given workload
-    // returns the minimum number of hosts required for workload
-    pub async fn find_hosts_meeting_workload_criteria(
-        &self,
-        workload: Workload,
-    ) -> Result<Vec<Host>, anyhow::Error> {
-        let pipeline = vec![
-            doc! {
-                "$match": {
-                    // verify there are enough system resources
-                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk },
-                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
-
-                    // limit how many workloads a single host can have
-                    "assigned_workloads": { "$lt": 1 }
-                }
-            },
-            doc! {
-                // the maximum number of hosts returned should be the minimum hosts required by workload
-                // sample randomized results and always return back atleast 1 result
-                "$sample": std::cmp::min(workload.min_hosts as i32, 1)
-            },
-            doc! {
-                "$project": {
-                    "_id": 1
-                }
-            }
-        ];
-        let results = self.host_collection.aggregate(pipeline).await?;
-        if results.is_empty() {
-            anyhow::bail!(
-                "Could not find a compatible host for this workload={:#?}",
-                workload._id
-            );
-        }
-        Ok(results)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    // trigger on mongodb [workload] collection (insert)
-    pub async fn handle_db_insertion(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.insert'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Assigned,
-            |workload: schemas::Workload| async move {
-                log::debug!("New workload to assign. Workload={:#?}", workload);
-
-                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
-                let workload_id = if let Some(id) = workload.clone()._id { id } else {
-                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
-                    return Err(anyhow!(err_msg));
-                };
-
-                // 1. Perform sanity check to ensure workload is not already assigned to a host
-                // ...and if so, exit fn
-                // todo: check for to ensure assigned host *still* has enough capacity for updated workload
-                if !workload.assigned_hosts.is_empty() {
-                    log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: Some(workload_id.to_hex()),
-                            desired: WorkloadState::Assigned,
-                            actual: WorkloadState::Assigned,
-                        },
-                        Some(
-                            workload.assigned_hosts
-                            .iter().map(|id| id.to_hex()).collect())
-                        )
-                    );
-                }
-
-                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
-                let eligible_hosts = self.find_hosts_meeting_workload_criteria(workload.clone()).await?;
-                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_hosts);
-
-                let host_ids: Vec<ObjectId> = eligible_hosts.iter().map(|host| host._id.to_owned().unwrap()).collect();
-
-                // 4. Update the Workload Collection with the assigned Host ID
-                let workload_query = doc! { "_id":  workload_id };
-                let updated_workload = &Workload {
-                    assigned_hosts: host_ids.clone(),
-                    ..workload.clone()
-                };
-                let updated_workload_doc = to_document(updated_workload)?;
-                let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
-                    updated_workload_result
-                );
-
-                // 5. Update the Host Collection with the assigned Workload ID
-                let host_query = doc! { "_id":  { "$in": host_ids } };
-                let updated_host_doc =  doc! {
-                    "$push": {
-                        "assigned_workloads": workload_id
-                    }
-                };
-                let updated_host_result = self.host_collection.update_many_within(
-                    host_query, 
-                    UpdateModifications::Document(updated_host_doc)
-                ).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
-                    updated_host_result
-                );
-
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
-                    },
-                    Some(
-                        updated_workload.assigned_hosts.to_owned()
-                        .iter().map(|host| host.to_hex()).collect()
-                    )
-                ))
-        },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    // triggers on mongodb [workload] collection (update)
-    pub async fn handle_db_update(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-
-        let payload_buf = msg.payload.to_vec();
-
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload to update. Workload={:#?}", workload.clone());
-
-        // 1. remove workloads from existing hosts
-        self.host_collection.mongo_error_handler(
-            self.host_collection
-                .collection
-                .update_many(
-                    doc! {},
-                    doc! { "$pull": { "assigned_workloads": workload._id } },
-                )
-                .await,
-        )?;
-        log::info!(
-            "Remove workload from previous hosts. Workload={:#?}",
-            workload._id
-        );
-
-        if !workload.metadata.is_deleted {
-            // 3. add workload to specific hosts
-            self.host_collection.mongo_error_handler(
-                self.host_collection
-                    .collection
-                    .update_one(
-                        doc! { "_id": { "$in": workload.clone().assigned_hosts } },
-                        doc! { "$push": { "assigned_workloads": workload._id } },
-                    )
-                    .await,
-            )?;
-            log::info!("Added workload to new hosts. Workload={:#?}", workload._id);
-        } else {
-            log::info!(
-                "Skipping (reason: deleted) - Added workload to new hosts. Workload={:#?}",
-                workload._id
-            );
-        }
-
-        let success_status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
-            desired: WorkloadState::Updating,
-            actual: WorkloadState::Updating,
-        };
-        log::info!("Workload update successful. Workload={:#?}", workload._id);
-
-        Ok(types::ApiResult(success_status, None))
-    }
-
-    // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status: WorkloadStatus = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload status to update. Status={:?}", workload_status);
-        if workload_status.id.is_none() {
-            return Err(anyhow!("Got a status update for workload without an id!"));
-        }
-        let workload_status_id = workload_status
-            .id
-            .clone()
-            .expect("workload is not provided");
-
-        self.workload_collection
-            .update_one_within(
-                doc! {
-                    "_id": ObjectId::parse_str(workload_status_id)?
-                },
-                UpdateModifications::Document(doc! {
-                    "$set": {
-                        "state": bson::to_bson(&workload_status.actual)?
-                    }
-                }),
-            )
-            .await?;
-
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    pub async fn uninstall_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_id = serde_json::from_slice::<String>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
-        // nix_uninstall_with(workload_id)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: Some(workload_id),
-            desired: WorkloadState::Uninstalled,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    // For host agent ? or elsewhere ?
-    // TODO: Talk through with Stefan
-    pub async fn send_workload_status(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!(
-            "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
-            msg
-        );
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status = serde_json::from_slice::<WorkloadStatus>(&payload_buf)?;
-
-        // Send updated status:
-        // NB: This will send the update to both the requester (if one exists)
-        // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************  Helper Fns  *********************************/
-    // Helper function to initialize mongodb collections
-    async fn init_collection<T>(
-        client: &MongoDBClient,
-        collection_name: &str,
-    ) -> Result<MongoCollection<T>>
+    fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
-        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+        T: for<'de> Deserialize<'de> + Send + Sync,
     {
-        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+        let payload_buf = msg.payload.to_vec();
+        serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
+            let err_msg = format!(
+                "Error: Failed to deserialize payload. Subject='{}' Err={}",
+                msg.subject.clone().into_string(),
+                e
+            );
+            log::error!("{}", err_msg);
+            ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
+        })
     }
 
     // Helper function to streamline the processing of incoming workload messages
@@ -485,33 +71,21 @@ impl WorkloadApi {
         &self,
         msg: Arc<Message>,
         desired_state: WorkloadState,
-        cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> types::ApiResult
+        cb_fn: impl Fn(T) -> Fut + Send + Sync,
+    ) -> Result<WorkloadApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = match serde_json::from_slice(&msg.payload) {
-            Ok(r) => r,
-            Err(e) => {
-                let err_msg = format!("Failed to deserialize payload for Workload Service Endpoint. Subject={} Error={:?}", msg.subject, e);
-                log::error!("{}", err_msg);
-                let status = WorkloadStatus {
-                    id: None,
-                    desired: desired_state,
-                    actual: error_state(err_msg),
-                };
-                return types::ApiResult(status, None);
-            }
-        };
+        let payload: T = Self::convert_msg_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
-        match cb_fn(payload.clone()).await {
+        Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
-                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
+                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject.clone().into_string(), payload, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
                     id: None,
@@ -520,8 +94,14 @@ impl WorkloadApi {
                 };
 
                 // 3. return response for stream
-                types::ApiResult(status, None)
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                }
             }
-        }
+        })
     }
 }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
new file mode 100644
index 0000000..9db39fa
--- /dev/null
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -0,0 +1,662 @@
+/*
+Endpoints & Managed Subjects:
+    - `add_workload`: handles the "WORKLOAD.add" subject
+    - `update_workload`: handles the "WORKLOAD.update" subject
+    - `remove_workload`: handles the "WORKLOAD.remove" subject
+    - `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
+    - `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
+    - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
+*/
+
+use crate::types::WorkloadResult;
+
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
+use anyhow::Result;
+use async_nats::Message;
+use bson::{self, doc, oid::ObjectId, to_document, DateTime};
+use core::option::Option::None;
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::{HashMap, HashSet},
+    fmt::Debug,
+    sync::Arc,
+};
+use util_libs::{
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
+    },
+    nats::types::ServiceError,
+};
+
+#[derive(Debug, Clone)]
+pub struct OrchestratorWorkloadApi {
+    pub workload_collection: MongoCollection<schemas::Workload>,
+    pub host_collection: MongoCollection<schemas::Host>,
+    pub user_collection: MongoCollection<schemas::User>,
+    pub developer_collection: MongoCollection<schemas::Developer>,
+}
+
+impl WorkloadServiceApi for OrchestratorWorkloadApi {}
+
+impl OrchestratorWorkloadApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        Ok(Self {
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
+                .await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME)
+                .await?,
+        })
+    }
+
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.add'");
+        self.process_request(
+            msg,
+            WorkloadState::Reported,
+            WorkloadState::Error,
+            |mut workload: schemas::Workload| async move {
+                let mut status = WorkloadStatus {
+                    id: None,
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Reported,
+                };
+                workload.status = status.clone();
+                workload.metadata.created_at = Some(DateTime::now());
+
+                let workload_id = self.workload_collection.insert_one_into(workload).await?;
+                status.id = Some(workload_id);
+
+                log::info!(
+                    "Successfully added workload. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                })
+            },
+        )
+        .await
+    }
+
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.update'");
+        self.process_request(
+            msg,
+            WorkloadState::Updating,
+            WorkloadState::Error,
+            |mut workload: schemas::Workload| async move {
+                let status = WorkloadStatus {
+                    id: workload._id,
+                    desired: WorkloadState::Updated,
+                    actual: WorkloadState::Updating,
+                };
+
+                workload.status = status.clone();
+                workload.metadata.updated_at = Some(DateTime::now());
+
+                // convert workload to document and submit to mongodb
+                let updated_workload_doc =
+                    to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection
+                    .update_one_within(
+                        doc! { "_id":  workload._id },
+                        UpdateModifications::Document(doc! { "$set": updated_workload_doc }),
+                    )
+                    .await?;
+
+                log::info!(
+                    "Successfully updated workload. MongodDB Workload ID={:?}",
+                    workload._id
+                );
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                })
+            },
+        )
+        .await
+    }
+
+    pub async fn remove_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.remove'");
+        self.process_request(
+            msg,
+            WorkloadState::Removed,
+            WorkloadState::Error,
+            |workload_id: ObjectId| async move {
+                let status = WorkloadStatus {
+                    id: Some(workload_id),
+                    desired: WorkloadState::Uninstalled,
+                    actual: WorkloadState::Removed,
+                };
+
+                let updated_status_doc = bson::to_bson(&status)
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection.update_one_within(
+                    doc! { "_id":  workload_id },
+                    UpdateModifications::Document(doc! {
+                        "$set": {
+                            "metadata.is_deleted": true,
+                            "metadata.deleted_at": DateTime::now(),
+                            "status": updated_status_doc
+                        }
+                    })
+                ).await?;
+                log::info!(
+                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None
+                    },
+                    maybe_response_tags: None
+                })
+            },
+        )
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    pub async fn handle_db_insertion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.insert'");
+        self.process_request(
+            msg,
+            WorkloadState::Assigned,
+            WorkloadState::Error,
+            |workload: schemas::Workload| async move {
+                log::debug!("New workload to assign. Workload={:#?}", workload);
+
+                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
+                let workload_id = if let Some(id) = workload.clone()._id { id } else {
+                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
+                    return Err(ServiceError::Internal(err_msg));
+                };
+
+                let status = WorkloadStatus {
+                    id: Some(workload_id),
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Assigned,
+                };
+
+                // 1. Perform sanity check to ensure workload is not already assigned to a host and if so, exit fn
+                if !workload.assigned_hosts.is_empty() {
+                    log::warn!("Attempted to assign host for new workload, but host already exists.");
+                    return Ok(WorkloadApiResult {
+                        result: WorkloadResult {
+                            status,
+                            workload: None
+                        },
+                        maybe_response_tags: None
+                    });
+                }
+
+                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
+                // & randomly choose host(s)
+                let eligible_host_ids = self.find_hosts_meeting_workload_criteria(workload.clone(), None).await?;
+                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_host_ids);
+
+                // 3. Update the selected host records with the assigned Workload ID
+                let assigned_host_ids = self.assign_workload_to_hosts(workload_id, eligible_host_ids, workload.min_hosts).await.map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 4. Update the Workload Collection with the assigned Host ID
+                let new_status = WorkloadStatus {
+                    id: None, // remove the id to avoid redundant saving of it in the db
+                    ..status.clone()
+                };
+                self.assign_hosts_to_workload(assigned_host_ids.clone(), workload_id, new_status).await.map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 5. Create tag map with host ids to inform nats to publish message to these hosts with workload install status                
+                let mut tag_map: HashMap<String, String> = HashMap::new();
+                for (index, host_pubkey) in assigned_host_ids.iter().cloned().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: Some(workload)
+                    },
+                    maybe_response_tags: Some(tag_map)
+                })
+        })
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    // triggers on mongodb [workload] collection (update)
+    pub async fn handle_db_modification(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.modify'");
+        let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
+
+        // Fail Safe: exit early if the workload provided does not include an `_id` field
+        let workload_id = if let Some(id) = workload.clone()._id {
+            id
+        } else {
+            let err_msg = format!(
+                "No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}",
+                workload
+            );
+            return Err(ServiceError::Internal(err_msg));
+        };
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        let log_msg = format!(
+            "Workload update in DB successful. Fwding update to assigned hosts. workload_id={}",
+            workload_id
+        );
+
+        // Match on state (updating or removed) and handle each case
+        let result = match workload.status.actual {
+            WorkloadState::Updating => {
+                log::trace!("Updated workload to handle. Workload={:#?}", workload);
+                // 1. Fetch current hosts
+                let hosts = self
+                    .fetch_hosts_assigned_to_workload(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 2. Remove workloads from existing hosts
+                self.remove_workload_from_hosts(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 3. Find eligible hosts
+                let eligible_host_ids = self
+                    .find_hosts_meeting_workload_criteria(workload.clone(), Some(hosts))
+                    .await?;
+                log::debug!(
+                    "Eligible hosts for new workload. MongodDB Host IDs={:?}",
+                    eligible_host_ids
+                );
+
+                // 4. Update the selected host records with the assigned Workload ID
+                let assigned_host_ids = self
+                    .assign_workload_to_hosts(workload_id, eligible_host_ids, workload.min_hosts)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 5. Update the Workload Collection with the assigned Host ID
+                // IMP: It is very important that the workload state changes to a state that is not `WorkloadState::Updating`,
+                // IMP: ...otherwise, this change will cause the workload update to loop between the db stream modification reads and this handler
+                let new_status = WorkloadStatus {
+                    id: None,
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Updated,
+                };
+                self.assign_hosts_to_workload(
+                    assigned_host_ids.clone(),
+                    workload_id,
+                    new_status.clone(),
+                )
+                .await
+                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 6. Create tag map with host ids to inform nats to publish message to these hosts with workload install status
+                for (index, host_pubkey) in assigned_host_ids.iter().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                log::info!("Added workload to new hosts. Workload={:#?}", workload_id);
+
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            ..new_status
+                        },
+                        workload: Some(workload),
+                    },
+                    maybe_response_tags: Some(tag_map),
+                }
+            }
+            WorkloadState::Removed => {
+                log::trace!("Removed workload to handle. Workload={:#?}", workload);
+                // 1. Fetch current hosts with `workload_id`` to know which
+                // hosts to send uninstall workload request to...
+                let hosts = self
+                    .fetch_hosts_assigned_to_workload(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 2. Remove workloads from existing hosts
+                self.remove_workload_from_hosts(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 3. Create tag map with host ids to inform nats to publish message to these hosts with workload install status
+                let host_ids = hosts
+                    .iter()
+                    .map(|h| {
+                        h._id
+                            .ok_or_else(|| ServiceError::Internal("Error".to_string()))
+                    })
+                    .collect::<Result<Vec<ObjectId>, ServiceError>>()?;
+                for (index, host_pubkey) in host_ids.iter().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                log::info!("{} Hosts={:?}", log_msg, hosts);
+
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Uninstalled,
+                            actual: WorkloadState::Removed,
+                        },
+                        workload: Some(workload),
+                    },
+                    maybe_response_tags: Some(tag_map),
+                }
+            }
+            _ => {
+                // Catches all other cases wherein a record in the workload collection was modified (not created),
+                // with a state other than "Updating" or "Removed".
+                // In this case, we don't want to do take any new action, so we return a default status without any updates or frowarding tags.
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: workload.status.desired,
+                            actual: workload.status.actual,
+                        },
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                }
+            }
+        };
+
+        Ok(result)
+    }
+
+    // NB: Published by the Hosting Agent whenever the status of a workload changes
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
+
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
+        log::trace!("Workload status to update. Status={:?}", workload_status);
+
+        let workload_status_id = workload_status
+            .id
+            .ok_or_else(|| ServiceError::Internal("Failed to read ._id from record".to_string()))?;
+
+        self.workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_status_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": {
+                        "status": bson::to_bson(&workload_status)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                    }
+                }),
+            )
+            .await?;
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    // Verifies that a host meets the workload criteria
+    fn verify_host_meets_workload_criteria(
+        &self,
+        assigned_host: &Host,
+        workload: &Workload,
+    ) -> bool {
+        if assigned_host.remaining_capacity.disk < workload.system_specs.capacity.disk {
+            return false;
+        }
+        if assigned_host.remaining_capacity.memory < workload.system_specs.capacity.memory {
+            return false;
+        }
+        if assigned_host.remaining_capacity.cores < workload.system_specs.capacity.cores {
+            return false;
+        }
+
+        true
+    }
+
+    async fn fetch_hosts_assigned_to_workload(&self, workload_id: ObjectId) -> Result<Vec<Host>> {
+        Ok(self
+            .host_collection
+            .get_many_from(doc! { "assigned_workloads": workload_id })
+            .await?)
+    }
+
+    async fn remove_workload_from_hosts(&self, workload_id: ObjectId) -> Result<()> {
+        self.host_collection
+            .inner
+            .update_many(
+                doc! {},
+                doc! { "$pull": { "assigned_workloads": workload_id } },
+            )
+            .await
+            .map_err(ServiceError::Database)?;
+        log::info!(
+            "Removed workload from previous hosts. Workload={:#?}",
+            workload_id
+        );
+        Ok(())
+    }
+
+    // Looks through existing hosts to find possible hosts for a given workload
+    // returns the minimum number of hosts required for workload
+    async fn find_hosts_meeting_workload_criteria(
+        &self,
+        workload: Workload,
+        maybe_existing_hosts: Option<Vec<Host>>,
+    ) -> Result<Vec<ObjectId>, ServiceError> {
+        let mut needed_host_count = workload.min_hosts;
+        let mut still_eligible_host_ids: Vec<ObjectId> = vec![];
+
+        if let Some(hosts) = maybe_existing_hosts {
+            still_eligible_host_ids = hosts.into_iter()
+                .filter_map(|h| {
+                    if self.verify_host_meets_workload_criteria(&h, &workload) {
+                        h._id.ok_or_else(|| {
+                            ServiceError::Internal(format!(
+                                "No `_id` found for workload. Unable to proceed verifying host eligibility. Workload={:?}",
+                                workload
+                            ))
+                        }).ok()
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+            needed_host_count -= still_eligible_host_ids.len() as u16;
+        }
+
+        let pipeline = vec![
+            doc! {
+                "$match": {
+                    // verify there are enough system resources
+                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk },
+                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
+                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
+
+                    // limit how many workloads a single host can have
+                    "assigned_workloads": { "$lt": 1 }
+                }
+            },
+            doc! {
+                // the maximum number of hosts returned should be the minimum hosts required by workload
+                // sample randomized results and always return back at least 1 result
+                "$sample": std::cmp::min( needed_host_count as i32, 1),
+
+                // only return the `host._id` feilds
+                "$project": { "_id": 1 }
+            },
+        ];
+        let host_ids = self.host_collection.aggregate::<ObjectId>(pipeline).await?;
+        if host_ids.is_empty() {
+            let err_msg = format!(
+                "Failed to locate a compatible host for workload. Workload_Id={:?}",
+                workload._id
+            );
+            return Err(ServiceError::Internal(err_msg));
+        } else if workload.min_hosts > host_ids.len() as u16 {
+            log::warn!(
+                "Failed to locate the the min required number of hosts for workload. Workload_Id={:?}",
+                workload._id
+            );
+        }
+
+        let mut eligible_host_ids = host_ids;
+        eligible_host_ids.extend(still_eligible_host_ids);
+
+        Ok(eligible_host_ids)
+    }
+
+    async fn assign_workload_to_hosts(
+        &self,
+        workload_id: ObjectId,
+        eligible_host_ids: Vec<ObjectId>,
+        needed_host_count: u16,
+    ) -> Result<Vec<ObjectId>> {
+        // NB: This will attempt to assign the hosts up to 5 times.. then exit loop with warning message
+        let assigned_host_ids: Vec<ObjectId>;
+        let mut unassigned_host_ids: Vec<ObjectId> = eligible_host_ids.clone();
+        let mut exit_flag = 0;
+        loop {
+            let updated_host_result = self
+                .host_collection
+                .update_many_within(
+                    doc! {
+                        "_id": { "$in": unassigned_host_ids.clone() },
+                        // Currently we only allow a single workload per host
+                        "assigned_workloads": { "$size": 0 }
+                    },
+                    UpdateModifications::Document(doc! {
+                        "$push": {
+                            "assigned_workloads": workload_id
+                        }
+                    }),
+                )
+                .await?;
+
+            if updated_host_result.matched_count == unassigned_host_ids.len() as u64 {
+                log::debug!(
+                    "Successfully updated Host records with the new workload id {}. Host_IDs={:?} Update_Result={:?}",
+                    workload_id,
+                    eligible_host_ids,
+                    updated_host_result
+                );
+                assigned_host_ids = eligible_host_ids;
+                break;
+            } else if exit_flag == 5 {
+                let unassigned_host_hashset: HashSet<ObjectId> =
+                    unassigned_host_ids.into_iter().collect();
+                assigned_host_ids = eligible_host_ids
+                    .into_iter()
+                    .filter(|id| !unassigned_host_hashset.contains(id))
+                    .collect();
+                log::warn!("Exiting loop after 5 attempts to assign the workload to the min number of hosts.
+                    Only able to assign {} hosts. Workload_ID={}, Assigned_Host_IDs={:?}",
+                    needed_host_count,
+                    workload_id,
+                    assigned_host_ids
+                );
+                break;
+            }
+
+            log::warn!("Failed to update all selected host records with workload_id.");
+            log::debug!("Fetching paired host records to see which one(s) still remain unassigned to workload...");
+            let unassigned_hosts = self
+                .host_collection
+                .get_many_from(doc! {
+                    "_id": { "$in": eligible_host_ids.clone() },
+                    "assigned_workloads": { "$size": 0 }
+                })
+                .await?;
+
+            unassigned_host_ids = unassigned_hosts
+                .into_iter()
+                .map(|h| h._id.unwrap_or_default())
+                .collect();
+            exit_flag += 1;
+        }
+
+        Ok(assigned_host_ids)
+    }
+
+    async fn assign_hosts_to_workload(
+        &self,
+        assigned_host_ids: Vec<ObjectId>,
+        workload_id: ObjectId,
+        new_status: WorkloadStatus,
+    ) -> Result<()> {
+        let updated_workload_result = self
+            .workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": [{
+                        "status": bson::to_bson(&new_status)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                        }, {
+                        "assigned_hosts": assigned_host_ids
+                    }]
+                }),
+            )
+            .await;
+
+        log::trace!(
+            "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
+            updated_workload_result
+        );
+
+        Ok(())
+    }
+
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+}
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 912ffb6..76e0f82 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,18 +1,49 @@
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use strum_macros::AsRefStr;
 use util_libs::{
-    db::schemas::WorkloadStatus,
-    js_stream_service::{CreateTag, EndpointTraits},
+    db::schemas::{self, WorkloadStatus},
+    nats::types::{CreateResponse, CreateTag, EndpointTraits},
 };
 
-pub use String as WorkloadId;
+#[derive(Serialize, Deserialize, Clone, Debug, AsRefStr)]
+#[serde(rename_all = "snake_case")]
+pub enum WorkloadServiceSubjects {
+    Add,
+    Update,
+    Remove,
+    Insert, // db change stream trigger
+    Modify, // db change stream trigger
+    HandleStatusUpdate,
+    SendStatus,
+    Install,
+    Uninstall,
+    UpdateInstalled,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult(pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct WorkloadResult {
+    pub status: WorkloadStatus,
+    pub workload: Option<schemas::Workload>,
+}
 
-impl CreateTag for ApiResult {
-    fn get_tags(&self) -> Option<Vec<String>> {
-        self.1.clone()
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadApiResult {
+    pub result: WorkloadResult,
+    pub maybe_response_tags: Option<HashMap<String, String>>,
+}
+impl EndpointTraits for WorkloadApiResult {}
+impl CreateTag for WorkloadApiResult {
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.maybe_response_tags.clone().unwrap_or_default()
+    }
+}
+impl CreateResponse for WorkloadApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        let r = self.result.clone();
+        match serde_json::to_vec(&r) {
+            Ok(r) => r.into(),
+            Err(e) => e.to_string().into(),
+        }
     }
 }
-
-impl EndpointTraits for ApiResult {}
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index b2974bb..332c741 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,47 +1,39 @@
-use anyhow::{anyhow, Context, Result};
+use crate::nats::types::ServiceError;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
-use bson::{self, doc, Document};
+use bson::oid::ObjectId;
+use bson::{self, Document};
 use futures::stream::TryStreamExt;
 use mongodb::options::UpdateModifications;
-use mongodb::results::{DeleteResult, UpdateResult};
+use mongodb::results::UpdateResult;
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
 
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Internal Error: {0}")]
-    Internal(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-}
-
 #[async_trait]
 pub trait MongoDbAPI<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync,
 {
-    fn mongo_error_handler<ReturnType>(
+    type Error;
+
+    async fn aggregate<R: for<'de> Deserialize<'de>>(
         &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType>;
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>>;
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>>;
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
-    async fn insert_one_into(&self, item: T) -> Result<String>;
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(
+        pipeline: Vec<Document>,
+    ) -> Result<Vec<R>, Self::Error>;
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error>;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error>;
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error>;
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn update_many_within(
+    ) -> Result<UpdateResult, Self::Error>;
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
-    async fn delete_all_from(&self) -> Result<DeleteResult>;
+    ) -> Result<UpdateResult, Self::Error>;
 }
 
 pub trait IntoIndexes {
@@ -53,7 +45,7 @@ pub struct MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
 {
-    pub collection: Collection<T>,
+    pub inner: Collection<T>,
     indices: Vec<IndexModel>,
 }
 
@@ -72,7 +64,7 @@ where
         let indices = vec![];
 
         Ok(MongoCollection {
-            collection,
+            inner: collection,
             indices,
         })
     }
@@ -94,7 +86,7 @@ where
         self.indices = indices.clone();
 
         // Apply the indices to the mongodb collection schema
-        self.collection.create_indexes(indices).await?;
+        self.inner.create_indexes(indices).await?;
         Ok(self)
     }
 }
@@ -104,109 +96,86 @@ impl<T> MongoDbAPI<T> for MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes + Debug,
 {
-    fn mongo_error_handler<ReturnType>(
-        &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType> {
-        let rtn = result.map_err(ServiceError::Database)?;
-        Ok(rtn)
-    }
+    type Error = ServiceError;
 
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>> {
-        log::info!("aggregate pipeline {:?}", pipeline);
-        let cursor = self.collection.aggregate(pipeline).await?;
+    async fn aggregate<R>(&self, pipeline: Vec<Document>) -> Result<Vec<R>, Self::Error>
+    where
+        R: for<'de> Deserialize<'de>,
+    {
+        log::trace!("Aggregate pipeline {:?}", pipeline);
+        let cursor = self.inner.aggregate(pipeline).await?;
 
         let results_doc: Vec<bson::Document> =
             cursor.try_collect().await.map_err(ServiceError::Database)?;
 
-        let results: Vec<T> = results_doc
+        let results: Vec<R> = results_doc
             .into_iter()
             .map(|doc| {
-                bson::from_document::<T>(doc).with_context(|| "failed to deserialize document")
+                bson::from_document::<R>(doc).with_context(|| "Failed to deserialize document")
             })
-            .collect::<Result<Vec<T>>>()?;
+            .collect::<Result<Vec<R>>>()
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         Ok(results)
     }
 
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>> {
-        log::info!("get_one_from filter {:?}", filter);
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error> {
+        log::trace!("Get_one_from filter {:?}", filter);
 
         let item = self
-            .collection
+            .inner
             .find_one(filter)
             .await
             .map_err(ServiceError::Database)?;
 
-        log::info!("item {:?}", item);
+        log::debug!("get_one_from item {:?}", item);
         Ok(item)
     }
 
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>> {
-        let cursor = self.collection.find(filter).await?;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error> {
+        let cursor = self.inner.find(filter).await?;
         let results: Vec<T> = cursor.try_collect().await.map_err(ServiceError::Database)?;
         Ok(results)
     }
 
-    async fn insert_one_into(&self, item: T) -> Result<String> {
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error> {
         let result = self
-            .collection
+            .inner
             .insert_one(item)
             .await
             .map_err(ServiceError::Database)?;
 
-        Ok(result.inserted_id.to_string())
-    }
+        let mongo_id = result
+            .inserted_id
+            .as_object_id()
+            .ok_or(ServiceError::Internal(format!(
+                "Failed to read the insert id after inserting item. insert_result={:?}.",
+                result
+            )))?;
 
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>> {
-        let result = self
-            .collection
-            .insert_many(items)
-            .await
-            .map_err(ServiceError::Database)?;
-
-        let ids = result
-            .inserted_ids
-            .values()
-            .map(|id| id.to_string())
-            .collect();
-        Ok(ids)
+        Ok(mongo_id)
     }
 
-    async fn update_one_within(
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_one(query, updated_doc)
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_many(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn update_many_within(
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_many(query, updated_doc)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult> {
-        self.collection
-            .delete_one(query)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_all_from(&self) -> Result<DeleteResult> {
-        self.collection
-            .delete_many(doc! {})
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_one(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 }
 
@@ -328,7 +297,7 @@ mod tests {
                     updated_at: Some(DateTime::now()),
                     deleted_at: None,
                 },
-                device_id: "Vf3IceiD".to_string(),
+                device_id: "placeholder_pubkey_host".to_string(),
                 ip_address: "127.0.0.1".to_string(),
                 remaining_capacity: Capacity {
                     memory: 16,
@@ -359,9 +328,9 @@ mod tests {
         let host_1 = get_mock_host();
         let host_2 = get_mock_host();
         let host_3 = get_mock_host();
-        host_api
-            .insert_many_into(vec![host_1.clone(), host_2.clone(), host_3.clone()])
-            .await?;
+        host_api.insert_one_into(host_1.clone()).await?;
+        host_api.insert_one_into(host_2.clone()).await?;
+        host_api.insert_one_into(host_3.clone()).await?;
 
         // get many docs
         let ids = vec![
@@ -383,13 +352,8 @@ mod tests {
         assert!(updated_ids.contains(&ids[1]));
         assert!(updated_ids.contains(&ids[2]));
 
-        // delete all documents
-        let DeleteResult { deleted_count, .. } = host_api.delete_all_from().await?;
-        assert_eq!(deleted_count, 4);
-        let fetched_host = host_api.get_one_from(filter_one).await?;
-        let fetched_hosts = host_api.get_many_from(filter_many).await?;
-        assert!(fetched_host.is_none());
-        assert!(fetched_hosts.is_empty());
+        // Delete collection and all documents therein.
+        let _ = host_api.inner.drop();
 
         Ok(())
     }
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 5a7945b..ed8cd92 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -14,18 +14,18 @@ pub const HOST_COLLECTION_NAME: &str = "host";
 pub const WORKLOAD_COLLECTION_NAME: &str = "workload";
 
 // Provide type Alias for HosterPubKey
-pub use String as HosterPubKey;
-
-// Provide type Alias for DeveloperPubkey
-pub use String as DeveloperPubkey;
-
-// Provide type Alias for DeveloperJWT
-pub use String as DeveloperJWT;
+pub use String as PubKey;
 
 // Provide type Alias for SemVer (semantic versioning)
 pub use String as SemVer;
 
 // ==================== User Schema ====================
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct RoleInfo {
+    pub collection_id: ObjectId, // Hoster/Developer colleciton Mongodb ID ref
+    pub pubkey: PubKey,          //  Hoster/Developer Pubkey *INDEXED*
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum UserPermission {
     Admin,
@@ -46,24 +46,24 @@ pub struct User {
     pub metadata: Metadata,
     pub jurisdiction: String,
     pub permissions: Vec<UserPermission>,
-    pub user_info_id: Option<ObjectId>,
-    pub developer: Option<ObjectId>,
-    pub hoster: Option<ObjectId>,
+    pub user_info_id: Option<ObjectId>, // *INDEXED*
+    pub developer: Option<RoleInfo>,    // *INDEXED*
+    pub hoster: Option<RoleInfo>,       // *INDEXED*
 }
 
-// No Additional Indexing for Developer
+// Indexing for User
 impl IntoIndexes for User {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
 
-        // add user_info index
-        let user_info_index_doc = doc! { "user_info": 1 };
-        let user_info_index_opts = Some(
+        // add user_info_id index
+        let user_info_id_index_doc = doc! { "user_info_id": 1 };
+        let user_info_id_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("user_info_index".to_string()))
+                .name(Some("user_info_id_index".to_string()))
                 .build(),
         );
-        indices.push((user_info_index_doc, user_info_index_opts));
+        indices.push((user_info_id_index_doc, user_info_id_index_opts));
 
         // add developer index
         let developer_index_doc = doc! { "developer": 1 };
@@ -75,10 +75,10 @@ impl IntoIndexes for User {
         indices.push((developer_index_doc, developer_index_opts));
 
         // add host index
-        let host_index_doc = doc! { "host": 1 };
+        let host_index_doc = doc! { "hoster": 1 };
         let host_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("host_index".to_string()))
+                .name(Some("hoster_index".to_string()))
                 .build(),
         );
         indices.push((host_index_doc, host_index_opts));
@@ -93,7 +93,7 @@ pub struct UserInfo {
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
     pub user_id: ObjectId,
-    pub email: String,
+    pub email: String, // *INDEXED*
     pub given_names: String,
     pub family_name: String,
 }
@@ -101,7 +101,6 @@ pub struct UserInfo {
 impl IntoIndexes for UserInfo {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         // add email index
         let email_index_doc = doc! { "email": 1 };
         let email_index_opts = Some(
@@ -110,7 +109,6 @@ impl IntoIndexes for UserInfo {
                 .build(),
         );
         indices.push((email_index_doc, email_index_opts));
-
         Ok(indices)
     }
 }
@@ -121,8 +119,8 @@ pub struct Developer {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user._id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub active_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub user_id: ObjectId,
+    pub active_workloads: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Developer
@@ -138,8 +136,8 @@ pub struct Hoster {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user.id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub assigned_hosts: Vec<ObjectId>, // MongoDB ID refs to `host._id`
+    pub user_id: ObjectId,
+    pub assigned_hosts: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Hoster
@@ -162,29 +160,27 @@ pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub assigned_hoster: ObjectId,
-    pub device_id: String, // *INDEXED*, Auto-generated Nats server ID
+    pub device_id: PubKey, // *INDEXED*
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
     pub avg_network_speed: i64,
     pub avg_latency: i64,
-    pub assigned_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub assigned_hoster: ObjectId,
+    pub assigned_workloads: Vec<ObjectId>,
 }
 
 impl IntoIndexes for Host {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         //  Add Device ID Index
-        let device_id_index_doc = doc! { "device_id": 1 };
-        let device_id_index_opts = Some(
+        let pubkey_index_doc = doc! { "device_id": 1 };
+        let pubkey_index_opts = Some(
             IndexOptions::builder()
                 .name(Some("device_id_index".to_string()))
                 .build(),
         );
-        indices.push((device_id_index_doc, device_id_index_opts));
-
+        indices.push((pubkey_index_doc, pubkey_index_opts));
         Ok(indices)
     }
 }
@@ -197,24 +193,27 @@ pub enum WorkloadState {
     Pending,
     Installed,
     Running,
+    Updating,
+    Updated,
     Removed,
     Uninstalled,
-    Updating,
     Error(String),   // String = error message
     Unknown(String), // String = context message
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
-    pub id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<ObjectId>,
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SystemSpecs {
-    pub capacity: Capacity, // network_speed: i64
-                            // uptime: i64
+    pub capacity: Capacity,
+    pub avg_network_speed: i64, // Mbps
+    pub avg_uptime: f64, //  decimal value between 0-1 representing avg uptime over past month
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -222,14 +221,14 @@ pub struct Workload {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub state: WorkloadState,
-    pub assigned_developer: ObjectId, // *INDEXED*, Developer Mongodb ID
+    pub assigned_developer: ObjectId, // *INDEXED*
     pub version: SemVer,
     pub nix_pkg: String, // (Includes everthing needed to deploy workload - ie: binary & env pkg & deps, etc)
     pub min_hosts: u16,
     pub system_specs: SystemSpecs,
-    pub assigned_hosts: Vec<ObjectId>, // Host Device IDs (eg: assigned nats server id)
-                                        // pub status: WorkloadStatus,
+    pub assigned_hosts: Vec<ObjectId>,
+    // pub state: WorkloadState,
+    pub status: WorkloadStatus,
 }
 
 impl Default for Workload {
@@ -252,7 +251,6 @@ impl Default for Workload {
                 updated_at: Some(DateTime::now()),
                 deleted_at: None,
             },
-            state: WorkloadState::Reported,
             version: semver,
             nix_pkg: String::new(),
             assigned_developer: ObjectId::new(),
@@ -263,8 +261,15 @@ impl Default for Workload {
                     disk: 400,
                     cores: 20,
                 },
+                avg_network_speed: 200,
+                avg_uptime: 0.8,
             },
             assigned_hosts: Vec::new(),
+            status: WorkloadStatus {
+                id: None, // skips serialization when `None`
+                desired: WorkloadState::Unknown("default state".to_string()),
+                actual: WorkloadState::Unknown("default state".to_string()),
+            },
         }
     }
 }
diff --git a/rust/util_libs/src/lib.rs b/rust/util_libs/src/lib.rs
index 861376d..f1b0265 100644
--- a/rust/util_libs/src/lib.rs
+++ b/rust/util_libs/src/lib.rs
@@ -1,5 +1,2 @@
 pub mod db;
-pub mod js_stream_service;
-pub mod nats_js_client;
-pub mod nats_server;
-pub mod nats_types;
+pub mod nats;
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats/jetstream_client.rs
similarity index 55%
rename from rust/util_libs/src/nats_js_client.rs
rename to rust/util_libs/src/nats/jetstream_client.rs
index cbd1fed..f8a2643 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats/jetstream_client.rs
@@ -1,73 +1,24 @@
-use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
-use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
-
+use super::{
+    jetstream_service::JsStreamService,
+    leaf_server::LEAF_SERVER_DEFAULT_LISTEN_PORT,
+    types::{
+        ErrClientDisconnected, EventHandler, EventListener, JsClientBuilder, JsServiceBuilder,
+        PublishInfo,
+    },
+};
 use anyhow::Result;
-use async_nats::{jetstream, Message, ServerInfo};
-use serde::{Deserialize, Serialize};
-use std::error::Error;
-use std::fmt;
-use std::fmt::Debug;
-use std::future::Future;
-use std::pin::Pin;
+use async_nats::{jetstream, ServerInfo};
+use core::option::Option::None;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
-pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
-        + Send
-        + Sync,
->;
-
-#[derive(Clone)]
-pub enum EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
-{
-    Sync(EndpointHandler<T>),
-    Async(AsyncEndpointHandler<T>),
-}
-
-impl<T> std::fmt::Debug for EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let t = match &self {
-            EndpointType::Async(_) => "EndpointType::Async(<function>)",
-            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
-        };
-
-        write!(f, "{}", t)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct SendRequest {
-    pub subject: String,
-    pub msg_id: String,
-    pub data: Vec<u8>,
-}
-
-#[derive(Debug)]
-pub struct ErrClientDisconnected;
-impl fmt::Display for ErrClientDisconnected {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Could not reach nats: connection closed")
-    }
-}
-impl Error for ErrClientDisconnected {}
-
 impl std::fmt::Debug for JsClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("JsClient")
             .field("url", &self.url)
             .field("name", &self.name)
             .field("client", &self.client)
-            .field("js", &self.js)
+            .field("js_context", &self.js_context)
             .field("js_services", &self.js_services)
             .field("service_log_prefix", &self.service_log_prefix)
             .finish()
@@ -80,30 +31,13 @@ pub struct JsClient {
     on_msg_published_event: Option<EventHandler>,
     on_msg_failed_event: Option<EventHandler>,
     client: async_nats::Client, // inner_client
-    pub js: jetstream::Context,
+    pub js_context: jetstream::Context,
     pub js_services: Option<Vec<JsStreamService>>,
     service_log_prefix: String,
 }
 
-#[derive(Deserialize, Default)]
-pub struct NewJsClientParams {
-    pub nats_url: String,
-    pub name: String,
-    pub inbox_prefix: String,
-    #[serde(default)]
-    pub service_params: Vec<JsServiceParamsPartial>,
-    #[serde(skip_deserializing)]
-    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
-    #[serde(default)]
-    pub credentials_path: Option<String>,
-    #[serde(default)]
-    pub ping_interval: Option<Duration>,
-    #[serde(default)]
-    pub request_timeout: Option<Duration>, // Defaults to 5s
-}
-
 impl JsClient {
-    pub async fn new(p: NewJsClientParams) -> Result<Self, async_nats::Error> {
+    pub async fn new(p: JsClientBuilder) -> Result<Self, async_nats::Error> {
         let connect_options = async_nats::ConnectOptions::new()
             // .require_tls(true)
             .name(&p.name)
@@ -123,77 +57,33 @@ impl JsClient {
             None => connect_options.connect(&p.nats_url).await?,
         };
 
-        let jetstream = jetstream::new(client.clone());
-        let mut services = vec![];
-        for params in p.service_params {
-            let service = JsStreamService::new(
-                jetstream.clone(),
-                &params.name,
-                &params.description,
-                &params.version,
-                &params.service_subject,
-            )
-            .await?;
-            services.push(service);
-        }
+        let log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
+        log::info!("{}Connected to NATS server at {}", log_prefix, p.nats_url);
 
-        let js_services = if services.is_empty() {
-            None
-        } else {
-            Some(services)
-        };
-
-        let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
-
-        let mut default_client = JsClient {
+        let mut js_client = JsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
             on_msg_failed_event: None,
+            js_services: None,
+            js_context: jetstream::new(client.clone()),
+            service_log_prefix: log_prefix,
             client,
-            js: jetstream,
-            js_services,
-            service_log_prefix: service_log_prefix.clone(),
         };
 
-        for opt in p.opts {
-            opt(&mut default_client);
+        for listener in p.listeners {
+            listener(&mut js_client);
         }
 
-        log::info!(
-            "{}Connected to NATS server at {}",
-            service_log_prefix,
-            default_client.url
-        );
-        Ok(default_client)
-    }
-
-    pub fn name(&self) -> &str {
-        &self.name
+        Ok(js_client)
     }
 
     pub fn get_server_info(&self) -> ServerInfo {
         self.client.server_info()
     }
 
-    pub async fn monitor(&self) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            Err(Box::new(ErrClientDisconnected))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub async fn close(&self) -> Result<(), async_nats::Error> {
-        self.client.drain().await?;
-        Ok(())
-    }
-
-    pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            return Err(Box::new(ErrClientDisconnected));
-        }
-        let stream = &self.js.get_stream(stream_name).await?;
+    pub async fn get_stream_info(&self, stream_name: &str) -> Result<(), async_nats::Error> {
+        let stream = &self.js_context.get_stream(stream_name).await?;
         let info = stream.get_info().await?;
         log::debug!(
             "{}JetStream info: stream:{}, info:{:?}",
@@ -204,43 +94,80 @@ impl JsClient {
         Ok(())
     }
 
-    pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
-        Ok(())
+    pub async fn check_connection(
+        &self,
+    ) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state = self.client.connection_state();
+        if let async_nats::connection::State::Disconnected = conn_state {
+            Err(Box::new(ErrClientDisconnected))
+        } else {
+            Ok(conn_state)
+        }
     }
 
-    pub async fn publish(&self, payload: &SendRequest) -> Result<(), async_nats::Error> {
+    pub async fn publish(
+        &self,
+        payload: PublishInfo,
+    ) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>>
+    {
+        log::debug!(
+            "{}Called Publish message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+
         let now = Instant::now();
-        let result = self
-            .js
-            .publish(payload.subject.clone(), payload.data.clone().into())
-            .await;
+        let result = match payload.headers {
+            Some(headers) => {
+                self.js_context
+                    .publish_with_headers(
+                        payload.subject.clone(),
+                        headers,
+                        payload.data.clone().into(),
+                    )
+                    .await
+            }
+            None => {
+                self.js_context
+                    .publish(payload.subject.clone(), payload.data.clone().into())
+                    .await
+            }
+        };
 
         let duration = now.elapsed();
         if let Err(err) = result {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(Box::new(err));
+            return Err(err);
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
         Ok(())
     }
 
-    pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
-        let mut current_services = self.js_services.unwrap_or_default();
-        current_services.extend(js_services);
+    pub async fn add_js_service(
+        &mut self,
+        params: JsServiceBuilder,
+    ) -> Result<(), async_nats::Error> {
+        let new_service = JsStreamService::new(
+            self.js_context.to_owned(),
+            &params.name,
+            &params.description,
+            &params.version,
+            &params.service_subject,
+        )
+        .await?;
+
+        let mut current_services = self.js_services.to_owned().unwrap_or_default();
+        current_services.push(new_service);
         self.js_services = Some(current_services);
-        self
+
+        Ok(())
     }
 
     pub async fn get_js_service(&self, js_service_name: String) -> Option<&JsStreamService> {
@@ -251,6 +178,11 @@ impl JsClient {
         }
         None
     }
+
+    pub async fn close(&self) -> Result<(), async_nats::Error> {
+        self.client.drain().await?;
+        Ok(())
+    }
 }
 
 // Client Options:
@@ -268,7 +200,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_published_event = Some(Box::pin(f.clone()));
+        c.on_msg_published_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -277,7 +209,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_failed_event = Some(Box::pin(f.clone()));
+        c.on_msg_failed_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -314,7 +246,7 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     };
 
     let event_listeners = vec![
-        on_msg_published_event(published_msg_handler),
+        on_msg_published_event(published_msg_handler), // Shouldn't this be the 'NATS_LISTEN_PORT'?
         on_msg_failed_event(failure_handler),
     ];
 
@@ -326,8 +258,8 @@ pub fn get_event_listeners() -> Vec<EventListener> {
 mod tests {
     use super::*;
 
-    pub fn get_default_params() -> NewJsClientParams {
-        NewJsClientParams {
+    pub fn get_default_params() -> JsClientBuilder {
+        JsClientBuilder {
             nats_url: "localhost:4222".to_string(),
             name: "test_client".to_string(),
             inbox_prefix: "_UNIQUE_INBOX".to_string(),
@@ -353,7 +285,7 @@ mod tests {
     async fn test_nats_js_client_publish() {
         let params = get_default_params();
         let client = JsClient::new(params).await.unwrap();
-        let payload = SendRequest {
+        let payload = PublishInfo {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
             data: b"Hello, NATS!".to_vec(),
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/nats/jetstream_service.rs
similarity index 78%
rename from rust/util_libs/src/js_stream_service.rs
rename to rust/util_libs/src/nats/jetstream_service.rs
index 0860c3b..e622e3a 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/nats/jetstream_service.rs
@@ -1,108 +1,17 @@
-use super::nats_js_client::EndpointType;
-
+use super::types::{
+    ConsumerBuilder, ConsumerExt, ConsumerExtTrait, EndpointTraits, EndpointType,
+    JsStreamServiceInfo, LogInfo, ResponseSubjectsGenerator,
+};
 use anyhow::{anyhow, Result};
-use std::any::Any;
-// use async_nats::jetstream::message::Message;
-use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
+use async_nats::jetstream::consumer::{self, AckPolicy};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
-use async_trait::async_trait;
 use futures::StreamExt;
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
-
-pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> Option<Vec<String>>;
-}
-
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
-{
-}
-
-#[async_trait]
-pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
-    fn get_name(&self) -> &str;
-    fn get_consumer(&self) -> PullConsumer;
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
-}
-
-impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
-where
-    T: EndpointTraits,
-{
-    type Error = anyhow::Error;
-
-    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
-        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
-            Ok(*endpoint)
-        } else {
-            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
-        }
-    }
-}
-
-#[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    name: String,
-    consumer: PullConsumer,
-    handler: EndpointType<T>,
-    #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>,
-}
-
-#[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    fn get_name(&self) -> &str {
-        &self.name
-    }
-    fn get_consumer(&self) -> PullConsumer {
-        self.consumer.clone()
-    }
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
-        Box::new(self.handler.clone())
-    }
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
-        self.response_subject_fn.clone()
-    }
-}
-
-#[allow(dead_code)]
-#[derive(Clone, Debug)]
-pub struct JsStreamServiceInfo<'a> {
-    pub name: &'a str,
-    pub version: &'a str,
-    pub service_subject: &'a str,
-}
-
-struct LogInfo {
-    prefix: String,
-    service_name: String,
-    service_subject: String,
-    endpoint_name: String,
-    endpoint_subject: String,
-}
-
-#[derive(Clone, Deserialize, Default)]
-pub struct JsServiceParamsPartial {
-    pub name: String,
-    pub description: String,
-    pub version: String,
-    pub service_subject: String,
-}
-
 /// Microservice for Jetstream Streams
 // This setup creates only one subject for the stream (eg: "WORKLOAD.>") and sets up
 // all consumers of the stream to listen to stream subjects beginning with that subject (eg: "WORKLOAD.start")
@@ -196,30 +105,30 @@ impl JsStreamService {
         let handler: EndpointType<T> = EndpointType::try_from(endpoint_trait_obj)?;
 
         Ok(ConsumerExt {
-            name: consumer_ext.get_name().to_string(),
             consumer: consumer_ext.get_consumer(),
             handler,
             response_subject_fn: consumer_ext.get_response(),
         })
     }
 
-    pub async fn add_local_consumer<T>(
+    pub async fn add_consumer<T>(
         &self,
-        consumer_name: &str,
-        endpoint_subject: &str,
-        endpoint_type: EndpointType<T>,
-        response_subject_fn: Option<ResponseSubjectsGenerator>,
+        builder_params: ConsumerBuilder<T>,
     ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
-        let full_subject = format!("{}.{}", self.service_subject, endpoint_subject);
+        // Add the Service Subject prefix
+        let consumer_subject = format!(
+            "{}.{}",
+            self.service_subject, builder_params.endpoint_subject
+        );
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
-            durable_name: Some(consumer_name.to_string()),
+            durable_name: Some(builder_params.name.to_string()),
             ack_policy: AckPolicy::Explicit,
-            filter_subject: full_subject,
+            filter_subject: consumer_subject,
             ..Default::default()
         };
 
@@ -227,28 +136,28 @@ impl JsStreamService {
             .stream
             .write()
             .await
-            .get_or_create_consumer(consumer_name, consumer_config)
+            .get_or_create_consumer(&builder_params.name, consumer_config)
             .await?;
 
         let consumer_with_handler = ConsumerExt {
-            name: consumer_name.to_string(),
             consumer,
-            handler: endpoint_type,
-            response_subject_fn,
+            handler: builder_params.handler,
+            response_subject_fn: builder_params.response_subject_fn,
         };
 
-        self.local_consumers
-            .write()
-            .await
-            .insert(consumer_name.to_string(), Arc::new(consumer_with_handler));
+        self.local_consumers.write().await.insert(
+            builder_params.name.to_string(),
+            Arc::new(consumer_with_handler),
+        );
 
-        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(consumer_name).await?;
-        self.spawn_consumer_handler::<T>(consumer_name).await?;
+        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(&builder_params.name).await?;
+        self.spawn_consumer_handler::<T>(&builder_params.name)
+            .await?;
 
         log::debug!(
             "{}Added the {} local consumer",
             self.service_log_prefix,
-            endpoint_consumer.name,
+            builder_params.name,
         );
 
         Ok(endpoint_consumer)
@@ -279,12 +188,19 @@ impl JsStreamService {
                 .messages()
                 .await?;
 
+            let consumer_info = consumer.info().await?;
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
-                endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
+                endpoint_name: consumer_info
+                    .config
+                    .durable_name
+                    .clone()
+                    .unwrap_or("Consumer Name Not Found".to_string())
+                    .clone(),
+                endpoint_subject: consumer_info.config.filter_subject.clone(),
             };
 
             let service_context = self.js_context.clone();
@@ -338,14 +254,11 @@ impl JsStreamService {
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
-                    let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
-                        Ok(r) => r.into(),
-                        Err(e) => e.to_string().into(),
-                    };
+                    let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 }
-                Err(err) => (err.to_string().into(), None),
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
@@ -498,7 +411,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_js_service_add_local_consumer() {
+    async fn test_js_service_add_consumer() {
         let context = setup_jetstream().await;
         let service = get_default_js_service(context).await;
 
@@ -508,7 +421,7 @@ mod tests {
         let response_subject = Some("response.subject".to_string());
 
         let consumer = service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
@@ -533,7 +446,7 @@ mod tests {
         let response_subject = None;
 
         service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats/leaf_server.rs
similarity index 98%
rename from rust/util_libs/src/nats_server.rs
rename to rust/util_libs/src/nats/leaf_server.rs
index 4e9030b..dfc40f1 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats/leaf_server.rs
@@ -143,7 +143,7 @@ impl LeafServer {
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .spawn()
-            .expect("Failed to start NATS server");
+            .context("Failed to start NATS server")?;
 
         // TODO: wait for a readiness indicator
         std::thread::sleep(std::time::Duration::from_millis(100));
@@ -192,8 +192,8 @@ mod tests {
     const TMP_JS_DIR: &str = "./tmp";
     const TEST_AUTH_DIR: &str = "./tmp/test-auth";
     const OPERATOR_NAME: &str = "test-operator";
-    const USER_ACCOUNT_NAME: &str = "hpos-account";
-    const USER_NAME: &str = "hpos-user";
+    const USER_ACCOUNT_NAME: &str = "host-account";
+    const USER_NAME: &str = "host-user";
     const NEW_LEAF_CONFIG_PATH: &str = "./test_configs/leaf_server.conf";
     // NB: if changed, the resolver file path must also be changed in the `hub-server.conf` iteself as well.
     const RESOLVER_FILE_PATH: &str = "./test_configs/resolver.conf";
@@ -227,7 +227,7 @@ mod tests {
             .output()
             .expect("Failed to create edit operator");
 
-        // Create hpos account (with js enabled)
+        // Create host account (with js enabled)
         Command::new("nsc")
             .args(["add", "account", USER_ACCOUNT_NAME])
             .output()
@@ -245,7 +245,7 @@ mod tests {
             .output()
             .expect("Failed to create edit account");
 
-        // Create user for hpos account
+        // Create user for host account
         Command::new("nsc")
             .args(["add", "user", USER_NAME])
             .args(["--account", USER_ACCOUNT_NAME])
diff --git a/rust/util_libs/src/nats/mod.rs b/rust/util_libs/src/nats/mod.rs
new file mode 100644
index 0000000..a320ee4
--- /dev/null
+++ b/rust/util_libs/src/nats/mod.rs
@@ -0,0 +1,4 @@
+pub mod jetstream_client;
+pub mod jetstream_service;
+pub mod leaf_server;
+pub mod types;
diff --git a/rust/util_libs/src/nats/types.rs b/rust/util_libs/src/nats/types.rs
new file mode 100644
index 0000000..d1b6fe1
--- /dev/null
+++ b/rust/util_libs/src/nats/types.rs
@@ -0,0 +1,200 @@
+use super::jetstream_client::JsClient;
+use anyhow::Result;
+use async_nats::jetstream::consumer::PullConsumer;
+use async_nats::{HeaderMap, Message};
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt;
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
+pub type AsyncEndpointHandler<T> = Arc<
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
+        + Send
+        + Sync,
+>;
+pub type ResponseSubjectsGenerator =
+    Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
+
+pub trait EndpointTraits:
+    Serialize
+    + for<'de> Deserialize<'de>
+    + Send
+    + Sync
+    + Clone
+    + Debug
+    + CreateTag
+    + CreateResponse
+    + 'static
+{
+}
+
+pub trait CreateTag: Send + Sync {
+    fn get_tags(&self) -> HashMap<String, String>;
+}
+
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
+}
+
+#[async_trait]
+pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
+    fn get_consumer(&self) -> PullConsumer;
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
+}
+
+#[async_trait]
+impl<T> ConsumerExtTrait for ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    fn get_consumer(&self) -> PullConsumer {
+        self.consumer.clone()
+    }
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
+        Box::new(self.handler.clone())
+    }
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
+        self.response_subject_fn.clone()
+    }
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    pub consumer: PullConsumer,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerBuilder<T>
+where
+    T: EndpointTraits,
+{
+    pub name: String,
+    pub endpoint_subject: String,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[allow(dead_code)]
+#[derive(Clone, Debug)]
+pub struct JsStreamServiceInfo<'a> {
+    pub name: &'a str,
+    pub version: &'a str,
+    pub service_subject: &'a str,
+}
+
+#[derive(Clone, Debug)]
+pub struct LogInfo {
+    pub prefix: String,
+    pub service_name: String,
+    pub service_subject: String,
+    pub endpoint_name: String,
+    pub endpoint_subject: String,
+}
+
+#[derive(Clone)]
+pub enum EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
+{
+    Sync(EndpointHandler<T>),
+    Async(AsyncEndpointHandler<T>),
+}
+
+impl<T> std::fmt::Debug for EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let t = match &self {
+            EndpointType::Async(_) => "EndpointType::Async(<function>)",
+            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
+        };
+
+        write!(f, "{}", t)
+    }
+}
+impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
+where
+    T: EndpointTraits,
+{
+    type Error = anyhow::Error;
+
+    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
+        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
+            Ok(*endpoint)
+        } else {
+            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
+        }
+    }
+}
+
+#[derive(Deserialize, Default)]
+pub struct JsClientBuilder {
+    pub nats_url: String,
+    pub name: String,
+    pub inbox_prefix: String,
+    #[serde(default)]
+    pub credentials_path: Option<String>,
+    #[serde(default)]
+    pub ping_interval: Option<Duration>,
+    #[serde(default)]
+    pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
+}
+
+#[derive(Clone, Deserialize, Default)]
+pub struct JsServiceBuilder {
+    pub name: String,
+    pub description: String,
+    pub version: String,
+    pub service_subject: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct PublishInfo {
+    pub subject: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+    pub headers: Option<HeaderMap>,
+}
+
+#[derive(Debug)]
+pub struct ErrClientDisconnected;
+impl fmt::Display for ErrClientDisconnected {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Could not reach nats: connection closed")
+    }
+}
+impl Error for ErrClientDisconnected {}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
deleted file mode 100644
index ece916b..0000000
--- a/rust/util_libs/src/nats_types.rs
+++ /dev/null
@@ -1,145 +0,0 @@
-/* --------
-NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
-IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
-TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */
-
-use serde::{Deserialize, Serialize};
-
-/// JWT claims for NATS compatible jwts
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Claims {
-    /// Time when the token was issued in seconds since the unix epoch
-    #[serde(rename = "iat")]
-    pub issued_at: i64,
-
-    /// Public key of the issuer signing nkey
-    #[serde(rename = "iss")]
-    pub issuer: String,
-
-    /// Base32 hash of the claims where this is empty
-    #[serde(rename = "jti")]
-    pub jwt_id: String,
-
-    /// Public key of the account or user the JWT is being issued to
-    pub sub: String,
-
-    /// Friendly name
-    pub name: String,
-
-    /// NATS claims
-    pub nats: NatsClaims,
-
-    /// Time when the token expires (in seconds since the unix epoch)
-    #[serde(rename = "exp", skip_serializing_if = "Option::is_none")]
-    pub expires: Option<i64>,
-}
-
-/// NATS claims describing settings for the user or account
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "lowercase")]
-pub enum NatsClaims {
-    /// Claims for NATS users
-    User {
-        /// Publish and subscribe permissions for the user
-        #[serde(flatten)]
-        permissions: NatsPermissionsMap,
-
-        /// Public key/id of the account that issued the JWT
-        issuer_account: String,
-
-        /// Maximum nuber of subscriptions the user can have
-        subs: i64,
-
-        /// Maximum size of the message data the user can send in bytes
-        data: i64,
-
-        /// Maximum size of the entire message payload the user can send in bytes
-        payload: i64,
-
-        /// If true, the user isn't challenged on connection. Typically used for websocket
-        /// connections as the browser won't have/want to have the user's private key.
-        bearer_token: bool,
-
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-    /// Claims for NATS accounts
-    Account {
-        /// Configuration for the limits for this account
-        limits: NatsAccountLimits,
-
-        /// List of signing keys (public key) this account uses
-        #[serde(skip_serializing_if = "Vec::is_empty")]
-        signing_keys: Vec<String>,
-
-        /// Default publish and subscribe permissions users under this account will have if not
-        /// specified otherwise
-        /// default_permissions: NatsPermissionsMap,
-        ///
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-}
-
-/// List of subjects that are allowed and/or denied
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissions {
-    /// List of subject patterns that are allowed
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub allow: Vec<String>,
-
-    /// List of subject patterns that are denied
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub deny: Vec<String>,
-}
-
-impl NatsPermissions {
-    /// Returns `true` if the allow and deny list are both empty
-    #[must_use]
-    pub fn is_empty(&self) -> bool {
-        self.allow.is_empty() && self.deny.is_empty()
-    }
-}
-
-/// Publish and subcribe permissons
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissionsMap {
-    /// Permissions for which subjects can be published to
-    #[serde(rename = "pub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub publish: NatsPermissions,
-
-    /// Permissions for which subjects can be subscribed to
-    #[serde(rename = "sub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub subscribe: NatsPermissions,
-}
-
-/// Limits on what an account or users in the account can do
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NatsAccountLimits {
-    /// Maximum nuber of subscriptions the account
-    pub subs: i64,
-
-    /// Maximum size of the message data a user can send in bytes
-    pub data: i64,
-
-    /// Maximum size of the entire message payload a user can send in bytes
-    pub payload: i64,
-
-    /// Maxiumum number of imports for the account
-    pub imports: i64,
-
-    /// Maxiumum number of exports for the account
-    pub exports: i64,
-
-    /// If true, exports can contain wildcards
-    pub wildcards: bool,
-
-    /// Maximum number of active connections
-    pub conn: i64,
-
-    /// Maximum number of leaf node connections
-    pub leaf: i64,
-}
diff --git a/rust/util_libs/test_configs/hub_server.conf b/rust/util_libs/test_configs/hub_server.conf
deleted file mode 100644
index 0184098..0000000
--- a/rust/util_libs/test_configs/hub_server.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-operator: "./test-auth/test-operator/test-operator.jwt"
-system_account: SYS
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-include ./resolver.conf
-
-# logging options
-debug:   true
-trace:   true
-logtime: false
diff --git a/rust/util_libs/test_configs/hub_server_pw_auth.conf b/rust/util_libs/test_configs/hub_server_pw_auth.conf
deleted file mode 100644
index 51eeb3f..0000000
--- a/rust/util_libs/test_configs/hub_server_pw_auth.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-authorization {
-    user: "test-user"
-    password: "pw-12345"
-}
-
-# logging options
-debug:   true
-trace:   true
-logtime: false

From 7e07dbd4b289ba2edc884a8591cf52ac164b76b7 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:04:33 -0600
Subject: [PATCH 86/91] update .gitignore

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 1812acf..dbb5350 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,7 +17,7 @@ target/
 rust/*/tmp
 rust/*/jwt
 rust/*/*/test_leaf_server/*
-rust/*/*/test_leaf_server.conf
+rust/*/*/*.conf
 rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf

From 5caac1fbec1c2f2190e62ec353bd994aa95f1bb9 Mon Sep 17 00:00:00 2001
From: Lisa Jetton <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:06:34 -0600
Subject: [PATCH 87/91] refactor-client-dir (#69)

* refactor host agent client structure
---
 .env.example                                  |  13 +-
 Cargo.lock                                    | 229 +++++-
 nix/lib/default.nix                           |   6 +-
 nix/modules/nixos/holo-nats-server.nix        |   2 +-
 rust/clients/host_agent/Cargo.toml            |  15 +-
 .../src/{ => hostd}/gen_leaf_server.rs        |  21 +-
 rust/clients/host_agent/src/hostd/mod.rs      |   2 +
 rust/clients/host_agent/src/hostd/workload.rs | 154 ++++
 rust/clients/host_agent/src/main.rs           |  23 +-
 .../host_agent/src/workload_manager.rs        | 129 ----
 rust/services/workload/Cargo.toml             |   3 +
 rust/services/workload/src/host_api.rs        | 169 +++++
 rust/services/workload/src/lib.rs             | 526 ++------------
 .../services/workload/src/orchestrator_api.rs | 662 ++++++++++++++++++
 rust/services/workload/src/types.rs           |  49 +-
 rust/util_libs/src/db/mongodb.rs              | 162 ++---
 rust/util_libs/src/db/schemas.rs              |  89 +--
 rust/util_libs/src/lib.rs                     |   5 +-
 .../jetstream_client.rs}                      | 250 +++----
 .../jetstream_service.rs}                     | 165 ++---
 .../{nats_server.rs => nats/leaf_server.rs}   |  10 +-
 rust/util_libs/src/nats/mod.rs                |   4 +
 rust/util_libs/src/nats/types.rs              | 200 ++++++
 rust/util_libs/src/nats_types.rs              | 145 ----
 rust/util_libs/test_configs/hub_server.conf   |  22 -
 .../test_configs/hub_server_pw_auth.conf      |  22 -
 26 files changed, 1779 insertions(+), 1298 deletions(-)
 rename rust/clients/host_agent/src/{ => hostd}/gen_leaf_server.rs (91%)
 create mode 100644 rust/clients/host_agent/src/hostd/mod.rs
 create mode 100644 rust/clients/host_agent/src/hostd/workload.rs
 delete mode 100644 rust/clients/host_agent/src/workload_manager.rs
 create mode 100644 rust/services/workload/src/host_api.rs
 create mode 100644 rust/services/workload/src/orchestrator_api.rs
 rename rust/util_libs/src/{nats_js_client.rs => nats/jetstream_client.rs} (55%)
 rename rust/util_libs/src/{js_stream_service.rs => nats/jetstream_service.rs} (78%)
 rename rust/util_libs/src/{nats_server.rs => nats/leaf_server.rs} (98%)
 create mode 100644 rust/util_libs/src/nats/mod.rs
 create mode 100644 rust/util_libs/src/nats/types.rs
 delete mode 100644 rust/util_libs/src/nats_types.rs
 delete mode 100644 rust/util_libs/test_configs/hub_server.conf
 delete mode 100644 rust/util_libs/test_configs/hub_server_pw_auth.conf

diff --git a/.env.example b/.env.example
index bfe8157..9473a3e 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,11 @@
-NSC_PATH = "<path/to/nsc_dir>"
+# ALL
+NSC_PATH="<path/to/nsc_dir>"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LOCAL_CREDS_PATH="<path/to/dir>"
+
+# HOSTING AGENT
 HOST_CREDS_FILE_PATH = "ops/admin.creds"
-MONGO_URI = "mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index aada039..30fba83 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -49,7 +49,7 @@ dependencies = [
  "mime",
  "percent-encoding",
  "pin-project-lite",
- "rand",
+ "rand 0.8.5",
  "sha1",
  "smallvec",
  "tokio",
@@ -220,7 +220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -366,7 +366,7 @@ dependencies = [
  "once_cell",
  "pin-project",
  "portable-atomic",
- "rand",
+ "rand 0.8.5",
  "regex",
  "ring 0.17.8",
  "rustls-native-certs 0.7.3",
@@ -445,6 +445,12 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -584,7 +590,7 @@ dependencies = [
  "indexmap 2.7.0",
  "js-sys",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -861,6 +867,7 @@ dependencies = [
  "fiat-crypto",
  "rustc_version",
  "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -911,9 +918,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.6.0"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
+checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010"
 
 [[package]]
 name = "der"
@@ -1055,6 +1062,7 @@ version = "2.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
 dependencies = [
+ "pkcs8",
  "signature",
 ]
 
@@ -1066,9 +1074,11 @@ checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
 dependencies = [
  "curve25519-dalek",
  "ed25519",
+ "serde",
  "sha2",
  "signature",
  "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -1285,6 +1295,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1292,8 +1313,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -1348,6 +1371,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1376,7 +1405,7 @@ dependencies = [
  "idna 1.0.3",
  "ipnet",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "thiserror 1.0.69",
  "tinyvec",
  "tokio",
@@ -1397,7 +1426,7 @@ dependencies = [
  "lru-cache",
  "once_cell",
  "parking_lot",
- "rand",
+ "rand 0.8.5",
  "resolv-conf",
  "smallvec",
  "thiserror 1.0.69",
@@ -1433,19 +1462,24 @@ dependencies = [
  "bytes",
  "chrono",
  "clap",
+ "data-encoding",
  "dotenv",
+ "ed25519-dalek",
  "env_logger",
  "futures",
  "hpos-hal",
+ "jsonwebtoken",
  "log",
  "machineid-rs",
- "mongodb",
+ "nats-jwt",
  "netdiag",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_json",
+ "sha2",
  "tempfile",
+ "textnonce",
  "thiserror 2.0.9",
  "tokio",
  "url",
@@ -1839,6 +1873,21 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonwebtoken"
+version = "9.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
+dependencies = [
+ "base64 0.22.1",
+ "js-sys",
+ "pem",
+ "ring 0.17.8",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
 [[package]]
 name = "language-tags"
 version = "0.3.2"
@@ -2026,7 +2075,7 @@ checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
  "libc",
  "log",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "windows-sys 0.52.0",
 ]
 
@@ -2056,7 +2105,7 @@ dependencies = [
  "once_cell",
  "pbkdf2",
  "percent-encoding",
- "rand",
+ "rand 0.8.5",
  "rustc_version_runtime",
  "rustls 0.21.12",
  "rustls-pemfile 1.0.4",
@@ -2128,9 +2177,9 @@ dependencies = [
  "data-encoding",
  "ed25519",
  "ed25519-dalek",
- "getrandom",
+ "getrandom 0.2.15",
  "log",
- "rand",
+ "rand 0.8.5",
  "signatory",
 ]
 
@@ -2159,7 +2208,17 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83"
 dependencies = [
- "rand",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
 ]
 
 [[package]]
@@ -2179,6 +2238,15 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -2253,6 +2321,16 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "pem"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae"
+dependencies = [
+ "base64 0.22.1",
+ "serde",
+]
+
 [[package]]
 name = "pem-rfc7468"
 version = "0.7.0"
@@ -2496,6 +2574,19 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -2503,8 +2594,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -2514,7 +2615,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
 ]
 
 [[package]]
@@ -2523,7 +2633,16 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -2623,7 +2742,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "libc",
  "spin 0.9.8",
  "untrusted 0.9.0",
@@ -2719,12 +2838,12 @@ dependencies = [
  "pest",
  "pest_consume",
  "pest_derive",
- "rand",
+ "rand 0.8.5",
  "regex",
  "serde",
  "serde_json",
  "strum 0.23.0",
- "strum_macros",
+ "strum_macros 0.23.1",
  "thiserror 1.0.69",
  "url",
 ]
@@ -3137,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31"
 dependencies = [
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "signature",
  "zeroize",
 ]
@@ -3149,7 +3268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -3158,6 +3277,18 @@ version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
 
+[[package]]
+name = "simple_asn1"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
+dependencies = [
+ "num-bigint",
+ "num-traits",
+ "thiserror 2.0.9",
+ "time",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -3240,6 +3371,12 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
 
+[[package]]
+name = "strum"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
+
 [[package]]
 name = "strum_macros"
 version = "0.23.1"
@@ -3253,6 +3390,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "strum_macros"
+version = "0.25.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.90",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -3327,7 +3477,7 @@ checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
 dependencies = [
  "cfg-if",
  "fastrand",
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "rustix",
  "windows-sys 0.59.0",
@@ -3350,6 +3500,16 @@ dependencies = [
  "touch",
 ]
 
+[[package]]
+name = "textnonce"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f8d70cd784ed1dc33106a18998d77758d281dc40dc3e6d050cf0f5286683"
+dependencies = [
+ "base64 0.12.3",
+ "rand 0.7.3",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -3488,7 +3648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
 dependencies = [
  "pin-project",
- "rand",
+ "rand 0.8.5",
  "tokio",
 ]
 
@@ -3549,7 +3709,7 @@ dependencies = [
  "futures-sink",
  "http 1.2.0",
  "httparse",
- "rand",
+ "rand 0.8.5",
  "ring 0.17.8",
  "rustls-native-certs 0.8.1",
  "rustls-pki-types",
@@ -3823,7 +3983,7 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
  "serde",
 ]
 
@@ -3861,6 +4021,12 @@ dependencies = [
  "try-lock",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -4261,6 +4427,7 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
+ "async-trait",
  "bson",
  "bytes",
  "chrono",
@@ -4270,10 +4437,12 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "semver",
  "serde",
  "serde_json",
+ "strum 0.25.0",
+ "strum_macros 0.25.3",
  "thiserror 2.0.9",
  "tokio",
  "url",
diff --git a/nix/lib/default.nix b/nix/lib/default.nix
index 3c1aa40..f2cd411 100644
--- a/nix/lib/default.nix
+++ b/nix/lib/default.nix
@@ -1,4 +1,8 @@
-{ inputs, flake, ... }:
+{
+  inputs,
+  flake,
+  ...
+}:
 
 {
   mkCraneLib =
diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index 5db4af6..9744ed4 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -125,4 +125,4 @@ in
       }
     );
   };
-}
+}
\ No newline at end of file
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 6b1c4aa..71d144a 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -14,17 +14,22 @@ log = { workspace = true }
 dotenv = { workspace = true }
 clap = { workspace = true }
 thiserror = { workspace = true }
+env_logger = { workspace = true }
 url = { version = "2", features = ["serde"] }
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
-env_logger = { workspace = true }
-mongodb = "3.1"
+ed25519-dalek = { version = "2.1.1" }
+nkeys = "=0.4.4"
+sha2 = "=0.10.8"
+nats-jwt = "0.3.0"
+data-encoding = "2.7.0"
+jsonwebtoken = "9.3.0"
+textnonce = "1.0.0"
 chrono = "0.4.0"
 bytes = "1.8.0"
-nkeys = "=0.4.4"
 rand = "0.8.5"
+tempfile = "3.15.0"
+machineid-rs = "1.2.4"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 hpos-hal = { path = "../../hpos-hal" }
 netdiag = { path = "../../netdiag" }
-tempfile = "3.15.0"
-machineid-rs = "1.2.4"
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
similarity index 91%
rename from rust/clients/host_agent/src/gen_leaf_server.rs
rename to rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 61dbd4d..38f36da 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -2,12 +2,13 @@ use std::{path::PathBuf, time::Duration};
 
 use anyhow::Context;
 use tempfile::tempdir;
-use util_libs::{
-    nats_js_client,
-    nats_server::{
+use util_libs::nats::{
+    jetstream_client,
+    leaf_server::{
         JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
         LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
     },
+    types::JsClientBuilder,
 };
 
 pub async fn run(
@@ -17,7 +18,7 @@ pub async fn run(
     hub_url: String,
     hub_tls_insecure: bool,
     nats_connect_timeout_secs: u64,
-) -> anyhow::Result<nats_js_client::JsClient> {
+) -> anyhow::Result<jetstream_client::JsClient> {
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
         .map(|var| var.parse().expect("can't parse into number"))
@@ -95,23 +96,21 @@ pub async fn run(
     // Spin up Nats Client
     // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
     const HOST_AGENT_CLIENT_NAME: &str = "Host Agent Bare";
 
     let nats_client = tokio::select! {
         client = async {loop {
-                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                let host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
                     nats_url:nats_url.clone(),
                     name:HOST_AGENT_CLIENT_NAME.to_string(),
+                    inbox_prefix: Default::default(),
+                    credentials_path: Default::default(),
                     ping_interval:Some(Duration::from_secs(10)),
                     request_timeout:Some(Duration::from_secs(29)),
-
-                    inbox_prefix: Default::default(),
-                    service_params:Default::default(),
-                    opts: Default::default(),
-                    credentials_path: Default::default()
+                    listeners: Default::default(),
                 })
                 .await
                 .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
new file mode 100644
index 0000000..3e7c23b
--- /dev/null
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -0,0 +1,2 @@
+pub mod gen_leaf_server;
+pub mod workload;
diff --git a/rust/clients/host_agent/src/hostd/workload.rs b/rust/clients/host_agent/src/hostd/workload.rs
new file mode 100644
index 0000000..3f76cda
--- /dev/null
+++ b/rust/clients/host_agent/src/hostd/workload.rs
@@ -0,0 +1,154 @@
+/*
+ This client is associated with the:
+    - HPOS account
+    - host user
+
+This client is responsible for subscribing to workload streams that handle:
+    - installing new workloads onto the hosting device
+    - removing workloads from the hosting device
+    - sending workload status upon request
+    - sending out active periodic workload reports
+*/
+
+use anyhow::{anyhow, Result};
+use async_nats::Message;
+use std::{path::PathBuf, sync::Arc, time::Duration};
+use util_libs::nats::{
+    jetstream_client,
+    types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
+};
+use workload::{
+    host_api::HostWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
+    WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+};
+
+const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
+const HOST_AGENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX";
+
+// TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
+pub async fn run(
+    host_pubkey: &str,
+    host_creds_path: &Option<PathBuf>,
+) -> Result<jetstream_client::JsClient, async_nats::Error> {
+    log::info!("Host Agent Client: Connecting to server...");
+    log::info!("host_creds_path : {:?}", host_creds_path);
+    log::info!("host_pubkey : {}", host_pubkey);
+
+    let pubkey_lowercase = host_pubkey.to_string().to_lowercase();
+
+    // ==================== Setup NATS ====================
+    // Connect to Nats server
+    let nats_url = jetstream_client::get_nats_url();
+    log::info!("nats_url : {}", nats_url);
+
+    let event_listeners = jetstream_client::get_event_listeners();
+
+    // Spin up Nats Client and loaded in the Js Stream Service
+    let mut host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
+        nats_url: nats_url.clone(),
+        name: HOST_AGENT_CLIENT_NAME.to_string(),
+        inbox_prefix: format!("{}.{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
+        credentials_path: host_creds_path
+            .as_ref()
+            .map(|path| path.to_string_lossy().to_string()),
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(29)),
+        listeners: vec![jetstream_client::with_event_listeners(
+            event_listeners.clone(),
+        )],
+    })
+    .await
+    .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
+    // ==================== Setup JS Stream Service ====================
+    // Instantiate the Workload API
+    let workload_api = HostWorkloadApi::default();
+
+    // Register Workload Streams for Host Agent to consume
+    // NB: Subjects are published by orchestrator or nats-db-connector
+    let workload_stream_service = JsServiceBuilder {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+    host_workload_client
+        .add_js_service(workload_stream_service)
+        .await?;
+
+    let workload_service = host_workload_client
+        .get_js_service(WORKLOAD_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate workload service. Unable to spin up Host Agent."
+        ))?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "install_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Install.as_ref()
+            ),
+            handler: EndpointType::Async(
+                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.install_workload(msg).await
+                }),
+            ),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "update_installed_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::UpdateInstalled.as_ref()
+            ),
+            handler: EndpointType::Async(
+                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.update_workload(msg).await
+                }),
+            ),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "uninstall_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Uninstall.as_ref()
+            ),
+            handler: EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.uninstall_workload(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "send_workload_status".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::SendStatus.as_ref()
+            ),
+            handler: EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.send_workload_status(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    Ok(host_workload_client)
+}
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 1495a47..6a67832 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -1,7 +1,7 @@
 /*
 This client is associated with the:
-  - WORKLOAD account
-  - hpos user
+  - HPOS account
+  - host user
 
 This client is responsible for subscribing the host agent to workload stream endpoints:
   - installing new workloads
@@ -10,15 +10,14 @@ This client is responsible for subscribing the host agent to workload stream end
   - sending workload status upon request
 */
 
-mod workload_manager;
+pub mod agent_cli;
+pub mod host_cmds;
+mod hostd;
+pub mod support_cmds;
 use agent_cli::DaemonzeArgs;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
-pub mod agent_cli;
-pub mod gen_leaf_server;
-pub mod host_cmds;
-pub mod support_cmds;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -48,8 +47,8 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
-    // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let bare_client = gen_leaf_server::run(
+    // let host_pubkey = auth::init_agent::run().await?;
+    let bare_client = hostd::gen_leaf_server::run(
         &args.nats_leafnode_server_name,
         &args.nats_leafnode_client_creds_path,
         &args.store_dir,
@@ -61,7 +60,7 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // TODO: would it be a good idea to reuse this client in the workload_manager and elsewhere later on?
     bare_client.close().await?;
 
-    let host_client = workload_manager::run(
+    let host_workload_client = hostd::workload::run(
         "host_id_placeholder>",
         &args.nats_leafnode_client_creds_path,
     )
@@ -70,6 +69,8 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    host_client.close().await?;
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    host_workload_client.close().await?;
+
     Ok(())
 }
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
deleted file mode 100644
index dd741d6..0000000
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- This client is associated with the:
-- WORKLOAD account
-- hpos user
-
-// This client is responsible for:
-  - subscribing to workload streams
-    - installing new workloads
-    - removing workloads
-    - sending workload status upon request
-    - sending active periodic workload reports
-*/
-
-use anyhow::{anyhow, Result};
-use async_nats::Message;
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::{path::PathBuf, sync::Arc, time::Duration};
-use util_libs::{
-    db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType},
-};
-use workload::{
-    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
-};
-
-const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
-const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
-
-// TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
-pub async fn run(
-    host_pubkey: &str,
-    host_creds_path: &Option<PathBuf>,
-) -> Result<nats_js_client::JsClient, async_nats::Error> {
-    log::info!("HPOS Agent Client: Connecting to server...");
-    log::info!("host_creds_path : {:?}", host_creds_path);
-    log::info!("host_pubkey : {}", host_pubkey);
-
-    // ==================== NATS Setup ====================
-    // Connect to Nats server
-    let nats_url = nats_js_client::get_nats_url();
-    log::info!("nats_url : {}", nats_url);
-
-    let event_listeners = nats_js_client::get_event_listeners();
-
-    // Setup JS Stream Service
-    let workload_stream_service_params = JsServiceParamsPartial {
-        name: WORKLOAD_SRV_NAME.to_string(),
-        description: WORKLOAD_SRV_DESC.to_string(),
-        version: WORKLOAD_SRV_VERSION.to_string(),
-        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
-    };
-
-    // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-        nats_url: nats_url.clone(),
-        name: HOST_AGENT_CLIENT_NAME.to_string(),
-        inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
-        service_params: vec![workload_stream_service_params.clone()],
-        credentials_path: host_creds_path
-            .as_ref()
-            .map(|path| path.to_string_lossy().to_string()),
-        opts: vec![nats_js_client::with_event_listeners(
-            event_listeners.clone(),
-        )],
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(29)),
-    })
-    .await
-    .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
-
-    // ==================== DB Setup ====================
-    // Create a new MongoDB Client and connect it to the cluster
-    let mongo_uri = get_mongodb_url();
-    let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-
-    // Generate the Workload API with access to db
-    let workload_api = WorkloadApi::new(&client).await?;
-
-    // ==================== API ENDPOINTS ====================
-    // Register Workload Streams for Host Agent to consume
-    // NB: Subjects are published by orchestrator or nats-db-connector
-    let workload_service = host_workload_client
-        .get_js_service(WORKLOAD_SRV_NAME.to_string())
-        .await
-        .ok_or(anyhow!(
-            "Failed to locate workload service. Unable to spin up Host Agent."
-        ))?;
-
-    workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "start_workload",
-            "start",
-            EndpointType::Async(workload_api.call(
-                |api: WorkloadApi, msg: Arc<Message>| async move { api.start_workload(msg).await },
-            )),
-            None,
-        )
-        .await?;
-
-    workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "send_workload_status",
-            "send_status",
-            EndpointType::Async(
-                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.send_workload_status(msg).await
-                }),
-            ),
-            None,
-        )
-        .await?;
-
-    workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "uninstall_workload",
-            "uninstall",
-            EndpointType::Async(
-                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.uninstall_workload(msg).await
-                }),
-            ),
-            None,
-        )
-        .await?;
-
-    Ok(host_workload_client)
-}
diff --git a/rust/services/workload/Cargo.toml b/rust/services/workload/Cargo.toml
index abc7708..60dfe59 100644
--- a/rust/services/workload/Cargo.toml
+++ b/rust/services/workload/Cargo.toml
@@ -14,6 +14,9 @@ env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
 thiserror = { workspace = true }
+strum = "0.25"
+strum_macros = "0.25"
+async-trait = "0.1.83"
 semver = "1.0.24"
 rand = "0.8.5"
 mongodb = "3.1"
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
new file mode 100644
index 0000000..00354ea
--- /dev/null
+++ b/rust/services/workload/src/host_api.rs
@@ -0,0 +1,169 @@
+/*
+Endpoints & Managed Subjects:
+    - `install_workload`: handles the "WORKLOAD.<host_pukey>.install." subject
+    - `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
+    - `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
+    - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
+*/
+
+use crate::types::WorkloadResult;
+
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
+use anyhow::Result;
+use async_nats::Message;
+use core::option::Option::None;
+use std::{fmt::Debug, sync::Arc};
+use util_libs::{
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::ServiceError,
+};
+
+#[derive(Debug, Clone, Default)]
+pub struct HostWorkloadApi {}
+
+impl WorkloadServiceApi for HostWorkloadApi {}
+
+impl HostWorkloadApi {
+    pub async fn install_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Running,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
+            // nix_uninstall_with(workload_id)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    // For host agent ? or elsewhere ?
+    // TODO: Talk through with Stefan
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
+
+        // Send updated status:
+        // NB: This will send the update to both the requester (if one exists)
+        // and will broadcast the update to for any `response_subject` address registred for the endpoint
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+}
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 75dad8c..e28d4d6 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -1,482 +1,68 @@
 /*
 Service Name: WORKLOAD
 Subject: "WORKLOAD.>"
-Provisioning Account: WORKLOAD
-Users: orchestrator & hpos
-Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
-- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
-- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
-- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+Provisioning Account: ADMIN
+Importing Account: HPOS
+Users: orchestrator & host
 */
 
+pub mod host_api;
+pub mod orchestrator_api;
 pub mod types;
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
+use async_nats::jetstream::ErrorCode;
 use async_nats::Message;
-use bson::oid::ObjectId;
-use bson::{doc, to_document, DateTime};
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use serde::{Deserialize, Serialize};
+use async_trait::async_trait;
+use core::option::Option::None;
+use serde::Deserialize;
 use std::future::Future;
-use std::{fmt::Debug, str::FromStr, sync::Arc};
+use std::{fmt::Debug, sync::Arc};
+use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
-    db::{
-        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
-    },
-    nats_js_client,
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::{AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 
-pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
+pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD_SERVICE";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
-pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
-
-#[derive(Debug, Clone)]
-pub struct WorkloadApi {
-    pub workload_collection: MongoCollection<schemas::Workload>,
-    pub host_collection: MongoCollection<schemas::Host>,
-    pub user_collection: MongoCollection<schemas::User>,
-    pub developer_collection: MongoCollection<schemas::Developer>,
-}
-
-impl WorkloadApi {
-    pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
-                .await?,
-            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
-            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME)
-                .await?,
-        })
-    }
-
-    pub fn call<F, Fut>(&self, handler: F) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
+
+#[async_trait]
+pub trait WorkloadServiceApi
+where
+    Self: std::fmt::Debug + Clone + 'static,
+{
+    fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<WorkloadApiResult>
     where
-        F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+        F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
+        Self: Send + Sync,
     {
         let api = self.to_owned();
         Arc::new(
-            move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+            move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
                 let api_clone = api.clone();
                 Box::pin(handler(api_clone, msg))
             },
         )
     }
 
-    /*******************************  For Orchestrator   *********************************/
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self
-            .process_request(
-                msg,
-                WorkloadState::Reported,
-                |workload: schemas::Workload| async move {
-                    let workload_id = self
-                        .workload_collection
-                        .insert_one_into(workload.clone())
-                        .await?;
-                    log::info!(
-                        "Successfully added workload. MongodDB Workload ID={:?}",
-                        workload_id
-                    );
-                    let updated_workload = schemas::Workload {
-                        _id: Some(ObjectId::from_str(&workload_id)?),
-                        ..workload
-                    };
-                    Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: updated_workload._id.map(|oid| oid.to_hex()),
-                            desired: WorkloadState::Reported,
-                            actual: WorkloadState::Reported,
-                        },
-                        None,
-                    ))
-                },
-                WorkloadState::Error,
-            )
-            .await)
-    }
-
-    pub async fn update_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self
-            .process_request(
-                msg,
-                WorkloadState::Running,
-                |workload: schemas::Workload| async move {
-                    let workload_query = doc! { "_id":  workload._id };
-
-                    // update workload updated_at
-                    let mut workload_doc = workload.clone();
-                    workload_doc.metadata.updated_at = Some(DateTime::now());
-
-                    // convert workload to document and submit to mongodb
-                    let updated_workload = to_document(&workload_doc)?;
-                    self.workload_collection
-                        .update_one_within(
-                            workload_query,
-                            UpdateModifications::Document(doc! { "$set": updated_workload }),
-                        )
-                        .await?;
-
-                    log::info!(
-                        "Successfully updated workload. MongodDB Workload ID={:?}",
-                        workload._id
-                    );
-                    Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: workload._id.map(|oid| oid.to_hex()),
-                            desired: WorkloadState::Reported,
-                            actual: WorkloadState::Reported,
-                        },
-                        None,
-                    ))
-                },
-                WorkloadState::Error,
-            )
-            .await)
-    }
-
-    pub async fn remove_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.remove'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Removed,
-            |workload_id: bson::oid::ObjectId| async move {
-                let workload_query = doc! { "_id":  workload_id };
-                self.workload_collection.update_one_within(
-                    workload_query,
-                    UpdateModifications::Document(doc! {
-                        "$set": {
-                            "metadata.is_deleted": true,
-                            "metadata.deleted_at": DateTime::now()
-                        }
-                    })
-                ).await?;
-                log::info!(
-                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
-                    workload_id
-                );
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
-                        desired: WorkloadState::Removed,
-                        actual: WorkloadState::Removed,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // looks through existing hosts to find possible hosts for a given workload
-    // returns the minimum number of hosts required for workload
-    pub async fn find_hosts_meeting_workload_criteria(
-        &self,
-        workload: Workload,
-    ) -> Result<Vec<Host>, anyhow::Error> {
-        let pipeline = vec![
-            doc! {
-                "$match": {
-                    // verify there are enough system resources
-                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk },
-                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
-
-                    // limit how many workloads a single host can have
-                    "assigned_workloads": { "$lt": 1 }
-                }
-            },
-            doc! {
-                // the maximum number of hosts returned should be the minimum hosts required by workload
-                // sample randomized results and always return back atleast 1 result
-                "$sample": std::cmp::min(workload.min_hosts as i32, 1)
-            },
-            doc! {
-                "$project": {
-                    "_id": 1
-                }
-            }
-        ];
-        let results = self.host_collection.aggregate(pipeline).await?;
-        if results.is_empty() {
-            anyhow::bail!(
-                "Could not find a compatible host for this workload={:#?}",
-                workload._id
-            );
-        }
-        Ok(results)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    // trigger on mongodb [workload] collection (insert)
-    pub async fn handle_db_insertion(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.insert'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Assigned,
-            |workload: schemas::Workload| async move {
-                log::debug!("New workload to assign. Workload={:#?}", workload);
-
-                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
-                let workload_id = if let Some(id) = workload.clone()._id { id } else {
-                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
-                    return Err(anyhow!(err_msg));
-                };
-
-                // 1. Perform sanity check to ensure workload is not already assigned to a host
-                // ...and if so, exit fn
-                // todo: check for to ensure assigned host *still* has enough capacity for updated workload
-                if !workload.assigned_hosts.is_empty() {
-                    log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: Some(workload_id.to_hex()),
-                            desired: WorkloadState::Assigned,
-                            actual: WorkloadState::Assigned,
-                        },
-                        Some(
-                            workload.assigned_hosts
-                            .iter().map(|id| id.to_hex()).collect())
-                        )
-                    );
-                }
-
-                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
-                let eligible_hosts = self.find_hosts_meeting_workload_criteria(workload.clone()).await?;
-                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_hosts);
-
-                let host_ids: Vec<ObjectId> = eligible_hosts.iter().map(|host| host._id.to_owned().unwrap()).collect();
-
-                // 4. Update the Workload Collection with the assigned Host ID
-                let workload_query = doc! { "_id":  workload_id };
-                let updated_workload = &Workload {
-                    assigned_hosts: host_ids.clone(),
-                    ..workload.clone()
-                };
-                let updated_workload_doc = to_document(updated_workload)?;
-                let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
-                    updated_workload_result
-                );
-
-                // 5. Update the Host Collection with the assigned Workload ID
-                let host_query = doc! { "_id":  { "$in": host_ids } };
-                let updated_host_doc =  doc! {
-                    "$push": {
-                        "assigned_workloads": workload_id
-                    }
-                };
-                let updated_host_result = self.host_collection.update_many_within(
-                    host_query, 
-                    UpdateModifications::Document(updated_host_doc)
-                ).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
-                    updated_host_result
-                );
-
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
-                    },
-                    Some(
-                        updated_workload.assigned_hosts.to_owned()
-                        .iter().map(|host| host.to_hex()).collect()
-                    )
-                ))
-        },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    // triggers on mongodb [workload] collection (update)
-    pub async fn handle_db_update(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-
-        let payload_buf = msg.payload.to_vec();
-
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload to update. Workload={:#?}", workload.clone());
-
-        // 1. remove workloads from existing hosts
-        self.host_collection.mongo_error_handler(
-            self.host_collection
-                .collection
-                .update_many(
-                    doc! {},
-                    doc! { "$pull": { "assigned_workloads": workload._id } },
-                )
-                .await,
-        )?;
-        log::info!(
-            "Remove workload from previous hosts. Workload={:#?}",
-            workload._id
-        );
-
-        if !workload.metadata.is_deleted {
-            // 3. add workload to specific hosts
-            self.host_collection.mongo_error_handler(
-                self.host_collection
-                    .collection
-                    .update_one(
-                        doc! { "_id": { "$in": workload.clone().assigned_hosts } },
-                        doc! { "$push": { "assigned_workloads": workload._id } },
-                    )
-                    .await,
-            )?;
-            log::info!("Added workload to new hosts. Workload={:#?}", workload._id);
-        } else {
-            log::info!(
-                "Skipping (reason: deleted) - Added workload to new hosts. Workload={:#?}",
-                workload._id
-            );
-        }
-
-        let success_status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
-            desired: WorkloadState::Updating,
-            actual: WorkloadState::Updating,
-        };
-        log::info!("Workload update successful. Workload={:#?}", workload._id);
-
-        Ok(types::ApiResult(success_status, None))
-    }
-
-    // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status: WorkloadStatus = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload status to update. Status={:?}", workload_status);
-        if workload_status.id.is_none() {
-            return Err(anyhow!("Got a status update for workload without an id!"));
-        }
-        let workload_status_id = workload_status
-            .id
-            .clone()
-            .expect("workload is not provided");
-
-        self.workload_collection
-            .update_one_within(
-                doc! {
-                    "_id": ObjectId::parse_str(workload_status_id)?
-                },
-                UpdateModifications::Document(doc! {
-                    "$set": {
-                        "state": bson::to_bson(&workload_status.actual)?
-                    }
-                }),
-            )
-            .await?;
-
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    pub async fn uninstall_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_id = serde_json::from_slice::<String>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
-        // nix_uninstall_with(workload_id)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: Some(workload_id),
-            desired: WorkloadState::Uninstalled,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    // For host agent ? or elsewhere ?
-    // TODO: Talk through with Stefan
-    pub async fn send_workload_status(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!(
-            "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
-            msg
-        );
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status = serde_json::from_slice::<WorkloadStatus>(&payload_buf)?;
-
-        // Send updated status:
-        // NB: This will send the update to both the requester (if one exists)
-        // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************  Helper Fns  *********************************/
-    // Helper function to initialize mongodb collections
-    async fn init_collection<T>(
-        client: &MongoDBClient,
-        collection_name: &str,
-    ) -> Result<MongoCollection<T>>
+    fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
-        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+        T: for<'de> Deserialize<'de> + Send + Sync,
     {
-        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+        let payload_buf = msg.payload.to_vec();
+        serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
+            let err_msg = format!(
+                "Error: Failed to deserialize payload. Subject='{}' Err={}",
+                msg.subject.clone().into_string(),
+                e
+            );
+            log::error!("{}", err_msg);
+            ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
+        })
     }
 
     // Helper function to streamline the processing of incoming workload messages
@@ -485,33 +71,21 @@ impl WorkloadApi {
         &self,
         msg: Arc<Message>,
         desired_state: WorkloadState,
-        cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> types::ApiResult
+        cb_fn: impl Fn(T) -> Fut + Send + Sync,
+    ) -> Result<WorkloadApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = match serde_json::from_slice(&msg.payload) {
-            Ok(r) => r,
-            Err(e) => {
-                let err_msg = format!("Failed to deserialize payload for Workload Service Endpoint. Subject={} Error={:?}", msg.subject, e);
-                log::error!("{}", err_msg);
-                let status = WorkloadStatus {
-                    id: None,
-                    desired: desired_state,
-                    actual: error_state(err_msg),
-                };
-                return types::ApiResult(status, None);
-            }
-        };
+        let payload: T = Self::convert_msg_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
-        match cb_fn(payload.clone()).await {
+        Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
-                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
+                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject.clone().into_string(), payload, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
                     id: None,
@@ -520,8 +94,14 @@ impl WorkloadApi {
                 };
 
                 // 3. return response for stream
-                types::ApiResult(status, None)
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                }
             }
-        }
+        })
     }
 }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
new file mode 100644
index 0000000..9db39fa
--- /dev/null
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -0,0 +1,662 @@
+/*
+Endpoints & Managed Subjects:
+    - `add_workload`: handles the "WORKLOAD.add" subject
+    - `update_workload`: handles the "WORKLOAD.update" subject
+    - `remove_workload`: handles the "WORKLOAD.remove" subject
+    - `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
+    - `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
+    - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
+*/
+
+use crate::types::WorkloadResult;
+
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
+use anyhow::Result;
+use async_nats::Message;
+use bson::{self, doc, oid::ObjectId, to_document, DateTime};
+use core::option::Option::None;
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::{HashMap, HashSet},
+    fmt::Debug,
+    sync::Arc,
+};
+use util_libs::{
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
+    },
+    nats::types::ServiceError,
+};
+
+#[derive(Debug, Clone)]
+pub struct OrchestratorWorkloadApi {
+    pub workload_collection: MongoCollection<schemas::Workload>,
+    pub host_collection: MongoCollection<schemas::Host>,
+    pub user_collection: MongoCollection<schemas::User>,
+    pub developer_collection: MongoCollection<schemas::Developer>,
+}
+
+impl WorkloadServiceApi for OrchestratorWorkloadApi {}
+
+impl OrchestratorWorkloadApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        Ok(Self {
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
+                .await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME)
+                .await?,
+        })
+    }
+
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.add'");
+        self.process_request(
+            msg,
+            WorkloadState::Reported,
+            WorkloadState::Error,
+            |mut workload: schemas::Workload| async move {
+                let mut status = WorkloadStatus {
+                    id: None,
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Reported,
+                };
+                workload.status = status.clone();
+                workload.metadata.created_at = Some(DateTime::now());
+
+                let workload_id = self.workload_collection.insert_one_into(workload).await?;
+                status.id = Some(workload_id);
+
+                log::info!(
+                    "Successfully added workload. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                })
+            },
+        )
+        .await
+    }
+
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.update'");
+        self.process_request(
+            msg,
+            WorkloadState::Updating,
+            WorkloadState::Error,
+            |mut workload: schemas::Workload| async move {
+                let status = WorkloadStatus {
+                    id: workload._id,
+                    desired: WorkloadState::Updated,
+                    actual: WorkloadState::Updating,
+                };
+
+                workload.status = status.clone();
+                workload.metadata.updated_at = Some(DateTime::now());
+
+                // convert workload to document and submit to mongodb
+                let updated_workload_doc =
+                    to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection
+                    .update_one_within(
+                        doc! { "_id":  workload._id },
+                        UpdateModifications::Document(doc! { "$set": updated_workload_doc }),
+                    )
+                    .await?;
+
+                log::info!(
+                    "Successfully updated workload. MongodDB Workload ID={:?}",
+                    workload._id
+                );
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                })
+            },
+        )
+        .await
+    }
+
+    pub async fn remove_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.remove'");
+        self.process_request(
+            msg,
+            WorkloadState::Removed,
+            WorkloadState::Error,
+            |workload_id: ObjectId| async move {
+                let status = WorkloadStatus {
+                    id: Some(workload_id),
+                    desired: WorkloadState::Uninstalled,
+                    actual: WorkloadState::Removed,
+                };
+
+                let updated_status_doc = bson::to_bson(&status)
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection.update_one_within(
+                    doc! { "_id":  workload_id },
+                    UpdateModifications::Document(doc! {
+                        "$set": {
+                            "metadata.is_deleted": true,
+                            "metadata.deleted_at": DateTime::now(),
+                            "status": updated_status_doc
+                        }
+                    })
+                ).await?;
+                log::info!(
+                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None
+                    },
+                    maybe_response_tags: None
+                })
+            },
+        )
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    pub async fn handle_db_insertion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.insert'");
+        self.process_request(
+            msg,
+            WorkloadState::Assigned,
+            WorkloadState::Error,
+            |workload: schemas::Workload| async move {
+                log::debug!("New workload to assign. Workload={:#?}", workload);
+
+                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
+                let workload_id = if let Some(id) = workload.clone()._id { id } else {
+                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
+                    return Err(ServiceError::Internal(err_msg));
+                };
+
+                let status = WorkloadStatus {
+                    id: Some(workload_id),
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Assigned,
+                };
+
+                // 1. Perform sanity check to ensure workload is not already assigned to a host and if so, exit fn
+                if !workload.assigned_hosts.is_empty() {
+                    log::warn!("Attempted to assign host for new workload, but host already exists.");
+                    return Ok(WorkloadApiResult {
+                        result: WorkloadResult {
+                            status,
+                            workload: None
+                        },
+                        maybe_response_tags: None
+                    });
+                }
+
+                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
+                // & randomly choose host(s)
+                let eligible_host_ids = self.find_hosts_meeting_workload_criteria(workload.clone(), None).await?;
+                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_host_ids);
+
+                // 3. Update the selected host records with the assigned Workload ID
+                let assigned_host_ids = self.assign_workload_to_hosts(workload_id, eligible_host_ids, workload.min_hosts).await.map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 4. Update the Workload Collection with the assigned Host ID
+                let new_status = WorkloadStatus {
+                    id: None, // remove the id to avoid redundant saving of it in the db
+                    ..status.clone()
+                };
+                self.assign_hosts_to_workload(assigned_host_ids.clone(), workload_id, new_status).await.map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 5. Create tag map with host ids to inform nats to publish message to these hosts with workload install status                
+                let mut tag_map: HashMap<String, String> = HashMap::new();
+                for (index, host_pubkey) in assigned_host_ids.iter().cloned().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: Some(workload)
+                    },
+                    maybe_response_tags: Some(tag_map)
+                })
+        })
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    // triggers on mongodb [workload] collection (update)
+    pub async fn handle_db_modification(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.modify'");
+        let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
+
+        // Fail Safe: exit early if the workload provided does not include an `_id` field
+        let workload_id = if let Some(id) = workload.clone()._id {
+            id
+        } else {
+            let err_msg = format!(
+                "No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}",
+                workload
+            );
+            return Err(ServiceError::Internal(err_msg));
+        };
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        let log_msg = format!(
+            "Workload update in DB successful. Fwding update to assigned hosts. workload_id={}",
+            workload_id
+        );
+
+        // Match on state (updating or removed) and handle each case
+        let result = match workload.status.actual {
+            WorkloadState::Updating => {
+                log::trace!("Updated workload to handle. Workload={:#?}", workload);
+                // 1. Fetch current hosts
+                let hosts = self
+                    .fetch_hosts_assigned_to_workload(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 2. Remove workloads from existing hosts
+                self.remove_workload_from_hosts(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 3. Find eligible hosts
+                let eligible_host_ids = self
+                    .find_hosts_meeting_workload_criteria(workload.clone(), Some(hosts))
+                    .await?;
+                log::debug!(
+                    "Eligible hosts for new workload. MongodDB Host IDs={:?}",
+                    eligible_host_ids
+                );
+
+                // 4. Update the selected host records with the assigned Workload ID
+                let assigned_host_ids = self
+                    .assign_workload_to_hosts(workload_id, eligible_host_ids, workload.min_hosts)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 5. Update the Workload Collection with the assigned Host ID
+                // IMP: It is very important that the workload state changes to a state that is not `WorkloadState::Updating`,
+                // IMP: ...otherwise, this change will cause the workload update to loop between the db stream modification reads and this handler
+                let new_status = WorkloadStatus {
+                    id: None,
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Updated,
+                };
+                self.assign_hosts_to_workload(
+                    assigned_host_ids.clone(),
+                    workload_id,
+                    new_status.clone(),
+                )
+                .await
+                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 6. Create tag map with host ids to inform nats to publish message to these hosts with workload install status
+                for (index, host_pubkey) in assigned_host_ids.iter().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                log::info!("Added workload to new hosts. Workload={:#?}", workload_id);
+
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            ..new_status
+                        },
+                        workload: Some(workload),
+                    },
+                    maybe_response_tags: Some(tag_map),
+                }
+            }
+            WorkloadState::Removed => {
+                log::trace!("Removed workload to handle. Workload={:#?}", workload);
+                // 1. Fetch current hosts with `workload_id`` to know which
+                // hosts to send uninstall workload request to...
+                let hosts = self
+                    .fetch_hosts_assigned_to_workload(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 2. Remove workloads from existing hosts
+                self.remove_workload_from_hosts(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 3. Create tag map with host ids to inform nats to publish message to these hosts with workload install status
+                let host_ids = hosts
+                    .iter()
+                    .map(|h| {
+                        h._id
+                            .ok_or_else(|| ServiceError::Internal("Error".to_string()))
+                    })
+                    .collect::<Result<Vec<ObjectId>, ServiceError>>()?;
+                for (index, host_pubkey) in host_ids.iter().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                log::info!("{} Hosts={:?}", log_msg, hosts);
+
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Uninstalled,
+                            actual: WorkloadState::Removed,
+                        },
+                        workload: Some(workload),
+                    },
+                    maybe_response_tags: Some(tag_map),
+                }
+            }
+            _ => {
+                // Catches all other cases wherein a record in the workload collection was modified (not created),
+                // with a state other than "Updating" or "Removed".
+                // In this case, we don't want to do take any new action, so we return a default status without any updates or frowarding tags.
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: workload.status.desired,
+                            actual: workload.status.actual,
+                        },
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                }
+            }
+        };
+
+        Ok(result)
+    }
+
+    // NB: Published by the Hosting Agent whenever the status of a workload changes
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
+
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
+        log::trace!("Workload status to update. Status={:?}", workload_status);
+
+        let workload_status_id = workload_status
+            .id
+            .ok_or_else(|| ServiceError::Internal("Failed to read ._id from record".to_string()))?;
+
+        self.workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_status_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": {
+                        "status": bson::to_bson(&workload_status)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                    }
+                }),
+            )
+            .await?;
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    // Verifies that a host meets the workload criteria
+    fn verify_host_meets_workload_criteria(
+        &self,
+        assigned_host: &Host,
+        workload: &Workload,
+    ) -> bool {
+        if assigned_host.remaining_capacity.disk < workload.system_specs.capacity.disk {
+            return false;
+        }
+        if assigned_host.remaining_capacity.memory < workload.system_specs.capacity.memory {
+            return false;
+        }
+        if assigned_host.remaining_capacity.cores < workload.system_specs.capacity.cores {
+            return false;
+        }
+
+        true
+    }
+
+    async fn fetch_hosts_assigned_to_workload(&self, workload_id: ObjectId) -> Result<Vec<Host>> {
+        Ok(self
+            .host_collection
+            .get_many_from(doc! { "assigned_workloads": workload_id })
+            .await?)
+    }
+
+    async fn remove_workload_from_hosts(&self, workload_id: ObjectId) -> Result<()> {
+        self.host_collection
+            .inner
+            .update_many(
+                doc! {},
+                doc! { "$pull": { "assigned_workloads": workload_id } },
+            )
+            .await
+            .map_err(ServiceError::Database)?;
+        log::info!(
+            "Removed workload from previous hosts. Workload={:#?}",
+            workload_id
+        );
+        Ok(())
+    }
+
+    // Looks through existing hosts to find possible hosts for a given workload
+    // returns the minimum number of hosts required for workload
+    async fn find_hosts_meeting_workload_criteria(
+        &self,
+        workload: Workload,
+        maybe_existing_hosts: Option<Vec<Host>>,
+    ) -> Result<Vec<ObjectId>, ServiceError> {
+        let mut needed_host_count = workload.min_hosts;
+        let mut still_eligible_host_ids: Vec<ObjectId> = vec![];
+
+        if let Some(hosts) = maybe_existing_hosts {
+            still_eligible_host_ids = hosts.into_iter()
+                .filter_map(|h| {
+                    if self.verify_host_meets_workload_criteria(&h, &workload) {
+                        h._id.ok_or_else(|| {
+                            ServiceError::Internal(format!(
+                                "No `_id` found for workload. Unable to proceed verifying host eligibility. Workload={:?}",
+                                workload
+                            ))
+                        }).ok()
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+            needed_host_count -= still_eligible_host_ids.len() as u16;
+        }
+
+        let pipeline = vec![
+            doc! {
+                "$match": {
+                    // verify there are enough system resources
+                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk },
+                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
+                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
+
+                    // limit how many workloads a single host can have
+                    "assigned_workloads": { "$lt": 1 }
+                }
+            },
+            doc! {
+                // the maximum number of hosts returned should be the minimum hosts required by workload
+                // sample randomized results and always return back at least 1 result
+                "$sample": std::cmp::min( needed_host_count as i32, 1),
+
+                // only return the `host._id` feilds
+                "$project": { "_id": 1 }
+            },
+        ];
+        let host_ids = self.host_collection.aggregate::<ObjectId>(pipeline).await?;
+        if host_ids.is_empty() {
+            let err_msg = format!(
+                "Failed to locate a compatible host for workload. Workload_Id={:?}",
+                workload._id
+            );
+            return Err(ServiceError::Internal(err_msg));
+        } else if workload.min_hosts > host_ids.len() as u16 {
+            log::warn!(
+                "Failed to locate the the min required number of hosts for workload. Workload_Id={:?}",
+                workload._id
+            );
+        }
+
+        let mut eligible_host_ids = host_ids;
+        eligible_host_ids.extend(still_eligible_host_ids);
+
+        Ok(eligible_host_ids)
+    }
+
+    async fn assign_workload_to_hosts(
+        &self,
+        workload_id: ObjectId,
+        eligible_host_ids: Vec<ObjectId>,
+        needed_host_count: u16,
+    ) -> Result<Vec<ObjectId>> {
+        // NB: This will attempt to assign the hosts up to 5 times.. then exit loop with warning message
+        let assigned_host_ids: Vec<ObjectId>;
+        let mut unassigned_host_ids: Vec<ObjectId> = eligible_host_ids.clone();
+        let mut exit_flag = 0;
+        loop {
+            let updated_host_result = self
+                .host_collection
+                .update_many_within(
+                    doc! {
+                        "_id": { "$in": unassigned_host_ids.clone() },
+                        // Currently we only allow a single workload per host
+                        "assigned_workloads": { "$size": 0 }
+                    },
+                    UpdateModifications::Document(doc! {
+                        "$push": {
+                            "assigned_workloads": workload_id
+                        }
+                    }),
+                )
+                .await?;
+
+            if updated_host_result.matched_count == unassigned_host_ids.len() as u64 {
+                log::debug!(
+                    "Successfully updated Host records with the new workload id {}. Host_IDs={:?} Update_Result={:?}",
+                    workload_id,
+                    eligible_host_ids,
+                    updated_host_result
+                );
+                assigned_host_ids = eligible_host_ids;
+                break;
+            } else if exit_flag == 5 {
+                let unassigned_host_hashset: HashSet<ObjectId> =
+                    unassigned_host_ids.into_iter().collect();
+                assigned_host_ids = eligible_host_ids
+                    .into_iter()
+                    .filter(|id| !unassigned_host_hashset.contains(id))
+                    .collect();
+                log::warn!("Exiting loop after 5 attempts to assign the workload to the min number of hosts.
+                    Only able to assign {} hosts. Workload_ID={}, Assigned_Host_IDs={:?}",
+                    needed_host_count,
+                    workload_id,
+                    assigned_host_ids
+                );
+                break;
+            }
+
+            log::warn!("Failed to update all selected host records with workload_id.");
+            log::debug!("Fetching paired host records to see which one(s) still remain unassigned to workload...");
+            let unassigned_hosts = self
+                .host_collection
+                .get_many_from(doc! {
+                    "_id": { "$in": eligible_host_ids.clone() },
+                    "assigned_workloads": { "$size": 0 }
+                })
+                .await?;
+
+            unassigned_host_ids = unassigned_hosts
+                .into_iter()
+                .map(|h| h._id.unwrap_or_default())
+                .collect();
+            exit_flag += 1;
+        }
+
+        Ok(assigned_host_ids)
+    }
+
+    async fn assign_hosts_to_workload(
+        &self,
+        assigned_host_ids: Vec<ObjectId>,
+        workload_id: ObjectId,
+        new_status: WorkloadStatus,
+    ) -> Result<()> {
+        let updated_workload_result = self
+            .workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": [{
+                        "status": bson::to_bson(&new_status)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                        }, {
+                        "assigned_hosts": assigned_host_ids
+                    }]
+                }),
+            )
+            .await;
+
+        log::trace!(
+            "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
+            updated_workload_result
+        );
+
+        Ok(())
+    }
+
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+}
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 912ffb6..76e0f82 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,18 +1,49 @@
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use strum_macros::AsRefStr;
 use util_libs::{
-    db::schemas::WorkloadStatus,
-    js_stream_service::{CreateTag, EndpointTraits},
+    db::schemas::{self, WorkloadStatus},
+    nats::types::{CreateResponse, CreateTag, EndpointTraits},
 };
 
-pub use String as WorkloadId;
+#[derive(Serialize, Deserialize, Clone, Debug, AsRefStr)]
+#[serde(rename_all = "snake_case")]
+pub enum WorkloadServiceSubjects {
+    Add,
+    Update,
+    Remove,
+    Insert, // db change stream trigger
+    Modify, // db change stream trigger
+    HandleStatusUpdate,
+    SendStatus,
+    Install,
+    Uninstall,
+    UpdateInstalled,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult(pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct WorkloadResult {
+    pub status: WorkloadStatus,
+    pub workload: Option<schemas::Workload>,
+}
 
-impl CreateTag for ApiResult {
-    fn get_tags(&self) -> Option<Vec<String>> {
-        self.1.clone()
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadApiResult {
+    pub result: WorkloadResult,
+    pub maybe_response_tags: Option<HashMap<String, String>>,
+}
+impl EndpointTraits for WorkloadApiResult {}
+impl CreateTag for WorkloadApiResult {
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.maybe_response_tags.clone().unwrap_or_default()
+    }
+}
+impl CreateResponse for WorkloadApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        let r = self.result.clone();
+        match serde_json::to_vec(&r) {
+            Ok(r) => r.into(),
+            Err(e) => e.to_string().into(),
+        }
     }
 }
-
-impl EndpointTraits for ApiResult {}
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index b2974bb..332c741 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,47 +1,39 @@
-use anyhow::{anyhow, Context, Result};
+use crate::nats::types::ServiceError;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
-use bson::{self, doc, Document};
+use bson::oid::ObjectId;
+use bson::{self, Document};
 use futures::stream::TryStreamExt;
 use mongodb::options::UpdateModifications;
-use mongodb::results::{DeleteResult, UpdateResult};
+use mongodb::results::UpdateResult;
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
 
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Internal Error: {0}")]
-    Internal(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-}
-
 #[async_trait]
 pub trait MongoDbAPI<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync,
 {
-    fn mongo_error_handler<ReturnType>(
+    type Error;
+
+    async fn aggregate<R: for<'de> Deserialize<'de>>(
         &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType>;
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>>;
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>>;
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
-    async fn insert_one_into(&self, item: T) -> Result<String>;
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(
+        pipeline: Vec<Document>,
+    ) -> Result<Vec<R>, Self::Error>;
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error>;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error>;
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error>;
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn update_many_within(
+    ) -> Result<UpdateResult, Self::Error>;
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
-    async fn delete_all_from(&self) -> Result<DeleteResult>;
+    ) -> Result<UpdateResult, Self::Error>;
 }
 
 pub trait IntoIndexes {
@@ -53,7 +45,7 @@ pub struct MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
 {
-    pub collection: Collection<T>,
+    pub inner: Collection<T>,
     indices: Vec<IndexModel>,
 }
 
@@ -72,7 +64,7 @@ where
         let indices = vec![];
 
         Ok(MongoCollection {
-            collection,
+            inner: collection,
             indices,
         })
     }
@@ -94,7 +86,7 @@ where
         self.indices = indices.clone();
 
         // Apply the indices to the mongodb collection schema
-        self.collection.create_indexes(indices).await?;
+        self.inner.create_indexes(indices).await?;
         Ok(self)
     }
 }
@@ -104,109 +96,86 @@ impl<T> MongoDbAPI<T> for MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes + Debug,
 {
-    fn mongo_error_handler<ReturnType>(
-        &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType> {
-        let rtn = result.map_err(ServiceError::Database)?;
-        Ok(rtn)
-    }
+    type Error = ServiceError;
 
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>> {
-        log::info!("aggregate pipeline {:?}", pipeline);
-        let cursor = self.collection.aggregate(pipeline).await?;
+    async fn aggregate<R>(&self, pipeline: Vec<Document>) -> Result<Vec<R>, Self::Error>
+    where
+        R: for<'de> Deserialize<'de>,
+    {
+        log::trace!("Aggregate pipeline {:?}", pipeline);
+        let cursor = self.inner.aggregate(pipeline).await?;
 
         let results_doc: Vec<bson::Document> =
             cursor.try_collect().await.map_err(ServiceError::Database)?;
 
-        let results: Vec<T> = results_doc
+        let results: Vec<R> = results_doc
             .into_iter()
             .map(|doc| {
-                bson::from_document::<T>(doc).with_context(|| "failed to deserialize document")
+                bson::from_document::<R>(doc).with_context(|| "Failed to deserialize document")
             })
-            .collect::<Result<Vec<T>>>()?;
+            .collect::<Result<Vec<R>>>()
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         Ok(results)
     }
 
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>> {
-        log::info!("get_one_from filter {:?}", filter);
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error> {
+        log::trace!("Get_one_from filter {:?}", filter);
 
         let item = self
-            .collection
+            .inner
             .find_one(filter)
             .await
             .map_err(ServiceError::Database)?;
 
-        log::info!("item {:?}", item);
+        log::debug!("get_one_from item {:?}", item);
         Ok(item)
     }
 
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>> {
-        let cursor = self.collection.find(filter).await?;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error> {
+        let cursor = self.inner.find(filter).await?;
         let results: Vec<T> = cursor.try_collect().await.map_err(ServiceError::Database)?;
         Ok(results)
     }
 
-    async fn insert_one_into(&self, item: T) -> Result<String> {
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error> {
         let result = self
-            .collection
+            .inner
             .insert_one(item)
             .await
             .map_err(ServiceError::Database)?;
 
-        Ok(result.inserted_id.to_string())
-    }
+        let mongo_id = result
+            .inserted_id
+            .as_object_id()
+            .ok_or(ServiceError::Internal(format!(
+                "Failed to read the insert id after inserting item. insert_result={:?}.",
+                result
+            )))?;
 
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>> {
-        let result = self
-            .collection
-            .insert_many(items)
-            .await
-            .map_err(ServiceError::Database)?;
-
-        let ids = result
-            .inserted_ids
-            .values()
-            .map(|id| id.to_string())
-            .collect();
-        Ok(ids)
+        Ok(mongo_id)
     }
 
-    async fn update_one_within(
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_one(query, updated_doc)
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_many(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn update_many_within(
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_many(query, updated_doc)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult> {
-        self.collection
-            .delete_one(query)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_all_from(&self) -> Result<DeleteResult> {
-        self.collection
-            .delete_many(doc! {})
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_one(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 }
 
@@ -328,7 +297,7 @@ mod tests {
                     updated_at: Some(DateTime::now()),
                     deleted_at: None,
                 },
-                device_id: "Vf3IceiD".to_string(),
+                device_id: "placeholder_pubkey_host".to_string(),
                 ip_address: "127.0.0.1".to_string(),
                 remaining_capacity: Capacity {
                     memory: 16,
@@ -359,9 +328,9 @@ mod tests {
         let host_1 = get_mock_host();
         let host_2 = get_mock_host();
         let host_3 = get_mock_host();
-        host_api
-            .insert_many_into(vec![host_1.clone(), host_2.clone(), host_3.clone()])
-            .await?;
+        host_api.insert_one_into(host_1.clone()).await?;
+        host_api.insert_one_into(host_2.clone()).await?;
+        host_api.insert_one_into(host_3.clone()).await?;
 
         // get many docs
         let ids = vec![
@@ -383,13 +352,8 @@ mod tests {
         assert!(updated_ids.contains(&ids[1]));
         assert!(updated_ids.contains(&ids[2]));
 
-        // delete all documents
-        let DeleteResult { deleted_count, .. } = host_api.delete_all_from().await?;
-        assert_eq!(deleted_count, 4);
-        let fetched_host = host_api.get_one_from(filter_one).await?;
-        let fetched_hosts = host_api.get_many_from(filter_many).await?;
-        assert!(fetched_host.is_none());
-        assert!(fetched_hosts.is_empty());
+        // Delete collection and all documents therein.
+        let _ = host_api.inner.drop();
 
         Ok(())
     }
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 5a7945b..ed8cd92 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -14,18 +14,18 @@ pub const HOST_COLLECTION_NAME: &str = "host";
 pub const WORKLOAD_COLLECTION_NAME: &str = "workload";
 
 // Provide type Alias for HosterPubKey
-pub use String as HosterPubKey;
-
-// Provide type Alias for DeveloperPubkey
-pub use String as DeveloperPubkey;
-
-// Provide type Alias for DeveloperJWT
-pub use String as DeveloperJWT;
+pub use String as PubKey;
 
 // Provide type Alias for SemVer (semantic versioning)
 pub use String as SemVer;
 
 // ==================== User Schema ====================
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct RoleInfo {
+    pub collection_id: ObjectId, // Hoster/Developer colleciton Mongodb ID ref
+    pub pubkey: PubKey,          //  Hoster/Developer Pubkey *INDEXED*
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum UserPermission {
     Admin,
@@ -46,24 +46,24 @@ pub struct User {
     pub metadata: Metadata,
     pub jurisdiction: String,
     pub permissions: Vec<UserPermission>,
-    pub user_info_id: Option<ObjectId>,
-    pub developer: Option<ObjectId>,
-    pub hoster: Option<ObjectId>,
+    pub user_info_id: Option<ObjectId>, // *INDEXED*
+    pub developer: Option<RoleInfo>,    // *INDEXED*
+    pub hoster: Option<RoleInfo>,       // *INDEXED*
 }
 
-// No Additional Indexing for Developer
+// Indexing for User
 impl IntoIndexes for User {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
 
-        // add user_info index
-        let user_info_index_doc = doc! { "user_info": 1 };
-        let user_info_index_opts = Some(
+        // add user_info_id index
+        let user_info_id_index_doc = doc! { "user_info_id": 1 };
+        let user_info_id_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("user_info_index".to_string()))
+                .name(Some("user_info_id_index".to_string()))
                 .build(),
         );
-        indices.push((user_info_index_doc, user_info_index_opts));
+        indices.push((user_info_id_index_doc, user_info_id_index_opts));
 
         // add developer index
         let developer_index_doc = doc! { "developer": 1 };
@@ -75,10 +75,10 @@ impl IntoIndexes for User {
         indices.push((developer_index_doc, developer_index_opts));
 
         // add host index
-        let host_index_doc = doc! { "host": 1 };
+        let host_index_doc = doc! { "hoster": 1 };
         let host_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("host_index".to_string()))
+                .name(Some("hoster_index".to_string()))
                 .build(),
         );
         indices.push((host_index_doc, host_index_opts));
@@ -93,7 +93,7 @@ pub struct UserInfo {
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
     pub user_id: ObjectId,
-    pub email: String,
+    pub email: String, // *INDEXED*
     pub given_names: String,
     pub family_name: String,
 }
@@ -101,7 +101,6 @@ pub struct UserInfo {
 impl IntoIndexes for UserInfo {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         // add email index
         let email_index_doc = doc! { "email": 1 };
         let email_index_opts = Some(
@@ -110,7 +109,6 @@ impl IntoIndexes for UserInfo {
                 .build(),
         );
         indices.push((email_index_doc, email_index_opts));
-
         Ok(indices)
     }
 }
@@ -121,8 +119,8 @@ pub struct Developer {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user._id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub active_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub user_id: ObjectId,
+    pub active_workloads: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Developer
@@ -138,8 +136,8 @@ pub struct Hoster {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user.id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub assigned_hosts: Vec<ObjectId>, // MongoDB ID refs to `host._id`
+    pub user_id: ObjectId,
+    pub assigned_hosts: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Hoster
@@ -162,29 +160,27 @@ pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub assigned_hoster: ObjectId,
-    pub device_id: String, // *INDEXED*, Auto-generated Nats server ID
+    pub device_id: PubKey, // *INDEXED*
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
     pub avg_network_speed: i64,
     pub avg_latency: i64,
-    pub assigned_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub assigned_hoster: ObjectId,
+    pub assigned_workloads: Vec<ObjectId>,
 }
 
 impl IntoIndexes for Host {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         //  Add Device ID Index
-        let device_id_index_doc = doc! { "device_id": 1 };
-        let device_id_index_opts = Some(
+        let pubkey_index_doc = doc! { "device_id": 1 };
+        let pubkey_index_opts = Some(
             IndexOptions::builder()
                 .name(Some("device_id_index".to_string()))
                 .build(),
         );
-        indices.push((device_id_index_doc, device_id_index_opts));
-
+        indices.push((pubkey_index_doc, pubkey_index_opts));
         Ok(indices)
     }
 }
@@ -197,24 +193,27 @@ pub enum WorkloadState {
     Pending,
     Installed,
     Running,
+    Updating,
+    Updated,
     Removed,
     Uninstalled,
-    Updating,
     Error(String),   // String = error message
     Unknown(String), // String = context message
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
-    pub id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<ObjectId>,
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SystemSpecs {
-    pub capacity: Capacity, // network_speed: i64
-                            // uptime: i64
+    pub capacity: Capacity,
+    pub avg_network_speed: i64, // Mbps
+    pub avg_uptime: f64, //  decimal value between 0-1 representing avg uptime over past month
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -222,14 +221,14 @@ pub struct Workload {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub state: WorkloadState,
-    pub assigned_developer: ObjectId, // *INDEXED*, Developer Mongodb ID
+    pub assigned_developer: ObjectId, // *INDEXED*
     pub version: SemVer,
     pub nix_pkg: String, // (Includes everthing needed to deploy workload - ie: binary & env pkg & deps, etc)
     pub min_hosts: u16,
     pub system_specs: SystemSpecs,
-    pub assigned_hosts: Vec<ObjectId>, // Host Device IDs (eg: assigned nats server id)
-                                        // pub status: WorkloadStatus,
+    pub assigned_hosts: Vec<ObjectId>,
+    // pub state: WorkloadState,
+    pub status: WorkloadStatus,
 }
 
 impl Default for Workload {
@@ -252,7 +251,6 @@ impl Default for Workload {
                 updated_at: Some(DateTime::now()),
                 deleted_at: None,
             },
-            state: WorkloadState::Reported,
             version: semver,
             nix_pkg: String::new(),
             assigned_developer: ObjectId::new(),
@@ -263,8 +261,15 @@ impl Default for Workload {
                     disk: 400,
                     cores: 20,
                 },
+                avg_network_speed: 200,
+                avg_uptime: 0.8,
             },
             assigned_hosts: Vec::new(),
+            status: WorkloadStatus {
+                id: None, // skips serialization when `None`
+                desired: WorkloadState::Unknown("default state".to_string()),
+                actual: WorkloadState::Unknown("default state".to_string()),
+            },
         }
     }
 }
diff --git a/rust/util_libs/src/lib.rs b/rust/util_libs/src/lib.rs
index 861376d..f1b0265 100644
--- a/rust/util_libs/src/lib.rs
+++ b/rust/util_libs/src/lib.rs
@@ -1,5 +1,2 @@
 pub mod db;
-pub mod js_stream_service;
-pub mod nats_js_client;
-pub mod nats_server;
-pub mod nats_types;
+pub mod nats;
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats/jetstream_client.rs
similarity index 55%
rename from rust/util_libs/src/nats_js_client.rs
rename to rust/util_libs/src/nats/jetstream_client.rs
index cbd1fed..f8a2643 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats/jetstream_client.rs
@@ -1,73 +1,24 @@
-use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
-use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
-
+use super::{
+    jetstream_service::JsStreamService,
+    leaf_server::LEAF_SERVER_DEFAULT_LISTEN_PORT,
+    types::{
+        ErrClientDisconnected, EventHandler, EventListener, JsClientBuilder, JsServiceBuilder,
+        PublishInfo,
+    },
+};
 use anyhow::Result;
-use async_nats::{jetstream, Message, ServerInfo};
-use serde::{Deserialize, Serialize};
-use std::error::Error;
-use std::fmt;
-use std::fmt::Debug;
-use std::future::Future;
-use std::pin::Pin;
+use async_nats::{jetstream, ServerInfo};
+use core::option::Option::None;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
-pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
-        + Send
-        + Sync,
->;
-
-#[derive(Clone)]
-pub enum EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
-{
-    Sync(EndpointHandler<T>),
-    Async(AsyncEndpointHandler<T>),
-}
-
-impl<T> std::fmt::Debug for EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let t = match &self {
-            EndpointType::Async(_) => "EndpointType::Async(<function>)",
-            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
-        };
-
-        write!(f, "{}", t)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct SendRequest {
-    pub subject: String,
-    pub msg_id: String,
-    pub data: Vec<u8>,
-}
-
-#[derive(Debug)]
-pub struct ErrClientDisconnected;
-impl fmt::Display for ErrClientDisconnected {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Could not reach nats: connection closed")
-    }
-}
-impl Error for ErrClientDisconnected {}
-
 impl std::fmt::Debug for JsClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("JsClient")
             .field("url", &self.url)
             .field("name", &self.name)
             .field("client", &self.client)
-            .field("js", &self.js)
+            .field("js_context", &self.js_context)
             .field("js_services", &self.js_services)
             .field("service_log_prefix", &self.service_log_prefix)
             .finish()
@@ -80,30 +31,13 @@ pub struct JsClient {
     on_msg_published_event: Option<EventHandler>,
     on_msg_failed_event: Option<EventHandler>,
     client: async_nats::Client, // inner_client
-    pub js: jetstream::Context,
+    pub js_context: jetstream::Context,
     pub js_services: Option<Vec<JsStreamService>>,
     service_log_prefix: String,
 }
 
-#[derive(Deserialize, Default)]
-pub struct NewJsClientParams {
-    pub nats_url: String,
-    pub name: String,
-    pub inbox_prefix: String,
-    #[serde(default)]
-    pub service_params: Vec<JsServiceParamsPartial>,
-    #[serde(skip_deserializing)]
-    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
-    #[serde(default)]
-    pub credentials_path: Option<String>,
-    #[serde(default)]
-    pub ping_interval: Option<Duration>,
-    #[serde(default)]
-    pub request_timeout: Option<Duration>, // Defaults to 5s
-}
-
 impl JsClient {
-    pub async fn new(p: NewJsClientParams) -> Result<Self, async_nats::Error> {
+    pub async fn new(p: JsClientBuilder) -> Result<Self, async_nats::Error> {
         let connect_options = async_nats::ConnectOptions::new()
             // .require_tls(true)
             .name(&p.name)
@@ -123,77 +57,33 @@ impl JsClient {
             None => connect_options.connect(&p.nats_url).await?,
         };
 
-        let jetstream = jetstream::new(client.clone());
-        let mut services = vec![];
-        for params in p.service_params {
-            let service = JsStreamService::new(
-                jetstream.clone(),
-                &params.name,
-                &params.description,
-                &params.version,
-                &params.service_subject,
-            )
-            .await?;
-            services.push(service);
-        }
+        let log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
+        log::info!("{}Connected to NATS server at {}", log_prefix, p.nats_url);
 
-        let js_services = if services.is_empty() {
-            None
-        } else {
-            Some(services)
-        };
-
-        let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
-
-        let mut default_client = JsClient {
+        let mut js_client = JsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
             on_msg_failed_event: None,
+            js_services: None,
+            js_context: jetstream::new(client.clone()),
+            service_log_prefix: log_prefix,
             client,
-            js: jetstream,
-            js_services,
-            service_log_prefix: service_log_prefix.clone(),
         };
 
-        for opt in p.opts {
-            opt(&mut default_client);
+        for listener in p.listeners {
+            listener(&mut js_client);
         }
 
-        log::info!(
-            "{}Connected to NATS server at {}",
-            service_log_prefix,
-            default_client.url
-        );
-        Ok(default_client)
-    }
-
-    pub fn name(&self) -> &str {
-        &self.name
+        Ok(js_client)
     }
 
     pub fn get_server_info(&self) -> ServerInfo {
         self.client.server_info()
     }
 
-    pub async fn monitor(&self) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            Err(Box::new(ErrClientDisconnected))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub async fn close(&self) -> Result<(), async_nats::Error> {
-        self.client.drain().await?;
-        Ok(())
-    }
-
-    pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            return Err(Box::new(ErrClientDisconnected));
-        }
-        let stream = &self.js.get_stream(stream_name).await?;
+    pub async fn get_stream_info(&self, stream_name: &str) -> Result<(), async_nats::Error> {
+        let stream = &self.js_context.get_stream(stream_name).await?;
         let info = stream.get_info().await?;
         log::debug!(
             "{}JetStream info: stream:{}, info:{:?}",
@@ -204,43 +94,80 @@ impl JsClient {
         Ok(())
     }
 
-    pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
-        Ok(())
+    pub async fn check_connection(
+        &self,
+    ) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state = self.client.connection_state();
+        if let async_nats::connection::State::Disconnected = conn_state {
+            Err(Box::new(ErrClientDisconnected))
+        } else {
+            Ok(conn_state)
+        }
     }
 
-    pub async fn publish(&self, payload: &SendRequest) -> Result<(), async_nats::Error> {
+    pub async fn publish(
+        &self,
+        payload: PublishInfo,
+    ) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>>
+    {
+        log::debug!(
+            "{}Called Publish message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+
         let now = Instant::now();
-        let result = self
-            .js
-            .publish(payload.subject.clone(), payload.data.clone().into())
-            .await;
+        let result = match payload.headers {
+            Some(headers) => {
+                self.js_context
+                    .publish_with_headers(
+                        payload.subject.clone(),
+                        headers,
+                        payload.data.clone().into(),
+                    )
+                    .await
+            }
+            None => {
+                self.js_context
+                    .publish(payload.subject.clone(), payload.data.clone().into())
+                    .await
+            }
+        };
 
         let duration = now.elapsed();
         if let Err(err) = result {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(Box::new(err));
+            return Err(err);
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
         Ok(())
     }
 
-    pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
-        let mut current_services = self.js_services.unwrap_or_default();
-        current_services.extend(js_services);
+    pub async fn add_js_service(
+        &mut self,
+        params: JsServiceBuilder,
+    ) -> Result<(), async_nats::Error> {
+        let new_service = JsStreamService::new(
+            self.js_context.to_owned(),
+            &params.name,
+            &params.description,
+            &params.version,
+            &params.service_subject,
+        )
+        .await?;
+
+        let mut current_services = self.js_services.to_owned().unwrap_or_default();
+        current_services.push(new_service);
         self.js_services = Some(current_services);
-        self
+
+        Ok(())
     }
 
     pub async fn get_js_service(&self, js_service_name: String) -> Option<&JsStreamService> {
@@ -251,6 +178,11 @@ impl JsClient {
         }
         None
     }
+
+    pub async fn close(&self) -> Result<(), async_nats::Error> {
+        self.client.drain().await?;
+        Ok(())
+    }
 }
 
 // Client Options:
@@ -268,7 +200,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_published_event = Some(Box::pin(f.clone()));
+        c.on_msg_published_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -277,7 +209,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_failed_event = Some(Box::pin(f.clone()));
+        c.on_msg_failed_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -314,7 +246,7 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     };
 
     let event_listeners = vec![
-        on_msg_published_event(published_msg_handler),
+        on_msg_published_event(published_msg_handler), // Shouldn't this be the 'NATS_LISTEN_PORT'?
         on_msg_failed_event(failure_handler),
     ];
 
@@ -326,8 +258,8 @@ pub fn get_event_listeners() -> Vec<EventListener> {
 mod tests {
     use super::*;
 
-    pub fn get_default_params() -> NewJsClientParams {
-        NewJsClientParams {
+    pub fn get_default_params() -> JsClientBuilder {
+        JsClientBuilder {
             nats_url: "localhost:4222".to_string(),
             name: "test_client".to_string(),
             inbox_prefix: "_UNIQUE_INBOX".to_string(),
@@ -353,7 +285,7 @@ mod tests {
     async fn test_nats_js_client_publish() {
         let params = get_default_params();
         let client = JsClient::new(params).await.unwrap();
-        let payload = SendRequest {
+        let payload = PublishInfo {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
             data: b"Hello, NATS!".to_vec(),
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/nats/jetstream_service.rs
similarity index 78%
rename from rust/util_libs/src/js_stream_service.rs
rename to rust/util_libs/src/nats/jetstream_service.rs
index 0860c3b..e622e3a 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/nats/jetstream_service.rs
@@ -1,108 +1,17 @@
-use super::nats_js_client::EndpointType;
-
+use super::types::{
+    ConsumerBuilder, ConsumerExt, ConsumerExtTrait, EndpointTraits, EndpointType,
+    JsStreamServiceInfo, LogInfo, ResponseSubjectsGenerator,
+};
 use anyhow::{anyhow, Result};
-use std::any::Any;
-// use async_nats::jetstream::message::Message;
-use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
+use async_nats::jetstream::consumer::{self, AckPolicy};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
-use async_trait::async_trait;
 use futures::StreamExt;
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
-
-pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> Option<Vec<String>>;
-}
-
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
-{
-}
-
-#[async_trait]
-pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
-    fn get_name(&self) -> &str;
-    fn get_consumer(&self) -> PullConsumer;
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
-}
-
-impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
-where
-    T: EndpointTraits,
-{
-    type Error = anyhow::Error;
-
-    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
-        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
-            Ok(*endpoint)
-        } else {
-            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
-        }
-    }
-}
-
-#[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    name: String,
-    consumer: PullConsumer,
-    handler: EndpointType<T>,
-    #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>,
-}
-
-#[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    fn get_name(&self) -> &str {
-        &self.name
-    }
-    fn get_consumer(&self) -> PullConsumer {
-        self.consumer.clone()
-    }
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
-        Box::new(self.handler.clone())
-    }
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
-        self.response_subject_fn.clone()
-    }
-}
-
-#[allow(dead_code)]
-#[derive(Clone, Debug)]
-pub struct JsStreamServiceInfo<'a> {
-    pub name: &'a str,
-    pub version: &'a str,
-    pub service_subject: &'a str,
-}
-
-struct LogInfo {
-    prefix: String,
-    service_name: String,
-    service_subject: String,
-    endpoint_name: String,
-    endpoint_subject: String,
-}
-
-#[derive(Clone, Deserialize, Default)]
-pub struct JsServiceParamsPartial {
-    pub name: String,
-    pub description: String,
-    pub version: String,
-    pub service_subject: String,
-}
-
 /// Microservice for Jetstream Streams
 // This setup creates only one subject for the stream (eg: "WORKLOAD.>") and sets up
 // all consumers of the stream to listen to stream subjects beginning with that subject (eg: "WORKLOAD.start")
@@ -196,30 +105,30 @@ impl JsStreamService {
         let handler: EndpointType<T> = EndpointType::try_from(endpoint_trait_obj)?;
 
         Ok(ConsumerExt {
-            name: consumer_ext.get_name().to_string(),
             consumer: consumer_ext.get_consumer(),
             handler,
             response_subject_fn: consumer_ext.get_response(),
         })
     }
 
-    pub async fn add_local_consumer<T>(
+    pub async fn add_consumer<T>(
         &self,
-        consumer_name: &str,
-        endpoint_subject: &str,
-        endpoint_type: EndpointType<T>,
-        response_subject_fn: Option<ResponseSubjectsGenerator>,
+        builder_params: ConsumerBuilder<T>,
     ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
-        let full_subject = format!("{}.{}", self.service_subject, endpoint_subject);
+        // Add the Service Subject prefix
+        let consumer_subject = format!(
+            "{}.{}",
+            self.service_subject, builder_params.endpoint_subject
+        );
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
-            durable_name: Some(consumer_name.to_string()),
+            durable_name: Some(builder_params.name.to_string()),
             ack_policy: AckPolicy::Explicit,
-            filter_subject: full_subject,
+            filter_subject: consumer_subject,
             ..Default::default()
         };
 
@@ -227,28 +136,28 @@ impl JsStreamService {
             .stream
             .write()
             .await
-            .get_or_create_consumer(consumer_name, consumer_config)
+            .get_or_create_consumer(&builder_params.name, consumer_config)
             .await?;
 
         let consumer_with_handler = ConsumerExt {
-            name: consumer_name.to_string(),
             consumer,
-            handler: endpoint_type,
-            response_subject_fn,
+            handler: builder_params.handler,
+            response_subject_fn: builder_params.response_subject_fn,
         };
 
-        self.local_consumers
-            .write()
-            .await
-            .insert(consumer_name.to_string(), Arc::new(consumer_with_handler));
+        self.local_consumers.write().await.insert(
+            builder_params.name.to_string(),
+            Arc::new(consumer_with_handler),
+        );
 
-        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(consumer_name).await?;
-        self.spawn_consumer_handler::<T>(consumer_name).await?;
+        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(&builder_params.name).await?;
+        self.spawn_consumer_handler::<T>(&builder_params.name)
+            .await?;
 
         log::debug!(
             "{}Added the {} local consumer",
             self.service_log_prefix,
-            endpoint_consumer.name,
+            builder_params.name,
         );
 
         Ok(endpoint_consumer)
@@ -279,12 +188,19 @@ impl JsStreamService {
                 .messages()
                 .await?;
 
+            let consumer_info = consumer.info().await?;
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
-                endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
+                endpoint_name: consumer_info
+                    .config
+                    .durable_name
+                    .clone()
+                    .unwrap_or("Consumer Name Not Found".to_string())
+                    .clone(),
+                endpoint_subject: consumer_info.config.filter_subject.clone(),
             };
 
             let service_context = self.js_context.clone();
@@ -338,14 +254,11 @@ impl JsStreamService {
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
-                    let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
-                        Ok(r) => r.into(),
-                        Err(e) => e.to_string().into(),
-                    };
+                    let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 }
-                Err(err) => (err.to_string().into(), None),
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
@@ -498,7 +411,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_js_service_add_local_consumer() {
+    async fn test_js_service_add_consumer() {
         let context = setup_jetstream().await;
         let service = get_default_js_service(context).await;
 
@@ -508,7 +421,7 @@ mod tests {
         let response_subject = Some("response.subject".to_string());
 
         let consumer = service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
@@ -533,7 +446,7 @@ mod tests {
         let response_subject = None;
 
         service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats/leaf_server.rs
similarity index 98%
rename from rust/util_libs/src/nats_server.rs
rename to rust/util_libs/src/nats/leaf_server.rs
index 4e9030b..dfc40f1 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats/leaf_server.rs
@@ -143,7 +143,7 @@ impl LeafServer {
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .spawn()
-            .expect("Failed to start NATS server");
+            .context("Failed to start NATS server")?;
 
         // TODO: wait for a readiness indicator
         std::thread::sleep(std::time::Duration::from_millis(100));
@@ -192,8 +192,8 @@ mod tests {
     const TMP_JS_DIR: &str = "./tmp";
     const TEST_AUTH_DIR: &str = "./tmp/test-auth";
     const OPERATOR_NAME: &str = "test-operator";
-    const USER_ACCOUNT_NAME: &str = "hpos-account";
-    const USER_NAME: &str = "hpos-user";
+    const USER_ACCOUNT_NAME: &str = "host-account";
+    const USER_NAME: &str = "host-user";
     const NEW_LEAF_CONFIG_PATH: &str = "./test_configs/leaf_server.conf";
     // NB: if changed, the resolver file path must also be changed in the `hub-server.conf` iteself as well.
     const RESOLVER_FILE_PATH: &str = "./test_configs/resolver.conf";
@@ -227,7 +227,7 @@ mod tests {
             .output()
             .expect("Failed to create edit operator");
 
-        // Create hpos account (with js enabled)
+        // Create host account (with js enabled)
         Command::new("nsc")
             .args(["add", "account", USER_ACCOUNT_NAME])
             .output()
@@ -245,7 +245,7 @@ mod tests {
             .output()
             .expect("Failed to create edit account");
 
-        // Create user for hpos account
+        // Create user for host account
         Command::new("nsc")
             .args(["add", "user", USER_NAME])
             .args(["--account", USER_ACCOUNT_NAME])
diff --git a/rust/util_libs/src/nats/mod.rs b/rust/util_libs/src/nats/mod.rs
new file mode 100644
index 0000000..a320ee4
--- /dev/null
+++ b/rust/util_libs/src/nats/mod.rs
@@ -0,0 +1,4 @@
+pub mod jetstream_client;
+pub mod jetstream_service;
+pub mod leaf_server;
+pub mod types;
diff --git a/rust/util_libs/src/nats/types.rs b/rust/util_libs/src/nats/types.rs
new file mode 100644
index 0000000..d1b6fe1
--- /dev/null
+++ b/rust/util_libs/src/nats/types.rs
@@ -0,0 +1,200 @@
+use super::jetstream_client::JsClient;
+use anyhow::Result;
+use async_nats::jetstream::consumer::PullConsumer;
+use async_nats::{HeaderMap, Message};
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt;
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
+pub type AsyncEndpointHandler<T> = Arc<
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
+        + Send
+        + Sync,
+>;
+pub type ResponseSubjectsGenerator =
+    Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
+
+pub trait EndpointTraits:
+    Serialize
+    + for<'de> Deserialize<'de>
+    + Send
+    + Sync
+    + Clone
+    + Debug
+    + CreateTag
+    + CreateResponse
+    + 'static
+{
+}
+
+pub trait CreateTag: Send + Sync {
+    fn get_tags(&self) -> HashMap<String, String>;
+}
+
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
+}
+
+#[async_trait]
+pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
+    fn get_consumer(&self) -> PullConsumer;
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
+}
+
+#[async_trait]
+impl<T> ConsumerExtTrait for ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    fn get_consumer(&self) -> PullConsumer {
+        self.consumer.clone()
+    }
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
+        Box::new(self.handler.clone())
+    }
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
+        self.response_subject_fn.clone()
+    }
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    pub consumer: PullConsumer,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerBuilder<T>
+where
+    T: EndpointTraits,
+{
+    pub name: String,
+    pub endpoint_subject: String,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[allow(dead_code)]
+#[derive(Clone, Debug)]
+pub struct JsStreamServiceInfo<'a> {
+    pub name: &'a str,
+    pub version: &'a str,
+    pub service_subject: &'a str,
+}
+
+#[derive(Clone, Debug)]
+pub struct LogInfo {
+    pub prefix: String,
+    pub service_name: String,
+    pub service_subject: String,
+    pub endpoint_name: String,
+    pub endpoint_subject: String,
+}
+
+#[derive(Clone)]
+pub enum EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
+{
+    Sync(EndpointHandler<T>),
+    Async(AsyncEndpointHandler<T>),
+}
+
+impl<T> std::fmt::Debug for EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let t = match &self {
+            EndpointType::Async(_) => "EndpointType::Async(<function>)",
+            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
+        };
+
+        write!(f, "{}", t)
+    }
+}
+impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
+where
+    T: EndpointTraits,
+{
+    type Error = anyhow::Error;
+
+    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
+        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
+            Ok(*endpoint)
+        } else {
+            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
+        }
+    }
+}
+
+#[derive(Deserialize, Default)]
+pub struct JsClientBuilder {
+    pub nats_url: String,
+    pub name: String,
+    pub inbox_prefix: String,
+    #[serde(default)]
+    pub credentials_path: Option<String>,
+    #[serde(default)]
+    pub ping_interval: Option<Duration>,
+    #[serde(default)]
+    pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
+}
+
+#[derive(Clone, Deserialize, Default)]
+pub struct JsServiceBuilder {
+    pub name: String,
+    pub description: String,
+    pub version: String,
+    pub service_subject: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct PublishInfo {
+    pub subject: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+    pub headers: Option<HeaderMap>,
+}
+
+#[derive(Debug)]
+pub struct ErrClientDisconnected;
+impl fmt::Display for ErrClientDisconnected {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Could not reach nats: connection closed")
+    }
+}
+impl Error for ErrClientDisconnected {}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
deleted file mode 100644
index ece916b..0000000
--- a/rust/util_libs/src/nats_types.rs
+++ /dev/null
@@ -1,145 +0,0 @@
-/* --------
-NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
-IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
-TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */
-
-use serde::{Deserialize, Serialize};
-
-/// JWT claims for NATS compatible jwts
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Claims {
-    /// Time when the token was issued in seconds since the unix epoch
-    #[serde(rename = "iat")]
-    pub issued_at: i64,
-
-    /// Public key of the issuer signing nkey
-    #[serde(rename = "iss")]
-    pub issuer: String,
-
-    /// Base32 hash of the claims where this is empty
-    #[serde(rename = "jti")]
-    pub jwt_id: String,
-
-    /// Public key of the account or user the JWT is being issued to
-    pub sub: String,
-
-    /// Friendly name
-    pub name: String,
-
-    /// NATS claims
-    pub nats: NatsClaims,
-
-    /// Time when the token expires (in seconds since the unix epoch)
-    #[serde(rename = "exp", skip_serializing_if = "Option::is_none")]
-    pub expires: Option<i64>,
-}
-
-/// NATS claims describing settings for the user or account
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "lowercase")]
-pub enum NatsClaims {
-    /// Claims for NATS users
-    User {
-        /// Publish and subscribe permissions for the user
-        #[serde(flatten)]
-        permissions: NatsPermissionsMap,
-
-        /// Public key/id of the account that issued the JWT
-        issuer_account: String,
-
-        /// Maximum nuber of subscriptions the user can have
-        subs: i64,
-
-        /// Maximum size of the message data the user can send in bytes
-        data: i64,
-
-        /// Maximum size of the entire message payload the user can send in bytes
-        payload: i64,
-
-        /// If true, the user isn't challenged on connection. Typically used for websocket
-        /// connections as the browser won't have/want to have the user's private key.
-        bearer_token: bool,
-
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-    /// Claims for NATS accounts
-    Account {
-        /// Configuration for the limits for this account
-        limits: NatsAccountLimits,
-
-        /// List of signing keys (public key) this account uses
-        #[serde(skip_serializing_if = "Vec::is_empty")]
-        signing_keys: Vec<String>,
-
-        /// Default publish and subscribe permissions users under this account will have if not
-        /// specified otherwise
-        /// default_permissions: NatsPermissionsMap,
-        ///
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-}
-
-/// List of subjects that are allowed and/or denied
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissions {
-    /// List of subject patterns that are allowed
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub allow: Vec<String>,
-
-    /// List of subject patterns that are denied
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub deny: Vec<String>,
-}
-
-impl NatsPermissions {
-    /// Returns `true` if the allow and deny list are both empty
-    #[must_use]
-    pub fn is_empty(&self) -> bool {
-        self.allow.is_empty() && self.deny.is_empty()
-    }
-}
-
-/// Publish and subcribe permissons
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissionsMap {
-    /// Permissions for which subjects can be published to
-    #[serde(rename = "pub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub publish: NatsPermissions,
-
-    /// Permissions for which subjects can be subscribed to
-    #[serde(rename = "sub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub subscribe: NatsPermissions,
-}
-
-/// Limits on what an account or users in the account can do
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NatsAccountLimits {
-    /// Maximum nuber of subscriptions the account
-    pub subs: i64,
-
-    /// Maximum size of the message data a user can send in bytes
-    pub data: i64,
-
-    /// Maximum size of the entire message payload a user can send in bytes
-    pub payload: i64,
-
-    /// Maxiumum number of imports for the account
-    pub imports: i64,
-
-    /// Maxiumum number of exports for the account
-    pub exports: i64,
-
-    /// If true, exports can contain wildcards
-    pub wildcards: bool,
-
-    /// Maximum number of active connections
-    pub conn: i64,
-
-    /// Maximum number of leaf node connections
-    pub leaf: i64,
-}
diff --git a/rust/util_libs/test_configs/hub_server.conf b/rust/util_libs/test_configs/hub_server.conf
deleted file mode 100644
index 0184098..0000000
--- a/rust/util_libs/test_configs/hub_server.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-operator: "./test-auth/test-operator/test-operator.jwt"
-system_account: SYS
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-include ./resolver.conf
-
-# logging options
-debug:   true
-trace:   true
-logtime: false
diff --git a/rust/util_libs/test_configs/hub_server_pw_auth.conf b/rust/util_libs/test_configs/hub_server_pw_auth.conf
deleted file mode 100644
index 51eeb3f..0000000
--- a/rust/util_libs/test_configs/hub_server_pw_auth.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-authorization {
-    user: "test-user"
-    password: "pw-12345"
-}
-
-# logging options
-debug:   true
-trace:   true
-logtime: false

From f13d089e771842dfd2834f28ad069c25ac2bfab3 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:32:34 -0600
Subject: [PATCH 88/91] update .gitignore

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 1812acf..dbb5350 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,7 +17,7 @@ target/
 rust/*/tmp
 rust/*/jwt
 rust/*/*/test_leaf_server/*
-rust/*/*/test_leaf_server.conf
+rust/*/*/*.conf
 rust/*/*/leaf_server.conf
 rust/*/*/resolver.conf
 leaf_server.conf

From 92c0c5387ea367feac2276871980de02aa244fbf Mon Sep 17 00:00:00 2001
From: Lisa Jetton <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:36:57 -0600
Subject: [PATCH 89/91] Orchestrator Workload Client (#53)

* temporary(flake): switch to blueprint fork

* feat(nix/lib): wrap runNixOSTest with defaults

this is required when VM tests use nixos modules that live in a
blueprint repository like this.

* feat(nix/packages/rust-workspace): expose rust binaries

previously it would only expose the target directory as an archive.

* WIP: feat(holo-agent): add nixos module with integration test

* FIXME: this commit needs splitting up

iterate on holo-agent-integration-nixos with code changes all over the place.

test can be run with:

nix build -vL .\#checks.x86_64-linux.holo-agent-integration-nixos

* fix(nix/modules/host-agent): wait for network connectivity

* holo-host-agent: use wantedBy and increase logging

* feat(host_agent): add leafnode creds CLI arg and improve handling consistency

this also takes out the hardoded path for the credentials path which has
been panicing in the integration tests.

* fix(host-agent): continously try to connect to spawned NATS leaf server

when running the host-agent on system startup there seems to be a race
condition that prevents the agent from connecting to the spawned NATS
instance.

the root cause for this _might_ be a race condition between the network
stack availability and spawning Nats, however that's a guess. it might
also just take a 100-200ms for Nats to start servicing the TCP port.

either way, the boot log in the integration test looks like this with
the fix applied. the loop fails once and then succeeds after waiting
100ms:

```
[    6.690765] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  util_libs::nats_server] NATS Leaf Server is running at 127.0.0.1:4222
[    6.692975] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] HPOS Agent Client: Connecting to server...
[    6.695163] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] host_creds_path : None
[    6.696391] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] host_pubkey : host_id_placeholder>
[    6.698881] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  host_agent::workload_manager] nats_url : 127.0.0.1:4222
[    6.720665] systemd-logind[707]: New seat seat0.
[    6.723219] holo-host-agent-start[695]: [2025-01-20T20:53:16Z WARN  host_agent::workload_manager] connecting to NATS via 127.0.0.1:4222: IO error: Connection refused (os error 111), retrying in 100ms
[    6.726726] systemd-logind[707]: Watching system buttons on /dev/input/event2 (Power Button)
[    6.727999] systemd-logind[707]: Watching system buttons on /dev/input/event3 (QEMU Virtio Keyboard)
[    6.731311] systemd-logind[707]: Watching system buttons on /dev/input/event0 (AT Translated Set 2 keyboard)
[    6.734306] systemd[1]: Started User Login Management.
[    6.762172] systemd[1]: Started Name Service Cache Daemon (nsncd).
[    6.764253] nsncd[750]: Jan 20 20:53:16.581 INFO started, config: Config { ignored_request_types: {}, worker_count: 8, handoff_timeout: 3s }, path: "/var/run/nscd/socket"
[    6.767328] systemd[1]: Reached target Host and Network Name Lookups.
[    6.768655] systemd[1]: Reached target User and Group Name Lookups.
[    6.771096] systemd[1]: Finished resolvconf update.
[    6.771760] systemd[1]: Reached target Preparation for Network.
[    6.776104] systemd[1]: Starting DHCP Client...
[    6.779801] systemd[1]: Starting Address configuration of eth1...
[    6.862637] network-addresses-eth1-start[775]: adding address 192.168.1.1/24... done
[    6.872977] holo-host-agent-start[695]: [2025-01-20T20:53:16Z INFO  util_libs::nats_js_client] NATS-CLIENT-LOG::Host Agent::Connected to NATS server at 127.0.0.1:4222
[    6.880800] network-addresses-eth1-start[775]: adding address 2001:db8:1::1/64... done
[    6.903973] systemd[1]: Finished Address configuration of eth1.
```

* separate out orchetrator client into own feature pr

* chore: nix fmt

* update hpos naming

* improve workload desc

* restore `WorkloadApiResult`

* remove host env var

* temporary(flake): bump blueprint for upstreamed fixes

* feat(nix/packages/rust-workspace): expose rust binaries

previously it would only expose the target directory as an archive.

* feat(nix): introduce holo-host-agent module with integration test

the holo-host agent also pulls in extra-container as that's going to be
the initial vehicle for defining and running host workloads.

* feat(nix/holo-nats-server): make port and leafnodeport configurable

* host-agent: improve resilience and configuration

* turn some hardcoded values into CLI arguments
* wait (with a timeout) for NATS to be ready to serve connections
* pass through NATS stdout/stderr
* provision (techdebt) TODOs

* feat(holo-nats-server): use lib.mkDefault for defaults

otherwise users will require `lib.mkForce` or similar to override

* feat(nix/modules/nixos): expose blueprint's publisherArgs

otherwise it uses `flake` from downstream consumers which will not work
as expected.

* feat(niox module holo-nats-server): add openFirewall cfg and use correct ports

* feat(nixos module holo-nats-server): configure TLS websockets via caddy

primarily this is motivated by TLS encryption. websockets are straight
forward to gate via a reverse TLS proxy like caddy.

as a nice side-effect, external clients and leafnodes can now connect
via the a shared port.

* feat,refactor(host-agent): TLS websocket connection, CLI args, config serialization

* feat(nixos module holo-host-agent): add hub TLS options and add extra args option

* test(holo-agent-integration-nixos): adapt to TLS via websocket

* feat(host-agent/cli): require command

* feat(host-agent): close NATS client connection before exiting the process

Co-authored-by: Lisa Jetton <ldjetton21@gmail.com>

* adjust codebase to new types

* add nix formatter updates

* update .env.example

* test with only 1 hpos

* correct log msg

* remove js prefix condition

* update README

* refactor/util-libs (#73)

* refactor workload service (#71)

* refactor-client-dir (#69)
---------

Co-authored-by: Stefan Junker <mail@stefanjunker.de>
Co-authored-by: Stefan Junker <1181362+steveej@users.noreply.github.com>
---
 .env.example                                  |  16 +-
 .gitignore                                    |   2 +-
 Cargo.lock                                    | 887 ++++++++++++------
 Cargo.toml                                    |   2 +-
 README.md                                     |  13 +-
 flake.lock                                    |   2 +-
 flake.nix                                     |   2 +-
 nix/checks/holo-agent-integration-nixos.nix   |  78 +-
 nix/lib/default.nix                           |   6 +-
 nix/modules/nixos/holo-nats-server.nix        |   2 +-
 rust/clients/host_agent/Cargo.toml            |  15 +-
 .../src/{ => hostd}/gen_leaf_server.rs        |  21 +-
 rust/clients/host_agent/src/hostd/mod.rs      |   2 +
 rust/clients/host_agent/src/hostd/workload.rs | 155 +++
 rust/clients/host_agent/src/main.rs           |  23 +-
 .../host_agent/src/workload_manager.rs        | 129 ---
 rust/clients/orchestrator/Cargo.toml          |  32 +
 .../orchestrator/src/extern_api/example.rs}   |   0
 .../orchestrator/src/extern_api/mod.rs}       |   8 +-
 rust/clients/orchestrator/src/main.rs         |  18 +
 rust/clients/orchestrator/src/utils.rs        |  24 +
 rust/clients/orchestrator/src/workloads.rs    | 183 ++++
 rust/orchestrator/Cargo.toml                  |  20 -
 rust/services/workload/Cargo.toml             |   3 +
 rust/services/workload/src/host_api.rs        | 169 ++++
 rust/services/workload/src/lib.rs             | 526 ++---------
 .../services/workload/src/orchestrator_api.rs | 662 +++++++++++++
 rust/services/workload/src/types.rs           |  49 +-
 rust/util_libs/src/db/mongodb.rs              | 162 ++--
 rust/util_libs/src/db/schemas.rs              |  89 +-
 rust/util_libs/src/lib.rs                     |   5 +-
 .../jetstream_client.rs}                      | 252 ++---
 .../jetstream_service.rs}                     | 165 +---
 .../{nats_server.rs => nats/leaf_server.rs}   |  10 +-
 rust/util_libs/src/nats/mod.rs                |   4 +
 rust/util_libs/src/nats/types.rs              | 200 ++++
 rust/util_libs/src/nats_types.rs              | 145 ---
 rust/util_libs/test_configs/hub_server.conf   |  22 -
 .../test_configs/hub_server_pw_auth.conf      |  22 -
 scripts/orchestrator_setup.sh                 |  59 +-
 40 files changed, 2524 insertions(+), 1660 deletions(-)
 rename rust/clients/host_agent/src/{ => hostd}/gen_leaf_server.rs (91%)
 create mode 100644 rust/clients/host_agent/src/hostd/mod.rs
 create mode 100644 rust/clients/host_agent/src/hostd/workload.rs
 delete mode 100644 rust/clients/host_agent/src/workload_manager.rs
 create mode 100644 rust/clients/orchestrator/Cargo.toml
 rename rust/{orchestrator/src/hello.rs => clients/orchestrator/src/extern_api/example.rs} (100%)
 rename rust/{orchestrator/src/main.rs => clients/orchestrator/src/extern_api/mod.rs} (92%)
 mode change 100755 => 100644
 create mode 100644 rust/clients/orchestrator/src/main.rs
 create mode 100644 rust/clients/orchestrator/src/utils.rs
 create mode 100644 rust/clients/orchestrator/src/workloads.rs
 delete mode 100755 rust/orchestrator/Cargo.toml
 create mode 100644 rust/services/workload/src/host_api.rs
 create mode 100644 rust/services/workload/src/orchestrator_api.rs
 rename rust/util_libs/src/{nats_js_client.rs => nats/jetstream_client.rs} (55%)
 rename rust/util_libs/src/{js_stream_service.rs => nats/jetstream_service.rs} (78%)
 rename rust/util_libs/src/{nats_server.rs => nats/leaf_server.rs} (98%)
 create mode 100644 rust/util_libs/src/nats/mod.rs
 create mode 100644 rust/util_libs/src/nats/types.rs
 delete mode 100644 rust/util_libs/src/nats_types.rs
 delete mode 100644 rust/util_libs/test_configs/hub_server.conf
 delete mode 100644 rust/util_libs/test_configs/hub_server_pw_auth.conf

diff --git a/.env.example b/.env.example
index bfe8157..3ca30fd 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,14 @@
-NSC_PATH = "<path/to/nsc_dir>"
+# ALL
+NSC_PATH="<path/to/nsc_dir>"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LOCAL_CREDS_PATH="<path/to/dir>"
+
+# ORCHESTRATOR
+MONGO_URI="mongodb://<host>:<port>"
+
+# HOSTING AGENT
 HOST_CREDS_FILE_PATH = "ops/admin.creds"
-MONGO_URI = "mongodb://<host>:<port>"
-NATS_HUB_SERVER_URL = "nats://<path/to/leaf>:<port>"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
 LEAF_SERVER_USER = "test-user"
-LEAF_SERVER_PW = "pw-123456789"
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index dbb5350..c4382e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,4 +23,4 @@ rust/*/*/resolver.conf
 leaf_server.conf
 .local
 rust/*/*/*/tmp/
-rust/*/*/*/*/tmp/
+rust/*/*/*/*/tmp/
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index aada039..4b66c95 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8,7 +8,7 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "bytes",
  "futures-core",
  "futures-sink",
@@ -31,11 +31,11 @@ dependencies = [
  "actix-utils",
  "ahash",
  "base64 0.22.1",
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "brotli",
  "bytes",
  "bytestring",
- "derive_more 0.99.18",
+ "derive_more 0.99.19",
  "encoding_rs",
  "flate2",
  "futures-core",
@@ -49,7 +49,7 @@ dependencies = [
  "mime",
  "percent-encoding",
  "pin-project-lite",
- "rand",
+ "rand 0.8.5",
  "sha1",
  "smallvec",
  "tokio",
@@ -65,7 +65,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
 dependencies = [
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -121,19 +121,6 @@ dependencies = [
  "pin-project-lite",
 ]
 
-[[package]]
-name = "actix-test"
-version = "0.1.0"
-dependencies = [
- "actix-web",
- "futures",
- "mongodb",
- "serde",
- "tokio",
- "utoipa",
- "utoipa-swagger-ui",
-]
-
 [[package]]
 name = "actix-utils"
 version = "3.0.1"
@@ -164,7 +151,7 @@ dependencies = [
  "bytestring",
  "cfg-if",
  "cookie",
- "derive_more 0.99.18",
+ "derive_more 0.99.19",
  "encoding_rs",
  "futures-core",
  "futures-util",
@@ -195,7 +182,7 @@ dependencies = [
  "actix-router",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -220,7 +207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -306,19 +293,20 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.6"
+version = "3.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
 dependencies = [
  "anstyle",
+ "once_cell",
  "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.94"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7"
+checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4"
 
 [[package]]
 name = "arbitrary"
@@ -366,9 +354,9 @@ dependencies = [
  "once_cell",
  "pin-project",
  "portable-atomic",
- "rand",
+ "rand 0.8.5",
  "regex",
- "ring 0.17.8",
+ "ring 0.17.10",
  "rustls-native-certs 0.7.3",
  "rustls-pemfile 2.2.0",
  "rustls-webpki 0.102.8",
@@ -389,13 +377,13 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.83"
+version = "0.1.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
+checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -406,9 +394,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.12.0"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148"
+checksum = "4cd755adf9707cf671e31d944a189be3deaaeee11c8bc1d669bb8022ac90fbd0"
 dependencies = [
  "aws-lc-sys",
  "paste",
@@ -417,16 +405,15 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.24.0"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8478a5c29ead3f3be14aff8a202ad965cf7da6856860041bfca271becf8ba48b"
+checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43"
 dependencies = [
  "bindgen",
  "cc",
  "cmake",
  "dunce",
  "fs_extra",
- "libc",
  "paste",
 ]
 
@@ -445,6 +432,12 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -475,7 +468,7 @@ version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "cexpr",
  "clang-sys",
  "itertools",
@@ -488,7 +481,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.90",
+ "syn 2.0.98",
  "which",
 ]
 
@@ -524,9 +517,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.6.0"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
 [[package]]
 name = "bitvec"
@@ -581,10 +574,10 @@ dependencies = [
  "bitvec",
  "chrono",
  "hex",
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "js-sys",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -594,9 +587,9 @@ dependencies = [
 
 [[package]]
 name = "bstr"
-version = "1.11.1"
+version = "1.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8"
+checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0"
 dependencies = [
  "memchr",
  "regex-automata",
@@ -605,15 +598,15 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.16.0"
+version = "3.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
 
 [[package]]
 name = "bytemuck"
-version = "1.20.0"
+version = "1.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a"
+checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3"
 
 [[package]]
 name = "byteorder"
@@ -623,9 +616,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
+checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
 dependencies = [
  "serde",
 ]
@@ -641,9 +634,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.4"
+version = "1.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf"
+checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af"
 dependencies = [
  "jobserver",
  "libc",
@@ -693,9 +686,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.23"
+version = "4.5.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84"
+checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -703,9 +696,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.23"
+version = "4.5.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838"
+checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c"
 dependencies = [
  "anstream",
  "anstyle",
@@ -715,14 +708,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.18"
+version = "4.5.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
+checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -733,9 +726,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
 
 [[package]]
 name = "cmake"
-version = "0.1.52"
+version = "0.1.54"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e"
+checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
 dependencies = [
  "cc",
 ]
@@ -752,6 +745,26 @@ version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
 
+[[package]]
+name = "const-random"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
+dependencies = [
+ "const-random-macro",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
+dependencies = [
+ "getrandom 0.2.15",
+ "once_cell",
+ "tiny-keccak",
+]
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -797,9 +810,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.16"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
 dependencies = [
  "libc",
 ]
@@ -838,6 +851,12 @@ version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
+[[package]]
+name = "crunchy"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -861,6 +880,7 @@ dependencies = [
  "fiat-crypto",
  "rustc_version",
  "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -871,7 +891,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -895,7 +915,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -906,14 +926,14 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "data-encoding"
-version = "2.6.0"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
+checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010"
 
 [[package]]
 name = "der"
@@ -947,6 +967,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive-syn-parse"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
 [[package]]
 name = "derive-where"
 version = "1.2.7"
@@ -955,7 +986,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -966,20 +997,20 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "derive_more"
-version = "0.99.18"
+version = "0.99.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
+checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f"
 dependencies = [
  "convert_case",
  "proc-macro2",
  "quote",
  "rustc_version",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -999,7 +1030,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
  "unicode-xid",
 ]
 
@@ -1028,7 +1059,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -1055,6 +1086,7 @@ version = "2.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
 dependencies = [
+ "pkcs8",
  "signature",
 ]
 
@@ -1066,9 +1098,11 @@ checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
 dependencies = [
  "curve25519-dalek",
  "ed25519",
+ "serde",
  "sha2",
  "signature",
  "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -1095,14 +1129,14 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "env_filter"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
 dependencies = [
  "log",
  "regex",
@@ -1110,9 +1144,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
+checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1123,9 +1157,9 @@ dependencies = [
 
 [[package]]
 name = "equivalent"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
 [[package]]
 name = "errno"
@@ -1242,7 +1276,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -1285,6 +1319,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1292,8 +1337,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.13.3+wasi-0.2.2",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -1304,9 +1363,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
 [[package]]
 name = "glob"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
 
 [[package]]
 name = "h2"
@@ -1320,7 +1379,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.12",
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "slab",
  "tokio",
  "tokio-util",
@@ -1348,6 +1407,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1362,9 +1427,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
 [[package]]
 name = "hickory-proto"
-version = "0.24.2"
+version = "0.24.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5"
+checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248"
 dependencies = [
  "async-trait",
  "cfg-if",
@@ -1376,7 +1441,7 @@ dependencies = [
  "idna 1.0.3",
  "ipnet",
  "once_cell",
- "rand",
+ "rand 0.8.5",
  "thiserror 1.0.69",
  "tinyvec",
  "tokio",
@@ -1386,9 +1451,9 @@ dependencies = [
 
 [[package]]
 name = "hickory-resolver"
-version = "0.24.2"
+version = "0.24.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4"
+checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e"
 dependencies = [
  "cfg-if",
  "futures-util",
@@ -1397,7 +1462,7 @@ dependencies = [
  "lru-cache",
  "once_cell",
  "parking_lot",
- "rand",
+ "rand 0.8.5",
  "resolv-conf",
  "smallvec",
  "thiserror 1.0.69",
@@ -1433,20 +1498,25 @@ dependencies = [
  "bytes",
  "chrono",
  "clap",
+ "data-encoding",
  "dotenv",
+ "ed25519-dalek",
  "env_logger",
  "futures",
  "hpos-hal",
+ "jsonwebtoken",
  "log",
  "machineid-rs",
- "mongodb",
+ "nats-jwt",
  "netdiag",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "serde",
  "serde_json",
+ "sha2",
  "tempfile",
- "thiserror 2.0.9",
+ "textnonce",
+ "thiserror 2.0.11",
  "tokio",
  "url",
  "util_libs",
@@ -1478,7 +1548,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "test-files",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "thiserror-context",
  "uuid",
 ]
@@ -1518,9 +1588,9 @@ dependencies = [
 
 [[package]]
 name = "httparse"
-version = "1.9.5"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
+checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a"
 
 [[package]]
 name = "httpdate"
@@ -1712,7 +1782,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -1772,9 +1842,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.7.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
+checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
 dependencies = [
  "equivalent",
  "hashbrown 0.15.2",
@@ -1795,9 +1865,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.10.1"
+version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708"
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
 
 [[package]]
 name = "is_terminal_polyfill"
@@ -1831,14 +1901,29 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.76"
+version = "0.3.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7"
+checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
 dependencies = [
  "once_cell",
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonwebtoken"
+version = "9.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
+dependencies = [
+ "base64 0.22.1",
+ "js-sys",
+ "pem",
+ "ring 0.17.10",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
 [[package]]
 name = "language-tags"
 version = "0.3.2"
@@ -1881,9 +1966,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
 
 [[package]]
 name = "litemap"
@@ -1926,9 +2011,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
 
 [[package]]
 name = "log"
-version = "0.4.22"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
+checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
 
 [[package]]
 name = "lru-cache"
@@ -1959,6 +2044,54 @@ dependencies = [
  "wmi",
 ]
 
+[[package]]
+name = "macro_magic"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d"
+dependencies = [
+ "macro_magic_core",
+ "macro_magic_macros",
+ "quote",
+ "syn 2.0.98",
+]
+
+[[package]]
+name = "macro_magic_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150"
+dependencies = [
+ "const-random",
+ "derive-syn-parse",
+ "macro_magic_core_macros",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
+[[package]]
+name = "macro_magic_core_macros"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
+[[package]]
+name = "macro_magic_macros"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869"
+dependencies = [
+ "macro_magic_core",
+ "quote",
+ "syn 2.0.98",
+]
+
 [[package]]
 name = "match_cfg"
 version = "0.1.0"
@@ -2011,9 +2144,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.8.2"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394"
+checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
 dependencies = [
  "adler2",
 ]
@@ -2026,15 +2159,15 @@ checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
  "libc",
  "log",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "mongodb"
-version = "3.1.1"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff1f6edf7fe8828429647a2200f684681ca6d5a33b45edc3140c81390d852301"
+checksum = "9a93560fa3ec754ed9aa0954ae8307c5997150dbba7aa735173b514660088475"
 dependencies = [
  "async-trait",
  "base64 0.13.1",
@@ -2042,7 +2175,7 @@ dependencies = [
  "bson",
  "chrono",
  "derive-where",
- "derive_more 0.99.18",
+ "derive_more 0.99.19",
  "futures-core",
  "futures-executor",
  "futures-io",
@@ -2051,12 +2184,13 @@ dependencies = [
  "hickory-proto",
  "hickory-resolver",
  "hmac",
+ "macro_magic",
  "md-5",
  "mongodb-internal-macros",
  "once_cell",
  "pbkdf2",
  "percent-encoding",
- "rand",
+ "rand 0.8.5",
  "rustc_version_runtime",
  "rustls 0.21.12",
  "rustls-pemfile 1.0.4",
@@ -2080,13 +2214,14 @@ dependencies = [
 
 [[package]]
 name = "mongodb-internal-macros"
-version = "3.1.1"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b07bfd601af78e39384707a8e80041946c98260e3e0190e294ee7435823e6bf"
+checksum = "79b3dace6c4f33db61d492b3d3b02f4358687a1eb59457ffef6f6cfe461cdb54"
 dependencies = [
+ "macro_magic",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -2111,12 +2246,12 @@ dependencies = [
  "env_logger",
  "log",
  "rustdns",
- "rustls 0.23.20",
+ "rustls 0.23.23",
  "serde",
  "serde_yaml",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "uuid",
- "webpki-roots 0.26.7",
+ "webpki-roots 0.26.8",
 ]
 
 [[package]]
@@ -2128,9 +2263,9 @@ dependencies = [
  "data-encoding",
  "ed25519",
  "ed25519-dalek",
- "getrandom",
+ "getrandom 0.2.15",
  "log",
- "rand",
+ "rand 0.8.5",
  "signatory",
 ]
 
@@ -2159,7 +2294,17 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83"
 dependencies = [
- "rand",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
 ]
 
 [[package]]
@@ -2179,6 +2324,15 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -2190,24 +2344,52 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.36.5"
+version = "0.36.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e"
+checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.20.2"
+version = "1.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
 
 [[package]]
 name = "openssl-probe"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
+
+[[package]]
+name = "orchestrator"
+version = "0.0.1"
+dependencies = [
+ "actix-web",
+ "anyhow",
+ "async-nats",
+ "bson",
+ "bytes",
+ "chrono",
+ "dotenv",
+ "env_logger",
+ "futures",
+ "log",
+ "mongodb",
+ "nkeys",
+ "rand 0.8.5",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.11",
+ "tokio",
+ "url",
+ "util_libs",
+ "utoipa",
+ "utoipa-swagger-ui",
+ "workload",
+]
 
 [[package]]
 name = "owo-colors"
@@ -2253,6 +2435,16 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "pem"
+version = "3.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3"
+dependencies = [
+ "base64 0.22.1",
+ "serde",
+]
+
 [[package]]
 name = "pem-rfc7468"
 version = "0.7.0"
@@ -2275,7 +2467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc"
 dependencies = [
  "memchr",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "ucd-trie",
 ]
 
@@ -2321,7 +2513,7 @@ dependencies = [
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -2337,29 +2529,29 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.1.7"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95"
+checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.7"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c"
+checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
 
 [[package]]
 name = "pin-utils"
@@ -2406,9 +2598,9 @@ dependencies = [
 
 [[package]]
 name = "predicates"
-version = "3.1.2"
+version = "3.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97"
+checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573"
 dependencies = [
  "anstyle",
  "difflib",
@@ -2417,15 +2609,15 @@ dependencies = [
 
 [[package]]
 name = "predicates-core"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931"
+checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa"
 
 [[package]]
 name = "predicates-tree"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13"
+checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c"
 dependencies = [
  "predicates-core",
  "termtree",
@@ -2433,19 +2625,19 @@ dependencies = [
 
 [[package]]
 name = "prettyplease"
-version = "0.2.25"
+version = "0.2.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033"
+checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac"
 dependencies = [
  "proc-macro2",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.92"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
 dependencies = [
  "unicode-ident",
 ]
@@ -2456,7 +2648,7 @@ version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "chrono",
  "flate2",
  "hex",
@@ -2470,7 +2662,7 @@ version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "chrono",
  "hex",
 ]
@@ -2483,9 +2675,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quote"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
 dependencies = [
  "proc-macro2",
 ]
@@ -2496,6 +2688,19 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -2503,8 +2708,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -2514,7 +2729,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
 ]
 
 [[package]]
@@ -2523,7 +2747,16 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -2548,11 +2781,11 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.8"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
+checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
 ]
 
 [[package]]
@@ -2609,7 +2842,7 @@ dependencies = [
  "cc",
  "libc",
  "once_cell",
- "spin 0.5.2",
+ "spin",
  "untrusted 0.7.1",
  "web-sys",
  "winapi",
@@ -2617,15 +2850,14 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.8"
+version = "0.17.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "libc",
- "spin 0.9.8",
  "untrusted 0.9.0",
  "windows-sys 0.52.0",
 ]
@@ -2650,7 +2882,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.90",
+ "syn 2.0.98",
  "walkdir",
 ]
 
@@ -2719,23 +2951,23 @@ dependencies = [
  "pest",
  "pest_consume",
  "pest_derive",
- "rand",
+ "rand 0.8.5",
  "regex",
  "serde",
  "serde_json",
  "strum 0.23.0",
- "strum_macros",
+ "strum_macros 0.23.1",
  "thiserror 1.0.69",
  "url",
 ]
 
 [[package]]
 name = "rustix"
-version = "0.38.42"
+version = "0.38.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -2762,21 +2994,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
 dependencies = [
  "log",
- "ring 0.17.8",
+ "ring 0.17.10",
  "rustls-webpki 0.101.7",
  "sct 0.7.1",
 ]
 
 [[package]]
 name = "rustls"
-version = "0.23.20"
+version = "0.23.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b"
+checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395"
 dependencies = [
  "aws-lc-rs",
  "log",
  "once_cell",
- "ring 0.17.8",
+ "ring 0.17.10",
  "rustls-pki-types",
  "rustls-webpki 0.102.8",
  "subtle",
@@ -2805,7 +3037,7 @@ dependencies = [
  "openssl-probe",
  "rustls-pki-types",
  "schannel",
- "security-framework 3.1.0",
+ "security-framework 3.2.0",
 ]
 
 [[package]]
@@ -2828,9 +3060,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37"
+checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
 
 [[package]]
 name = "rustls-webpki"
@@ -2838,7 +3070,7 @@ version = "0.101.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
 dependencies = [
- "ring 0.17.8",
+ "ring 0.17.10",
  "untrusted 0.9.0",
 ]
 
@@ -2849,22 +3081,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
 dependencies = [
  "aws-lc-rs",
- "ring 0.17.8",
+ "ring 0.17.10",
  "rustls-pki-types",
  "untrusted 0.9.0",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.18"
+version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
+checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
 
 [[package]]
 name = "ryu"
-version = "1.0.18"
+version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
 
 [[package]]
 name = "same-file"
@@ -2906,7 +3138,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
 dependencies = [
- "ring 0.17.8",
+ "ring 0.17.10",
  "untrusted 0.9.0",
 ]
 
@@ -2929,7 +3161,7 @@ version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "core-foundation 0.9.4",
  "core-foundation-sys",
  "libc",
@@ -2938,11 +3170,11 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "3.1.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc"
+checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
 dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
  "core-foundation 0.10.0",
  "core-foundation-sys",
  "libc",
@@ -2951,9 +3183,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.13.0"
+version = "2.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5"
+checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -2961,15 +3193,15 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba"
+checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03"
 
 [[package]]
 name = "serde"
-version = "1.0.216"
+version = "1.0.218"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
+checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60"
 dependencies = [
  "serde_derive",
 ]
@@ -2985,22 +3217,22 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.216"
+version = "1.0.218"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
+checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.133"
+version = "1.0.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
+checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6"
 dependencies = [
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "itoa",
  "memchr",
  "ryu",
@@ -3024,7 +3256,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3041,15 +3273,15 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.11.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817"
+checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
 dependencies = [
  "base64 0.22.1",
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "serde",
  "serde_derive",
  "serde_json",
@@ -3059,14 +3291,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.11.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d"
+checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3075,7 +3307,7 @@ version = "0.9.34+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
 dependencies = [
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "itoa",
  "ryu",
  "serde",
@@ -3137,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31"
 dependencies = [
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "signature",
  "zeroize",
 ]
@@ -3149,7 +3381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -3158,6 +3390,18 @@ version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
 
+[[package]]
+name = "simple_asn1"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
+dependencies = [
+ "num-bigint",
+ "num-traits",
+ "thiserror 2.0.11",
+ "time",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -3169,9 +3413,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.13.2"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
 
 [[package]]
 name = "socket2"
@@ -3189,12 +3433,6 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
-[[package]]
-name = "spin"
-version = "0.9.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
-
 [[package]]
 name = "spki"
 version = "0.7.3"
@@ -3240,6 +3478,12 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
 
+[[package]]
+name = "strum"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
+
 [[package]]
 name = "strum_macros"
 version = "0.23.1"
@@ -3253,6 +3497,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "strum_macros"
+version = "0.25.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.98",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -3272,9 +3529,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.90"
+version = "2.0.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
+checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3289,7 +3546,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3321,13 +3578,13 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tempfile"
-version = "3.15.0"
+version = "3.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
+checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230"
 dependencies = [
  "cfg-if",
  "fastrand",
- "getrandom",
+ "getrandom 0.3.1",
  "once_cell",
  "rustix",
  "windows-sys 0.59.0",
@@ -3335,9 +3592,9 @@ dependencies = [
 
 [[package]]
 name = "termtree"
-version = "0.4.1"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
+checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
 
 [[package]]
 name = "test-files"
@@ -3350,6 +3607,16 @@ dependencies = [
  "touch",
 ]
 
+[[package]]
+name = "textnonce"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f8d70cd784ed1dc33106a18998d77758d281dc40dc3e6d050cf0f5286683"
+dependencies = [
+ "base64 0.12.3",
+ "rand 0.7.3",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -3361,11 +3628,11 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.9"
+version = "2.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
 dependencies = [
- "thiserror-impl 2.0.9",
+ "thiserror-impl 2.0.11",
 ]
 
 [[package]]
@@ -3382,18 +3649,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.9"
+version = "2.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3427,6 +3694,15 @@ dependencies = [
  "time-core",
 ]
 
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.7.6"
@@ -3439,9 +3715,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
+checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -3454,9 +3730,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.42.0"
+version = "1.43.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
+checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
 dependencies = [
  "backtrace",
  "bytes",
@@ -3472,13 +3748,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3488,7 +3764,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
 dependencies = [
  "pin-project",
- "rand",
+ "rand 0.8.5",
  "tokio",
 ]
 
@@ -3519,7 +3795,7 @@ version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
 dependencies = [
- "rustls 0.23.20",
+ "rustls 0.23.23",
  "tokio",
 ]
 
@@ -3549,8 +3825,8 @@ dependencies = [
  "futures-sink",
  "http 1.2.0",
  "httparse",
- "rand",
- "ring 0.17.8",
+ "rand 0.8.5",
+ "ring 0.17.10",
  "rustls-native-certs 0.8.1",
  "rustls-pki-types",
  "tokio",
@@ -3593,7 +3869,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3635,9 +3911,9 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.17.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
+checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
 
 [[package]]
 name = "ucd-trie"
@@ -3659,9 +3935,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.14"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
+checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
 
 [[package]]
 name = "unicode-normalization"
@@ -3762,7 +4038,7 @@ dependencies = [
  "serde_with",
  "strum 0.24.1",
  "tempfile",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "tokio",
  "tokio-retry",
  "url",
@@ -3774,7 +4050,7 @@ version = "5.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "435c6f69ef38c9017b4b4eea965dfb91e71e53d869e896db40d1cf2441dd75c0"
 dependencies = [
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "serde",
  "serde_json",
  "utoipa-gen",
@@ -3789,7 +4065,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -3819,11 +4095,11 @@ checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d"
 
 [[package]]
 name = "uuid"
-version = "1.11.0"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
+checksum = "93d59ca99a559661b96bf898d8fce28ed87935fd2bea9f05983c1464dd6c71b1"
 dependencies = [
- "getrandom",
+ "getrandom 0.3.1",
  "serde",
 ]
 
@@ -3835,9 +4111,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "wait-timeout"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11"
 dependencies = [
  "libc",
 ]
@@ -3861,12 +4137,27 @@ dependencies = [
  "try-lock",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasi"
+version = "0.13.3+wasi-0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
 [[package]]
 name = "wasite"
 version = "0.1.0"
@@ -3875,34 +4166,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.99"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
+checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
 dependencies = [
  "cfg-if",
  "once_cell",
+ "rustversion",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.99"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
+checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
 dependencies = [
  "bumpalo",
  "log",
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.99"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
+checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -3910,28 +4202,31 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.99"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
+checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.99"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
+checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
+dependencies = [
+ "unicode-ident",
+]
 
 [[package]]
 name = "web-sys"
-version = "0.3.76"
+version = "0.3.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
+checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3964,9 +4259,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
 [[package]]
 name = "webpki-roots"
-version = "0.26.7"
+version = "0.26.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e"
+checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -4241,6 +4536,15 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+dependencies = [
+ "bitflags 2.8.0",
+]
+
 [[package]]
 name = "wmi"
 version = "0.12.2"
@@ -4261,6 +4565,7 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-nats",
+ "async-trait",
  "bson",
  "bytes",
  "chrono",
@@ -4270,11 +4575,13 @@ dependencies = [
  "log",
  "mongodb",
  "nkeys",
- "rand",
+ "rand 0.8.5",
  "semver",
  "serde",
  "serde_json",
- "thiserror 2.0.9",
+ "strum 0.25.0",
+ "strum_macros 0.25.3",
+ "thiserror 2.0.11",
  "tokio",
  "url",
  "util_libs",
@@ -4321,7 +4628,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
  "synstructure",
 ]
 
@@ -4343,7 +4650,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -4363,7 +4670,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
  "synstructure",
 ]
 
@@ -4392,7 +4699,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.90",
+ "syn 2.0.98",
 ]
 
 [[package]]
@@ -4406,9 +4713,9 @@ dependencies = [
  "crossbeam-utils",
  "displaydoc",
  "flate2",
- "indexmap 2.7.0",
+ "indexmap 2.7.1",
  "memchr",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "zopfli",
 ]
 
@@ -4428,27 +4735,27 @@ dependencies = [
 
 [[package]]
 name = "zstd"
-version = "0.13.2"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "7.2.1"
+version = "7.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
+checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722"
 dependencies = [
  "zstd-sys",
 ]
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.13+zstd.1.5.6"
+version = "2.0.14+zstd.1.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
+checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5"
 dependencies = [
  "cc",
  "pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index d6b664c..faf6e0b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,9 +6,9 @@ metadata.crane.name = "holo-host-workspace"
 members = [
     "rust/hpos-hal",
     "rust/clients/host_agent",
+    "rust/clients/orchestrator",
     "rust/services/workload",
     "rust/util_libs",
-    "rust/orchestrator",
     "rust/netdiag",
 ]
 
diff --git a/README.md b/README.md
index 8914767..be84d2e 100644
--- a/README.md
+++ b/README.md
@@ -22,14 +22,11 @@ The code is grouped by language or framework name.
 /Cargo.toml
 /Cargo.lock
 /rust/ # all rust code lives here
-/rust/common/Cargo.toml
-/rust/common/src/lib.rs
-/rust/holo-agentctl/Cargo.toml
-/rust/holo-agentctl/src/main.rs
-/rust/holo-agentd/Cargo.toml
-/rust/holo-agentd/src/main.rs
-/rust/holo-hqd/Cargo.toml
-/rust/holo-hqd/src/main.rs
+/rust/clients/
+/rust/services/
+/rust/hpos-hal/
+/rust/netdiag/
+/rust/util_libs/
 ```
 
 ### Pulumi for Infrastructure-as-Code
diff --git a/flake.lock b/flake.lock
index 453d91d..aeee488 100644
--- a/flake.lock
+++ b/flake.lock
@@ -463,4 +463,4 @@
   },
   "root": "root",
   "version": 7
-}
+}
\ No newline at end of file
diff --git a/flake.nix b/flake.nix
index dcecf64..2d0c191 100644
--- a/flake.nix
+++ b/flake.nix
@@ -35,4 +35,4 @@
       prefix = "nix/";
       nixpkgs.config.allowUnfree = true;
     };
-}
+}
\ No newline at end of file
diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index 5e63704..ccb4953 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -180,32 +180,32 @@ pkgs.testers.runNixOSTest (
 
         with subtest("start the hosts and ensure they have TCP level connectivity to the hub"):
           host1.start()
-          host2.start()
-          host3.start()
-          host4.start()
-          host5.start()
+          # host2.start()
+          # host3.start()
+          # host4.start()
+          # host5.start()
 
           host1.wait_for_open_port(addr = "${nodes.hub.networking.fqdn}", port = ${builtins.toString nodes.hub.holo.nats-server.websocket.externalPort}, timeout = 10)
 
           host1.wait_for_unit('holo-host-agent')
-          host2.wait_for_unit('holo-host-agent')
-          host3.wait_for_unit('holo-host-agent')
-          host4.wait_for_unit('holo-host-agent')
-          host5.wait_for_unit('holo-host-agent')
+          # host2.wait_for_unit('holo-host-agent')
+          # host3.wait_for_unit('holo-host-agent')
+          # host4.wait_for_unit('holo-host-agent')
+          # host5.wait_for_unit('holo-host-agent')
 
         with subtest("running the setup script on the hosts"):
           host1.succeed("${hostSetupScript}", timeout = 1)
-          host2.succeed("${hostSetupScript}", timeout = 1)
-          host3.succeed("${hostSetupScript}", timeout = 1)
-          host4.succeed("${hostSetupScript}", timeout = 1)
-          host5.succeed("${hostSetupScript}", timeout = 1)
+          # host2.succeed("${hostSetupScript}", timeout = 1)
+          # host3.succeed("${hostSetupScript}", timeout = 1)
+          # host4.succeed("${hostSetupScript}", timeout = 1)
+          # host5.succeed("${hostSetupScript}", timeout = 1)
 
         with subtest("wait until all hosts receive all published messages"):
           host1.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          host2.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          host3.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          host4.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          host5.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          # host2.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          # host3.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          # host4.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          # host5.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
 
         with subtest("publish more messages from the hub and ensure they arrive on all hosts"):
           hub.succeed("${pkgs.writeShellScript "script" ''
@@ -216,29 +216,29 @@ pkgs.testers.runNixOSTest (
           ''}", timeout = 1)
 
           host1.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host1' --count=10''}", timeout = 5)
-          host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
-          host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
-          host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
-          host5.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
-
-        with subtest("bring a host down, publish messages, bring it back up, make sure it receives all messages"):
-          host5.shutdown()
-
-          hub.succeed("${pkgs.writeShellScript "script" ''
-            set -xeE
-            for i in `seq 1 5`; do
-              ${natsCmdHub} pub --count=10 "${testStreamName}.host''${i}" --js-domain ${hubJsDomain} "{\"message\":\"hello host''${i}\"}"
-            done
-          ''}", timeout = 2)
-
-          host1.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host1' --count=10''}", timeout = 5)
-          host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
-          host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
-          host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
-
-          host5.start()
-          host5.wait_for_unit('holo-host-agent')
-          host5.wait_until_succeeds("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
+          # host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
+          # host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
+          # host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
+          # host5.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
+
+        # with subtest("bring a host down, publish messages, bring it back up, make sure it receives all messages"):
+        #   host5.shutdown()
+
+        #   hub.succeed("${pkgs.writeShellScript "script" ''
+        #     set -xeE
+        #     for i in `seq 1 5`; do
+        #       ${natsCmdHub} pub --count=10 "${testStreamName}.host''${i}" --js-domain ${hubJsDomain} "{\"message\":\"hello host''${i}\"}"
+        #     done
+        #   ''}", timeout = 2)
+
+        #   host1.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host1' --count=10''}", timeout = 5)
+        #   host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
+        #   host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
+        #   host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
+
+        #   host5.start()
+        #   host5.wait_for_unit('holo-host-agent')
+        #   host5.wait_until_succeeds("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
       '';
   }
 )
diff --git a/nix/lib/default.nix b/nix/lib/default.nix
index 3c1aa40..f2cd411 100644
--- a/nix/lib/default.nix
+++ b/nix/lib/default.nix
@@ -1,4 +1,8 @@
-{ inputs, flake, ... }:
+{
+  inputs,
+  flake,
+  ...
+}:
 
 {
   mkCraneLib =
diff --git a/nix/modules/nixos/holo-nats-server.nix b/nix/modules/nixos/holo-nats-server.nix
index 5db4af6..9744ed4 100644
--- a/nix/modules/nixos/holo-nats-server.nix
+++ b/nix/modules/nixos/holo-nats-server.nix
@@ -125,4 +125,4 @@ in
       }
     );
   };
-}
+}
\ No newline at end of file
diff --git a/rust/clients/host_agent/Cargo.toml b/rust/clients/host_agent/Cargo.toml
index 6b1c4aa..71d144a 100644
--- a/rust/clients/host_agent/Cargo.toml
+++ b/rust/clients/host_agent/Cargo.toml
@@ -14,17 +14,22 @@ log = { workspace = true }
 dotenv = { workspace = true }
 clap = { workspace = true }
 thiserror = { workspace = true }
+env_logger = { workspace = true }
 url = { version = "2", features = ["serde"] }
 bson = { version = "2.6.1", features = ["chrono-0_4"] }
-env_logger = { workspace = true }
-mongodb = "3.1"
+ed25519-dalek = { version = "2.1.1" }
+nkeys = "=0.4.4"
+sha2 = "=0.10.8"
+nats-jwt = "0.3.0"
+data-encoding = "2.7.0"
+jsonwebtoken = "9.3.0"
+textnonce = "1.0.0"
 chrono = "0.4.0"
 bytes = "1.8.0"
-nkeys = "=0.4.4"
 rand = "0.8.5"
+tempfile = "3.15.0"
+machineid-rs = "1.2.4"
 util_libs = { path = "../../util_libs" }
 workload = { path = "../../services/workload" }
 hpos-hal = { path = "../../hpos-hal" }
 netdiag = { path = "../../netdiag" }
-tempfile = "3.15.0"
-machineid-rs = "1.2.4"
diff --git a/rust/clients/host_agent/src/gen_leaf_server.rs b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
similarity index 91%
rename from rust/clients/host_agent/src/gen_leaf_server.rs
rename to rust/clients/host_agent/src/hostd/gen_leaf_server.rs
index 61dbd4d..38f36da 100644
--- a/rust/clients/host_agent/src/gen_leaf_server.rs
+++ b/rust/clients/host_agent/src/hostd/gen_leaf_server.rs
@@ -2,12 +2,13 @@ use std::{path::PathBuf, time::Duration};
 
 use anyhow::Context;
 use tempfile::tempdir;
-use util_libs::{
-    nats_js_client,
-    nats_server::{
+use util_libs::nats::{
+    jetstream_client,
+    leaf_server::{
         JetStreamConfig, LeafNodeRemote, LeafNodeRemoteTlsConfig, LeafServer, LoggingOptions,
         LEAF_SERVER_CONFIG_PATH, LEAF_SERVER_DEFAULT_LISTEN_PORT,
     },
+    types::JsClientBuilder,
 };
 
 pub async fn run(
@@ -17,7 +18,7 @@ pub async fn run(
     hub_url: String,
     hub_tls_insecure: bool,
     nats_connect_timeout_secs: u64,
-) -> anyhow::Result<nats_js_client::JsClient> {
+) -> anyhow::Result<jetstream_client::JsClient> {
     let leaf_client_conn_domain = "127.0.0.1";
     let leaf_client_conn_port = std::env::var("NATS_LISTEN_PORT")
         .map(|var| var.parse().expect("can't parse into number"))
@@ -95,23 +96,21 @@ pub async fn run(
     // Spin up Nats Client
     // Nats takes a moment to become responsive, so we try to connecti in a loop for a few seconds.
     // TODO: how do we recover from a connection loss to Nats in case it crashes or something else?
-    let nats_url = nats_js_client::get_nats_url();
+    let nats_url = jetstream_client::get_nats_url();
     log::info!("nats_url : {}", nats_url);
 
     const HOST_AGENT_CLIENT_NAME: &str = "Host Agent Bare";
 
     let nats_client = tokio::select! {
         client = async {loop {
-                let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
+                let host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
                     nats_url:nats_url.clone(),
                     name:HOST_AGENT_CLIENT_NAME.to_string(),
+                    inbox_prefix: Default::default(),
+                    credentials_path: Default::default(),
                     ping_interval:Some(Duration::from_secs(10)),
                     request_timeout:Some(Duration::from_secs(29)),
-
-                    inbox_prefix: Default::default(),
-                    service_params:Default::default(),
-                    opts: Default::default(),
-                    credentials_path: Default::default()
+                    listeners: Default::default(),
                 })
                 .await
                 .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"));
diff --git a/rust/clients/host_agent/src/hostd/mod.rs b/rust/clients/host_agent/src/hostd/mod.rs
new file mode 100644
index 0000000..3e7c23b
--- /dev/null
+++ b/rust/clients/host_agent/src/hostd/mod.rs
@@ -0,0 +1,2 @@
+pub mod gen_leaf_server;
+pub mod workload;
diff --git a/rust/clients/host_agent/src/hostd/workload.rs b/rust/clients/host_agent/src/hostd/workload.rs
new file mode 100644
index 0000000..5d23d49
--- /dev/null
+++ b/rust/clients/host_agent/src/hostd/workload.rs
@@ -0,0 +1,155 @@
+/*
+ This client is associated with the:
+    - HPOS account
+    - host user
+
+This client is responsible for subscribing to workload streams that handle:
+    - installing new workloads onto the hosting device
+    - removing workloads from the hosting device
+    - sending workload status upon request
+    - sending out active periodic workload reports
+*/
+
+use anyhow::{anyhow, Result};
+use async_nats::Message;
+use std::{path::PathBuf, sync::Arc, time::Duration};
+use util_libs::nats::{
+    jetstream_client,
+    types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
+};
+use workload::{
+    host_api::HostWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
+    WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+};
+
+const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
+const HOST_AGENT_INBOX_PREFIX: &str = "_WORKLOAD_INBOX";
+
+// TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
+pub async fn run(
+    host_pubkey: &str,
+    host_creds_path: &Option<PathBuf>,
+) -> Result<jetstream_client::JsClient, async_nats::Error> {
+    log::info!("Host Agent Client: Connecting to server...");
+    log::info!("host_creds_path : {:?}", host_creds_path);
+    log::info!("host_pubkey : {}", host_pubkey);
+
+    let pubkey_lowercase = host_pubkey.to_string().to_lowercase();
+
+    // ==================== Setup NATS ====================
+    // Connect to Nats server
+    let nats_url = jetstream_client::get_nats_url();
+    log::info!("nats_url : {}", nats_url);
+
+    let event_listeners = jetstream_client::get_event_listeners();
+
+    // Spin up Nats Client and loaded in the Js Stream Service
+    let mut host_workload_client = jetstream_client::JsClient::new(JsClientBuilder {
+        nats_url: nats_url.clone(),
+        name: HOST_AGENT_CLIENT_NAME.to_string(),
+        inbox_prefix: format!("{}.{}", HOST_AGENT_INBOX_PREFIX, pubkey_lowercase),
+        credentials_path: host_creds_path
+            .as_ref()
+            .map(|path| path.to_string_lossy().to_string()),
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(29)),
+        listeners: vec![jetstream_client::with_event_listeners(
+            event_listeners.clone(),
+        )],
+    })
+    .await
+    .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
+
+    // ==================== Setup JS Stream Service ====================
+    // Instantiate the Workload API
+    let workload_api = HostWorkloadApi::default();
+
+    // Register Workload Streams for Host Agent to consume
+    // NB: Subjects are published by orchestrator or nats-db-connector
+    let workload_stream_service = JsServiceBuilder {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+    host_workload_client
+        .add_js_service(workload_stream_service)
+        .await?;
+
+    let workload_service = host_workload_client
+        .get_js_service(WORKLOAD_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate workload service. Unable to run holo agent workload service."
+        ))?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "install_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Install.as_ref()
+            ),
+            handler: EndpointType::Async(
+                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.install_workload(msg).await
+                }),
+            ),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "update_installed_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::UpdateInstalled.as_ref()
+            ),
+            handler: EndpointType::Async(
+                workload_api.call(|api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.update_workload(msg).await
+                }),
+            ),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "uninstall_workload".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::Uninstall.as_ref()
+            ),
+            handler: EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.uninstall_workload(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "send_workload_status".to_string(),
+            endpoint_subject: format!(
+                "{}.{}",
+                pubkey_lowercase,
+                WorkloadServiceSubjects::SendStatus.as_ref()
+            ),
+            handler: EndpointType::Async(workload_api.call(
+                |api: HostWorkloadApi, msg: Arc<Message>| async move {
+                    api.send_workload_status(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    Ok(host_workload_client)
+}
diff --git a/rust/clients/host_agent/src/main.rs b/rust/clients/host_agent/src/main.rs
index 1495a47..6a67832 100644
--- a/rust/clients/host_agent/src/main.rs
+++ b/rust/clients/host_agent/src/main.rs
@@ -1,7 +1,7 @@
 /*
 This client is associated with the:
-  - WORKLOAD account
-  - hpos user
+  - HPOS account
+  - host user
 
 This client is responsible for subscribing the host agent to workload stream endpoints:
   - installing new workloads
@@ -10,15 +10,14 @@ This client is responsible for subscribing the host agent to workload stream end
   - sending workload status upon request
 */
 
-mod workload_manager;
+pub mod agent_cli;
+pub mod host_cmds;
+mod hostd;
+pub mod support_cmds;
 use agent_cli::DaemonzeArgs;
 use anyhow::Result;
 use clap::Parser;
 use dotenv::dotenv;
-pub mod agent_cli;
-pub mod gen_leaf_server;
-pub mod host_cmds;
-pub mod support_cmds;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -48,8 +47,8 @@ async fn main() -> Result<(), AgentCliError> {
 }
 
 async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
-    // let (host_pubkey, host_creds_path) = auth::initializer::run().await?;
-    let bare_client = gen_leaf_server::run(
+    // let host_pubkey = auth::init_agent::run().await?;
+    let bare_client = hostd::gen_leaf_server::run(
         &args.nats_leafnode_server_name,
         &args.nats_leafnode_client_creds_path,
         &args.store_dir,
@@ -61,7 +60,7 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // TODO: would it be a good idea to reuse this client in the workload_manager and elsewhere later on?
     bare_client.close().await?;
 
-    let host_client = workload_manager::run(
+    let host_workload_client = hostd::workload::run(
         "host_id_placeholder>",
         &args.nats_leafnode_client_creds_path,
     )
@@ -70,6 +69,8 @@ async fn daemonize(args: &DaemonzeArgs) -> Result<(), async_nats::Error> {
     // Only exit program when explicitly requested
     tokio::signal::ctrl_c().await?;
 
-    host_client.close().await?;
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    host_workload_client.close().await?;
+
     Ok(())
 }
diff --git a/rust/clients/host_agent/src/workload_manager.rs b/rust/clients/host_agent/src/workload_manager.rs
deleted file mode 100644
index dd741d6..0000000
--- a/rust/clients/host_agent/src/workload_manager.rs
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- This client is associated with the:
-- WORKLOAD account
-- hpos user
-
-// This client is responsible for:
-  - subscribing to workload streams
-    - installing new workloads
-    - removing workloads
-    - sending workload status upon request
-    - sending active periodic workload reports
-*/
-
-use anyhow::{anyhow, Result};
-use async_nats::Message;
-use mongodb::{options::ClientOptions, Client as MongoDBClient};
-use std::{path::PathBuf, sync::Arc, time::Duration};
-use util_libs::{
-    db::mongodb::get_mongodb_url,
-    js_stream_service::JsServiceParamsPartial,
-    nats_js_client::{self, EndpointType},
-};
-use workload::{
-    WorkloadApi, WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
-};
-
-const HOST_AGENT_CLIENT_NAME: &str = "Host Agent";
-const HOST_AGENT_INBOX_PREFIX: &str = "_host_inbox";
-
-// TODO: Use _host_creds_path for auth once we add in the more resilient auth pattern.
-pub async fn run(
-    host_pubkey: &str,
-    host_creds_path: &Option<PathBuf>,
-) -> Result<nats_js_client::JsClient, async_nats::Error> {
-    log::info!("HPOS Agent Client: Connecting to server...");
-    log::info!("host_creds_path : {:?}", host_creds_path);
-    log::info!("host_pubkey : {}", host_pubkey);
-
-    // ==================== NATS Setup ====================
-    // Connect to Nats server
-    let nats_url = nats_js_client::get_nats_url();
-    log::info!("nats_url : {}", nats_url);
-
-    let event_listeners = nats_js_client::get_event_listeners();
-
-    // Setup JS Stream Service
-    let workload_stream_service_params = JsServiceParamsPartial {
-        name: WORKLOAD_SRV_NAME.to_string(),
-        description: WORKLOAD_SRV_DESC.to_string(),
-        version: WORKLOAD_SRV_VERSION.to_string(),
-        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
-    };
-
-    // Spin up Nats Client and loaded in the Js Stream Service
-    let host_workload_client = nats_js_client::JsClient::new(nats_js_client::NewJsClientParams {
-        nats_url: nats_url.clone(),
-        name: HOST_AGENT_CLIENT_NAME.to_string(),
-        inbox_prefix: format!("{}_{}", HOST_AGENT_INBOX_PREFIX, host_pubkey),
-        service_params: vec![workload_stream_service_params.clone()],
-        credentials_path: host_creds_path
-            .as_ref()
-            .map(|path| path.to_string_lossy().to_string()),
-        opts: vec![nats_js_client::with_event_listeners(
-            event_listeners.clone(),
-        )],
-        ping_interval: Some(Duration::from_secs(10)),
-        request_timeout: Some(Duration::from_secs(29)),
-    })
-    .await
-    .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
-
-    // ==================== DB Setup ====================
-    // Create a new MongoDB Client and connect it to the cluster
-    let mongo_uri = get_mongodb_url();
-    let client_options = ClientOptions::parse(mongo_uri).await?;
-    let client = MongoDBClient::with_options(client_options)?;
-
-    // Generate the Workload API with access to db
-    let workload_api = WorkloadApi::new(&client).await?;
-
-    // ==================== API ENDPOINTS ====================
-    // Register Workload Streams for Host Agent to consume
-    // NB: Subjects are published by orchestrator or nats-db-connector
-    let workload_service = host_workload_client
-        .get_js_service(WORKLOAD_SRV_NAME.to_string())
-        .await
-        .ok_or(anyhow!(
-            "Failed to locate workload service. Unable to spin up Host Agent."
-        ))?;
-
-    workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "start_workload",
-            "start",
-            EndpointType::Async(workload_api.call(
-                |api: WorkloadApi, msg: Arc<Message>| async move { api.start_workload(msg).await },
-            )),
-            None,
-        )
-        .await?;
-
-    workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "send_workload_status",
-            "send_status",
-            EndpointType::Async(
-                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.send_workload_status(msg).await
-                }),
-            ),
-            None,
-        )
-        .await?;
-
-    workload_service
-        .add_local_consumer::<workload::types::ApiResult>(
-            "uninstall_workload",
-            "uninstall",
-            EndpointType::Async(
-                workload_api.call(|api: WorkloadApi, msg: Arc<Message>| async move {
-                    api.uninstall_workload(msg).await
-                }),
-            ),
-            None,
-        )
-        .await?;
-
-    Ok(host_workload_client)
-}
diff --git a/rust/clients/orchestrator/Cargo.toml b/rust/clients/orchestrator/Cargo.toml
new file mode 100644
index 0000000..c961b0c
--- /dev/null
+++ b/rust/clients/orchestrator/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "orchestrator"
+version = "0.0.1"
+edition = "2021"
+
+[dependencies]
+async-nats = { workspace = true }
+anyhow = { workspace = true }
+tokio = { workspace = true }
+futures = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+log = { workspace = true }
+dotenv = { workspace = true }
+thiserror = { workspace = true }
+url = { version = "2", features = ["serde"] }
+bson = { version = "2.6.1", features = ["chrono-0_4"] }
+env_logger = { workspace = true }
+mongodb = "3.1"
+chrono = "0.4.0"
+bytes = "1.8.0"
+nkeys = "=0.4.4"
+rand = "0.8.5"
+actix-web = "4"
+utoipa-swagger-ui = { version = "9", features = [
+    "actix-web",
+    # Enables vendored Swagger UI via utoipa-swagger-ui-vendored crate.
+    "vendored"
+]}
+utoipa = { version = "5", features = ["actix_extras"] }
+util_libs = { path = "../../util_libs" }
+workload = { path = "../../services/workload" }
diff --git a/rust/orchestrator/src/hello.rs b/rust/clients/orchestrator/src/extern_api/example.rs
similarity index 100%
rename from rust/orchestrator/src/hello.rs
rename to rust/clients/orchestrator/src/extern_api/example.rs
diff --git a/rust/orchestrator/src/main.rs b/rust/clients/orchestrator/src/extern_api/mod.rs
old mode 100755
new mode 100644
similarity index 92%
rename from rust/orchestrator/src/main.rs
rename to rust/clients/orchestrator/src/extern_api/mod.rs
index 9add73b..6c7ecef
--- a/rust/orchestrator/src/main.rs
+++ b/rust/clients/orchestrator/src/extern_api/mod.rs
@@ -3,10 +3,11 @@ use mongodb::Client;
 use std::env;
 use utoipa::{openapi::Server, OpenApi};
 use utoipa_swagger_ui::{SwaggerUi, Url};
-mod hello;
+
+mod example;
 
 #[derive(OpenApi)]
-#[openapi(paths(hello::hello))]
+#[openapi(paths(example::hello))]
 struct ApiDoc;
 
 #[get("/api-docs/json")]
@@ -19,6 +20,7 @@ async fn docs() -> impl Responder {
         .body(docs.to_pretty_json().unwrap())
 }
 
+#[allow(dead_code)]
 async fn db() -> mongodb::error::Result<()> {
     let connection_uri = env::var("DB_CONNECTION_STRING").unwrap();
     let client = Client::with_uri_str(connection_uri).await?;
@@ -38,7 +40,7 @@ async fn main() -> std::io::Result<()> {
                 SwaggerUi::new("/api-docs/ui/{_:.*}")
                     .url(Url::new("api1", "/api-docs/json"), ApiDoc::openapi()),
             )
-            .service(hello::hello)
+            .service(example::hello)
             .service(docs)
     })
     .bind(("0.0.0.0", 3000))?
diff --git a/rust/clients/orchestrator/src/main.rs b/rust/clients/orchestrator/src/main.rs
new file mode 100644
index 0000000..19aa705
--- /dev/null
+++ b/rust/clients/orchestrator/src/main.rs
@@ -0,0 +1,18 @@
+mod extern_api;
+mod utils;
+mod workloads;
+use anyhow::Result;
+use dotenv::dotenv;
+
+#[tokio::main]
+async fn main() -> Result<(), async_nats::Error> {
+    dotenv().ok();
+    env_logger::init();
+    // Run auth service
+    // TODO: invoke auth service (once ready)
+
+    // Run workload service
+    workloads::run().await?;
+
+    Ok(())
+}
diff --git a/rust/clients/orchestrator/src/utils.rs b/rust/clients/orchestrator/src/utils.rs
new file mode 100644
index 0000000..5694213
--- /dev/null
+++ b/rust/clients/orchestrator/src/utils.rs
@@ -0,0 +1,24 @@
+use std::{collections::HashMap, sync::Arc};
+use util_libs::nats::types::ResponseSubjectsGenerator;
+
+pub fn create_callback_subject_to_host(
+    is_prefix: bool,
+    tag_name: String,
+    sub_subject_name: String,
+) -> ResponseSubjectsGenerator {
+    Arc::new(move |tag_map: HashMap<String, String>| -> Vec<String> {
+        if is_prefix {
+            let matching_tags = tag_map.into_iter().fold(vec![], |mut acc, (k, v)| {
+                if k.starts_with(&tag_name) {
+                    acc.push(v)
+                }
+                acc
+            });
+            return matching_tags;
+        } else if let Some(tag) = tag_map.get(&tag_name) {
+            return vec![format!("{}.{}", tag, sub_subject_name)];
+        }
+        log::error!("WORKLOAD Error: Failed to find {}. Unable to send orchestrator response to hosting agent for subject {}. Fwding response to `WORKLOAD.ERROR.INBOX`.", tag_name, sub_subject_name);
+        vec!["WORKLOAD.ERROR.INBOX".to_string()]
+    })
+}
diff --git a/rust/clients/orchestrator/src/workloads.rs b/rust/clients/orchestrator/src/workloads.rs
new file mode 100644
index 0000000..d9a42df
--- /dev/null
+++ b/rust/clients/orchestrator/src/workloads.rs
@@ -0,0 +1,183 @@
+/*
+This client is associated with the:
+    - ADMIN account
+    - admin user
+
+This client is responsible for:
+    - initalizing connection and handling interface with db
+    - registering with the host worklload service to:
+        - handling requests to add workloads
+        - handling requests to update workloads
+        - handling requests to remove workloads
+        - handling workload status updates
+    - interfacing with mongodb DB
+    - keeping service running until explicitly cancelled out
+*/
+
+use super::utils;
+use anyhow::{anyhow, Result};
+use async_nats::Message;
+use mongodb::{options::ClientOptions, Client as MongoDBClient};
+use std::{sync::Arc, time::Duration};
+use util_libs::{
+    db::mongodb::get_mongodb_url,
+    nats::{
+        jetstream_client::{self, JsClient},
+        types::{ConsumerBuilder, EndpointType, JsClientBuilder, JsServiceBuilder},
+    },
+};
+use workload::{
+    orchestrator_api::OrchestratorWorkloadApi, types::WorkloadServiceSubjects, WorkloadServiceApi,
+    WORKLOAD_SRV_DESC, WORKLOAD_SRV_NAME, WORKLOAD_SRV_SUBJ, WORKLOAD_SRV_VERSION,
+};
+
+const ORCHESTRATOR_WORKLOAD_CLIENT_NAME: &str = "Orchestrator Workload Agent";
+const ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX: &str = "ORCHESTRATOR._WORKLOAD_INBOX";
+
+pub async fn run() -> Result<(), async_nats::Error> {
+    // ==================== Setup NATS ====================
+    let nats_url = jetstream_client::get_nats_url();
+    let creds_path = jetstream_client::get_nats_client_creds("HOLO", "WORKLOAD", "orchestrator");
+    let event_listeners = jetstream_client::get_event_listeners();
+
+    let mut orchestrator_workload_client = JsClient::new(JsClientBuilder {
+        nats_url,
+        name: ORCHESTRATOR_WORKLOAD_CLIENT_NAME.to_string(),
+        inbox_prefix: ORCHESTRATOR_WORKLOAD_CLIENT_INBOX_PREFIX.to_string(),
+        credentials_path: Some(creds_path),
+        ping_interval: Some(Duration::from_secs(10)),
+        request_timeout: Some(Duration::from_secs(5)),
+        listeners: vec![jetstream_client::with_event_listeners(event_listeners)],
+    })
+    .await?;
+
+    // ==================== Setup DB ====================
+    // Create a new MongoDB Client and connect it to the cluster
+    let mongo_uri = get_mongodb_url();
+    let client_options = ClientOptions::parse(mongo_uri).await?;
+    let client = MongoDBClient::with_options(client_options)?;
+
+    // ==================== Setup JS Stream Service ====================
+    // Instantiate the Workload API (requires access to db client)
+    let workload_api = OrchestratorWorkloadApi::new(&client).await?;
+
+    let workload_stream_service = JsServiceBuilder {
+        name: WORKLOAD_SRV_NAME.to_string(),
+        description: WORKLOAD_SRV_DESC.to_string(),
+        version: WORKLOAD_SRV_VERSION.to_string(),
+        service_subject: WORKLOAD_SRV_SUBJ.to_string(),
+    };
+    orchestrator_workload_client
+        .add_js_service(workload_stream_service)
+        .await?;
+
+    // Register Workload Streams for Orchestrator to consume and proceess
+    // NB: These subjects are published by external Developer (via external api), the Nats-DB-Connector, or the Hosting Agent
+    let workload_service = orchestrator_workload_client
+        .get_js_service(WORKLOAD_SRV_NAME.to_string())
+        .await
+        .ok_or(anyhow!(
+            "Failed to locate Workload Service. Unable to spin up Orchestrator Workload Service."
+        ))?;
+
+    // Published by Developer
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "add_workload".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Add.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.add_workload(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "update_workload".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Update.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.update_workload(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "remove_workload".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Remove.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.remove_workload(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    // Automatically published by the Nats-DB-Connector
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "handle_db_insertion".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Insert.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.handle_db_insertion(msg).await
+                },
+            )),
+            response_subject_fn: Some(utils::create_callback_subject_to_host(
+                true,
+                "assigned_hosts".to_string(),
+                WorkloadServiceSubjects::Install.as_ref().to_string(),
+            )),
+        })
+        .await?;
+
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "handle_db_modification".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::Modify.as_ref().to_string(),
+            handler: EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.handle_db_modification(msg).await
+                },
+            )),
+            response_subject_fn: Some(utils::create_callback_subject_to_host(
+                true,
+                "assigned_hosts".to_string(),
+                WorkloadServiceSubjects::UpdateInstalled
+                    .as_ref()
+                    .to_string(),
+            )),
+        })
+        .await?;
+
+    // Published by the Host Agent
+    workload_service
+        .add_consumer(ConsumerBuilder {
+            name: "handle_status_update".to_string(),
+            endpoint_subject: WorkloadServiceSubjects::HandleStatusUpdate
+                .as_ref()
+                .to_string(),
+            handler: EndpointType::Async(workload_api.call(
+                |api: OrchestratorWorkloadApi, msg: Arc<Message>| async move {
+                    api.handle_status_update(msg).await
+                },
+            )),
+            response_subject_fn: None,
+        })
+        .await?;
+
+    // ==================== Close and Clean Client ====================
+    // Only exit program when explicitly requested
+    tokio::signal::ctrl_c().await?;
+
+    // Close client and drain internal buffer before exiting to make sure all messages are sent
+    orchestrator_workload_client.close().await?;
+    Ok(())
+}
diff --git a/rust/orchestrator/Cargo.toml b/rust/orchestrator/Cargo.toml
deleted file mode 100755
index 2ea58f7..0000000
--- a/rust/orchestrator/Cargo.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-[package]
-name = "actix-test"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-actix-web = "4"
-utoipa = { version = "5", features = ["actix_extras"] }
-utoipa-swagger-ui = { version = "9", features = [
-    "actix-web",
-
-    # Enables vendored Swagger UI via utoipa-swagger-ui-vendored crate.
-    "vendored"
-]}
-serde = "1.0.188"
-futures = "0.3.28"
-tokio = {version = "1.32.0", features = ["full"]}
-
-[dependencies.mongodb]
-version = "3.1.0"
diff --git a/rust/services/workload/Cargo.toml b/rust/services/workload/Cargo.toml
index abc7708..60dfe59 100644
--- a/rust/services/workload/Cargo.toml
+++ b/rust/services/workload/Cargo.toml
@@ -14,6 +14,9 @@ env_logger = { workspace = true }
 log = { workspace = true }
 dotenv = { workspace = true }
 thiserror = { workspace = true }
+strum = "0.25"
+strum_macros = "0.25"
+async-trait = "0.1.83"
 semver = "1.0.24"
 rand = "0.8.5"
 mongodb = "3.1"
diff --git a/rust/services/workload/src/host_api.rs b/rust/services/workload/src/host_api.rs
new file mode 100644
index 0000000..00354ea
--- /dev/null
+++ b/rust/services/workload/src/host_api.rs
@@ -0,0 +1,169 @@
+/*
+Endpoints & Managed Subjects:
+    - `install_workload`: handles the "WORKLOAD.<host_pukey>.install." subject
+    - `update_workload`: handles the "WORKLOAD.<host_pukey>.update_installed" subject
+    - `uninstall_workload`: handles the "WORKLOAD.<host_pukey>.uninstall." subject
+    - `send_workload_status`: handles the "WORKLOAD.<host_pukey>.send_status" subject
+*/
+
+use crate::types::WorkloadResult;
+
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
+use anyhow::Result;
+use async_nats::Message;
+use core::option::Option::None;
+use std::{fmt::Debug, sync::Arc};
+use util_libs::{
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::ServiceError,
+};
+
+#[derive(Debug, Clone, Default)]
+pub struct HostWorkloadApi {}
+
+impl WorkloadServiceApi for HostWorkloadApi {}
+
+impl HostWorkloadApi {
+    pub async fn install_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Running,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to install workload...
+            // eg: nix_install_with(workload)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Updating,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    pub async fn uninstall_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let message_payload = Self::convert_msg_to_type::<WorkloadResult>(msg)?;
+        log::debug!("Message payload '{}' : {:?}", msg_subject, message_payload);
+
+        let status = if let Some(workload) = message_payload.workload {
+            // TODO: Talk through with Stefan
+            // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
+            // nix_uninstall_with(workload_id)
+
+            // 2. Respond to endpoint request
+            WorkloadStatus {
+                id: workload._id,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Unknown("..".to_string()),
+            }
+        } else {
+            let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Error=No workload found in message.", msg_subject);
+            log::error!("{}", err_msg);
+            WorkloadStatus {
+                id: None,
+                desired: WorkloadState::Uninstalled,
+                actual: WorkloadState::Error(err_msg),
+            }
+        };
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    // For host agent ? or elsewhere ?
+    // TODO: Talk through with Stefan
+    pub async fn send_workload_status(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        let msg_subject = msg.subject.clone().into_string();
+        log::trace!("Incoming message for '{}'", msg_subject);
+
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
+
+        // Send updated status:
+        // NB: This will send the update to both the requester (if one exists)
+        // and will broadcast the update to for any `response_subject` address registred for the endpoint
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+}
diff --git a/rust/services/workload/src/lib.rs b/rust/services/workload/src/lib.rs
index 75dad8c..e28d4d6 100644
--- a/rust/services/workload/src/lib.rs
+++ b/rust/services/workload/src/lib.rs
@@ -1,482 +1,68 @@
 /*
 Service Name: WORKLOAD
 Subject: "WORKLOAD.>"
-Provisioning Account: WORKLOAD
-Users: orchestrator & hpos
-Endpoints & Managed Subjects:
-- `add_workload`: handles the "WORKLOAD.add" subject
-- `remove_workload`: handles the "WORKLOAD.remove" subject
-- Partial: `handle_db_change`: handles the "WORKLOAD.handle_change" subject // the stream changed output by the mongo<>nats connector (stream eg: DB_COLL_CHANGE_WORKLOAD).
-- TODO: `start_workload`: handles the "WORKLOAD.start.{{hpos_id}}" subject
-- TODO: `send_workload_status`: handles the "WORKLOAD.send_status.{{hpos_id}}" subject
-- TODO: `uninstall_workload`: handles the "WORKLOAD.uninstall.{{hpos_id}}" subject
+Provisioning Account: ADMIN
+Importing Account: HPOS
+Users: orchestrator & host
 */
 
+pub mod host_api;
+pub mod orchestrator_api;
 pub mod types;
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
+use async_nats::jetstream::ErrorCode;
 use async_nats::Message;
-use bson::oid::ObjectId;
-use bson::{doc, to_document, DateTime};
-use mongodb::{options::UpdateModifications, Client as MongoDBClient};
-use serde::{Deserialize, Serialize};
+use async_trait::async_trait;
+use core::option::Option::None;
+use serde::Deserialize;
 use std::future::Future;
-use std::{fmt::Debug, str::FromStr, sync::Arc};
+use std::{fmt::Debug, sync::Arc};
+use types::{WorkloadApiResult, WorkloadResult};
 use util_libs::{
-    db::{
-        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
-        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
-    },
-    nats_js_client,
+    db::schemas::{WorkloadState, WorkloadStatus},
+    nats::types::{AsyncEndpointHandler, JsServiceResponse, ServiceError},
 };
 
-pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD";
+pub const WORKLOAD_SRV_NAME: &str = "WORKLOAD_SERVICE";
 pub const WORKLOAD_SRV_SUBJ: &str = "WORKLOAD";
 pub const WORKLOAD_SRV_VERSION: &str = "0.0.1";
-pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and HPOS.";
-
-#[derive(Debug, Clone)]
-pub struct WorkloadApi {
-    pub workload_collection: MongoCollection<schemas::Workload>,
-    pub host_collection: MongoCollection<schemas::Host>,
-    pub user_collection: MongoCollection<schemas::User>,
-    pub developer_collection: MongoCollection<schemas::Developer>,
-}
-
-impl WorkloadApi {
-    pub async fn new(client: &MongoDBClient) -> Result<Self> {
-        Ok(Self {
-            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
-                .await?,
-            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
-            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
-            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME)
-                .await?,
-        })
-    }
-
-    pub fn call<F, Fut>(&self, handler: F) -> nats_js_client::AsyncEndpointHandler<types::ApiResult>
+pub const WORKLOAD_SRV_DESC: &str = "This service handles the flow of Workload requests between the Developer and the Orchestrator, and between the Orchestrator and Host.";
+
+#[async_trait]
+pub trait WorkloadServiceApi
+where
+    Self: std::fmt::Debug + Clone + 'static,
+{
+    fn call<F, Fut>(&self, handler: F) -> AsyncEndpointHandler<WorkloadApiResult>
     where
-        F: Fn(WorkloadApi, Arc<Message>) -> Fut + Send + Sync + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send + 'static,
+        F: Fn(Self, Arc<Message>) -> Fut + Send + Sync + 'static,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send + 'static,
+        Self: Send + Sync,
     {
         let api = self.to_owned();
         Arc::new(
-            move |msg: Arc<Message>| -> nats_js_client::JsServiceResponse<types::ApiResult> {
+            move |msg: Arc<Message>| -> JsServiceResponse<WorkloadApiResult> {
                 let api_clone = api.clone();
                 Box::pin(handler(api_clone, msg))
             },
         )
     }
 
-    /*******************************  For Orchestrator   *********************************/
-    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.add'");
-        Ok(self
-            .process_request(
-                msg,
-                WorkloadState::Reported,
-                |workload: schemas::Workload| async move {
-                    let workload_id = self
-                        .workload_collection
-                        .insert_one_into(workload.clone())
-                        .await?;
-                    log::info!(
-                        "Successfully added workload. MongodDB Workload ID={:?}",
-                        workload_id
-                    );
-                    let updated_workload = schemas::Workload {
-                        _id: Some(ObjectId::from_str(&workload_id)?),
-                        ..workload
-                    };
-                    Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: updated_workload._id.map(|oid| oid.to_hex()),
-                            desired: WorkloadState::Reported,
-                            actual: WorkloadState::Reported,
-                        },
-                        None,
-                    ))
-                },
-                WorkloadState::Error,
-            )
-            .await)
-    }
-
-    pub async fn update_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-        Ok(self
-            .process_request(
-                msg,
-                WorkloadState::Running,
-                |workload: schemas::Workload| async move {
-                    let workload_query = doc! { "_id":  workload._id };
-
-                    // update workload updated_at
-                    let mut workload_doc = workload.clone();
-                    workload_doc.metadata.updated_at = Some(DateTime::now());
-
-                    // convert workload to document and submit to mongodb
-                    let updated_workload = to_document(&workload_doc)?;
-                    self.workload_collection
-                        .update_one_within(
-                            workload_query,
-                            UpdateModifications::Document(doc! { "$set": updated_workload }),
-                        )
-                        .await?;
-
-                    log::info!(
-                        "Successfully updated workload. MongodDB Workload ID={:?}",
-                        workload._id
-                    );
-                    Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: workload._id.map(|oid| oid.to_hex()),
-                            desired: WorkloadState::Reported,
-                            actual: WorkloadState::Reported,
-                        },
-                        None,
-                    ))
-                },
-                WorkloadState::Error,
-            )
-            .await)
-    }
-
-    pub async fn remove_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.remove'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Removed,
-            |workload_id: bson::oid::ObjectId| async move {
-                let workload_query = doc! { "_id":  workload_id };
-                self.workload_collection.update_one_within(
-                    workload_query,
-                    UpdateModifications::Document(doc! {
-                        "$set": {
-                            "metadata.is_deleted": true,
-                            "metadata.deleted_at": DateTime::now()
-                        }
-                    })
-                ).await?;
-                log::info!(
-                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
-                    workload_id
-                );
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
-                        desired: WorkloadState::Removed,
-                        actual: WorkloadState::Removed,
-                    },
-                    None
-                ))
-            },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // looks through existing hosts to find possible hosts for a given workload
-    // returns the minimum number of hosts required for workload
-    pub async fn find_hosts_meeting_workload_criteria(
-        &self,
-        workload: Workload,
-    ) -> Result<Vec<Host>, anyhow::Error> {
-        let pipeline = vec![
-            doc! {
-                "$match": {
-                    // verify there are enough system resources
-                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk },
-                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
-                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
-
-                    // limit how many workloads a single host can have
-                    "assigned_workloads": { "$lt": 1 }
-                }
-            },
-            doc! {
-                // the maximum number of hosts returned should be the minimum hosts required by workload
-                // sample randomized results and always return back atleast 1 result
-                "$sample": std::cmp::min(workload.min_hosts as i32, 1)
-            },
-            doc! {
-                "$project": {
-                    "_id": 1
-                }
-            }
-        ];
-        let results = self.host_collection.aggregate(pipeline).await?;
-        if results.is_empty() {
-            anyhow::bail!(
-                "Could not find a compatible host for this workload={:#?}",
-                workload._id
-            );
-        }
-        Ok(results)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    // trigger on mongodb [workload] collection (insert)
-    pub async fn handle_db_insertion(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.insert'");
-        Ok(self.process_request(
-            msg,
-            WorkloadState::Assigned,
-            |workload: schemas::Workload| async move {
-                log::debug!("New workload to assign. Workload={:#?}", workload);
-
-                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
-                let workload_id = if let Some(id) = workload.clone()._id { id } else {
-                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
-                    return Err(anyhow!(err_msg));
-                };
-
-                // 1. Perform sanity check to ensure workload is not already assigned to a host
-                // ...and if so, exit fn
-                // todo: check for to ensure assigned host *still* has enough capacity for updated workload
-                if !workload.assigned_hosts.is_empty() {
-                    log::warn!("Attempted to assign host for new workload, but host already exists.");
-                    return Ok(types::ApiResult(
-                        WorkloadStatus {
-                            id: Some(workload_id.to_hex()),
-                            desired: WorkloadState::Assigned,
-                            actual: WorkloadState::Assigned,
-                        },
-                        Some(
-                            workload.assigned_hosts
-                            .iter().map(|id| id.to_hex()).collect())
-                        )
-                    );
-                }
-
-                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
-                let eligible_hosts = self.find_hosts_meeting_workload_criteria(workload.clone()).await?;
-                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_hosts);
-
-                let host_ids: Vec<ObjectId> = eligible_hosts.iter().map(|host| host._id.to_owned().unwrap()).collect();
-
-                // 4. Update the Workload Collection with the assigned Host ID
-                let workload_query = doc! { "_id":  workload_id };
-                let updated_workload = &Workload {
-                    assigned_hosts: host_ids.clone(),
-                    ..workload.clone()
-                };
-                let updated_workload_doc = to_document(updated_workload)?;
-                let updated_workload_result = self.workload_collection.update_one_within(workload_query, UpdateModifications::Document(updated_workload_doc)).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
-                    updated_workload_result
-                );
-
-                // 5. Update the Host Collection with the assigned Workload ID
-                let host_query = doc! { "_id":  { "$in": host_ids } };
-                let updated_host_doc =  doc! {
-                    "$push": {
-                        "assigned_workloads": workload_id
-                    }
-                };
-                let updated_host_result = self.host_collection.update_many_within(
-                    host_query, 
-                    UpdateModifications::Document(updated_host_doc)
-                ).await?;
-                log::trace!(
-                    "Successfully added new workload into the Workload Collection. MongodDB Host ID={:?}",
-                    updated_host_result
-                );
-
-                Ok(types::ApiResult(
-                    WorkloadStatus {
-                        id: Some(workload_id.to_hex()),
-                        desired: WorkloadState::Assigned,
-                        actual: WorkloadState::Assigned,
-                    },
-                    Some(
-                        updated_workload.assigned_hosts.to_owned()
-                        .iter().map(|host| host.to_hex()).collect()
-                    )
-                ))
-        },
-            WorkloadState::Error,
-        )
-        .await)
-    }
-
-    // NB: Automatically published by the nats-db-connector
-    // triggers on mongodb [workload] collection (update)
-    pub async fn handle_db_update(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.update'");
-
-        let payload_buf = msg.payload.to_vec();
-
-        let workload: schemas::Workload = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload to update. Workload={:#?}", workload.clone());
-
-        // 1. remove workloads from existing hosts
-        self.host_collection.mongo_error_handler(
-            self.host_collection
-                .collection
-                .update_many(
-                    doc! {},
-                    doc! { "$pull": { "assigned_workloads": workload._id } },
-                )
-                .await,
-        )?;
-        log::info!(
-            "Remove workload from previous hosts. Workload={:#?}",
-            workload._id
-        );
-
-        if !workload.metadata.is_deleted {
-            // 3. add workload to specific hosts
-            self.host_collection.mongo_error_handler(
-                self.host_collection
-                    .collection
-                    .update_one(
-                        doc! { "_id": { "$in": workload.clone().assigned_hosts } },
-                        doc! { "$push": { "assigned_workloads": workload._id } },
-                    )
-                    .await,
-            )?;
-            log::info!("Added workload to new hosts. Workload={:#?}", workload._id);
-        } else {
-            log::info!(
-                "Skipping (reason: deleted) - Added workload to new hosts. Workload={:#?}",
-                workload._id
-            );
-        }
-
-        let success_status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
-            desired: WorkloadState::Updating,
-            actual: WorkloadState::Updating,
-        };
-        log::info!("Workload update successful. Workload={:#?}", workload._id);
-
-        Ok(types::ApiResult(success_status, None))
-    }
-
-    // NB: Published by the Hosting Agent whenever the status of a workload changes
-    pub async fn handle_status_update(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.read_status_update'");
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status: WorkloadStatus = serde_json::from_slice(&payload_buf)?;
-        log::trace!("Workload status to update. Status={:?}", workload_status);
-        if workload_status.id.is_none() {
-            return Err(anyhow!("Got a status update for workload without an id!"));
-        }
-        let workload_status_id = workload_status
-            .id
-            .clone()
-            .expect("workload is not provided");
-
-        self.workload_collection
-            .update_one_within(
-                doc! {
-                    "_id": ObjectId::parse_str(workload_status_id)?
-                },
-                UpdateModifications::Document(doc! {
-                    "$set": {
-                        "state": bson::to_bson(&workload_status.actual)?
-                    }
-                }),
-            )
-            .await?;
-
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************   For Host Agent   *********************************/
-    pub async fn start_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.start' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload = serde_json::from_slice::<schemas::Workload>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to install workload...
-        // eg: nix_install_with(workload)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: workload._id.map(|oid| oid.to_hex()),
-            desired: WorkloadState::Running,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    pub async fn uninstall_workload(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!("Incoming message for 'WORKLOAD.uninstall' : {:?}", msg);
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_id = serde_json::from_slice::<String>(&payload_buf)?;
-
-        // TODO: Talk through with Stefan
-        // 1. Connect to interface for Nix and instruct systemd to UNinstall workload...
-        // nix_uninstall_with(workload_id)
-
-        // 2. Respond to endpoint request
-        let status = WorkloadStatus {
-            id: Some(workload_id),
-            desired: WorkloadState::Uninstalled,
-            actual: WorkloadState::Unknown("..".to_string()),
-        };
-        Ok(types::ApiResult(status, None))
-    }
-
-    // For host agent ? or elsewhere ?
-    // TODO: Talk through with Stefan
-    pub async fn send_workload_status(
-        &self,
-        msg: Arc<Message>,
-    ) -> Result<types::ApiResult, anyhow::Error> {
-        log::debug!(
-            "Incoming message for 'WORKLOAD.send_workload_status' : {:?}",
-            msg
-        );
-
-        let payload_buf = msg.payload.to_vec();
-        let workload_status = serde_json::from_slice::<WorkloadStatus>(&payload_buf)?;
-
-        // Send updated status:
-        // NB: This will send the update to both the requester (if one exists)
-        // and will broadcast the update to for any `response_subject` address registred for the endpoint
-        Ok(types::ApiResult(workload_status, None))
-    }
-
-    /*******************************  Helper Fns  *********************************/
-    // Helper function to initialize mongodb collections
-    async fn init_collection<T>(
-        client: &MongoDBClient,
-        collection_name: &str,
-    ) -> Result<MongoCollection<T>>
+    fn convert_msg_to_type<T>(msg: Arc<Message>) -> Result<T, ServiceError>
     where
-        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+        T: for<'de> Deserialize<'de> + Send + Sync,
     {
-        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+        let payload_buf = msg.payload.to_vec();
+        serde_json::from_slice::<T>(&payload_buf).map_err(|e| {
+            let err_msg = format!(
+                "Error: Failed to deserialize payload. Subject='{}' Err={}",
+                msg.subject.clone().into_string(),
+                e
+            );
+            log::error!("{}", err_msg);
+            ServiceError::Request(format!("{} Code={:?}", err_msg, ErrorCode::BAD_REQUEST))
+        })
     }
 
     // Helper function to streamline the processing of incoming workload messages
@@ -485,33 +71,21 @@ impl WorkloadApi {
         &self,
         msg: Arc<Message>,
         desired_state: WorkloadState,
-        cb_fn: impl Fn(T) -> Fut + Send + Sync,
         error_state: impl Fn(String) -> WorkloadState + Send + Sync,
-    ) -> types::ApiResult
+        cb_fn: impl Fn(T) -> Fut + Send + Sync,
+    ) -> Result<WorkloadApiResult, ServiceError>
     where
         T: for<'de> Deserialize<'de> + Clone + Send + Sync + Debug + 'static,
-        Fut: Future<Output = Result<types::ApiResult, anyhow::Error>> + Send,
+        Fut: Future<Output = Result<WorkloadApiResult, ServiceError>> + Send,
     {
         // 1. Deserialize payload into the expected type
-        let payload: T = match serde_json::from_slice(&msg.payload) {
-            Ok(r) => r,
-            Err(e) => {
-                let err_msg = format!("Failed to deserialize payload for Workload Service Endpoint. Subject={} Error={:?}", msg.subject, e);
-                log::error!("{}", err_msg);
-                let status = WorkloadStatus {
-                    id: None,
-                    desired: desired_state,
-                    actual: error_state(err_msg),
-                };
-                return types::ApiResult(status, None);
-            }
-        };
+        let payload: T = Self::convert_msg_to_type::<T>(msg.clone())?;
 
         // 2. Call callback handler
-        match cb_fn(payload.clone()).await {
+        Ok(match cb_fn(payload.clone()).await {
             Ok(r) => r,
             Err(e) => {
-                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject, payload, e);
+                let err_msg = format!("Failed to process Workload Service Endpoint. Subject={} Payload={:?}, Error={:?}", msg.subject.clone().into_string(), payload, e);
                 log::error!("{}", err_msg);
                 let status = WorkloadStatus {
                     id: None,
@@ -520,8 +94,14 @@ impl WorkloadApi {
                 };
 
                 // 3. return response for stream
-                types::ApiResult(status, None)
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                }
             }
-        }
+        })
     }
 }
diff --git a/rust/services/workload/src/orchestrator_api.rs b/rust/services/workload/src/orchestrator_api.rs
new file mode 100644
index 0000000..9db39fa
--- /dev/null
+++ b/rust/services/workload/src/orchestrator_api.rs
@@ -0,0 +1,662 @@
+/*
+Endpoints & Managed Subjects:
+    - `add_workload`: handles the "WORKLOAD.add" subject
+    - `update_workload`: handles the "WORKLOAD.update" subject
+    - `remove_workload`: handles the "WORKLOAD.remove" subject
+    - `handle_db_insertion`: handles the "WORKLOAD.insert" subject // published by mongo<>nats connector
+    - `handle_db_modification`: handles the "WORKLOAD.modify" subject // published by mongo<>nats connector
+    - `handle_status_update`: handles the "WORKLOAD.handle_status_update" subject // published by hosting agent
+*/
+
+use crate::types::WorkloadResult;
+
+use super::{types::WorkloadApiResult, WorkloadServiceApi};
+use anyhow::Result;
+use async_nats::Message;
+use bson::{self, doc, oid::ObjectId, to_document, DateTime};
+use core::option::Option::None;
+use mongodb::{options::UpdateModifications, Client as MongoDBClient};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::{HashMap, HashSet},
+    fmt::Debug,
+    sync::Arc,
+};
+use util_libs::{
+    db::{
+        mongodb::{IntoIndexes, MongoCollection, MongoDbAPI},
+        schemas::{self, Host, Workload, WorkloadState, WorkloadStatus},
+    },
+    nats::types::ServiceError,
+};
+
+#[derive(Debug, Clone)]
+pub struct OrchestratorWorkloadApi {
+    pub workload_collection: MongoCollection<schemas::Workload>,
+    pub host_collection: MongoCollection<schemas::Host>,
+    pub user_collection: MongoCollection<schemas::User>,
+    pub developer_collection: MongoCollection<schemas::Developer>,
+}
+
+impl WorkloadServiceApi for OrchestratorWorkloadApi {}
+
+impl OrchestratorWorkloadApi {
+    pub async fn new(client: &MongoDBClient) -> Result<Self> {
+        Ok(Self {
+            workload_collection: Self::init_collection(client, schemas::WORKLOAD_COLLECTION_NAME)
+                .await?,
+            host_collection: Self::init_collection(client, schemas::HOST_COLLECTION_NAME).await?,
+            user_collection: Self::init_collection(client, schemas::USER_COLLECTION_NAME).await?,
+            developer_collection: Self::init_collection(client, schemas::DEVELOPER_COLLECTION_NAME)
+                .await?,
+        })
+    }
+
+    pub async fn add_workload(&self, msg: Arc<Message>) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.add'");
+        self.process_request(
+            msg,
+            WorkloadState::Reported,
+            WorkloadState::Error,
+            |mut workload: schemas::Workload| async move {
+                let mut status = WorkloadStatus {
+                    id: None,
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Reported,
+                };
+                workload.status = status.clone();
+                workload.metadata.created_at = Some(DateTime::now());
+
+                let workload_id = self.workload_collection.insert_one_into(workload).await?;
+                status.id = Some(workload_id);
+
+                log::info!(
+                    "Successfully added workload. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                })
+            },
+        )
+        .await
+    }
+
+    pub async fn update_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.update'");
+        self.process_request(
+            msg,
+            WorkloadState::Updating,
+            WorkloadState::Error,
+            |mut workload: schemas::Workload| async move {
+                let status = WorkloadStatus {
+                    id: workload._id,
+                    desired: WorkloadState::Updated,
+                    actual: WorkloadState::Updating,
+                };
+
+                workload.status = status.clone();
+                workload.metadata.updated_at = Some(DateTime::now());
+
+                // convert workload to document and submit to mongodb
+                let updated_workload_doc =
+                    to_document(&workload).map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection
+                    .update_one_within(
+                        doc! { "_id":  workload._id },
+                        UpdateModifications::Document(doc! { "$set": updated_workload_doc }),
+                    )
+                    .await?;
+
+                log::info!(
+                    "Successfully updated workload. MongodDB Workload ID={:?}",
+                    workload._id
+                );
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                })
+            },
+        )
+        .await
+    }
+
+    pub async fn remove_workload(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.remove'");
+        self.process_request(
+            msg,
+            WorkloadState::Removed,
+            WorkloadState::Error,
+            |workload_id: ObjectId| async move {
+                let status = WorkloadStatus {
+                    id: Some(workload_id),
+                    desired: WorkloadState::Uninstalled,
+                    actual: WorkloadState::Removed,
+                };
+
+                let updated_status_doc = bson::to_bson(&status)
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                self.workload_collection.update_one_within(
+                    doc! { "_id":  workload_id },
+                    UpdateModifications::Document(doc! {
+                        "$set": {
+                            "metadata.is_deleted": true,
+                            "metadata.deleted_at": DateTime::now(),
+                            "status": updated_status_doc
+                        }
+                    })
+                ).await?;
+                log::info!(
+                    "Successfully removed workload from the Workload Collection. MongodDB Workload ID={:?}",
+                    workload_id
+                );
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: None
+                    },
+                    maybe_response_tags: None
+                })
+            },
+        )
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    pub async fn handle_db_insertion(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.insert'");
+        self.process_request(
+            msg,
+            WorkloadState::Assigned,
+            WorkloadState::Error,
+            |workload: schemas::Workload| async move {
+                log::debug!("New workload to assign. Workload={:#?}", workload);
+
+                // 0. Fail Safe: exit early if the workload provided does not include an `_id` field
+                let workload_id = if let Some(id) = workload.clone()._id { id } else {
+                    let err_msg = format!("No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}", workload);
+                    return Err(ServiceError::Internal(err_msg));
+                };
+
+                let status = WorkloadStatus {
+                    id: Some(workload_id),
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Assigned,
+                };
+
+                // 1. Perform sanity check to ensure workload is not already assigned to a host and if so, exit fn
+                if !workload.assigned_hosts.is_empty() {
+                    log::warn!("Attempted to assign host for new workload, but host already exists.");
+                    return Ok(WorkloadApiResult {
+                        result: WorkloadResult {
+                            status,
+                            workload: None
+                        },
+                        maybe_response_tags: None
+                    });
+                }
+
+                // 2. Otherwise call mongodb to get host collection to get hosts that meet the capacity requirements
+                // & randomly choose host(s)
+                let eligible_host_ids = self.find_hosts_meeting_workload_criteria(workload.clone(), None).await?;
+                log::debug!("Eligible hosts for new workload. MongodDB Host IDs={:?}", eligible_host_ids);
+
+                // 3. Update the selected host records with the assigned Workload ID
+                let assigned_host_ids = self.assign_workload_to_hosts(workload_id, eligible_host_ids, workload.min_hosts).await.map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 4. Update the Workload Collection with the assigned Host ID
+                let new_status = WorkloadStatus {
+                    id: None, // remove the id to avoid redundant saving of it in the db
+                    ..status.clone()
+                };
+                self.assign_hosts_to_workload(assigned_host_ids.clone(), workload_id, new_status).await.map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 5. Create tag map with host ids to inform nats to publish message to these hosts with workload install status                
+                let mut tag_map: HashMap<String, String> = HashMap::new();
+                for (index, host_pubkey) in assigned_host_ids.iter().cloned().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                Ok(WorkloadApiResult {
+                    result: WorkloadResult {
+                        status,
+                        workload: Some(workload)
+                    },
+                    maybe_response_tags: Some(tag_map)
+                })
+        })
+        .await
+    }
+
+    // NB: Automatically published by the nats-db-connector
+    // triggers on mongodb [workload] collection (update)
+    pub async fn handle_db_modification(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.modify'");
+        let workload = Self::convert_msg_to_type::<schemas::Workload>(msg)?;
+
+        // Fail Safe: exit early if the workload provided does not include an `_id` field
+        let workload_id = if let Some(id) = workload.clone()._id {
+            id
+        } else {
+            let err_msg = format!(
+                "No `_id` found for workload.  Unable to proceed assigning a host. Workload={:?}",
+                workload
+            );
+            return Err(ServiceError::Internal(err_msg));
+        };
+
+        let mut tag_map: HashMap<String, String> = HashMap::new();
+        let log_msg = format!(
+            "Workload update in DB successful. Fwding update to assigned hosts. workload_id={}",
+            workload_id
+        );
+
+        // Match on state (updating or removed) and handle each case
+        let result = match workload.status.actual {
+            WorkloadState::Updating => {
+                log::trace!("Updated workload to handle. Workload={:#?}", workload);
+                // 1. Fetch current hosts
+                let hosts = self
+                    .fetch_hosts_assigned_to_workload(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 2. Remove workloads from existing hosts
+                self.remove_workload_from_hosts(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 3. Find eligible hosts
+                let eligible_host_ids = self
+                    .find_hosts_meeting_workload_criteria(workload.clone(), Some(hosts))
+                    .await?;
+                log::debug!(
+                    "Eligible hosts for new workload. MongodDB Host IDs={:?}",
+                    eligible_host_ids
+                );
+
+                // 4. Update the selected host records with the assigned Workload ID
+                let assigned_host_ids = self
+                    .assign_workload_to_hosts(workload_id, eligible_host_ids, workload.min_hosts)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 5. Update the Workload Collection with the assigned Host ID
+                // IMP: It is very important that the workload state changes to a state that is not `WorkloadState::Updating`,
+                // IMP: ...otherwise, this change will cause the workload update to loop between the db stream modification reads and this handler
+                let new_status = WorkloadStatus {
+                    id: None,
+                    desired: WorkloadState::Running,
+                    actual: WorkloadState::Updated,
+                };
+                self.assign_hosts_to_workload(
+                    assigned_host_ids.clone(),
+                    workload_id,
+                    new_status.clone(),
+                )
+                .await
+                .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 6. Create tag map with host ids to inform nats to publish message to these hosts with workload install status
+                for (index, host_pubkey) in assigned_host_ids.iter().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                log::info!("Added workload to new hosts. Workload={:#?}", workload_id);
+
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            ..new_status
+                        },
+                        workload: Some(workload),
+                    },
+                    maybe_response_tags: Some(tag_map),
+                }
+            }
+            WorkloadState::Removed => {
+                log::trace!("Removed workload to handle. Workload={:#?}", workload);
+                // 1. Fetch current hosts with `workload_id`` to know which
+                // hosts to send uninstall workload request to...
+                let hosts = self
+                    .fetch_hosts_assigned_to_workload(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 2. Remove workloads from existing hosts
+                self.remove_workload_from_hosts(workload_id)
+                    .await
+                    .map_err(|e| ServiceError::Internal(e.to_string()))?;
+
+                // 3. Create tag map with host ids to inform nats to publish message to these hosts with workload install status
+                let host_ids = hosts
+                    .iter()
+                    .map(|h| {
+                        h._id
+                            .ok_or_else(|| ServiceError::Internal("Error".to_string()))
+                    })
+                    .collect::<Result<Vec<ObjectId>, ServiceError>>()?;
+                for (index, host_pubkey) in host_ids.iter().enumerate() {
+                    tag_map.insert(format!("assigned_host_{}", index), host_pubkey.to_hex());
+                }
+
+                log::info!("{} Hosts={:?}", log_msg, hosts);
+
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: WorkloadState::Uninstalled,
+                            actual: WorkloadState::Removed,
+                        },
+                        workload: Some(workload),
+                    },
+                    maybe_response_tags: Some(tag_map),
+                }
+            }
+            _ => {
+                // Catches all other cases wherein a record in the workload collection was modified (not created),
+                // with a state other than "Updating" or "Removed".
+                // In this case, we don't want to do take any new action, so we return a default status without any updates or frowarding tags.
+                WorkloadApiResult {
+                    result: WorkloadResult {
+                        status: WorkloadStatus {
+                            id: Some(workload_id),
+                            desired: workload.status.desired,
+                            actual: workload.status.actual,
+                        },
+                        workload: None,
+                    },
+                    maybe_response_tags: None,
+                }
+            }
+        };
+
+        Ok(result)
+    }
+
+    // NB: Published by the Hosting Agent whenever the status of a workload changes
+    pub async fn handle_status_update(
+        &self,
+        msg: Arc<Message>,
+    ) -> Result<WorkloadApiResult, ServiceError> {
+        log::debug!("Incoming message for 'WORKLOAD.handle_status_update'");
+
+        let workload_status = Self::convert_msg_to_type::<WorkloadResult>(msg)?.status;
+        log::trace!("Workload status to update. Status={:?}", workload_status);
+
+        let workload_status_id = workload_status
+            .id
+            .ok_or_else(|| ServiceError::Internal("Failed to read ._id from record".to_string()))?;
+
+        self.workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_status_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": {
+                        "status": bson::to_bson(&workload_status)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                    }
+                }),
+            )
+            .await?;
+
+        Ok(WorkloadApiResult {
+            result: WorkloadResult {
+                status: workload_status,
+                workload: None,
+            },
+            maybe_response_tags: None,
+        })
+    }
+
+    // Verifies that a host meets the workload criteria
+    fn verify_host_meets_workload_criteria(
+        &self,
+        assigned_host: &Host,
+        workload: &Workload,
+    ) -> bool {
+        if assigned_host.remaining_capacity.disk < workload.system_specs.capacity.disk {
+            return false;
+        }
+        if assigned_host.remaining_capacity.memory < workload.system_specs.capacity.memory {
+            return false;
+        }
+        if assigned_host.remaining_capacity.cores < workload.system_specs.capacity.cores {
+            return false;
+        }
+
+        true
+    }
+
+    async fn fetch_hosts_assigned_to_workload(&self, workload_id: ObjectId) -> Result<Vec<Host>> {
+        Ok(self
+            .host_collection
+            .get_many_from(doc! { "assigned_workloads": workload_id })
+            .await?)
+    }
+
+    async fn remove_workload_from_hosts(&self, workload_id: ObjectId) -> Result<()> {
+        self.host_collection
+            .inner
+            .update_many(
+                doc! {},
+                doc! { "$pull": { "assigned_workloads": workload_id } },
+            )
+            .await
+            .map_err(ServiceError::Database)?;
+        log::info!(
+            "Removed workload from previous hosts. Workload={:#?}",
+            workload_id
+        );
+        Ok(())
+    }
+
+    // Looks through existing hosts to find possible hosts for a given workload
+    // returns the minimum number of hosts required for workload
+    async fn find_hosts_meeting_workload_criteria(
+        &self,
+        workload: Workload,
+        maybe_existing_hosts: Option<Vec<Host>>,
+    ) -> Result<Vec<ObjectId>, ServiceError> {
+        let mut needed_host_count = workload.min_hosts;
+        let mut still_eligible_host_ids: Vec<ObjectId> = vec![];
+
+        if let Some(hosts) = maybe_existing_hosts {
+            still_eligible_host_ids = hosts.into_iter()
+                .filter_map(|h| {
+                    if self.verify_host_meets_workload_criteria(&h, &workload) {
+                        h._id.ok_or_else(|| {
+                            ServiceError::Internal(format!(
+                                "No `_id` found for workload. Unable to proceed verifying host eligibility. Workload={:?}",
+                                workload
+                            ))
+                        }).ok()
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+            needed_host_count -= still_eligible_host_ids.len() as u16;
+        }
+
+        let pipeline = vec![
+            doc! {
+                "$match": {
+                    // verify there are enough system resources
+                    "remaining_capacity.disk": { "$gte": workload.system_specs.capacity.disk },
+                    "remaining_capacity.memory": { "$gte": workload.system_specs.capacity.memory },
+                    "remaining_capacity.cores": { "$gte": workload.system_specs.capacity.cores },
+
+                    // limit how many workloads a single host can have
+                    "assigned_workloads": { "$lt": 1 }
+                }
+            },
+            doc! {
+                // the maximum number of hosts returned should be the minimum hosts required by workload
+                // sample randomized results and always return back at least 1 result
+                "$sample": std::cmp::min( needed_host_count as i32, 1),
+
+                // only return the `host._id` feilds
+                "$project": { "_id": 1 }
+            },
+        ];
+        let host_ids = self.host_collection.aggregate::<ObjectId>(pipeline).await?;
+        if host_ids.is_empty() {
+            let err_msg = format!(
+                "Failed to locate a compatible host for workload. Workload_Id={:?}",
+                workload._id
+            );
+            return Err(ServiceError::Internal(err_msg));
+        } else if workload.min_hosts > host_ids.len() as u16 {
+            log::warn!(
+                "Failed to locate the the min required number of hosts for workload. Workload_Id={:?}",
+                workload._id
+            );
+        }
+
+        let mut eligible_host_ids = host_ids;
+        eligible_host_ids.extend(still_eligible_host_ids);
+
+        Ok(eligible_host_ids)
+    }
+
+    async fn assign_workload_to_hosts(
+        &self,
+        workload_id: ObjectId,
+        eligible_host_ids: Vec<ObjectId>,
+        needed_host_count: u16,
+    ) -> Result<Vec<ObjectId>> {
+        // NB: This will attempt to assign the hosts up to 5 times.. then exit loop with warning message
+        let assigned_host_ids: Vec<ObjectId>;
+        let mut unassigned_host_ids: Vec<ObjectId> = eligible_host_ids.clone();
+        let mut exit_flag = 0;
+        loop {
+            let updated_host_result = self
+                .host_collection
+                .update_many_within(
+                    doc! {
+                        "_id": { "$in": unassigned_host_ids.clone() },
+                        // Currently we only allow a single workload per host
+                        "assigned_workloads": { "$size": 0 }
+                    },
+                    UpdateModifications::Document(doc! {
+                        "$push": {
+                            "assigned_workloads": workload_id
+                        }
+                    }),
+                )
+                .await?;
+
+            if updated_host_result.matched_count == unassigned_host_ids.len() as u64 {
+                log::debug!(
+                    "Successfully updated Host records with the new workload id {}. Host_IDs={:?} Update_Result={:?}",
+                    workload_id,
+                    eligible_host_ids,
+                    updated_host_result
+                );
+                assigned_host_ids = eligible_host_ids;
+                break;
+            } else if exit_flag == 5 {
+                let unassigned_host_hashset: HashSet<ObjectId> =
+                    unassigned_host_ids.into_iter().collect();
+                assigned_host_ids = eligible_host_ids
+                    .into_iter()
+                    .filter(|id| !unassigned_host_hashset.contains(id))
+                    .collect();
+                log::warn!("Exiting loop after 5 attempts to assign the workload to the min number of hosts.
+                    Only able to assign {} hosts. Workload_ID={}, Assigned_Host_IDs={:?}",
+                    needed_host_count,
+                    workload_id,
+                    assigned_host_ids
+                );
+                break;
+            }
+
+            log::warn!("Failed to update all selected host records with workload_id.");
+            log::debug!("Fetching paired host records to see which one(s) still remain unassigned to workload...");
+            let unassigned_hosts = self
+                .host_collection
+                .get_many_from(doc! {
+                    "_id": { "$in": eligible_host_ids.clone() },
+                    "assigned_workloads": { "$size": 0 }
+                })
+                .await?;
+
+            unassigned_host_ids = unassigned_hosts
+                .into_iter()
+                .map(|h| h._id.unwrap_or_default())
+                .collect();
+            exit_flag += 1;
+        }
+
+        Ok(assigned_host_ids)
+    }
+
+    async fn assign_hosts_to_workload(
+        &self,
+        assigned_host_ids: Vec<ObjectId>,
+        workload_id: ObjectId,
+        new_status: WorkloadStatus,
+    ) -> Result<()> {
+        let updated_workload_result = self
+            .workload_collection
+            .update_one_within(
+                doc! {
+                    "_id": workload_id
+                },
+                UpdateModifications::Document(doc! {
+                    "$set": [{
+                        "status": bson::to_bson(&new_status)
+                            .map_err(|e| ServiceError::Internal(e.to_string()))?
+                        }, {
+                        "assigned_hosts": assigned_host_ids
+                    }]
+                }),
+            )
+            .await;
+
+        log::trace!(
+            "Successfully added new workload into the Workload Collection. MongodDB Workload ID={:?}",
+            updated_workload_result
+        );
+
+        Ok(())
+    }
+
+    // Helper function to initialize mongodb collections
+    async fn init_collection<T>(
+        client: &MongoDBClient,
+        collection_name: &str,
+    ) -> Result<MongoCollection<T>>
+    where
+        T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
+    {
+        Ok(MongoCollection::<T>::new(client, schemas::DATABASE_NAME, collection_name).await?)
+    }
+}
diff --git a/rust/services/workload/src/types.rs b/rust/services/workload/src/types.rs
index 912ffb6..76e0f82 100644
--- a/rust/services/workload/src/types.rs
+++ b/rust/services/workload/src/types.rs
@@ -1,18 +1,49 @@
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use strum_macros::AsRefStr;
 use util_libs::{
-    db::schemas::WorkloadStatus,
-    js_stream_service::{CreateTag, EndpointTraits},
+    db::schemas::{self, WorkloadStatus},
+    nats::types::{CreateResponse, CreateTag, EndpointTraits},
 };
 
-pub use String as WorkloadId;
+#[derive(Serialize, Deserialize, Clone, Debug, AsRefStr)]
+#[serde(rename_all = "snake_case")]
+pub enum WorkloadServiceSubjects {
+    Add,
+    Update,
+    Remove,
+    Insert, // db change stream trigger
+    Modify, // db change stream trigger
+    HandleStatusUpdate,
+    SendStatus,
+    Install,
+    Uninstall,
+    UpdateInstalled,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ApiResult(pub WorkloadStatus, pub Option<Vec<String>>);
+pub struct WorkloadResult {
+    pub status: WorkloadStatus,
+    pub workload: Option<schemas::Workload>,
+}
 
-impl CreateTag for ApiResult {
-    fn get_tags(&self) -> Option<Vec<String>> {
-        self.1.clone()
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadApiResult {
+    pub result: WorkloadResult,
+    pub maybe_response_tags: Option<HashMap<String, String>>,
+}
+impl EndpointTraits for WorkloadApiResult {}
+impl CreateTag for WorkloadApiResult {
+    fn get_tags(&self) -> HashMap<String, String> {
+        self.maybe_response_tags.clone().unwrap_or_default()
+    }
+}
+impl CreateResponse for WorkloadApiResult {
+    fn get_response(&self) -> bytes::Bytes {
+        let r = self.result.clone();
+        match serde_json::to_vec(&r) {
+            Ok(r) => r.into(),
+            Err(e) => e.to_string().into(),
+        }
     }
 }
-
-impl EndpointTraits for ApiResult {}
diff --git a/rust/util_libs/src/db/mongodb.rs b/rust/util_libs/src/db/mongodb.rs
index b2974bb..332c741 100644
--- a/rust/util_libs/src/db/mongodb.rs
+++ b/rust/util_libs/src/db/mongodb.rs
@@ -1,47 +1,39 @@
-use anyhow::{anyhow, Context, Result};
+use crate::nats::types::ServiceError;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
-use bson::{self, doc, Document};
+use bson::oid::ObjectId;
+use bson::{self, Document};
 use futures::stream::TryStreamExt;
 use mongodb::options::UpdateModifications;
-use mongodb::results::{DeleteResult, UpdateResult};
+use mongodb::results::UpdateResult;
 use mongodb::{options::IndexOptions, Client, Collection, IndexModel};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
 
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum ServiceError {
-    #[error("Internal Error: {0}")]
-    Internal(String),
-    #[error(transparent)]
-    Database(#[from] mongodb::error::Error),
-}
-
 #[async_trait]
 pub trait MongoDbAPI<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync,
 {
-    fn mongo_error_handler<ReturnType>(
+    type Error;
+
+    async fn aggregate<R: for<'de> Deserialize<'de>>(
         &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType>;
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>>;
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>>;
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>>;
-    async fn insert_one_into(&self, item: T) -> Result<String>;
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>>;
-    async fn update_one_within(
+        pipeline: Vec<Document>,
+    ) -> Result<Vec<R>, Self::Error>;
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error>;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error>;
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error>;
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn update_many_within(
+    ) -> Result<UpdateResult, Self::Error>;
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult>;
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult>;
-    async fn delete_all_from(&self) -> Result<DeleteResult>;
+    ) -> Result<UpdateResult, Self::Error>;
 }
 
 pub trait IntoIndexes {
@@ -53,7 +45,7 @@ pub struct MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes,
 {
-    pub collection: Collection<T>,
+    pub inner: Collection<T>,
     indices: Vec<IndexModel>,
 }
 
@@ -72,7 +64,7 @@ where
         let indices = vec![];
 
         Ok(MongoCollection {
-            collection,
+            inner: collection,
             indices,
         })
     }
@@ -94,7 +86,7 @@ where
         self.indices = indices.clone();
 
         // Apply the indices to the mongodb collection schema
-        self.collection.create_indexes(indices).await?;
+        self.inner.create_indexes(indices).await?;
         Ok(self)
     }
 }
@@ -104,109 +96,86 @@ impl<T> MongoDbAPI<T> for MongoCollection<T>
 where
     T: Serialize + for<'de> Deserialize<'de> + Unpin + Send + Sync + Default + IntoIndexes + Debug,
 {
-    fn mongo_error_handler<ReturnType>(
-        &self,
-        result: Result<ReturnType, mongodb::error::Error>,
-    ) -> Result<ReturnType> {
-        let rtn = result.map_err(ServiceError::Database)?;
-        Ok(rtn)
-    }
+    type Error = ServiceError;
 
-    async fn aggregate(&self, pipeline: Vec<Document>) -> Result<Vec<T>> {
-        log::info!("aggregate pipeline {:?}", pipeline);
-        let cursor = self.collection.aggregate(pipeline).await?;
+    async fn aggregate<R>(&self, pipeline: Vec<Document>) -> Result<Vec<R>, Self::Error>
+    where
+        R: for<'de> Deserialize<'de>,
+    {
+        log::trace!("Aggregate pipeline {:?}", pipeline);
+        let cursor = self.inner.aggregate(pipeline).await?;
 
         let results_doc: Vec<bson::Document> =
             cursor.try_collect().await.map_err(ServiceError::Database)?;
 
-        let results: Vec<T> = results_doc
+        let results: Vec<R> = results_doc
             .into_iter()
             .map(|doc| {
-                bson::from_document::<T>(doc).with_context(|| "failed to deserialize document")
+                bson::from_document::<R>(doc).with_context(|| "Failed to deserialize document")
             })
-            .collect::<Result<Vec<T>>>()?;
+            .collect::<Result<Vec<R>>>()
+            .map_err(|e| ServiceError::Internal(e.to_string()))?;
 
         Ok(results)
     }
 
-    async fn get_one_from(&self, filter: Document) -> Result<Option<T>> {
-        log::info!("get_one_from filter {:?}", filter);
+    async fn get_one_from(&self, filter: Document) -> Result<Option<T>, Self::Error> {
+        log::trace!("Get_one_from filter {:?}", filter);
 
         let item = self
-            .collection
+            .inner
             .find_one(filter)
             .await
             .map_err(ServiceError::Database)?;
 
-        log::info!("item {:?}", item);
+        log::debug!("get_one_from item {:?}", item);
         Ok(item)
     }
 
-    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>> {
-        let cursor = self.collection.find(filter).await?;
+    async fn get_many_from(&self, filter: Document) -> Result<Vec<T>, Self::Error> {
+        let cursor = self.inner.find(filter).await?;
         let results: Vec<T> = cursor.try_collect().await.map_err(ServiceError::Database)?;
         Ok(results)
     }
 
-    async fn insert_one_into(&self, item: T) -> Result<String> {
+    async fn insert_one_into(&self, item: T) -> Result<ObjectId, Self::Error> {
         let result = self
-            .collection
+            .inner
             .insert_one(item)
             .await
             .map_err(ServiceError::Database)?;
 
-        Ok(result.inserted_id.to_string())
-    }
+        let mongo_id = result
+            .inserted_id
+            .as_object_id()
+            .ok_or(ServiceError::Internal(format!(
+                "Failed to read the insert id after inserting item. insert_result={:?}.",
+                result
+            )))?;
 
-    async fn insert_many_into(&self, items: Vec<T>) -> Result<Vec<String>> {
-        let result = self
-            .collection
-            .insert_many(items)
-            .await
-            .map_err(ServiceError::Database)?;
-
-        let ids = result
-            .inserted_ids
-            .values()
-            .map(|id| id.to_string())
-            .collect();
-        Ok(ids)
+        Ok(mongo_id)
     }
 
-    async fn update_one_within(
+    async fn update_many_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_one(query, updated_doc)
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_many(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 
-    async fn update_many_within(
+    async fn update_one_within(
         &self,
         query: Document,
         updated_doc: UpdateModifications,
-    ) -> Result<UpdateResult> {
-        self.collection
-            .update_many(query, updated_doc)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_one_from(&self, query: Document) -> Result<DeleteResult> {
-        self.collection
-            .delete_one(query)
-            .await
-            .map_err(|e| anyhow!(e))
-    }
-
-    async fn delete_all_from(&self) -> Result<DeleteResult> {
-        self.collection
-            .delete_many(doc! {})
+    ) -> Result<UpdateResult, Self::Error> {
+        self.inner
+            .update_one(query, updated_doc)
             .await
-            .map_err(|e| anyhow!(e))
+            .map_err(ServiceError::Database)
     }
 }
 
@@ -328,7 +297,7 @@ mod tests {
                     updated_at: Some(DateTime::now()),
                     deleted_at: None,
                 },
-                device_id: "Vf3IceiD".to_string(),
+                device_id: "placeholder_pubkey_host".to_string(),
                 ip_address: "127.0.0.1".to_string(),
                 remaining_capacity: Capacity {
                     memory: 16,
@@ -359,9 +328,9 @@ mod tests {
         let host_1 = get_mock_host();
         let host_2 = get_mock_host();
         let host_3 = get_mock_host();
-        host_api
-            .insert_many_into(vec![host_1.clone(), host_2.clone(), host_3.clone()])
-            .await?;
+        host_api.insert_one_into(host_1.clone()).await?;
+        host_api.insert_one_into(host_2.clone()).await?;
+        host_api.insert_one_into(host_3.clone()).await?;
 
         // get many docs
         let ids = vec![
@@ -383,13 +352,8 @@ mod tests {
         assert!(updated_ids.contains(&ids[1]));
         assert!(updated_ids.contains(&ids[2]));
 
-        // delete all documents
-        let DeleteResult { deleted_count, .. } = host_api.delete_all_from().await?;
-        assert_eq!(deleted_count, 4);
-        let fetched_host = host_api.get_one_from(filter_one).await?;
-        let fetched_hosts = host_api.get_many_from(filter_many).await?;
-        assert!(fetched_host.is_none());
-        assert!(fetched_hosts.is_empty());
+        // Delete collection and all documents therein.
+        let _ = host_api.inner.drop();
 
         Ok(())
     }
diff --git a/rust/util_libs/src/db/schemas.rs b/rust/util_libs/src/db/schemas.rs
index 5a7945b..ed8cd92 100644
--- a/rust/util_libs/src/db/schemas.rs
+++ b/rust/util_libs/src/db/schemas.rs
@@ -14,18 +14,18 @@ pub const HOST_COLLECTION_NAME: &str = "host";
 pub const WORKLOAD_COLLECTION_NAME: &str = "workload";
 
 // Provide type Alias for HosterPubKey
-pub use String as HosterPubKey;
-
-// Provide type Alias for DeveloperPubkey
-pub use String as DeveloperPubkey;
-
-// Provide type Alias for DeveloperJWT
-pub use String as DeveloperJWT;
+pub use String as PubKey;
 
 // Provide type Alias for SemVer (semantic versioning)
 pub use String as SemVer;
 
 // ==================== User Schema ====================
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct RoleInfo {
+    pub collection_id: ObjectId, // Hoster/Developer colleciton Mongodb ID ref
+    pub pubkey: PubKey,          //  Hoster/Developer Pubkey *INDEXED*
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum UserPermission {
     Admin,
@@ -46,24 +46,24 @@ pub struct User {
     pub metadata: Metadata,
     pub jurisdiction: String,
     pub permissions: Vec<UserPermission>,
-    pub user_info_id: Option<ObjectId>,
-    pub developer: Option<ObjectId>,
-    pub hoster: Option<ObjectId>,
+    pub user_info_id: Option<ObjectId>, // *INDEXED*
+    pub developer: Option<RoleInfo>,    // *INDEXED*
+    pub hoster: Option<RoleInfo>,       // *INDEXED*
 }
 
-// No Additional Indexing for Developer
+// Indexing for User
 impl IntoIndexes for User {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
 
-        // add user_info index
-        let user_info_index_doc = doc! { "user_info": 1 };
-        let user_info_index_opts = Some(
+        // add user_info_id index
+        let user_info_id_index_doc = doc! { "user_info_id": 1 };
+        let user_info_id_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("user_info_index".to_string()))
+                .name(Some("user_info_id_index".to_string()))
                 .build(),
         );
-        indices.push((user_info_index_doc, user_info_index_opts));
+        indices.push((user_info_id_index_doc, user_info_id_index_opts));
 
         // add developer index
         let developer_index_doc = doc! { "developer": 1 };
@@ -75,10 +75,10 @@ impl IntoIndexes for User {
         indices.push((developer_index_doc, developer_index_opts));
 
         // add host index
-        let host_index_doc = doc! { "host": 1 };
+        let host_index_doc = doc! { "hoster": 1 };
         let host_index_opts = Some(
             IndexOptions::builder()
-                .name(Some("host_index".to_string()))
+                .name(Some("hoster_index".to_string()))
                 .build(),
         );
         indices.push((host_index_doc, host_index_opts));
@@ -93,7 +93,7 @@ pub struct UserInfo {
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
     pub user_id: ObjectId,
-    pub email: String,
+    pub email: String, // *INDEXED*
     pub given_names: String,
     pub family_name: String,
 }
@@ -101,7 +101,6 @@ pub struct UserInfo {
 impl IntoIndexes for UserInfo {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         // add email index
         let email_index_doc = doc! { "email": 1 };
         let email_index_opts = Some(
@@ -110,7 +109,6 @@ impl IntoIndexes for UserInfo {
                 .build(),
         );
         indices.push((email_index_doc, email_index_opts));
-
         Ok(indices)
     }
 }
@@ -121,8 +119,8 @@ pub struct Developer {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user._id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub active_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub user_id: ObjectId,
+    pub active_workloads: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Developer
@@ -138,8 +136,8 @@ pub struct Hoster {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub user_id: String, // MongoDB ID ref to `user.id` (which stores the hoster's pubkey, jurisdiction and email)
-    pub assigned_hosts: Vec<ObjectId>, // MongoDB ID refs to `host._id`
+    pub user_id: ObjectId,
+    pub assigned_hosts: Vec<ObjectId>,
 }
 
 // No Additional Indexing for Hoster
@@ -162,29 +160,27 @@ pub struct Host {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub assigned_hoster: ObjectId,
-    pub device_id: String, // *INDEXED*, Auto-generated Nats server ID
+    pub device_id: PubKey, // *INDEXED*
     pub ip_address: String,
     pub remaining_capacity: Capacity,
     pub avg_uptime: i64,
     pub avg_network_speed: i64,
     pub avg_latency: i64,
-    pub assigned_workloads: Vec<ObjectId>, // MongoDB ID refs to `workload._id`
+    pub assigned_hoster: ObjectId,
+    pub assigned_workloads: Vec<ObjectId>,
 }
 
 impl IntoIndexes for Host {
     fn into_indices(self) -> Result<Vec<(Document, Option<IndexOptions>)>> {
         let mut indices = vec![];
-
         //  Add Device ID Index
-        let device_id_index_doc = doc! { "device_id": 1 };
-        let device_id_index_opts = Some(
+        let pubkey_index_doc = doc! { "device_id": 1 };
+        let pubkey_index_opts = Some(
             IndexOptions::builder()
                 .name(Some("device_id_index".to_string()))
                 .build(),
         );
-        indices.push((device_id_index_doc, device_id_index_opts));
-
+        indices.push((pubkey_index_doc, pubkey_index_opts));
         Ok(indices)
     }
 }
@@ -197,24 +193,27 @@ pub enum WorkloadState {
     Pending,
     Installed,
     Running,
+    Updating,
+    Updated,
     Removed,
     Uninstalled,
-    Updating,
     Error(String),   // String = error message
     Unknown(String), // String = context message
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadStatus {
-    pub id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<ObjectId>,
     pub desired: WorkloadState,
     pub actual: WorkloadState,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SystemSpecs {
-    pub capacity: Capacity, // network_speed: i64
-                            // uptime: i64
+    pub capacity: Capacity,
+    pub avg_network_speed: i64, // Mbps
+    pub avg_uptime: f64, //  decimal value between 0-1 representing avg uptime over past month
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -222,14 +221,14 @@ pub struct Workload {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub _id: Option<ObjectId>,
     pub metadata: Metadata,
-    pub state: WorkloadState,
-    pub assigned_developer: ObjectId, // *INDEXED*, Developer Mongodb ID
+    pub assigned_developer: ObjectId, // *INDEXED*
     pub version: SemVer,
     pub nix_pkg: String, // (Includes everthing needed to deploy workload - ie: binary & env pkg & deps, etc)
     pub min_hosts: u16,
     pub system_specs: SystemSpecs,
-    pub assigned_hosts: Vec<ObjectId>, // Host Device IDs (eg: assigned nats server id)
-                                        // pub status: WorkloadStatus,
+    pub assigned_hosts: Vec<ObjectId>,
+    // pub state: WorkloadState,
+    pub status: WorkloadStatus,
 }
 
 impl Default for Workload {
@@ -252,7 +251,6 @@ impl Default for Workload {
                 updated_at: Some(DateTime::now()),
                 deleted_at: None,
             },
-            state: WorkloadState::Reported,
             version: semver,
             nix_pkg: String::new(),
             assigned_developer: ObjectId::new(),
@@ -263,8 +261,15 @@ impl Default for Workload {
                     disk: 400,
                     cores: 20,
                 },
+                avg_network_speed: 200,
+                avg_uptime: 0.8,
             },
             assigned_hosts: Vec::new(),
+            status: WorkloadStatus {
+                id: None, // skips serialization when `None`
+                desired: WorkloadState::Unknown("default state".to_string()),
+                actual: WorkloadState::Unknown("default state".to_string()),
+            },
         }
     }
 }
diff --git a/rust/util_libs/src/lib.rs b/rust/util_libs/src/lib.rs
index 861376d..f1b0265 100644
--- a/rust/util_libs/src/lib.rs
+++ b/rust/util_libs/src/lib.rs
@@ -1,5 +1,2 @@
 pub mod db;
-pub mod js_stream_service;
-pub mod nats_js_client;
-pub mod nats_server;
-pub mod nats_types;
+pub mod nats;
diff --git a/rust/util_libs/src/nats_js_client.rs b/rust/util_libs/src/nats/jetstream_client.rs
similarity index 55%
rename from rust/util_libs/src/nats_js_client.rs
rename to rust/util_libs/src/nats/jetstream_client.rs
index cbd1fed..1e6a000 100644
--- a/rust/util_libs/src/nats_js_client.rs
+++ b/rust/util_libs/src/nats/jetstream_client.rs
@@ -1,73 +1,24 @@
-use super::js_stream_service::{CreateTag, JsServiceParamsPartial, JsStreamService};
-use crate::nats_server::LEAF_SERVER_DEFAULT_LISTEN_PORT;
-
+use super::{
+    jetstream_service::JsStreamService,
+    leaf_server::LEAF_SERVER_DEFAULT_LISTEN_PORT,
+    types::{
+        ErrClientDisconnected, EventHandler, EventListener, JsClientBuilder, JsServiceBuilder,
+        PublishInfo,
+    },
+};
 use anyhow::Result;
-use async_nats::{jetstream, Message, ServerInfo};
-use serde::{Deserialize, Serialize};
-use std::error::Error;
-use std::fmt;
-use std::fmt::Debug;
-use std::future::Future;
-use std::pin::Pin;
+use async_nats::{jetstream, ServerInfo};
+use core::option::Option::None;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
-pub type EventHandler = Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>;
-pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>;
-pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, anyhow::Error> + Send + Sync>;
-pub type AsyncEndpointHandler<T> = Arc<
-    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, anyhow::Error>> + Send>>
-        + Send
-        + Sync,
->;
-
-#[derive(Clone)]
-pub enum EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
-{
-    Sync(EndpointHandler<T>),
-    Async(AsyncEndpointHandler<T>),
-}
-
-impl<T> std::fmt::Debug for EndpointType<T>
-where
-    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let t = match &self {
-            EndpointType::Async(_) => "EndpointType::Async(<function>)",
-            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
-        };
-
-        write!(f, "{}", t)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct SendRequest {
-    pub subject: String,
-    pub msg_id: String,
-    pub data: Vec<u8>,
-}
-
-#[derive(Debug)]
-pub struct ErrClientDisconnected;
-impl fmt::Display for ErrClientDisconnected {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Could not reach nats: connection closed")
-    }
-}
-impl Error for ErrClientDisconnected {}
-
 impl std::fmt::Debug for JsClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("JsClient")
             .field("url", &self.url)
             .field("name", &self.name)
             .field("client", &self.client)
-            .field("js", &self.js)
+            .field("js_context", &self.js_context)
             .field("js_services", &self.js_services)
             .field("service_log_prefix", &self.service_log_prefix)
             .finish()
@@ -80,36 +31,19 @@ pub struct JsClient {
     on_msg_published_event: Option<EventHandler>,
     on_msg_failed_event: Option<EventHandler>,
     client: async_nats::Client, // inner_client
-    pub js: jetstream::Context,
+    pub js_context: jetstream::Context,
     pub js_services: Option<Vec<JsStreamService>>,
     service_log_prefix: String,
 }
 
-#[derive(Deserialize, Default)]
-pub struct NewJsClientParams {
-    pub nats_url: String,
-    pub name: String,
-    pub inbox_prefix: String,
-    #[serde(default)]
-    pub service_params: Vec<JsServiceParamsPartial>,
-    #[serde(skip_deserializing)]
-    pub opts: Vec<EventListener>, // NB: These opts should not be required for client instantiation
-    #[serde(default)]
-    pub credentials_path: Option<String>,
-    #[serde(default)]
-    pub ping_interval: Option<Duration>,
-    #[serde(default)]
-    pub request_timeout: Option<Duration>, // Defaults to 5s
-}
-
 impl JsClient {
-    pub async fn new(p: NewJsClientParams) -> Result<Self, async_nats::Error> {
+    pub async fn new(p: JsClientBuilder) -> Result<Self, async_nats::Error> {
         let connect_options = async_nats::ConnectOptions::new()
-            // .require_tls(true)
             .name(&p.name)
             .ping_interval(p.ping_interval.unwrap_or(Duration::from_secs(120)))
             .request_timeout(Some(p.request_timeout.unwrap_or(Duration::from_secs(10))))
             .custom_inbox_prefix(&p.inbox_prefix);
+        // .require_tls(true)
 
         let client = match p.credentials_path {
             Some(cp) => {
@@ -123,77 +57,33 @@ impl JsClient {
             None => connect_options.connect(&p.nats_url).await?,
         };
 
-        let jetstream = jetstream::new(client.clone());
-        let mut services = vec![];
-        for params in p.service_params {
-            let service = JsStreamService::new(
-                jetstream.clone(),
-                &params.name,
-                &params.description,
-                &params.version,
-                &params.service_subject,
-            )
-            .await?;
-            services.push(service);
-        }
+        let log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
+        log::info!("{}Connected to NATS server at {}", log_prefix, p.nats_url);
 
-        let js_services = if services.is_empty() {
-            None
-        } else {
-            Some(services)
-        };
-
-        let service_log_prefix = format!("NATS-CLIENT-LOG::{}::", p.name);
-
-        let mut default_client = JsClient {
+        let mut js_client = JsClient {
             url: p.nats_url,
             name: p.name,
             on_msg_published_event: None,
             on_msg_failed_event: None,
+            js_services: None,
+            js_context: jetstream::new(client.clone()),
+            service_log_prefix: log_prefix,
             client,
-            js: jetstream,
-            js_services,
-            service_log_prefix: service_log_prefix.clone(),
         };
 
-        for opt in p.opts {
-            opt(&mut default_client);
+        for listener in p.listeners {
+            listener(&mut js_client);
         }
 
-        log::info!(
-            "{}Connected to NATS server at {}",
-            service_log_prefix,
-            default_client.url
-        );
-        Ok(default_client)
-    }
-
-    pub fn name(&self) -> &str {
-        &self.name
+        Ok(js_client)
     }
 
     pub fn get_server_info(&self) -> ServerInfo {
         self.client.server_info()
     }
 
-    pub async fn monitor(&self) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            Err(Box::new(ErrClientDisconnected))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub async fn close(&self) -> Result<(), async_nats::Error> {
-        self.client.drain().await?;
-        Ok(())
-    }
-
-    pub async fn health_check_stream(&self, stream_name: &str) -> Result<(), async_nats::Error> {
-        if let async_nats::connection::State::Disconnected = self.client.connection_state() {
-            return Err(Box::new(ErrClientDisconnected));
-        }
-        let stream = &self.js.get_stream(stream_name).await?;
+    pub async fn get_stream_info(&self, stream_name: &str) -> Result<(), async_nats::Error> {
+        let stream = &self.js_context.get_stream(stream_name).await?;
         let info = stream.get_info().await?;
         log::debug!(
             "{}JetStream info: stream:{}, info:{:?}",
@@ -204,43 +94,80 @@ impl JsClient {
         Ok(())
     }
 
-    pub async fn request(&self, _payload: &SendRequest) -> Result<(), async_nats::Error> {
-        Ok(())
+    pub async fn check_connection(
+        &self,
+    ) -> Result<async_nats::connection::State, async_nats::Error> {
+        let conn_state = self.client.connection_state();
+        if let async_nats::connection::State::Disconnected = conn_state {
+            Err(Box::new(ErrClientDisconnected))
+        } else {
+            Ok(conn_state)
+        }
     }
 
-    pub async fn publish(&self, payload: &SendRequest) -> Result<(), async_nats::Error> {
+    pub async fn publish(
+        &self,
+        payload: PublishInfo,
+    ) -> Result<(), async_nats::error::Error<async_nats::jetstream::context::PublishErrorKind>>
+    {
+        log::debug!(
+            "{}Called Publish message: subj={}, msg_id={} data={:?}",
+            self.service_log_prefix,
+            payload.subject,
+            payload.msg_id,
+            payload.data
+        );
+
         let now = Instant::now();
-        let result = self
-            .js
-            .publish(payload.subject.clone(), payload.data.clone().into())
-            .await;
+        let result = match payload.headers {
+            Some(headers) => {
+                self.js_context
+                    .publish_with_headers(
+                        payload.subject.clone(),
+                        headers,
+                        payload.data.clone().into(),
+                    )
+                    .await
+            }
+            None => {
+                self.js_context
+                    .publish(payload.subject.clone(), payload.data.clone().into())
+                    .await
+            }
+        };
 
         let duration = now.elapsed();
         if let Err(err) = result {
             if let Some(ref on_failed) = self.on_msg_failed_event {
                 on_failed(&payload.subject, &self.name, duration); // todo: add msg_id
             }
-            return Err(Box::new(err));
+            return Err(err);
         }
 
-        log::debug!(
-            "{}Published message: subj={}, msg_id={} data={:?}",
-            self.service_log_prefix,
-            payload.subject,
-            payload.msg_id,
-            payload.data
-        );
         if let Some(ref on_published) = self.on_msg_published_event {
             on_published(&payload.subject, &self.name, duration);
         }
         Ok(())
     }
 
-    pub async fn add_js_services(mut self, js_services: Vec<JsStreamService>) -> Self {
-        let mut current_services = self.js_services.unwrap_or_default();
-        current_services.extend(js_services);
+    pub async fn add_js_service(
+        &mut self,
+        params: JsServiceBuilder,
+    ) -> Result<(), async_nats::Error> {
+        let new_service = JsStreamService::new(
+            self.js_context.to_owned(),
+            &params.name,
+            &params.description,
+            &params.version,
+            &params.service_subject,
+        )
+        .await?;
+
+        let mut current_services = self.js_services.to_owned().unwrap_or_default();
+        current_services.push(new_service);
         self.js_services = Some(current_services);
-        self
+
+        Ok(())
     }
 
     pub async fn get_js_service(&self, js_service_name: String) -> Option<&JsStreamService> {
@@ -251,6 +178,11 @@ impl JsClient {
         }
         None
     }
+
+    pub async fn close(&self) -> Result<(), async_nats::Error> {
+        self.client.drain().await?;
+        Ok(())
+    }
 }
 
 // Client Options:
@@ -268,7 +200,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_published_event = Some(Box::pin(f.clone()));
+        c.on_msg_published_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -277,7 +209,7 @@ where
     F: Fn(&str, &str, Duration) + Send + Sync + Clone + 'static,
 {
     Arc::new(Box::new(move |c: &mut JsClient| {
-        c.on_msg_failed_event = Some(Box::pin(f.clone()));
+        c.on_msg_failed_event = Some(Arc::new(Box::pin(f.clone())));
     }))
 }
 
@@ -314,7 +246,7 @@ pub fn get_event_listeners() -> Vec<EventListener> {
     };
 
     let event_listeners = vec![
-        on_msg_published_event(published_msg_handler),
+        on_msg_published_event(published_msg_handler), // Shouldn't this be the 'NATS_LISTEN_PORT'?
         on_msg_failed_event(failure_handler),
     ];
 
@@ -326,8 +258,8 @@ pub fn get_event_listeners() -> Vec<EventListener> {
 mod tests {
     use super::*;
 
-    pub fn get_default_params() -> NewJsClientParams {
-        NewJsClientParams {
+    pub fn get_default_params() -> JsClientBuilder {
+        JsClientBuilder {
             nats_url: "localhost:4222".to_string(),
             name: "test_client".to_string(),
             inbox_prefix: "_UNIQUE_INBOX".to_string(),
@@ -353,7 +285,7 @@ mod tests {
     async fn test_nats_js_client_publish() {
         let params = get_default_params();
         let client = JsClient::new(params).await.unwrap();
-        let payload = SendRequest {
+        let payload = PublishInfo {
             subject: "test_subject".to_string(),
             msg_id: "test_msg".to_string(),
             data: b"Hello, NATS!".to_vec(),
diff --git a/rust/util_libs/src/js_stream_service.rs b/rust/util_libs/src/nats/jetstream_service.rs
similarity index 78%
rename from rust/util_libs/src/js_stream_service.rs
rename to rust/util_libs/src/nats/jetstream_service.rs
index 0860c3b..e622e3a 100644
--- a/rust/util_libs/src/js_stream_service.rs
+++ b/rust/util_libs/src/nats/jetstream_service.rs
@@ -1,108 +1,17 @@
-use super::nats_js_client::EndpointType;
-
+use super::types::{
+    ConsumerBuilder, ConsumerExt, ConsumerExtTrait, EndpointTraits, EndpointType,
+    JsStreamServiceInfo, LogInfo, ResponseSubjectsGenerator,
+};
 use anyhow::{anyhow, Result};
-use std::any::Any;
-// use async_nats::jetstream::message::Message;
-use async_nats::jetstream::consumer::{self, AckPolicy, PullConsumer};
+use async_nats::jetstream::consumer::{self, AckPolicy};
 use async_nats::jetstream::stream::{self, Info, Stream};
 use async_nats::jetstream::Context;
-use async_trait::async_trait;
 use futures::StreamExt;
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-type ResponseSubjectsGenerator = Arc<dyn Fn(Option<Vec<String>>) -> Vec<String> + Send + Sync>;
-
-pub trait CreateTag: Send + Sync {
-    fn get_tags(&self) -> Option<Vec<String>>;
-}
-
-pub trait EndpointTraits:
-    Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static
-{
-}
-
-#[async_trait]
-pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
-    fn get_name(&self) -> &str;
-    fn get_consumer(&self) -> PullConsumer;
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
-}
-
-impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
-where
-    T: EndpointTraits,
-{
-    type Error = anyhow::Error;
-
-    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
-        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
-            Ok(*endpoint)
-        } else {
-            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
-        }
-    }
-}
-
-#[derive(Clone, derive_more::Debug)]
-pub struct ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    name: String,
-    consumer: PullConsumer,
-    handler: EndpointType<T>,
-    #[debug(skip)]
-    response_subject_fn: Option<ResponseSubjectsGenerator>,
-}
-
-#[async_trait]
-impl<T> ConsumerExtTrait for ConsumerExt<T>
-where
-    T: EndpointTraits,
-{
-    fn get_name(&self) -> &str {
-        &self.name
-    }
-    fn get_consumer(&self) -> PullConsumer {
-        self.consumer.clone()
-    }
-    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
-        Box::new(self.handler.clone())
-    }
-    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
-        self.response_subject_fn.clone()
-    }
-}
-
-#[allow(dead_code)]
-#[derive(Clone, Debug)]
-pub struct JsStreamServiceInfo<'a> {
-    pub name: &'a str,
-    pub version: &'a str,
-    pub service_subject: &'a str,
-}
-
-struct LogInfo {
-    prefix: String,
-    service_name: String,
-    service_subject: String,
-    endpoint_name: String,
-    endpoint_subject: String,
-}
-
-#[derive(Clone, Deserialize, Default)]
-pub struct JsServiceParamsPartial {
-    pub name: String,
-    pub description: String,
-    pub version: String,
-    pub service_subject: String,
-}
-
 /// Microservice for Jetstream Streams
 // This setup creates only one subject for the stream (eg: "WORKLOAD.>") and sets up
 // all consumers of the stream to listen to stream subjects beginning with that subject (eg: "WORKLOAD.start")
@@ -196,30 +105,30 @@ impl JsStreamService {
         let handler: EndpointType<T> = EndpointType::try_from(endpoint_trait_obj)?;
 
         Ok(ConsumerExt {
-            name: consumer_ext.get_name().to_string(),
             consumer: consumer_ext.get_consumer(),
             handler,
             response_subject_fn: consumer_ext.get_response(),
         })
     }
 
-    pub async fn add_local_consumer<T>(
+    pub async fn add_consumer<T>(
         &self,
-        consumer_name: &str,
-        endpoint_subject: &str,
-        endpoint_type: EndpointType<T>,
-        response_subject_fn: Option<ResponseSubjectsGenerator>,
+        builder_params: ConsumerBuilder<T>,
     ) -> Result<ConsumerExt<T>, async_nats::Error>
     where
         T: EndpointTraits,
     {
-        let full_subject = format!("{}.{}", self.service_subject, endpoint_subject);
+        // Add the Service Subject prefix
+        let consumer_subject = format!(
+            "{}.{}",
+            self.service_subject, builder_params.endpoint_subject
+        );
 
         // Register JS Subject Consumer
         let consumer_config = consumer::pull::Config {
-            durable_name: Some(consumer_name.to_string()),
+            durable_name: Some(builder_params.name.to_string()),
             ack_policy: AckPolicy::Explicit,
-            filter_subject: full_subject,
+            filter_subject: consumer_subject,
             ..Default::default()
         };
 
@@ -227,28 +136,28 @@ impl JsStreamService {
             .stream
             .write()
             .await
-            .get_or_create_consumer(consumer_name, consumer_config)
+            .get_or_create_consumer(&builder_params.name, consumer_config)
             .await?;
 
         let consumer_with_handler = ConsumerExt {
-            name: consumer_name.to_string(),
             consumer,
-            handler: endpoint_type,
-            response_subject_fn,
+            handler: builder_params.handler,
+            response_subject_fn: builder_params.response_subject_fn,
         };
 
-        self.local_consumers
-            .write()
-            .await
-            .insert(consumer_name.to_string(), Arc::new(consumer_with_handler));
+        self.local_consumers.write().await.insert(
+            builder_params.name.to_string(),
+            Arc::new(consumer_with_handler),
+        );
 
-        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(consumer_name).await?;
-        self.spawn_consumer_handler::<T>(consumer_name).await?;
+        let endpoint_consumer: ConsumerExt<T> = self.get_consumer(&builder_params.name).await?;
+        self.spawn_consumer_handler::<T>(&builder_params.name)
+            .await?;
 
         log::debug!(
             "{}Added the {} local consumer",
             self.service_log_prefix,
-            endpoint_consumer.name,
+            builder_params.name,
         );
 
         Ok(endpoint_consumer)
@@ -279,12 +188,19 @@ impl JsStreamService {
                 .messages()
                 .await?;
 
+            let consumer_info = consumer.info().await?;
+
             let log_info = LogInfo {
                 prefix: self.service_log_prefix.clone(),
                 service_name: self.name.clone(),
                 service_subject: self.service_subject.clone(),
-                endpoint_name: consumer_details.get_name().to_owned(),
-                endpoint_subject: consumer.info().await?.config.filter_subject.clone(),
+                endpoint_name: consumer_info
+                    .config
+                    .durable_name
+                    .clone()
+                    .unwrap_or("Consumer Name Not Found".to_string())
+                    .clone(),
+                endpoint_subject: consumer_info.config.filter_subject.clone(),
             };
 
             let service_context = self.js_context.clone();
@@ -338,14 +254,11 @@ impl JsStreamService {
 
             let (response_bytes, maybe_subject_tags) = match result {
                 Ok(r) => {
-                    let bytes: bytes::Bytes = match serde_json::to_vec(&r) {
-                        Ok(r) => r.into(),
-                        Err(e) => e.to_string().into(),
-                    };
+                    let bytes = r.get_response();
                     let maybe_subject_tags = r.get_tags();
                     (bytes, maybe_subject_tags)
                 }
-                Err(err) => (err.to_string().into(), None),
+                Err(err) => (err.to_string().into(), HashMap::new()),
             };
 
             // Returns a response if a reply address exists.
@@ -498,7 +411,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_js_service_add_local_consumer() {
+    async fn test_js_service_add_consumer() {
         let context = setup_jetstream().await;
         let service = get_default_js_service(context).await;
 
@@ -508,7 +421,7 @@ mod tests {
         let response_subject = Some("response.subject".to_string());
 
         let consumer = service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
@@ -533,7 +446,7 @@ mod tests {
         let response_subject = None;
 
         service
-            .add_local_consumer(
+            .add_consumer(
                 consumer_name,
                 endpoint_subject,
                 endpoint_type,
diff --git a/rust/util_libs/src/nats_server.rs b/rust/util_libs/src/nats/leaf_server.rs
similarity index 98%
rename from rust/util_libs/src/nats_server.rs
rename to rust/util_libs/src/nats/leaf_server.rs
index 4e9030b..dfc40f1 100644
--- a/rust/util_libs/src/nats_server.rs
+++ b/rust/util_libs/src/nats/leaf_server.rs
@@ -143,7 +143,7 @@ impl LeafServer {
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .spawn()
-            .expect("Failed to start NATS server");
+            .context("Failed to start NATS server")?;
 
         // TODO: wait for a readiness indicator
         std::thread::sleep(std::time::Duration::from_millis(100));
@@ -192,8 +192,8 @@ mod tests {
     const TMP_JS_DIR: &str = "./tmp";
     const TEST_AUTH_DIR: &str = "./tmp/test-auth";
     const OPERATOR_NAME: &str = "test-operator";
-    const USER_ACCOUNT_NAME: &str = "hpos-account";
-    const USER_NAME: &str = "hpos-user";
+    const USER_ACCOUNT_NAME: &str = "host-account";
+    const USER_NAME: &str = "host-user";
     const NEW_LEAF_CONFIG_PATH: &str = "./test_configs/leaf_server.conf";
     // NB: if changed, the resolver file path must also be changed in the `hub-server.conf` iteself as well.
     const RESOLVER_FILE_PATH: &str = "./test_configs/resolver.conf";
@@ -227,7 +227,7 @@ mod tests {
             .output()
             .expect("Failed to create edit operator");
 
-        // Create hpos account (with js enabled)
+        // Create host account (with js enabled)
         Command::new("nsc")
             .args(["add", "account", USER_ACCOUNT_NAME])
             .output()
@@ -245,7 +245,7 @@ mod tests {
             .output()
             .expect("Failed to create edit account");
 
-        // Create user for hpos account
+        // Create user for host account
         Command::new("nsc")
             .args(["add", "user", USER_NAME])
             .args(["--account", USER_ACCOUNT_NAME])
diff --git a/rust/util_libs/src/nats/mod.rs b/rust/util_libs/src/nats/mod.rs
new file mode 100644
index 0000000..a320ee4
--- /dev/null
+++ b/rust/util_libs/src/nats/mod.rs
@@ -0,0 +1,4 @@
+pub mod jetstream_client;
+pub mod jetstream_service;
+pub mod leaf_server;
+pub mod types;
diff --git a/rust/util_libs/src/nats/types.rs b/rust/util_libs/src/nats/types.rs
new file mode 100644
index 0000000..d1b6fe1
--- /dev/null
+++ b/rust/util_libs/src/nats/types.rs
@@ -0,0 +1,200 @@
+use super::jetstream_client::JsClient;
+use anyhow::Result;
+use async_nats::jetstream::consumer::PullConsumer;
+use async_nats::{HeaderMap, Message};
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt;
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type EventListener = Arc<Box<dyn Fn(&mut JsClient) + Send + Sync>>;
+pub type EventHandler = Arc<Pin<Box<dyn Fn(&str, &str, Duration) + Send + Sync>>>;
+pub type JsServiceResponse<T> = Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>;
+pub type EndpointHandler<T> = Arc<dyn Fn(&Message) -> Result<T, ServiceError> + Send + Sync>;
+pub type AsyncEndpointHandler<T> = Arc<
+    dyn Fn(Arc<Message>) -> Pin<Box<dyn Future<Output = Result<T, ServiceError>> + Send>>
+        + Send
+        + Sync,
+>;
+pub type ResponseSubjectsGenerator =
+    Arc<dyn Fn(HashMap<String, String>) -> Vec<String> + Send + Sync>;
+
+pub trait EndpointTraits:
+    Serialize
+    + for<'de> Deserialize<'de>
+    + Send
+    + Sync
+    + Clone
+    + Debug
+    + CreateTag
+    + CreateResponse
+    + 'static
+{
+}
+
+pub trait CreateTag: Send + Sync {
+    fn get_tags(&self) -> HashMap<String, String>;
+}
+
+pub trait CreateResponse: Send + Sync {
+    fn get_response(&self) -> bytes::Bytes;
+}
+
+#[async_trait]
+pub trait ConsumerExtTrait: Send + Sync + Debug + 'static {
+    fn get_consumer(&self) -> PullConsumer;
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync>;
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator>;
+}
+
+#[async_trait]
+impl<T> ConsumerExtTrait for ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    fn get_consumer(&self) -> PullConsumer {
+        self.consumer.clone()
+    }
+    fn get_endpoint(&self) -> Box<dyn Any + Send + Sync> {
+        Box::new(self.handler.clone())
+    }
+    fn get_response(&self) -> Option<ResponseSubjectsGenerator> {
+        self.response_subject_fn.clone()
+    }
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerExt<T>
+where
+    T: EndpointTraits,
+{
+    pub consumer: PullConsumer,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[derive(Clone, derive_more::Debug)]
+pub struct ConsumerBuilder<T>
+where
+    T: EndpointTraits,
+{
+    pub name: String,
+    pub endpoint_subject: String,
+    pub handler: EndpointType<T>,
+    #[debug(skip)]
+    pub response_subject_fn: Option<ResponseSubjectsGenerator>,
+}
+
+#[allow(dead_code)]
+#[derive(Clone, Debug)]
+pub struct JsStreamServiceInfo<'a> {
+    pub name: &'a str,
+    pub version: &'a str,
+    pub service_subject: &'a str,
+}
+
+#[derive(Clone, Debug)]
+pub struct LogInfo {
+    pub prefix: String,
+    pub service_name: String,
+    pub service_subject: String,
+    pub endpoint_name: String,
+    pub endpoint_subject: String,
+}
+
+#[derive(Clone)]
+pub enum EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + CreateTag,
+{
+    Sync(EndpointHandler<T>),
+    Async(AsyncEndpointHandler<T>),
+}
+
+impl<T> std::fmt::Debug for EndpointType<T>
+where
+    T: Serialize + for<'de> Deserialize<'de> + Send + Sync + Clone + Debug + CreateTag + 'static,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let t = match &self {
+            EndpointType::Async(_) => "EndpointType::Async(<function>)",
+            EndpointType::Sync(_) => "EndpointType::Sync(<function>)",
+        };
+
+        write!(f, "{}", t)
+    }
+}
+impl<T> TryFrom<Box<dyn Any + Send + Sync>> for EndpointType<T>
+where
+    T: EndpointTraits,
+{
+    type Error = anyhow::Error;
+
+    fn try_from(value: Box<dyn Any + Send + Sync>) -> Result<Self, Self::Error> {
+        if let Ok(endpoint) = value.downcast::<EndpointType<T>>() {
+            Ok(*endpoint)
+        } else {
+            Err(anyhow::anyhow!("Failed to downcast to EndpointType"))
+        }
+    }
+}
+
+#[derive(Deserialize, Default)]
+pub struct JsClientBuilder {
+    pub nats_url: String,
+    pub name: String,
+    pub inbox_prefix: String,
+    #[serde(default)]
+    pub credentials_path: Option<String>,
+    #[serde(default)]
+    pub ping_interval: Option<Duration>,
+    #[serde(default)]
+    pub request_timeout: Option<Duration>, // Defaults to 5s
+    #[serde(skip_deserializing)]
+    pub listeners: Vec<EventListener>,
+}
+
+#[derive(Clone, Deserialize, Default)]
+pub struct JsServiceBuilder {
+    pub name: String,
+    pub description: String,
+    pub version: String,
+    pub service_subject: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct PublishInfo {
+    pub subject: String,
+    pub msg_id: String,
+    pub data: Vec<u8>,
+    pub headers: Option<HeaderMap>,
+}
+
+#[derive(Debug)]
+pub struct ErrClientDisconnected;
+impl fmt::Display for ErrClientDisconnected {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Could not reach nats: connection closed")
+    }
+}
+impl Error for ErrClientDisconnected {}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum ServiceError {
+    #[error("Request Error: {0}")]
+    Request(String),
+    #[error(transparent)]
+    Database(#[from] mongodb::error::Error),
+    #[error("Nats Error: {0}")]
+    NATS(String),
+    #[error("Internal Error: {0}")]
+    Internal(String),
+}
diff --git a/rust/util_libs/src/nats_types.rs b/rust/util_libs/src/nats_types.rs
deleted file mode 100644
index ece916b..0000000
--- a/rust/util_libs/src/nats_types.rs
+++ /dev/null
@@ -1,145 +0,0 @@
-/* --------
-NOTE: These types are the standaried types from NATS and are already made available as rust structs via the `nats-jwt` crate.
-IMP: Currently there is an issue serizialing claims that were generated without any permissions. This file removes one of the serialization traits that was causing the issue, but consequently required us to copy down all the related nats claim types.
-TODO: Make PR into `nats-jwt` repo to properly fix the serialization issue with the Permissions Map, so we can import these structs from thhe `nats-jwt` crate, rather than re-implmenting them here.
--------- */
-
-use serde::{Deserialize, Serialize};
-
-/// JWT claims for NATS compatible jwts
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Claims {
-    /// Time when the token was issued in seconds since the unix epoch
-    #[serde(rename = "iat")]
-    pub issued_at: i64,
-
-    /// Public key of the issuer signing nkey
-    #[serde(rename = "iss")]
-    pub issuer: String,
-
-    /// Base32 hash of the claims where this is empty
-    #[serde(rename = "jti")]
-    pub jwt_id: String,
-
-    /// Public key of the account or user the JWT is being issued to
-    pub sub: String,
-
-    /// Friendly name
-    pub name: String,
-
-    /// NATS claims
-    pub nats: NatsClaims,
-
-    /// Time when the token expires (in seconds since the unix epoch)
-    #[serde(rename = "exp", skip_serializing_if = "Option::is_none")]
-    pub expires: Option<i64>,
-}
-
-/// NATS claims describing settings for the user or account
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "lowercase")]
-pub enum NatsClaims {
-    /// Claims for NATS users
-    User {
-        /// Publish and subscribe permissions for the user
-        #[serde(flatten)]
-        permissions: NatsPermissionsMap,
-
-        /// Public key/id of the account that issued the JWT
-        issuer_account: String,
-
-        /// Maximum nuber of subscriptions the user can have
-        subs: i64,
-
-        /// Maximum size of the message data the user can send in bytes
-        data: i64,
-
-        /// Maximum size of the entire message payload the user can send in bytes
-        payload: i64,
-
-        /// If true, the user isn't challenged on connection. Typically used for websocket
-        /// connections as the browser won't have/want to have the user's private key.
-        bearer_token: bool,
-
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-    /// Claims for NATS accounts
-    Account {
-        /// Configuration for the limits for this account
-        limits: NatsAccountLimits,
-
-        /// List of signing keys (public key) this account uses
-        #[serde(skip_serializing_if = "Vec::is_empty")]
-        signing_keys: Vec<String>,
-
-        /// Default publish and subscribe permissions users under this account will have if not
-        /// specified otherwise
-        /// default_permissions: NatsPermissionsMap,
-        ///
-        /// Version of the nats claims object, always 2 in this crate
-        version: i64,
-    },
-}
-
-/// List of subjects that are allowed and/or denied
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissions {
-    /// List of subject patterns that are allowed
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub allow: Vec<String>,
-
-    /// List of subject patterns that are denied
-    /// #[serde(skip_serializing_if = "Vec::is_empty")]
-    /// ^^ causes the serialization to fail when tyring to seralize raw json into this struct...
-    pub deny: Vec<String>,
-}
-
-impl NatsPermissions {
-    /// Returns `true` if the allow and deny list are both empty
-    #[must_use]
-    pub fn is_empty(&self) -> bool {
-        self.allow.is_empty() && self.deny.is_empty()
-    }
-}
-
-/// Publish and subcribe permissons
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct NatsPermissionsMap {
-    /// Permissions for which subjects can be published to
-    #[serde(rename = "pub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub publish: NatsPermissions,
-
-    /// Permissions for which subjects can be subscribed to
-    #[serde(rename = "sub", skip_serializing_if = "NatsPermissions::is_empty")]
-    pub subscribe: NatsPermissions,
-}
-
-/// Limits on what an account or users in the account can do
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NatsAccountLimits {
-    /// Maximum nuber of subscriptions the account
-    pub subs: i64,
-
-    /// Maximum size of the message data a user can send in bytes
-    pub data: i64,
-
-    /// Maximum size of the entire message payload a user can send in bytes
-    pub payload: i64,
-
-    /// Maxiumum number of imports for the account
-    pub imports: i64,
-
-    /// Maxiumum number of exports for the account
-    pub exports: i64,
-
-    /// If true, exports can contain wildcards
-    pub wildcards: bool,
-
-    /// Maximum number of active connections
-    pub conn: i64,
-
-    /// Maximum number of leaf node connections
-    pub leaf: i64,
-}
diff --git a/rust/util_libs/test_configs/hub_server.conf b/rust/util_libs/test_configs/hub_server.conf
deleted file mode 100644
index 0184098..0000000
--- a/rust/util_libs/test_configs/hub_server.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-operator: "./test-auth/test-operator/test-operator.jwt"
-system_account: SYS
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-include ./resolver.conf
-
-# logging options
-debug:   true
-trace:   true
-logtime: false
diff --git a/rust/util_libs/test_configs/hub_server_pw_auth.conf b/rust/util_libs/test_configs/hub_server_pw_auth.conf
deleted file mode 100644
index 51eeb3f..0000000
--- a/rust/util_libs/test_configs/hub_server_pw_auth.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-server_name: test_hub_server
-listen: localhost:4333
-
-jetstream {
-    enabled: true
-    domain: "hub" 
-    store_dir: "./tmp/hub_store"
-}
-
-leafnodes {
-    port: 7422
-}
-
-authorization {
-    user: "test-user"
-    password: "pw-12345"
-}
-
-# logging options
-debug:   true
-trace:   true
-logtime: false
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index 347ea6c..1e7ef8c 100644
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -11,12 +11,12 @@
 # The operator is generated and named "HOLO" and a system account is generated and named SYS.  Both are assigned two signing keys and are associated with the JWT server.  The JWT server URL set to nats://0.0.0.0:4222.
 
 # Account Creation:
-# Two accounts, named "ADMIN" and "WORKLOAD", are created with JetStream enabled.  Both are associated with the HOLO Operator.
+# Two accounts, named "ADMIN" and "HPOS", are created with JetStream enabled.  Both are associated with the HOLO Operator.
 # Each account has a signing key with a randomly generated role name, which is assigned scoped permissions to allow only users assigned to the signing key to publish and subscribe to their respective streams.
 
 # User Creation:
 # One user named "admin" is created under the "ADMIN" account.
-# One user named "orchestrator" is created under the "WORKLOAD" account.
+# One user named "orchestrator" is created under the "HPOS" account.
 
 # JWT Generation:
 # JWT files are generated for the operator and both accounts, saved in the jwt_output/ directory.
@@ -25,16 +25,16 @@
 #   - OPERATOR_NAME
 #   - SYS_ACCOUNT
 #   - ACCOUNT_JWT_SERVER
-#   - WORKLOAD_ACCOUNT
+#   - HPOS_ACCOUNT
 #   - ADMIN_ACCOUNT
 #   - JWT_OUTPUT_DIR
 #   - RESOLVER_FILE
 
 # Output:
 # One Operator: HOLO
-# Two accounts: HOLO/ADMIN & `HOLO/WORKLOAD`
-# Two Users: /HOLO/ADMIN/admin & HOLO/WORKLOAD/orchestrator
-# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and workload_account.jwt in the `jwt_output` directory.
+# Two accounts: HOLO/ADMIN & `HOLO/HPOS`
+# One Users: /HOLO/ADMIN/admin
+# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and hpos_account.jwt in the `jwt_output` directory.
 # --------
 
 set -e # Exit on any error
@@ -50,11 +50,13 @@ for cmd in nsc nats; do
 done
 
 # Variables
+NATS_SERVER_HOST=$1
+NATS_PORT="4222"
+OPERATOR_SERVICE_URL="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
+ACCOUNT_JWT_SERVER="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
 OPERATOR_NAME="HOLO"
 SYS_ACCOUNT="SYS"
-ACCOUNT_JWT_SERVER="nats://143.244.144.52:4222"
-OPERATOR_SERVICE_URL="nats://143.244.144.52:4222"
-WORKLOAD_ACCOUNT="WORKLOAD"
+HPOS_ACCOUNT="HPOS"
 ADMIN_ACCOUNT="ADMIN"
 JWT_OUTPUT_DIR="jwt_output"
 RESOLVER_FILE="main-resolver.conf"
@@ -70,33 +72,36 @@ nsc edit operator --sk generate
 # Step 2: Create ADMIN_Account with JetStream and scoped signing key
 nsc add account --name $ADMIN_ACCOUNT
 nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_ADMIN="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_ADMIN="admin_role"
-nsc edit signing-key --sk $SIGNING_KEY_ADMIN --role $ROLE_NAME_ADMIN --allow-pub "ADMIN_>" --allow-sub "ADMIN_>" --allow-pub-response
-
-# Step 3: Create WORKLOAD Account with JetStream and scoped signing key
-nsc add account --name $WORKLOAD_ACCOUNT
-nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_WORKLOAD="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_WORKLOAD="workload-role"
-nsc edit signing-key --sk $SIGNING_KEY_WORKLOAD --role $ROLE_NAME_WORKLOAD --allow-pub "WORKLOAD.>" --allow-sub "WORKLOAD.>" --allow-pub-response
-
-# Step 4: Create User "orchestrator" in ADMIN Account // noauth
+ADMIN_SIGNING_KEY="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+ADMIN_ROLE_NAME="admin_role"
+nsc edit signing-key --sk $ADMIN_SIGNING_KEY --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","*._WORKLOAD_INBOX.>" --allow-sub "ADMIN.>""WORKLOAD.>","\$JS.API.>","\$SYS.>","_INBOX.>","_INBOX_*.>","ORCHESTRATOR._WORKLOAD_INBOX.>" --allow-pub-response
+
+# Step 3: Create HPOS Account with JetStream and scoped signing key
+nsc add account --name $HPOS_ACCOUNT
+nsc edit account --name $HPOS_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
+HPOS_SIGNING_KEY="$(echo "$(nsc edit account -n $HPOS_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+WORKLOAD_ROLE_NAME="workload-role"
+nsc edit signing-key --sk $HPOS_SIGNING_KEY --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","{{tag(pubkey)}}._WORKLOAD_INBOX.>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","{{tag(pubkey)}}._WORKLOAD_INBOX.>" --allow-pub-response
+
+# Step 4: Create User "admin" in ADMIN Account
 nsc add user --name admin --account $ADMIN_ACCOUNT
 
-# Step 5: Create User "orchestrator" in WORKLOAD Account
-nsc add user --name orchestrator --account $WORKLOAD_ACCOUNT
+# Step 5: Export/Import WORKLOAD Service Stream between ADMIN and HPOS accounts
+# Share orchestrator (as admin user) workload streams with host
+nsc add export --name "WORKLOAD_SERVICE" --subject "WORKLOAD.>" --account ADMIN
+nsc add import --src-account ADMIN --name "WORKLOAD_SERVICE" --local-subject "WORKLOAD.>" --account HPOS
+# Share host workload streams with orchestrator (as admin user)
+nsc add export --name "WORKLOAD_SERVICE" --subject "WORKLOAD.>" --account HPOS
+nsc add import --src-account HPOS --name "WORKLOAD_SERVICE" --local-subject "WORKLOAD.>" --account ADMIN
 
 # Step 6: Generate JWT files
 nsc describe operator --raw --output-file $JWT_OUTPUT_DIR/holo_operator.jwt
 nsc describe account --name SYS --raw --output-file $JWT_OUTPUT_DIR/sys_account.jwt
-nsc describe account --name $WORKLOAD_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/workload_account.jwt
+nsc describe account --name $HPOS_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/hpos_account.jwt
 nsc describe account --name $ADMIN_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/admin_account.jwt
 
 # Step 7: Generate Resolver Config
 nsc generate config --nats-resolver --sys-account $SYS_ACCOUNT --force --config-file $RESOLVER_FILE
 
-# Step 8: Push credentials to NATS server
-nsc push -A
-
 echo "Setup complete. JWTs and resolver file are in the $JWT_OUTPUT_DIR/ directory."
+echo "!! Don't forget to start the NATS server and push the credentials to the server with 'nsc push -A' !!"

From ffa16a5202f4324ca74fd75967311915ee4d656d Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 17:48:40 -0600
Subject: [PATCH 90/91] restore 5 host test

---
 .env.example                                | 14 ++++
 .gitignore                                  |  2 +-
 nix/checks/holo-agent-integration-nixos.nix | 78 ++++++++++-----------
 3 files changed, 54 insertions(+), 40 deletions(-)
 create mode 100644 .env.example

diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..3ca30fd
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,14 @@
+# ALL
+NSC_PATH="<path/to/nsc_dir>"
+NATS_URL="nats:/<domain>:<port>"
+NATS_LISTEN_PORT="<port>"
+LOCAL_CREDS_PATH="<path/to/dir>"
+
+# ORCHESTRATOR
+MONGO_URI="mongodb://<host>:<port>"
+
+# HOSTING AGENT
+HOST_CREDS_FILE_PATH = "ops/admin.creds"
+LEAF_SERVER_DEFAULT_LISTEN_PORT="4111"
+LEAF_SERVER_USER = "test-user"
+LEAF_SERVER_PW = "pw-123456789"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index c4382e7..dbb5350 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,4 +23,4 @@ rust/*/*/resolver.conf
 leaf_server.conf
 .local
 rust/*/*/*/tmp/
-rust/*/*/*/*/tmp/
\ No newline at end of file
+rust/*/*/*/*/tmp/
diff --git a/nix/checks/holo-agent-integration-nixos.nix b/nix/checks/holo-agent-integration-nixos.nix
index ccb4953..5e63704 100644
--- a/nix/checks/holo-agent-integration-nixos.nix
+++ b/nix/checks/holo-agent-integration-nixos.nix
@@ -180,32 +180,32 @@ pkgs.testers.runNixOSTest (
 
         with subtest("start the hosts and ensure they have TCP level connectivity to the hub"):
           host1.start()
-          # host2.start()
-          # host3.start()
-          # host4.start()
-          # host5.start()
+          host2.start()
+          host3.start()
+          host4.start()
+          host5.start()
 
           host1.wait_for_open_port(addr = "${nodes.hub.networking.fqdn}", port = ${builtins.toString nodes.hub.holo.nats-server.websocket.externalPort}, timeout = 10)
 
           host1.wait_for_unit('holo-host-agent')
-          # host2.wait_for_unit('holo-host-agent')
-          # host3.wait_for_unit('holo-host-agent')
-          # host4.wait_for_unit('holo-host-agent')
-          # host5.wait_for_unit('holo-host-agent')
+          host2.wait_for_unit('holo-host-agent')
+          host3.wait_for_unit('holo-host-agent')
+          host4.wait_for_unit('holo-host-agent')
+          host5.wait_for_unit('holo-host-agent')
 
         with subtest("running the setup script on the hosts"):
           host1.succeed("${hostSetupScript}", timeout = 1)
-          # host2.succeed("${hostSetupScript}", timeout = 1)
-          # host3.succeed("${hostSetupScript}", timeout = 1)
-          # host4.succeed("${hostSetupScript}", timeout = 1)
-          # host5.succeed("${hostSetupScript}", timeout = 1)
+          host2.succeed("${hostSetupScript}", timeout = 1)
+          host3.succeed("${hostSetupScript}", timeout = 1)
+          host4.succeed("${hostSetupScript}", timeout = 1)
+          host5.succeed("${hostSetupScript}", timeout = 1)
 
         with subtest("wait until all hosts receive all published messages"):
           host1.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          # host2.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          # host3.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          # host4.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
-          # host5.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          host2.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          host3.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          host4.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
+          host5.succeed("${pkgs.writeShellScript "receive-all-msgs" ''set -x; ${natsCmdHosts} --trace sub --stream "${testStreamName}" '${testStreamName}.integrate' --count=10''}", timeout = 5)
 
         with subtest("publish more messages from the hub and ensure they arrive on all hosts"):
           hub.succeed("${pkgs.writeShellScript "script" ''
@@ -216,29 +216,29 @@ pkgs.testers.runNixOSTest (
           ''}", timeout = 1)
 
           host1.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host1' --count=10''}", timeout = 5)
-          # host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
-          # host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
-          # host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
-          # host5.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
-
-        # with subtest("bring a host down, publish messages, bring it back up, make sure it receives all messages"):
-        #   host5.shutdown()
-
-        #   hub.succeed("${pkgs.writeShellScript "script" ''
-        #     set -xeE
-        #     for i in `seq 1 5`; do
-        #       ${natsCmdHub} pub --count=10 "${testStreamName}.host''${i}" --js-domain ${hubJsDomain} "{\"message\":\"hello host''${i}\"}"
-        #     done
-        #   ''}", timeout = 2)
-
-        #   host1.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host1' --count=10''}", timeout = 5)
-        #   host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
-        #   host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
-        #   host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
-
-        #   host5.start()
-        #   host5.wait_for_unit('holo-host-agent')
-        #   host5.wait_until_succeeds("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
+          host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
+          host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
+          host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
+          host5.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
+
+        with subtest("bring a host down, publish messages, bring it back up, make sure it receives all messages"):
+          host5.shutdown()
+
+          hub.succeed("${pkgs.writeShellScript "script" ''
+            set -xeE
+            for i in `seq 1 5`; do
+              ${natsCmdHub} pub --count=10 "${testStreamName}.host''${i}" --js-domain ${hubJsDomain} "{\"message\":\"hello host''${i}\"}"
+            done
+          ''}", timeout = 2)
+
+          host1.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host1' --count=10''}", timeout = 5)
+          host2.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host2' --count=10''}", timeout = 5)
+          host3.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host3' --count=10''}", timeout = 5)
+          host4.succeed("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host4' --count=10''}", timeout = 5)
+
+          host5.start()
+          host5.wait_for_unit('holo-host-agent')
+          host5.wait_until_succeeds("${pkgs.writeShellScript "receive-specific-msgs" ''${natsCmdHosts} sub --stream "${testStreamName}" '${testStreamName}.host5' --count=10''}", timeout = 5)
       '';
   }
 )

From a534e588b35bcc09157882c715a6841e839e7522 Mon Sep 17 00:00:00 2001
From: JettTech <ldjetton21@gmail.com>
Date: Fri, 21 Feb 2025 18:22:27 -0600
Subject: [PATCH 91/91] update host client log, update script

---
 rust/clients/host_agent/src/hostd/workload.rs |  3 +-
 scripts/orchestrator_setup.sh                 | 59 ++++++++++---------
 2 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/rust/clients/host_agent/src/hostd/workload.rs b/rust/clients/host_agent/src/hostd/workload.rs
index 3f76cda..5d23d49 100644
--- a/rust/clients/host_agent/src/hostd/workload.rs
+++ b/rust/clients/host_agent/src/hostd/workload.rs
@@ -59,6 +59,7 @@ pub async fn run(
     })
     .await
     .map_err(|e| anyhow::anyhow!("connecting to NATS via {nats_url}: {e}"))?;
+
     // ==================== Setup JS Stream Service ====================
     // Instantiate the Workload API
     let workload_api = HostWorkloadApi::default();
@@ -79,7 +80,7 @@ pub async fn run(
         .get_js_service(WORKLOAD_SRV_NAME.to_string())
         .await
         .ok_or(anyhow!(
-            "Failed to locate workload service. Unable to spin up Host Agent."
+            "Failed to locate workload service. Unable to run holo agent workload service."
         ))?;
 
     workload_service
diff --git a/scripts/orchestrator_setup.sh b/scripts/orchestrator_setup.sh
index 347ea6c..3d28c9e 100644
--- a/scripts/orchestrator_setup.sh
+++ b/scripts/orchestrator_setup.sh
@@ -11,12 +11,12 @@
 # The operator is generated and named "HOLO" and a system account is generated and named SYS.  Both are assigned two signing keys and are associated with the JWT server.  The JWT server URL set to nats://0.0.0.0:4222.
 
 # Account Creation:
-# Two accounts, named "ADMIN" and "WORKLOAD", are created with JetStream enabled.  Both are associated with the HOLO Operator.
+# Two accounts, named "ADMIN" and "HPOS", are created with JetStream enabled.  Both are associated with the HOLO Operator.
 # Each account has a signing key with a randomly generated role name, which is assigned scoped permissions to allow only users assigned to the signing key to publish and subscribe to their respective streams.
 
 # User Creation:
 # One user named "admin" is created under the "ADMIN" account.
-# One user named "orchestrator" is created under the "WORKLOAD" account.
+# One user named "orchestrator" is created under the "HPOS" account.
 
 # JWT Generation:
 # JWT files are generated for the operator and both accounts, saved in the jwt_output/ directory.
@@ -25,16 +25,16 @@
 #   - OPERATOR_NAME
 #   - SYS_ACCOUNT
 #   - ACCOUNT_JWT_SERVER
-#   - WORKLOAD_ACCOUNT
+#   - HPOS_ACCOUNT
 #   - ADMIN_ACCOUNT
 #   - JWT_OUTPUT_DIR
 #   - RESOLVER_FILE
 
 # Output:
 # One Operator: HOLO
-# Two accounts: HOLO/ADMIN & `HOLO/WORKLOAD`
-# Two Users: /HOLO/ADMIN/admin & HOLO/WORKLOAD/orchestrator
-# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and workload_account.jwt in the `jwt_output` directory.
+# Two accounts: HOLO/ADMIN & `HOLO/HPOS`
+# One Users: /HOLO/ADMIN/admin
+# JWT Files: holo-operator.jwt, sys_account.jwt, admin_account.jwt and hpos_account.jwt in the `jwt_output` directory.
 # --------
 
 set -e # Exit on any error
@@ -50,11 +50,13 @@ for cmd in nsc nats; do
 done
 
 # Variables
+NATS_SERVER_HOST=$1
+NATS_PORT="4222"
+OPERATOR_SERVICE_URL="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
+ACCOUNT_JWT_SERVER="nats://{$NATS_SERVER_HOST}:$NATS_PORT"
 OPERATOR_NAME="HOLO"
 SYS_ACCOUNT="SYS"
-ACCOUNT_JWT_SERVER="nats://143.244.144.52:4222"
-OPERATOR_SERVICE_URL="nats://143.244.144.52:4222"
-WORKLOAD_ACCOUNT="WORKLOAD"
+HPOS_ACCOUNT="HPOS"
 ADMIN_ACCOUNT="ADMIN"
 JWT_OUTPUT_DIR="jwt_output"
 RESOLVER_FILE="main-resolver.conf"
@@ -70,33 +72,36 @@ nsc edit operator --sk generate
 # Step 2: Create ADMIN_Account with JetStream and scoped signing key
 nsc add account --name $ADMIN_ACCOUNT
 nsc edit account --name $ADMIN_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_ADMIN="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_ADMIN="admin_role"
-nsc edit signing-key --sk $SIGNING_KEY_ADMIN --role $ROLE_NAME_ADMIN --allow-pub "ADMIN_>" --allow-sub "ADMIN_>" --allow-pub-response
-
-# Step 3: Create WORKLOAD Account with JetStream and scoped signing key
-nsc add account --name $WORKLOAD_ACCOUNT
-nsc edit account --name $WORKLOAD_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
-SIGNING_KEY_WORKLOAD="$(echo "$(nsc edit account -n $WORKLOAD_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
-ROLE_NAME_WORKLOAD="workload-role"
-nsc edit signing-key --sk $SIGNING_KEY_WORKLOAD --role $ROLE_NAME_WORKLOAD --allow-pub "WORKLOAD.>" --allow-sub "WORKLOAD.>" --allow-pub-response
-
-# Step 4: Create User "orchestrator" in ADMIN Account // noauth
+ADMIN_SIGNING_KEY="$(echo "$(nsc edit account -n $ADMIN_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+ADMIN_ROLE_NAME="admin-role"
+nsc edit signing-key --sk $ADMIN_SIGNING_KEY --role $ADMIN_ROLE_NAME --allow-pub "ADMIN.>","WORKLOAD.>","\$JS.>","\$SYS.>","_INBOX.>","_INBOX_*.>","*._WORKLOAD_INBOX.>" --allow-sub "ADMIN.>""WORKLOAD.>","\$JS.>","\$SYS.>","_INBOX.>","_INBOX_*.>","ORCHESTRATOR._WORKLOAD_INBOX.>" --allow-pub-response
+
+# Step 3: Create HPOS Account with JetStream and scoped signing key
+nsc add account --name $HPOS_ACCOUNT
+nsc edit account --name $HPOS_ACCOUNT --js-streams -1 --js-consumer -1 --js-mem-storage 1G --js-disk-storage 5G
+HPOS_SIGNING_KEY="$(echo "$(nsc edit account -n $HPOS_ACCOUNT --sk generate 2>&1)" | grep -oP "signing key\s*\K\S+")"
+WORKLOAD_ROLE_NAME="workload-role"
+nsc edit signing-key --sk $HPOS_SIGNING_KEY --role $WORKLOAD_ROLE_NAME --allow-pub "WORKLOAD.>","{{tag(pubkey)}}._WORKLOAD_INBOX.>","\$JS.API>" --allow-sub "WORKLOAD.{{tag(pubkey)}}.*","{{tag(pubkey)}}._WORKLOAD_INBOX.>","\$JS.API>" --allow-pub-response
+
+# Step 4: Create User "admin" in ADMIN Account
 nsc add user --name admin --account $ADMIN_ACCOUNT
 
-# Step 5: Create User "orchestrator" in WORKLOAD Account
-nsc add user --name orchestrator --account $WORKLOAD_ACCOUNT
+# Step 5: Export/Import WORKLOAD Service Stream between ADMIN and HPOS accounts
+# Share orchestrator (as admin user) workload streams with host
+nsc add export --name "WORKLOAD_SERVICE" --subject "WORKLOAD.>" --account ADMIN
+nsc add import --src-account ADMIN --name "WORKLOAD_SERVICE" --remote-subject "WORKLOAD.>" --local-subject "WORKLOAD.>" --account HPOS
+# Share host workload streams with orchestrator (as admin user)
+nsc add export --name "WORKLOAD_SERVICE" --subject "WORKLOAD.>" --account HPOS
+nsc add import --src-account HPOS --name "WORKLOAD_SERVICE" --remote-subject "WORKLOAD.>" --local-subject "WORKLOAD.>" --account ADMIN
 
 # Step 6: Generate JWT files
 nsc describe operator --raw --output-file $JWT_OUTPUT_DIR/holo_operator.jwt
 nsc describe account --name SYS --raw --output-file $JWT_OUTPUT_DIR/sys_account.jwt
-nsc describe account --name $WORKLOAD_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/workload_account.jwt
+nsc describe account --name $HPOS_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/hpos_account.jwt
 nsc describe account --name $ADMIN_ACCOUNT --raw --output-file $JWT_OUTPUT_DIR/admin_account.jwt
 
 # Step 7: Generate Resolver Config
 nsc generate config --nats-resolver --sys-account $SYS_ACCOUNT --force --config-file $RESOLVER_FILE
 
-# Step 8: Push credentials to NATS server
-nsc push -A
-
 echo "Setup complete. JWTs and resolver file are in the $JWT_OUTPUT_DIR/ directory."
+echo "!! Don't forget to start the NATS server and push the credentials to the server with 'nsc push -A' !!"