From da6b6075aaac37bb74d7b048c23e35f925aa1969 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Sat, 7 Dec 2024 02:26:28 +0800 Subject: [PATCH 1/6] KMS: Add GetMeta API for heartbeat. --- kms/rpc/proto/kms_rpc.proto | 8 ++++++++ kms/src/main_service.rs | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/kms/rpc/proto/kms_rpc.proto b/kms/rpc/proto/kms_rpc.proto index 6af014e..365acc4 100644 --- a/kms/rpc/proto/kms_rpc.proto +++ b/kms/rpc/proto/kms_rpc.proto @@ -20,6 +20,9 @@ service KMS { rpc GetAppEnvEncryptPubKey(AppId) returns (PublicKeyResponse) { // Retrieves the app environment encryption public key given the app id } + // Request the KMS instance metadata for use as a probe and health check. + rpc GetMeta(google.protobuf.Empty) returns (GetMetaResponse) { + } } message AppId { @@ -36,3 +39,8 @@ message AppKeyResponse { bytes env_crypt_key = 3; repeated string certificate_chain = 4; } + +message GetMetaResponse { + string ca_cert = 1; + bool allow_any_upgrade = 2; +} \ No newline at end of file diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 2262c08..7f6d488 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::{bail, Context, Result}; use kms_rpc::{ kms_server::{KmsRpc, KmsServer}, - AppId, AppKeyResponse, GetAppKeyRequest, PublicKeyResponse, + AppId, AppKeyResponse, GetAppKeyRequest, PublicKeyResponse, GetMetaResponse, }; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ @@ -193,6 +193,13 @@ impl KmsRpc for RpcHandler { public_key: pubkey.to_bytes().to_vec(), }) } + + async fn get_meta(self) -> Result { + Ok(GetMetaResponse { + ca_cert: self.state.inner.root_ca.cert.pem(), + allow_any_upgrade: self.state.inner.config.allow_any_upgrade, + }) + } } impl RpcCall for RpcHandler { From 5809c4eb64e924c7487d4e75b1b6768afd2a28df Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Mon, 9 Dec 2024 22:20:19 +0800 Subject: [PATCH 2/6] Tproxy: Add GetMeta API for heartbeat. --- tproxy/rpc/proto/tproxy_rpc.proto | 7 +++++++ tproxy/src/main_service.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tproxy/rpc/proto/tproxy_rpc.proto b/tproxy/rpc/proto/tproxy_rpc.proto index f4eb4e2..2c2fa8b 100644 --- a/tproxy/rpc/proto/tproxy_rpc.proto +++ b/tproxy/rpc/proto/tproxy_rpc.proto @@ -81,6 +81,11 @@ message GetInfoResponse { optional HostInfo info = 2; } +message GetMetaResponse { + uint32 registered = 1; + uint32 online = 2; +} + service Tproxy { // Register a new proxied CVM. rpc RegisterCvm(RegisterCvmRequest) returns (RegisterCvmResponse) {} @@ -90,4 +95,6 @@ service Tproxy { rpc AcmeInfo(google.protobuf.Empty) returns (AcmeInfoResponse) {} // Find Proxied HostInfo by instance ID rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) {} + // Summary API for inspect. + rpc GetMeta(google.protobuf.Empty) returns (GetMetaResponse); } diff --git a/tproxy/src/main_service.rs b/tproxy/src/main_service.rs index 9821748..18da193 100644 --- a/tproxy/src/main_service.rs +++ b/tproxy/src/main_service.rs @@ -18,7 +18,7 @@ use smallvec::{smallvec, SmallVec}; use tproxy_rpc::{ tproxy_server::{TproxyRpc, TproxyServer}, AcmeInfoResponse, GetInfoRequest, GetInfoResponse, HostInfo as PbHostInfo, ListResponse, - RegisterCvmRequest, RegisterCvmResponse, TappdConfig, WireGuardConfig, + RegisterCvmRequest, RegisterCvmResponse, TappdConfig, WireGuardConfig, GetMetaResponse, }; use tracing::{debug, error, info, warn}; @@ -446,6 +446,34 @@ impl TproxyRpc for RpcHandler { hist_keys: keys.into_iter().collect(), }) } + + async fn get_meta(self) -> Result { + let state = self.state.lock(); + let handshakes = state.latest_handshakes(None)?; + + // Total registered instances + let registered = state.state.instances.len(); + + // Get current timestamp + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system time before Unix epoch")? + .as_secs(); + + // Count online instances (those with handshakes in last 5 minutes) + let online = handshakes + .values() + .filter(|(ts, _)| { + // Skip instances that never connected (ts == 0) + *ts != 0 && (now - *ts) < 300 + }) + .count(); + + Ok(GetMetaResponse { + registered: registered as u32, + online: online as u32, + }) + } } impl RpcCall for RpcHandler { From f16bed57af828b7a347477e70eda9f228ee286b5 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Mon, 9 Dec 2024 22:20:48 +0800 Subject: [PATCH 3/6] Teepod: Add GetMeta API for heartbeat. --- teepod/rpc/proto/teepod_rpc.proto | 29 ++++++++++++++++++++++++++++- teepod/src/config.rs | 3 +++ teepod/src/main_service.rs | 21 +++++++++++++++++++++ teepod/teepod.toml | 2 ++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/teepod/rpc/proto/teepod_rpc.proto b/teepod/rpc/proto/teepod_rpc.proto index e59e9d4..ad9423f 100644 --- a/teepod/rpc/proto/teepod_rpc.proto +++ b/teepod/rpc/proto/teepod_rpc.proto @@ -128,6 +128,30 @@ message ResizeVmRequest { optional string image = 5; } +message KmsSettings { + string url = 1; +} + +message TProxySettings { + string url = 1; + string base_domain = 2; + uint32 port = 3; + uint32 tappd_port = 4; +} + +message ResourcesSettings { + uint32 max_cvm_number = 1; // equals to the cid pool size. + uint32 max_allocable_vcpu = 2; + uint32 max_allocable_memory_in_mb = 3; // in MB. + uint32 max_disk_size_in_gb = 4; // in GB. +} + +message GetMetaResponse { + KmsSettings kms = 1; + TProxySettings tproxy = 2; + ResourcesSettings resources = 3; +} + message VersionResponse { string version = 1; string rev = 2; @@ -163,4 +187,7 @@ service Teepod { // Get version info of the Teepod rpc Version(google.protobuf.Empty) returns (VersionResponse); -} + + // Get version info of the Teepod + rpc GetMeta(google.protobuf.Empty) returns (GetMetaResponse); +} \ No newline at end of file diff --git a/teepod/src/config.rs b/teepod/src/config.rs index 25f72f8..b522703 100644 --- a/teepod/src/config.rs +++ b/teepod/src/config.rs @@ -89,6 +89,9 @@ pub struct CvmConfig { pub cid_pool_size: u32, /// Port mapping configuration pub port_mapping: PortMappingConfig, + /// Max allocable resources. Not yet implement fully, only for inspect API `GetMeta` + pub max_allocable_vcpu: u32, + pub max_allocable_memory_in_mb: u32, /// Enable qmp socket pub qmp_socket: bool, } diff --git a/teepod/src/main_service.rs b/teepod/src/main_service.rs index 621d7ae..9132b7a 100644 --- a/teepod/src/main_service.rs +++ b/teepod/src/main_service.rs @@ -8,6 +8,7 @@ use teepod_rpc::teepod_server::{TeepodRpc, TeepodServer}; use teepod_rpc::{ AppId, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, PublicKeyResponse, ResizeVmRequest, StatusResponse, UpgradeAppRequest, VersionResponse, VmConfiguration, + GetMetaResponse, KmsSettings, TProxySettings, ResourcesSettings, }; use tracing::{info, warn}; @@ -308,6 +309,26 @@ impl TeepodRpc for RpcHandler { rev: crate::GIT_REV.to_string(), }) } + + async fn get_meta(self) -> Result { + Ok(GetMetaResponse { + kms: Some(KmsSettings { + url: self.app.config.cvm.kms_url.clone(), + }), + tproxy: Some(TProxySettings { + url: self.app.config.cvm.tproxy_url.clone(), + base_domain: self.app.config.gateway.base_domain.clone(), + port: self.app.config.gateway.port.into(), + tappd_port: self.app.config.gateway.tappd_port.into(), + }), + resources: Some(ResourcesSettings { + max_cvm_number: self.app.config.cvm.cid_pool_size, + max_allocable_vcpu: self.app.config.cvm.max_allocable_vcpu, + max_allocable_memory_in_mb: self.app.config.cvm.max_allocable_memory_in_mb, + max_disk_size_in_gb: self.app.config.cvm.max_disk_size, + }), + }) + } } impl RpcCall for RpcHandler { diff --git a/teepod/teepod.toml b/teepod/teepod.toml index 73064dd..ebe37d9 100644 --- a/teepod/teepod.toml +++ b/teepod/teepod.toml @@ -24,6 +24,8 @@ docker_registry = "" max_disk_size = 100 cid_start = 1000 cid_pool_size = 1000 +max_allocable_vcpu = 20 +max_allocable_memory_in_mb = 100_000 # MB qmp_socket = true [cvm.port_mapping] From 38017c7c3e662255056be327f28727212ec28a40 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Tue, 10 Dec 2024 22:09:25 +0800 Subject: [PATCH 4/6] Tproxy: Rename 'id' to 'instance_id' to avoid potential confusion --- tproxy/rpc/proto/tproxy_rpc.proto | 2 +- tproxy/src/main_service.rs | 4 ++-- tproxy/templates/cvmlist.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tproxy/rpc/proto/tproxy_rpc.proto b/tproxy/rpc/proto/tproxy_rpc.proto index 2c2fa8b..eef305b 100644 --- a/tproxy/rpc/proto/tproxy_rpc.proto +++ b/tproxy/rpc/proto/tproxy_rpc.proto @@ -49,7 +49,7 @@ message ListResponse { // HostInfo is the information of a host. message HostInfo { // The Instance id - string id = 1; + string instance_id = 1; // The IP address of the host. string ip = 2; // The app id of the host. diff --git a/tproxy/src/main_service.rs b/tproxy/src/main_service.rs index 18da193..657a586 100644 --- a/tproxy/src/main_service.rs +++ b/tproxy/src/main_service.rs @@ -387,7 +387,7 @@ impl TproxyRpc for RpcHandler { .instances .values() .map(|instance| PbHostInfo { - id: instance.id.clone(), + instance_id: instance.id.clone(), ip: instance.ip.to_string(), app_id: instance.app_id.clone(), base_domain: base_domain.clone(), @@ -411,7 +411,7 @@ impl TproxyRpc for RpcHandler { if let Some(instance) = state.state.instances.get(&request.id) { let host_info = PbHostInfo { - id: instance.id.clone(), + instance_id: instance.id.clone(), ip: instance.ip.to_string(), app_id: instance.app_id.clone(), base_domain: base_domain.clone(), diff --git a/tproxy/templates/cvmlist.html b/tproxy/templates/cvmlist.html index afd869f..3b6e0a9 100644 --- a/tproxy/templates/cvmlist.html +++ b/tproxy/templates/cvmlist.html @@ -128,7 +128,7 @@

CVM List

{% for host in hosts %} - {{ host.id }} + {{ host.instance_id }} {{ host.app_id }} {{ host.ip }} {{ host.latest_handshake }} From 19f1704f8af9d3d2452b9082cce8438173b4e68b Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Tue, 24 Dec 2024 05:56:48 +0800 Subject: [PATCH 5/6] chore: format --- kms/src/main_service.rs | 2 +- teepod/src/main_service.rs | 6 +++--- tproxy/src/main_service.rs | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 7f6d488..c12b26e 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::{bail, Context, Result}; use kms_rpc::{ kms_server::{KmsRpc, KmsServer}, - AppId, AppKeyResponse, GetAppKeyRequest, PublicKeyResponse, GetMetaResponse, + AppId, AppKeyResponse, GetAppKeyRequest, GetMetaResponse, PublicKeyResponse, }; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ diff --git a/teepod/src/main_service.rs b/teepod/src/main_service.rs index 9132b7a..0e7c135 100644 --- a/teepod/src/main_service.rs +++ b/teepod/src/main_service.rs @@ -6,9 +6,9 @@ use fs_err as fs; use ra_rpc::{CallContext, RpcCall}; use teepod_rpc::teepod_server::{TeepodRpc, TeepodServer}; use teepod_rpc::{ - AppId, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, PublicKeyResponse, - ResizeVmRequest, StatusResponse, UpgradeAppRequest, VersionResponse, VmConfiguration, - GetMetaResponse, KmsSettings, TProxySettings, ResourcesSettings, + AppId, GetInfoResponse, GetMetaResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, + KmsSettings, PublicKeyResponse, ResizeVmRequest, ResourcesSettings, StatusResponse, + TProxySettings, UpgradeAppRequest, VersionResponse, VmConfiguration, }; use tracing::{info, warn}; diff --git a/tproxy/src/main_service.rs b/tproxy/src/main_service.rs index 657a586..eb5714a 100644 --- a/tproxy/src/main_service.rs +++ b/tproxy/src/main_service.rs @@ -17,8 +17,8 @@ use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; use tproxy_rpc::{ tproxy_server::{TproxyRpc, TproxyServer}, - AcmeInfoResponse, GetInfoRequest, GetInfoResponse, HostInfo as PbHostInfo, ListResponse, - RegisterCvmRequest, RegisterCvmResponse, TappdConfig, WireGuardConfig, GetMetaResponse, + AcmeInfoResponse, GetInfoRequest, GetInfoResponse, GetMetaResponse, HostInfo as PbHostInfo, + ListResponse, RegisterCvmRequest, RegisterCvmResponse, TappdConfig, WireGuardConfig, }; use tracing::{debug, error, info, warn}; @@ -450,16 +450,16 @@ impl TproxyRpc for RpcHandler { async fn get_meta(self) -> Result { let state = self.state.lock(); let handshakes = state.latest_handshakes(None)?; - + // Total registered instances let registered = state.state.instances.len(); - + // Get current timestamp let now = SystemTime::now() .duration_since(UNIX_EPOCH) .context("system time before Unix epoch")? .as_secs(); - + // Count online instances (those with handshakes in last 5 minutes) let online = handshakes .values() From b30b17461d44e34948c580df22f326994450e629 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Tue, 24 Dec 2024 06:20:14 +0800 Subject: [PATCH 6/6] Inspect public_logs and public_sysinfo option in Worker.Info --- tappd/rpc/proto/tappd_rpc.proto | 4 ++++ tappd/src/http_routes.rs | 2 ++ tappd/src/rpc_service.rs | 2 ++ 3 files changed, 8 insertions(+) diff --git a/tappd/rpc/proto/tappd_rpc.proto b/tappd/rpc/proto/tappd_rpc.proto index 2c62ecb..1bfc4af 100644 --- a/tappd/rpc/proto/tappd_rpc.proto +++ b/tappd/rpc/proto/tappd_rpc.proto @@ -97,6 +97,10 @@ message WorkerInfo { string tcb_info = 4; // App name string app_name = 5; + // Whether the app logs are public + bool public_logs = 6; + // Whether the app sysinfo is public + bool public_sysinfo = 7; } // The response to a WorkerInfo request diff --git a/tappd/src/http_routes.rs b/tappd/src/http_routes.rs index b743934..f864ad7 100644 --- a/tappd/src/http_routes.rs +++ b/tappd/src/http_routes.rs @@ -31,6 +31,8 @@ async fn index(state: &State) -> Result, String> { instance_id, tcb_info, app_cert, + public_logs, + public_sysinfo, } = handler .info() .await diff --git a/tappd/src/rpc_service.rs b/tappd/src/rpc_service.rs index 8febb59..e8d0f12 100644 --- a/tappd/src/rpc_service.rs +++ b/tappd/src/rpc_service.rs @@ -182,6 +182,8 @@ impl WorkerRpc for ExternalRpcHandler { instance_id, app_cert: ca.pem_cert.clone(), tcb_info, + public_logs: self.state.config().public_logs, + public_sysinfo: self.state.config().public_sysinfo, }) }