Merge pull request #850 from scrtlabs/throttle-enclave-access

WIP: Put enclave access behind a reentrant mutex

CHANGELOG.md

@@ -1,14 +1,30 @@
 # CHANGELOG
 
-# 1.2.0-beta1
+# 1.2.2
 
-Version 1.2.0-beta1 has been released - Supernova upgrade testnet v1!
+## Secretd
+
+* Fixed issue where queries would try to access the Enclave in parallel from multiple threads,
+  causing `SGX_ERROR_OUT_OF_TCS` to be returned to users when a node was under sufficient load.
+  Queries now access the enclave one-at-a-time again.
+
+# 1.2.1
+
+This is a minor non-breaking version release. 
+
+## SecretCLI
+
+- Migrate the `secretcli tx sign-doc` command from v1. See [this](https://github.com/enigmampc/snip20-reference-impl/pull/22) for more info.
+
+# 1.2.0
+
+Version 1.2.0 has been released - the Supernova upgrade!
 
 ## Highlights
 
-* Upgraded to Cosmos SDK 0.43. Full changelog can be found [here](https://github.com/cosmos/cosmos-sdk/blob/v0.43.0/CHANGELOG.md)
+* Upgraded to Cosmos SDK 0.44.3. Full changelog can be found [here](https://github.com/cosmos/cosmos-sdk/blob/v0.44.3/CHANGELOG.md)
 
-* Gas prices are lower - as a result of performance upgrades and optimizations, gas amounts required will be much lower. We will be monitoring these metrics during the testnet period, so the numbers may not be final
+* Gas prices are lower - as a result of performance upgrades and optimizations, gas amounts required will be much lower.
 * GRPC for cosmos-sdk modules in addition to legacy REST API. See API [here](http://bootstrap.supernova.enigma.co/swagger/)
 
 * New modules:
@@ -44,7 +60,7 @@ modifying /home/\<account\>/.secretd/config/app.toml and looking for the `api` c
 
 ## SecretJS
 
-Version 0.17.0-beta1 has been released!
+Version 0.17.3 has been released!
 SecretJS has been upgraded to support the Supernova upgrade.
 All APIs remain unchanged, although the versions are NOT backwards compatible.
 
@@ -57,7 +73,7 @@ Secret-CosmWasm remains in a version that is compatabile with the v0.10 of vanil
 
 A new feature has been added - plaintext logs. To send an unencrypted log (contract output), use `plaintext_log` instead of `log`.
 This allows contracts to emit public events, and attach websockets to listen to specific events. To take advantage of this feature, compile contracts with
-`cosmwasm-std = { git = "https://github.com/enigmampc/SecretNetwork", tag = "v1.2.0-beta1" }`
+`cosmwasm-std = { git = "https://github.com/enigmampc/SecretNetwork", tag = "v1.2.0" }`
 
 ## Known Issues
 

cosmwasm/packages/sgx-vm/Cargo.toml

@@ -64,6 +64,7 @@ sgx_types = { git = "https://github.com/apache/teaclave-sgx-sdk.git", rev = "a37
 sgx_urts = { git = "https://github.com/apache/teaclave-sgx-sdk.git", rev = "a37ffb9449ba6d5b6e4a9d586bbab864ae732269" }
 log = "0.4.8"
 base64 = "0.12.0"
+parking_lot = "0.11"
 
 [dev-dependencies]
 tempfile = "3.1.0"

cosmwasm/packages/sgx-vm/src/attestation.rs

@@ -127,7 +127,7 @@ pub extern "C" fn ocall_get_update_info(
 }
 
 pub fn create_attestation_report_u(spid: &[u8], api_key: &[u8]) -> SgxResult<()> {
-    let enclave = get_enclave()?;
+    let enclave = get_enclave()?.ok_or(sgx_status_t::SGX_ERROR_OUT_OF_TCS)?;
 
     let eid = enclave.geteid();
     let mut retval = sgx_status_t::SGX_SUCCESS;
@@ -156,7 +156,7 @@ pub fn create_attestation_report_u(spid: &[u8], api_key: &[u8]) -> SgxResult<()>
 pub fn untrusted_get_encrypted_seed(
     cert: &[u8],
 ) -> SgxResult<Result<[u8; ENCRYPTED_SEED_SIZE], NodeAuthResult>> {
-    let enclave = get_enclave()?;
+    let enclave = get_enclave()?.ok_or(sgx_status_t::SGX_ERROR_OUT_OF_TCS)?;
     let eid = enclave.geteid();
     let mut retval = NodeAuthResult::Success;
     let mut seed = [0u8; ENCRYPTED_SEED_SIZE];

cosmwasm/packages/sgx-vm/src/enclave.rs

@@ -1,6 +1,9 @@
-use std::{env, path::Path, sync::Mutex};
+use enclave_ffi_types::{EnclaveBuffer, RuntimeConfiguration};
+use std::env;
+use std::ops::Deref;
+use std::path::Path;
+use std::time::Duration;
 
-use enclave_ffi_types::RuntimeConfiguration;
 use sgx_types::{
     sgx_attributes_t, sgx_enclave_id_t, sgx_launch_token_t, sgx_misc_attribute_t, sgx_status_t,
     SgxResult,
@@ -9,6 +12,9 @@ use sgx_urts::SgxEnclave;
 
 use lazy_static::lazy_static;
 use log::*;
+use parking_lot::{Mutex, ReentrantMutex, ReentrantMutexGuard};
+
+use crate::wasmi::imports;
 
 static ENCLAVE_FILE: &str = "librust_cosmwasm_enclave.signed.so";
 
@@ -18,6 +24,34 @@ const ENCLAVE_DEBUG: i32 = 0;
 #[cfg(not(feature = "production"))]
 const ENCLAVE_DEBUG: i32 = 1;
 
+struct EnclaveMutex {
+    enclave: ReentrantMutex<SgxEnclave>,
+}
+
+impl EnclaveMutex {
+    fn new() -> SgxResult<EnclaveMutex> {
+        let enclave = ReentrantMutex::new(init_enclave()?);
+        Ok(Self { enclave })
+    }
+
+    fn get_enclave(&'static self, timeout: Duration) -> Option<EnclaveGuard> {
+        let guard = self.enclave.try_lock_for(timeout);
+        guard.map(|guard| EnclaveGuard { guard })
+    }
+}
+
+pub struct EnclaveGuard {
+    guard: ReentrantMutexGuard<'static, SgxEnclave>,
+}
+
+impl Deref for EnclaveGuard {
+    type Target = SgxEnclave;
+
+    fn deref(&self) -> &Self::Target {
+        self.guard.deref()
+    }
+}
+
 fn init_enclave() -> SgxResult<SgxEnclave> {
     let mut launch_token: sgx_launch_token_t = [0; 1024];
     let mut launch_token_updated: i32 = 0;
@@ -68,16 +102,24 @@ fn init_enclave() -> SgxResult<SgxEnclave> {
 
 #[allow(clippy::mutex_atomic)]
 lazy_static! {
-    static ref SGX_ENCLAVE: SgxResult<SgxEnclave> = init_enclave();
+    static ref SGX_ENCLAVE_MUTEX: SgxResult<EnclaveMutex> = EnclaveMutex::new();
+
     /// This variable indicates if the enclave configuration has already been set
     static ref SGX_ENCLAVE_CONFIGURED: Mutex<bool> = Mutex::new(false);
 }
 
+/// This const determines how many seconds we wait when trying to get access to the enclave
+/// before giving up.
+const ENCLAVE_LOCK_TIMEOUT: u64 = 6;
+
 /// Use this method when trying to get access to the enclave.
 /// You can unwrap the result when you are certain that the enclave
 /// must have been initialized if you even reached that point in the code.
-pub fn get_enclave() -> SgxResult<&'static SgxEnclave> {
-    SGX_ENCLAVE.as_ref().map_err(|status| *status)
+/// If `Ok(None)` is returned, that means that the enclave is currently busy.
+pub fn get_enclave() -> SgxResult<Option<EnclaveGuard>> {
+    let mutex = SGX_ENCLAVE_MUTEX.as_ref().map_err(|status| *status)?;
+    let maybe_guard = mutex.get_enclave(Duration::from_secs(ENCLAVE_LOCK_TIMEOUT));
+    Ok(maybe_guard)
 }
 
 extern "C" {
@@ -101,14 +143,15 @@ impl EnclaveRuntimeConfig {
 }
 
 pub fn configure_enclave(config: EnclaveRuntimeConfig) -> SgxResult<()> {
-    let mut configured = SGX_ENCLAVE_CONFIGURED.lock().unwrap();
+    let mut configured = SGX_ENCLAVE_CONFIGURED.lock();
     if *configured {
         return Ok(());
     }
     *configured = true;
     drop(configured);
 
-    let enclave = get_enclave()?;
+    let enclave = get_enclave()?
+        .expect("This function should only be called once when the node is initializing");
 
     let mut retval = sgx_status_t::SGX_SUCCESS;
 
@@ -125,3 +168,30 @@ pub fn configure_enclave(config: EnclaveRuntimeConfig) -> SgxResult<()> {
 
     Ok(())
 }
+
+/// This is a safe wrapper for allocating buffers inside the enclave.
+///
+/// It must be called after the enclave has been initialized, and can not be called
+/// while another thread is using the enclave, or it will panic.
+pub(super) fn allocate_enclave_buffer(buffer: &[u8]) -> SgxResult<EnclaveBuffer> {
+    let ptr = buffer.as_ptr();
+    let len = buffer.len();
+    let mut enclave_buffer = EnclaveBuffer::default();
+
+    let enclave_id = get_enclave()
+        .expect("If we got here, surely the enclave has been loaded")
+        .expect("If we got here, surely we are the thread that holds the enclave")
+        .geteid();
+
+    trace!(
+        target: module_path!(),
+        "allocate_enclave_buffer() called with len: {:?} enclave_id: {:?}",
+        len,
+        enclave_id,
+    );
+
+    match unsafe { imports::ecall_allocate(enclave_id, &mut enclave_buffer, ptr, len) } {
+        sgx_status_t::SGX_SUCCESS => Ok(enclave_buffer),
+        failure_status => Err(failure_status),
+    }
+}

cosmwasm/packages/sgx-vm/src/enclave_tests.rs

@@ -8,7 +8,7 @@ extern "C" {
 }
 
 pub fn run_tests() -> SgxResult<u32> {
-    let enclave = get_enclave()?;
+    let enclave = get_enclave()?.ok_or(sgx_status_t::SGX_ERROR_OUT_OF_TCS)?;
     let mut failed_tests = 0;
     let status = unsafe { ecall_run_tests(enclave.geteid(), &mut failed_tests) };
     match status {