Merge openenclave#4276

4276: SGX thread interrupt support on Linux r=mingweishih a=mingweishih

This PR adds the OE internal support for the thread interrupt on Linux. The support allows an sophisticated application built on top of OE to implement the thread preemption mechanism. The highlight of this PR is as follows.

- Now the host is forwarding the host signals to the enclave (require the enclave to opt-in)
- Introduce an "interrupt request" logic in the enclave enter flow
  - The host needs to pass an input (value=signo) when entering the enclave (after an AEX occurred)
  - Upon such entry, the enclave checks whether it's an interrupt request. If it's then checking states on the td to determine whether to serve the request
- Introduce the enclave "direct return" flow
  - Now when the enclave logic detects invalid conditions, the enclave will directly "return" to the host (EENTER -> EEXIT) instead of aborting. This allows the enclave runtime to robust handle interrupt requests that could arrive at any time.
- Introduce state machine on the td, which is mainly used to indicate whether the enclave can serve an interrupt request

The PR also adds an end-to-end test that demonstrates the interrupt flow (say `thread 1` wants to interrupt `thread 2`):

a. `thread 1`
   - Make an ocall to create `thread 2` 
   - Wait for the `thread 2` to initialize

b. `thread 2`
   - Register an exception handler
   - Unmask the interrupt

c. `thread 1`
   - Register the host signal (e.g.,  `SIGUSR1`) for `thread 2`
   - Make an ocall to send `SIGUSR1` signal to the `thread 2`

d. `thread 2`
   - The exception handler is invoked after receiving an interrupt

Signed-off-by: Ming-Wei Shih <[email protected]>

Co-authored-by: Ming-Wei Shih <[email protected]>

enclave/core/sgx/asmdefs.h

@@ -17,7 +17,25 @@
 #define STATIC_STACK_SIZE 8 * 100
 #define OE_WORD_SIZE 8
 
-#define CODE_ERET 0x200000000
+/* Defined in oe_result_t (result.h) */
+#define CODE_ENCLAVE_ABORTING 0x13
+
+/* Defined in exception.h */
+#define CODE_EXCEPTION_CONTINUE_EXECUTION 0xFFFFFFFF
+
+/* Assembly code cannot use enum values directly,
+ * define them here to match oe_td_state_t in
+ * internal/sgx/td.h */
+#define TD_STATE_NULL 0
+#define TD_STATE_ENTERED 1
+#define TD_STATE_RUNNING 2
+#define TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING 3
+#define TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING 4
+#define TD_STATE_EXITED 5
+#define TD_STATE_ABORTED 6
+
+/* Set the max signal number based on Linux (i.e., SIGRTMAX) */
+#define MAX_SIGNAL_NUMBER 64
 
 /* Use GS register if this flag is set */
 #ifdef __ASSEMBLER__
@@ -39,6 +57,13 @@
 #define td_host_previous_ecall_context (td_host_ecall_context + 8)
 #define td_exception_handler_stack (td_host_previous_ecall_context + 8)
 #define td_exception_handler_stack_size (td_exception_handler_stack + 8)
+#define td_state (td_exception_handler_stack_size + 8)
+#define td_previous_state (td_state + 8)
+#define td_exception_nesting_level (td_previous_state + 8)
+#define td_host_signal_unmasked (td_exception_nesting_level + 8)
+#define td_is_handling_host_signal (td_host_signal_unmasked + 8)
+#define td_host_signal (td_is_handling_host_signal + 8)
+#define td_host_signal_bitmask (td_host_signal + 8)
 
 #define oe_exit_enclave __morestack
 #ifndef __ASSEMBLER__

enclave/core/sgx/calls.c

@@ -725,7 +725,7 @@ static void _exit_enclave(uint64_t arg1, uint64_t arg2)
             host_ecall_context->debug_eexit_rip = frame[1];
         }
     }
-    oe_asm_exit(arg1, arg2, td, 0 /* aborting */);
+    oe_asm_exit(arg1, arg2, td, 0 /* direct_return */);
 }
 
 /*
@@ -814,6 +814,15 @@ oe_result_t oe_ocall(uint16_t func, uint64_t arg_in, uint64_t* arg_out)
         if (arg_out)
             *arg_out = td->oret_arg;
 
+        if (td->state != OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING)
+        {
+            /* State machine check */
+            if (td->state != OE_TD_STATE_ENTERED)
+                oe_abort();
+
+            td->state = OE_TD_STATE_RUNNING;
+        }
+
         /* ORET here */
     }
 
@@ -1166,6 +1175,14 @@ void __oe_handle_main(
                 if (func == OE_ECALL_VIRTUAL_EXCEPTION_HANDLER)
                     oe_abort();
 
+                /* State machine check */
+                if (td->state != OE_TD_STATE_ENTERED)
+                    oe_abort();
+
+                /* At this point, we are ready to execute the ecall.
+                 * Update the state to RUNNING */
+                td->state = OE_TD_STATE_RUNNING;
+
                 _handle_ecall(td, func, arg_in, output_arg1, output_arg2);
                 break;
             }
@@ -1204,6 +1221,10 @@ void __oe_handle_main(
 
 void oe_abort(void)
 {
+    oe_sgx_td_t* td = oe_sgx_get_td();
+
+    td->state = OE_TD_STATE_ABORTED;
+
     // Once it starts to crash, the state can only transit forward, not
     // backward.
     if (__oe_enclave_status < OE_ENCLAVE_ABORTING)

enclave/core/sgx/enter.S

@@ -50,6 +50,10 @@ oe_enter:
     mov _td_from_tcs_offset(%rip), %r11
     add %rbx, %r11
 
+.check_aborted:
+    cmpq $TD_STATE_ABORTED, td_state(%r11)
+    je .abort
+
     // Get the first ssa address from tcs
     lea OE_SSA_FROM_TCS_BYTE_OFFSET(%rbx), %r10
 
@@ -65,16 +69,28 @@ oe_enter:
 
 .determine_entry_type:
     // Check if this is exception dispatching request
-    // Abort on the eenter if cssa greater than one, which
+    // Return on the eenter if cssa greater than one, which
     // should not occur because OE assumes the enclave with nssa=2
     cmp $1, %rax
     je .exception_entry
-    ja .abort
+    ja .return
 
     // Stop speculative execution at fallthrough of conditional
     // exception-dispatching-request-check.
     lfence
 
+.update_td_state_on_normal_entry:
+    // Do not update the state if the enclave enters in the middle
+    // the exception handling (e.g., making an ocall)
+    cmpq $TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING, td_state(%r11)
+    je .check_entry_nesting_level
+
+    // Update state and clear previous state on normal entries
+    movq $TD_STATE_NULL, td_previous_state(%r11)
+    movq $TD_STATE_ENTERED, td_state(%r11)
+
+.check_entry_nesting_level:
+    lfence
     // Check whether this is a clean entry or a nested entry
     // clean-entry-check.
     mov td_depth(%r11), %r8
@@ -91,7 +107,7 @@ oe_enter:
     mov SGX_SSA_RSP_OFFSET(%r10), %r8
     mov SGX_SSA_URSP_OFFSET(%r10), %r9
     cmp %r8, %r9
-    je .abort
+    je .return
 
     // Calculate the base address of the enclave
     lea _enclave_rva(%rip), %r12
@@ -108,7 +124,7 @@ oe_enter:
     jb .exception_handler_stack_check
     cmp %r13, %r9
     jae .exception_handler_stack_check
-    jmp .abort
+    jmp .return
 
     // Reaching this point implies SSA[0].GPRSGX.RSP is within the enclave
     // memory range so we do not need additional checks.
@@ -140,7 +156,7 @@ oe_enter:
     and $-16, %r8
 
     // Proceed without the red zone
-    jmp .call_function
+    jmp .state_machine_check
 
 .exception_stack_setup:
     // Stop speculative execution at target of conditional jump
@@ -151,6 +167,78 @@ oe_enter:
 
     // Start the new stack under the red zone
     sub $ABI_REDZONE_BYTE_SIZE, %r8
+
+.state_machine_check:
+    cmpq $0, td_exception_nesting_level(%r11)
+    jne .state_machine_check_nested_exception
+
+.state_machine_check_non_nested_exception:
+    // Expect the state to be RUNNING on a non-nested exception
+    // entry
+    cmpq $TD_STATE_RUNNING, td_state(%r11)
+    jne .return
+    jmp .check_host_signal_request
+
+.state_machine_check_nested_exception:
+    lfence
+    // Expect the state to be SECOND_LEVEL_EXCEPTION_HANDLING
+    // on a nested exception entry
+    cmpq $TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING, td_state(%r11)
+    jne .return
+
+.check_host_signal_request:
+    movq td_state(%r11), %r12
+
+    // Input value falls in the range of [1, 64] indicates
+    // a host signal request
+    cmp $0, %rsi
+    je .update_td_state
+    cmp $MAX_SIGNAL_NUMBER, %rsi
+    ja .update_td_state
+
+    // Proceed if the host_signal_unmasked flag is set
+    cmpq $1, td_host_signal_unmasked(%r11)
+    jne .return
+
+    // Proceed if the corresponding bit of the signal
+    // (i.e., signal number - 1) is set in the bitmask
+    mov td_host_signal_bitmask(%r11), %r13
+    mov %rsi, %r14
+    dec %r14
+    bt %r14, %r13
+    jnc .return
+
+    // Proceed only if the state is RUNNING
+    cmp $TD_STATE_RUNNING, %r12
+    jne .return
+
+    // Proceed if the thread is currently not handling a host signal
+    cmpq $1, td_is_handling_host_signal(%r11)
+    je .return
+
+    // Proceed if the exception entry is not nested
+    cmpq $0, td_exception_nesting_level(%r11)
+    jne .return
+
+    lfence
+
+    // Set the flag if the request is accepted
+    movq $1, td_is_handling_host_signal(%r11)
+
+    // Store the host-passed signal number
+    mov %rsi, td_host_signal(%r11)
+
+.update_td_state:
+    lfence
+
+    // Keep the state before the exception so that we can restore the
+    // state in the illegal instruction emulation flow
+    mov %r12, td_previous_state(%r11)
+    movq $TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING, td_state(%r11)
+
+    // Increase the nesting level, which will be decreased before resuming
+    // the execution (see exception.c)
+    incq td_exception_nesting_level(%r11)
     jmp .call_function
 
 .nested_entry:
@@ -244,17 +332,31 @@ oe_enter:
     jmp .eexit
 
 .abort:
+    lfence
+
+    // Set argument 2 for oe_asm_exit
+    mov $CODE_ENCLAVE_ABORTING, %rsi
+
+    // Update the global enclave status
+    mov %rsi, __oe_enclave_status(%rip)
+
+    jmp .prepare_eexit
+
+.return:
+    lfence
+
+    // Set argument 2 for oe_asm_exit
+    mov $CODE_EXCEPTION_CONTINUE_EXECUTION, %rsi
+
+.prepare_eexit:
 #define ARG1_CODE_ERET        0x2 // OE_CODE_ERET in oe_code_t
 #define ARG1_CODE_BIT_OFFSET  0x30 // Refer to oe_make_call_arg1 in calls.h
-#define ARG2_ENCLAVE_ABORTING 0x13 // OE_ENCLAVE_ABORTING in oe_result_t
 
-    // Set arguments for oe_asm_exit
+    // Set argument 1 for oe_asm_exit
     mov $ARG1_CODE_ERET, %rdi
     shl $ARG1_CODE_BIT_OFFSET, %rdi
-    mov $ARG2_ENCLAVE_ABORTING, %rsi
-    mov %rsi, __oe_enclave_status(%rip)
     mov %r11, %rdx
-    mov $1, %rcx // aborting=1
+    mov $1, %rcx // direct_return=1
 
 .eexit:
     // Invoke oe_asm_exit with (ARG1=RDI, ARG2=RSI, TD=RDX, ABORTING=RCX)