From 6f88fafa08b3f18163c7288a73a7f5aeb6af1a13 Mon Sep 17 00:00:00 2001 From: Ming-Wei Shih Date: Wed, 3 Nov 2021 18:06:12 +0000 Subject: [PATCH] SGX thread interrupt support on Linux Signed-off-by: Ming-Wei Shih --- enclave/core/sgx/asmdefs.h | 27 +- enclave/core/sgx/calls.c | 23 +- enclave/core/sgx/enter.S | 122 +++++- enclave/core/sgx/exception.c | 138 +++++-- enclave/core/sgx/exit.S | 33 +- enclave/core/sgx/td.c | 119 +++++- host/sgx/exception.c | 11 +- host/sgx/exception.h | 1 + host/sgx/linux/exception.c | 91 +++-- include/openenclave/bits/exception.h | 6 + include/openenclave/internal/sgx/td.h | 56 ++- tests/CMakeLists.txt | 7 + tests/sgx/CMakeLists.txt | 7 + tests/sgx/td_state/CMakeLists.txt | 11 + tests/sgx/td_state/enc/CMakeLists.txt | 18 + tests/sgx/td_state/enc/enc.c | 355 +++++++++++++++++ tests/sgx/td_state/host/CMakeLists.txt | 16 + tests/sgx/td_state/host/host.c | 115 ++++++ tests/sgx/td_state/td_state.edl | 23 ++ tests/sgx/thread_interrupt/CMakeLists.txt | 18 + tests/sgx/thread_interrupt/enc/CMakeLists.txt | 17 + tests/sgx/thread_interrupt/enc/enc.c | 366 ++++++++++++++++++ .../sgx/thread_interrupt/host/CMakeLists.txt | 17 + tests/sgx/thread_interrupt/host/host.c | 116 ++++++ .../sgx/thread_interrupt/thread_interrupt.edl | 24 ++ 25 files changed, 1660 insertions(+), 77 deletions(-) create mode 100644 tests/sgx/CMakeLists.txt create mode 100644 tests/sgx/td_state/CMakeLists.txt create mode 100644 tests/sgx/td_state/enc/CMakeLists.txt create mode 100644 tests/sgx/td_state/enc/enc.c create mode 100644 tests/sgx/td_state/host/CMakeLists.txt create mode 100644 tests/sgx/td_state/host/host.c create mode 100644 tests/sgx/td_state/td_state.edl create mode 100644 tests/sgx/thread_interrupt/CMakeLists.txt create mode 100644 tests/sgx/thread_interrupt/enc/CMakeLists.txt create mode 100644 tests/sgx/thread_interrupt/enc/enc.c create mode 100644 tests/sgx/thread_interrupt/host/CMakeLists.txt create mode 100644 tests/sgx/thread_interrupt/host/host.c create mode 100644 tests/sgx/thread_interrupt/thread_interrupt.edl diff --git a/enclave/core/sgx/asmdefs.h b/enclave/core/sgx/asmdefs.h index 2f2c3adca2..9c36faac53 100644 --- a/enclave/core/sgx/asmdefs.h +++ b/enclave/core/sgx/asmdefs.h @@ -17,7 +17,25 @@ #define STATIC_STACK_SIZE 8 * 100 #define OE_WORD_SIZE 8 -#define CODE_ERET 0x200000000 +/* Defined in oe_result_t (result.h) */ +#define CODE_ENCLAVE_ABORTING 0x13 + +/* Defined in exception.h */ +#define CODE_EXCEPTION_CONTINUE_EXECUTION 0xFFFFFFFF + +/* Assembly code cannot use enum values directly, + * define them here to match oe_td_state_t in + * internal/sgx/td.h */ +#define TD_STATE_NULL 0 +#define TD_STATE_ENTERED 1 +#define TD_STATE_RUNNING 2 +#define TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING 3 +#define TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING 4 +#define TD_STATE_EXITED 5 +#define TD_STATE_ABORTED 6 + +/* Set the max signal number based on Linux (i.e., SIGRTMAX) */ +#define MAX_SIGNAL_NUMBER 64 /* Use GS register if this flag is set */ #ifdef __ASSEMBLER__ @@ -39,6 +57,13 @@ #define td_host_previous_ecall_context (td_host_ecall_context + 8) #define td_exception_handler_stack (td_host_previous_ecall_context + 8) #define td_exception_handler_stack_size (td_exception_handler_stack + 8) +#define td_state (td_exception_handler_stack_size + 8) +#define td_previous_state (td_state + 8) +#define td_exception_nesting_level (td_previous_state + 8) +#define td_host_signal_unmasked (td_exception_nesting_level + 8) +#define td_is_handling_host_signal (td_host_signal_unmasked + 8) +#define td_host_signal (td_is_handling_host_signal + 8) +#define td_host_signal_bitmask (td_host_signal + 8) #define oe_exit_enclave __morestack #ifndef __ASSEMBLER__ diff --git a/enclave/core/sgx/calls.c b/enclave/core/sgx/calls.c index 6f8ee30f1b..ace58631a9 100644 --- a/enclave/core/sgx/calls.c +++ b/enclave/core/sgx/calls.c @@ -725,7 +725,7 @@ static void _exit_enclave(uint64_t arg1, uint64_t arg2) host_ecall_context->debug_eexit_rip = frame[1]; } } - oe_asm_exit(arg1, arg2, td, 0 /* aborting */); + oe_asm_exit(arg1, arg2, td, 0 /* direct_return */); } /* @@ -814,6 +814,15 @@ oe_result_t oe_ocall(uint16_t func, uint64_t arg_in, uint64_t* arg_out) if (arg_out) *arg_out = td->oret_arg; + if (td->state != OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING) + { + /* State machine check */ + if (td->state != OE_TD_STATE_ENTERED) + oe_abort(); + + td->state = OE_TD_STATE_RUNNING; + } + /* ORET here */ } @@ -1166,6 +1175,14 @@ void __oe_handle_main( if (func == OE_ECALL_VIRTUAL_EXCEPTION_HANDLER) oe_abort(); + /* State machine check */ + if (td->state != OE_TD_STATE_ENTERED) + oe_abort(); + + /* At this point, we are ready to execute the ecall. + * Update the state to RUNNING */ + td->state = OE_TD_STATE_RUNNING; + _handle_ecall(td, func, arg_in, output_arg1, output_arg2); break; } @@ -1204,6 +1221,10 @@ void __oe_handle_main( void oe_abort(void) { + oe_sgx_td_t* td = oe_sgx_get_td(); + + td->state = OE_TD_STATE_ABORTED; + // Once it starts to crash, the state can only transit forward, not // backward. if (__oe_enclave_status < OE_ENCLAVE_ABORTING) diff --git a/enclave/core/sgx/enter.S b/enclave/core/sgx/enter.S index c852feac6f..b4c7c9d1a2 100644 --- a/enclave/core/sgx/enter.S +++ b/enclave/core/sgx/enter.S @@ -50,6 +50,10 @@ oe_enter: mov _td_from_tcs_offset(%rip), %r11 add %rbx, %r11 +.check_aborted: + cmpq $TD_STATE_ABORTED, td_state(%r11) + je .abort + // Get the first ssa address from tcs lea OE_SSA_FROM_TCS_BYTE_OFFSET(%rbx), %r10 @@ -65,16 +69,28 @@ oe_enter: .determine_entry_type: // Check if this is exception dispatching request - // Abort on the eenter if cssa greater than one, which + // Return on the eenter if cssa greater than one, which // should not occur because OE assumes the enclave with nssa=2 cmp $1, %rax je .exception_entry - ja .abort + ja .return // Stop speculative execution at fallthrough of conditional // exception-dispatching-request-check. lfence +.update_td_state_on_normal_entry: + // Do not update the state if the enclave enters in the middle + // the exception handling (e.g., making an ocall) + cmpq $TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING, td_state(%r11) + je .check_entry_nesting_level + + // Update state and clear previous state on normal entries + movq $TD_STATE_NULL, td_previous_state(%r11) + movq $TD_STATE_ENTERED, td_state(%r11) + +.check_entry_nesting_level: + lfence // Check whether this is a clean entry or a nested entry // clean-entry-check. mov td_depth(%r11), %r8 @@ -91,7 +107,7 @@ oe_enter: mov SGX_SSA_RSP_OFFSET(%r10), %r8 mov SGX_SSA_URSP_OFFSET(%r10), %r9 cmp %r8, %r9 - je .abort + je .return // Calculate the base address of the enclave lea _enclave_rva(%rip), %r12 @@ -108,7 +124,7 @@ oe_enter: jb .exception_handler_stack_check cmp %r13, %r9 jae .exception_handler_stack_check - jmp .abort + jmp .return // Reaching this point implies SSA[0].GPRSGX.RSP is within the enclave // memory range so we do not need additional checks. @@ -140,7 +156,7 @@ oe_enter: and $-16, %r8 // Proceed without the red zone - jmp .call_function + jmp .state_machine_check .exception_stack_setup: // Stop speculative execution at target of conditional jump @@ -151,6 +167,78 @@ oe_enter: // Start the new stack under the red zone sub $ABI_REDZONE_BYTE_SIZE, %r8 + +.state_machine_check: + cmpq $0, td_exception_nesting_level(%r11) + jne .state_machine_check_nested_exception + +.state_machine_check_non_nested_exception: + // Expect the state to be RUNNING on a non-nested exception + // entry + cmpq $TD_STATE_RUNNING, td_state(%r11) + jne .return + jmp .check_host_signal_request + +.state_machine_check_nested_exception: + lfence + // Expect the state to be SECOND_LEVEL_EXCEPTION_HANDLING + // on a nested exception entry + cmpq $TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING, td_state(%r11) + jne .return + +.check_host_signal_request: + movq td_state(%r11), %r12 + + // Input value falls in the range of [1, 64] indicates + // a host signal request + cmp $0, %rsi + je .update_td_state + cmp $MAX_SIGNAL_NUMBER, %rsi + ja .update_td_state + + // Proceed if the host_signal_unmasked flag is set + cmpq $1, td_host_signal_unmasked(%r11) + jne .return + + // Proceed if the corresponding bit of the signal + // (i.e., signal number - 1) is set in the bitmask + mov td_host_signal_bitmask(%r11), %r13 + mov %rsi, %r14 + dec %r14 + bt %r14, %r13 + jnc .return + + // Proceed only if the state is RUNNING + cmp $TD_STATE_RUNNING, %r12 + jne .return + + // Proceed if the thread is currently not handling a host signal + cmpq $1, td_is_handling_host_signal(%r11) + je .return + + // Proceed if the exception entry is not nested + cmpq $0, td_exception_nesting_level(%r11) + jne .return + + lfence + + // Set the flag if the request is accepted + movq $1, td_is_handling_host_signal(%r11) + + // Store the host-passed signal number + mov %rsi, td_host_signal(%r11) + +.update_td_state: + lfence + + // Keep the state before the exception so that we can restore the + // state in the illegal instruction emulation flow + mov %r12, td_previous_state(%r11) + movq $TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING, td_state(%r11) + + // Increase the nesting level, which will be decreased before resuming + // the execution (see exception.c) + incq td_exception_nesting_level(%r11) jmp .call_function .nested_entry: @@ -244,17 +332,31 @@ oe_enter: jmp .eexit .abort: + lfence + + // Set argument 2 for oe_asm_exit + mov $CODE_ENCLAVE_ABORTING, %rsi + + // Update the global enclave status + mov %rsi, __oe_enclave_status(%rip) + + jmp .prepare_eexit + +.return: + lfence + + // Set argument 2 for oe_asm_exit + mov $CODE_EXCEPTION_CONTINUE_EXECUTION, %rsi + +.prepare_eexit: #define ARG1_CODE_ERET 0x2 // OE_CODE_ERET in oe_code_t #define ARG1_CODE_BIT_OFFSET 0x30 // Refer to oe_make_call_arg1 in calls.h -#define ARG2_ENCLAVE_ABORTING 0x13 // OE_ENCLAVE_ABORTING in oe_result_t - // Set arguments for oe_asm_exit + // Set argument 1 for oe_asm_exit mov $ARG1_CODE_ERET, %rdi shl $ARG1_CODE_BIT_OFFSET, %rdi - mov $ARG2_ENCLAVE_ABORTING, %rsi - mov %rsi, __oe_enclave_status(%rip) mov %r11, %rdx - mov $1, %rcx // aborting=1 + mov $1, %rcx // direct_return=1 .eexit: // Invoke oe_asm_exit with (ARG1=RDI, ARG2=RSI, TD=RDX, ABORTING=RCX) diff --git a/enclave/core/sgx/exception.c b/enclave/core/sgx/exception.c index 4ffb7339a5..de384442df 100644 --- a/enclave/core/sgx/exception.c +++ b/enclave/core/sgx/exception.c @@ -239,6 +239,14 @@ void oe_real_exception_dispatcher(oe_context_t* oe_context) { oe_sgx_td_t* td = oe_sgx_get_td(); + /* Validate the td state, which ensures the function + * is only invoked after oe_virtual_exception_dispatcher */ + if (td->state != OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING) + { + oe_abort(); + return; + } + // Change the rip of oe_context to the real exception address. oe_context->rip = td->exception_address; @@ -252,6 +260,11 @@ void oe_real_exception_dispatcher(oe_context_t* oe_context) oe_exception_record.faulting_address = td->faulting_address; oe_exception_record.error_code = td->error_code; oe_exception_record.context = oe_context; + /* Only pass the host signal for non-nested exceptions */ + if (td->exception_nesting_level == 1) + oe_exception_record.host_signal_number = (uint16_t)td->host_signal; + else + oe_exception_record.host_signal_number = 0; // Refer to oe_enter in host/sgx/enter.c. // Restore host_ecall_context from the first EENTER (cssa=0) that allows for @@ -296,6 +309,31 @@ void oe_real_exception_dispatcher(oe_context_t* oe_context) td->last_ssa_rsp = 0; td->last_ssa_rbp = 0; + /* Validate and decrease the nesting level (increased by enter.S) + * after all the handlers finish */ + if (td->exception_nesting_level == 0) + { + oe_abort(); + return; + } + td->exception_nesting_level--; + + if (td->exception_nesting_level == 0) + { + /* Clear the flag if it is set after non-nested exception handling + * is done */ + if (td->is_handling_host_signal == 1) + { + td->is_handling_host_signal = 0; + } + + td->host_signal = 0; + + /* Retore the state */ + td->state = OE_TD_STATE_RUNNING; + } + td->previous_state = OE_TD_STATE_NULL; + // Jump to the point where oe_context refers to and continue. if (handler_ret == OE_EXCEPTION_CONTINUE_EXECUTION) { @@ -337,7 +375,15 @@ void oe_virtual_exception_dispatcher( uint64_t* arg_out) { SSA_Info ssa_info = {0}; - OE_UNUSED(arg_in); + + /* Validate the td state, which ensures the function + * is only invoked by the exception entry code path (see enter.S) */ + if (td->state != OE_TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING) + { + td->state = OE_TD_STATE_ABORTED; + *arg_out = OE_EXCEPTION_CONTINUE_SEARCH; + return; + } // Verify if the first SSA has valid exception info. if (_get_enclave_thread_first_ssa_info(td, &ssa_info) != 0) @@ -346,50 +392,83 @@ void oe_virtual_exception_dispatcher( return; } + /* Only keep the host signal for non-nested exceptions */ + if (td->exception_nesting_level == 1) + td->host_signal = arg_in; + uint64_t gprsgx_offset = (uint64_t)ssa_info.base_address + ssa_info.frame_byte_size - OE_SGX_GPR_BYTE_SIZE; sgx_ssa_gpr_t* ssa_gpr = (sgx_ssa_gpr_t*)gprsgx_offset; - if (!ssa_gpr->exit_info.as_fields.valid) - { - // Not a valid/expected enclave exception; - *arg_out = OE_EXCEPTION_CONTINUE_SEARCH; - return; - } - // Get the exception address, code, and flags. td->exception_address = ssa_gpr->rip; td->exception_code = OE_EXCEPTION_UNKNOWN; - for (uint32_t i = 0; i < OE_COUNTOF(g_vector_to_exception_code_mapping); - i++) + + /* Get the exception code and flags only if the exception type + * is recognized by the SGX hardware */ + if (ssa_gpr->exit_info.as_fields.valid) { - if (g_vector_to_exception_code_mapping[i].sgx_vector == - ssa_gpr->exit_info.as_fields.vector) + for (uint32_t i = 0; i < OE_COUNTOF(g_vector_to_exception_code_mapping); + i++) { - td->exception_code = - g_vector_to_exception_code_mapping[i].exception_code; - break; + if (g_vector_to_exception_code_mapping[i].sgx_vector == + ssa_gpr->exit_info.as_fields.vector) + { + td->exception_code = + g_vector_to_exception_code_mapping[i].exception_code; + break; + } } - } - td->exception_flags = 0; - if (ssa_gpr->exit_info.as_fields.exit_type == SGX_EXIT_TYPE_HARDWARE) - { - td->exception_flags |= OE_EXCEPTION_FLAGS_HARDWARE; + td->exception_flags = 0; + if (ssa_gpr->exit_info.as_fields.exit_type == SGX_EXIT_TYPE_HARDWARE) + { + td->exception_flags |= OE_EXCEPTION_FLAGS_HARDWARE; + } + else if ( + ssa_gpr->exit_info.as_fields.exit_type == SGX_EXIT_TYPE_SOFTWARE) + { + td->exception_flags |= OE_EXCEPTION_FLAGS_SOFTWARE; + } } - else if (ssa_gpr->exit_info.as_fields.exit_type == SGX_EXIT_TYPE_SOFTWARE) + else { - td->exception_flags |= OE_EXCEPTION_FLAGS_SOFTWARE; + /* The unknown exception type indicates a host signal request. Validate + * the states on the td to ensure that the thread is handling the + * host signal */ + if (!oe_sgx_td_host_signal_registered(td, (int)td->host_signal) || + td->exception_nesting_level != 1 || + td->is_handling_host_signal != 1) + { + *arg_out = OE_EXCEPTION_CONTINUE_SEARCH; + return; + } } if (td->exception_code == OE_EXCEPTION_ILLEGAL_INSTRUCTION && _emulate_illegal_instruction(ssa_gpr) == 0) { - // Refer to oe_enter in host/sgx/enter.c. - // Restore host_ecall_context from the first EENTER (cssa=0) that allows - // for correctly stitching the stack on EEXIT when the enclave is in the - // debug mode + /* Refer to oe_enter in host/sgx/enter.c. + * Restore host_ecall_context from the first EENTER (cssa=0) that allows + * for correctly stitching the stack on EEXIT when the enclave is in the + * debug mode */ td->host_ecall_context = td->host_previous_ecall_context; + /* Restore the state using the previous_state and update + * the previous_state. The latter allows the exiting flow + * to skip updating the state. */ + td->state = td->previous_state; + td->previous_state = OE_TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING; + + /* Validate and decrease the nesting level (increased by enter.S) + * after all the handlers finish */ + if (td->exception_nesting_level == 0) + { + td->state = OE_TD_STATE_ABORTED; + *arg_out = OE_EXCEPTION_CONTINUE_SEARCH; + return; + } + td->exception_nesting_level--; + // Advance RIP to the next instruction for continuation ssa_gpr->rip += 2; } @@ -405,6 +484,13 @@ void oe_virtual_exception_dispatcher( td->faulting_address = exinfo->maddr; td->error_code = exinfo->errcd; } + + /* Update the state here to indicate the second-level exception + * handler is running next. This allows both exiting and the following + * entering flows to skip updating the state; i.e., the second-level + * exception handler can run with this state. */ + td->state = OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING; + // Modify the ssa_gpr so that e_resume will go to second pass exception // handler. ssa_gpr->rip = (uint64_t)oe_exception_dispatcher; diff --git a/enclave/core/sgx/exit.S b/enclave/core/sgx/exit.S index c6c4a5697f..ede93b1d88 100644 --- a/enclave/core/sgx/exit.S +++ b/enclave/core/sgx/exit.S @@ -7,13 +7,13 @@ //============================================================================== // -// void oe_asm_exit(uint64_t arg1, uint64_t arg2, oe_sgx_td_t* td, uint64_t aborting) +// void oe_asm_exit(uint64_t arg1, uint64_t arg2, oe_sgx_td_t* td, uint64_t direct_return) // // Registers: // RDI - arg1 // RSI - arg2 // RDX - td -// RCX - aborting +// RCX - direct_return // // Purpose: // Restores user registers and executes the EEXIT instruction to leave the @@ -58,9 +58,9 @@ oe_asm_exit: mov %rdx, %r11 .determine_exit_type: - // Check if the argument aborting is set + // Check if the argument direct_return is set cmp $1, %rcx - je .clean_exit + je .return // Check the depth of the ECALL stack (zero for clean exit) // exit-type-check. @@ -83,6 +83,14 @@ oe_asm_exit: // Clear the oe_sgx_td_t.last_sp field (force oe_enter to calculate stack pointer) movq $0, td_last_sp(%r11) + jmp .prepare_eexit + +.return: + lfence + + // Restore host ecall context + mov td_host_previous_ecall_context(%r11), %r8 + mov %r8, td_host_ecall_context(%r11) .prepare_eexit: mov _td_from_tcs_offset(%rip), %r8 @@ -100,19 +108,32 @@ oe_asm_exit: lfence add $PAGE_SIZE, %r12 - xor %r8, %r8 // Reset the saved cssa to zero to match the value on // the previous oe_enter (which ERESUME will return to) - mov %r8, td_eenter_rax(%r11) + movq $0, td_eenter_rax(%r11) .restore_host_registers: mov td_host_rcx(%r11), %rcx mov SGX_SSA_URSP_OFFSET(%r12), %rsp mov SGX_SSA_URBP_OFFSET(%r12), %rbp + // Do not update the state if the enclave exits in the middle + // the exception handling (e.g., exiting from the first-level + // exception handler) or exits after an illegal instruction + // emulation + cmpq $TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING, td_state(%r11) + je .execute_eexit + cmpq $TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING, td_previous_state(%r11) + je .execute_eexit + + // Update the state to indicate that the enclave is returning + // to the host + movq $TD_STATE_EXITED, td_state(%r11) + oe_cleanup_registers .execute_eexit: + lfence // Check oe_sgx_td_t.simulate flag // simulate-flag-check. mov td_simulate(%r11), %rax diff --git a/enclave/core/sgx/td.c b/enclave/core/sgx/td.c index 698c55686f..5f66e1c452 100644 --- a/enclave/core/sgx/td.c +++ b/enclave/core/sgx/td.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "asmdefs.h" #include "thread.h" @@ -192,14 +193,128 @@ bool oe_sgx_set_td_exception_handler_stack(void* stack, uint64_t size) return true; } +/* +**============================================================================== +** +** oe_sgx_td_mask_host_signal() +** oe_sgx_td_unmask_host_signal() +** +** Internal APIs that allows a thread to self-mask or unmask host signals +** +**============================================================================== +*/ + +OE_INLINE void _set_td_host_signal_unmasked(uint64_t value) +{ + oe_sgx_td_t* td = oe_sgx_get_td(); + + td->host_signal_unmasked = value; +} + +void oe_sgx_td_mask_host_signal() +{ + _set_td_host_signal_unmasked(0); +} + +void oe_sgx_td_unmask_host_signal() +{ + _set_td_host_signal_unmasked(1); +} + +/* +**============================================================================== +** +** oe_sgx_register_td_host_signal() +** oe_sgx_unregister_td_host_signal() +** +** Internal APIs that allows an enclave to register or unregister signals +** raised by the host for itself or a target thread +** +**============================================================================== +*/ + +OE_INLINE bool _set_td_host_signal_bitmask( + oe_sgx_td_t* td, + int signal_number, + bool set_bit) +{ + if (!td) + return false; + + /* only allow number 1-64 */ + if (signal_number <= 0 || signal_number > 64) + return false; + + oe_spin_lock(&td->lock); + + if (set_bit) + td->host_signal_bitmask |= 1UL << (signal_number - 1); + else + td->host_signal_bitmask &= ~(1UL << (signal_number - 1)); + + oe_spin_unlock(&td->lock); + + return true; +} + +bool oe_sgx_register_td_host_signal(oe_sgx_td_t* td, int signal_number) +{ + return _set_td_host_signal_bitmask(td, signal_number, 1 /* set */); +} + +bool oe_sgx_unregister_td_host_signal(oe_sgx_td_t* td, int signal_number) +{ + return _set_td_host_signal_bitmask(td, signal_number, 0 /* clear */); +} + +/* +**============================================================================== +** +** oe_sgx_td_host_signal_registered +** +** Internal API for querying whether the thread registers the given +** host signal +** +**============================================================================== +*/ +bool oe_sgx_td_host_signal_registered(oe_sgx_td_t* td, int signal_number) +{ + if (!td) + return false; + + /* only allow number 1-64 */ + if (signal_number <= 0 || signal_number > 64) + return false; + + return (td->host_signal_bitmask & (1UL << (signal_number - 1))) != 0; +} + +/* +**============================================================================== +** +** oe_sgx_td_is_handling_host_signal() +** +** Internal API for querying whether the thread is handling a host signal +** +**============================================================================== +*/ + +bool oe_sgx_td_is_handling_host_signal(oe_sgx_td_t* td) +{ + if (!td) + return false; + + return td->is_handling_host_signal; +} + /* **============================================================================== ** ** td_initialized() ** ** Returns TRUE if this thread data structure (oe_sgx_td_t) is initialized. -*An -** initialized oe_sgx_td_t meets the following conditions: +** +** An initialized oe_sgx_td_t meets the following conditions: ** ** (1) td is not null ** (2) td->base.self_addr == td diff --git a/host/sgx/exception.c b/host/sgx/exception.c index 273e68e5e8..acd407a6b1 100644 --- a/host/sgx/exception.c +++ b/host/sgx/exception.c @@ -23,6 +23,7 @@ uint64_t oe_host_handle_exception(oe_host_exception_context_t* context) uint64_t exit_code = context->rax; uint64_t tcs_address = context->rbx; uint64_t exit_address = context->rip; + uint64_t signal_number = context->signal_number; // Check if the signal happens inside the enclave. if ((exit_address == OE_AEP_ADDRESS) && (exit_code == ENCLU_ERESUME)) @@ -32,7 +33,8 @@ uint64_t oe_host_handle_exception(oe_host_exception_context_t* context) oe_thread_binding_t* thread_data = oe_get_thread_binding(); if (thread_data->flags & _OE_THREAD_HANDLING_EXCEPTION) { - abort(); + // Return directly. + return OE_EXCEPTION_CONTINUE_EXECUTION; } // Call-in enclave to handle the exception. @@ -47,8 +49,11 @@ uint64_t oe_host_handle_exception(oe_host_exception_context_t* context) // Call into enclave first pass exception handler. uint64_t arg_out = 0; - oe_result_t result = - oe_ecall(enclave, OE_ECALL_VIRTUAL_EXCEPTION_HANDLER, 0, &arg_out); + oe_result_t result = oe_ecall( + enclave, + OE_ECALL_VIRTUAL_EXCEPTION_HANDLER, + signal_number, + &arg_out); // Reset the flag thread_data->flags &= (~_OE_THREAD_HANDLING_EXCEPTION); diff --git a/host/sgx/exception.h b/host/sgx/exception.h index 0552194e72..eb7ba4dc07 100644 --- a/host/sgx/exception.h +++ b/host/sgx/exception.h @@ -12,6 +12,7 @@ typedef struct _host_exception_context uint64_t rax; uint64_t rbx; uint64_t rip; + uint64_t signal_number; } oe_host_exception_context_t; /* Initialize the exception processing. */ diff --git a/host/sgx/linux/exception.c b/host/sgx/linux/exception.c index 6242e15f5d..a872ff085b 100644 --- a/host/sgx/linux/exception.c +++ b/host/sgx/linux/exception.c @@ -27,6 +27,22 @@ static struct sigaction g_previous_sigaction[_NSIG]; +/* The set of default signals that we always forward to the + * enclave and is recognizable by the SGX hardware */ +static int default_signals[] = {SIGBUS, SIGFPE, SIGILL, SIGSEGV, SIGTRAP}; + +#define DEFAULT_SIGNALS_NUMBER \ + (sizeof(default_signals) / sizeof(default_signals[0])) + +/* The set of optional signals that we forward to the enclave + * but requires the enclave to explicitly register (otherwise + * the signal will be blocked) */ +static int optional_signals[] = + {SIGHUP, SIGABRT, SIGALRM, SIGPIPE, SIGPOLL, SIGUSR1, SIGUSR2}; + +#define OPTIONAL_SIGNALS_NUMBER \ + (sizeof(optional_signals) / sizeof(optional_signals[0])) + static void _host_signal_handler( int sig_num, siginfo_t* sig_info, @@ -38,6 +54,17 @@ static void _host_signal_handler( host_context.rbx = (uint64_t)context->uc_mcontext.gregs[REG_RBX]; host_context.rip = (uint64_t)context->uc_mcontext.gregs[REG_RIP]; + host_context.signal_number = (uint64_t)sig_num; + for (size_t i = 0; i < DEFAULT_SIGNALS_NUMBER; i++) + { + if (sig_num == default_signals[i]) + { + /* Do not pass the number of default signals */ + host_context.signal_number = 0; + break; + } + } + // Call platform neutral handler. uint64_t action = oe_host_handle_exception(&host_context); @@ -48,6 +75,17 @@ static void _host_signal_handler( } else if (g_previous_sigaction[sig_num].sa_handler == SIG_DFL) { + // Bypass if the signal is part of the optional set and is + // sent to the host and the host does not install the corresponding + // handler + for (size_t i = 0; i < OPTIONAL_SIGNALS_NUMBER; i++) + { + // Do not bypass SIGABRT, which is expected to abort + // the host process + if (sig_num == optional_signals[i] && sig_num != SIGABRT) + return; + } + // If not an enclave exception, and no valid previous signal handler is // set, raise it again, and let the default signal handler handle it. signal(sig_num, SIG_DFL); @@ -111,37 +149,36 @@ static void _register_signal_handlers(void) } // Unmask the signals we want to receive. - sigdelset(&sig_action.sa_mask, SIGSEGV); - sigdelset(&sig_action.sa_mask, SIGFPE); - sigdelset(&sig_action.sa_mask, SIGILL); - sigdelset(&sig_action.sa_mask, SIGBUS); - sigdelset(&sig_action.sa_mask, SIGTRAP); - - // Set the signal handlers, and store the previous signal action into a - // global array. - if (sigaction(SIGSEGV, &sig_action, &g_previous_sigaction[SIGSEGV]) != 0) - { - abort(); - } - - if (sigaction(SIGFPE, &sig_action, &g_previous_sigaction[SIGFPE]) != 0) - { - abort(); - } - - if (sigaction(SIGILL, &sig_action, &g_previous_sigaction[SIGILL]) != 0) + for (size_t i = 0; i < DEFAULT_SIGNALS_NUMBER; i++) { - abort(); - } - - if (sigaction(SIGBUS, &sig_action, &g_previous_sigaction[SIGBUS]) != 0) - { - abort(); + int signal_number = default_signals[i]; + sigdelset(&sig_action.sa_mask, signal_number); + + // Set the signal handlers, and store the previous signal action into a + // global array. + if (sigaction( + signal_number, + &sig_action, + &g_previous_sigaction[signal_number]) != 0) + { + abort(); + } } - if (sigaction(SIGTRAP, &sig_action, &g_previous_sigaction[SIGTRAP]) != 0) + for (size_t i = 0; i < OPTIONAL_SIGNALS_NUMBER; i++) { - abort(); + int signal_number = optional_signals[i]; + sigdelset(&sig_action.sa_mask, signal_number); + + // Set the signal handlers, and store the previous signal action into a + // global array. + if (sigaction( + signal_number, + &sig_action, + &g_previous_sigaction[signal_number]) != 0) + { + abort(); + } } return; diff --git a/include/openenclave/bits/exception.h b/include/openenclave/bits/exception.h index 5ee44eae70..16e09a4594 100644 --- a/include/openenclave/bits/exception.h +++ b/include/openenclave/bits/exception.h @@ -214,6 +214,12 @@ typedef struct _oe_exception_record uint64_t faulting_address; uint32_t error_code; + /* The signal number passed by the host. Some applications might + * want to implement special exception handling logic based + * on this information. However, please use with caution as the + * host-passed information is untrusted in the SGX threat model. */ + uint16_t host_signal_number; + oe_context_t* context; /**< Exception context */ } oe_exception_record_t; /**< typedef struct _oe_exception_record oe_exception_record_t*/ diff --git a/include/openenclave/internal/sgx/td.h b/include/openenclave/internal/sgx/td.h index 052045ddda..71543e7748 100644 --- a/include/openenclave/internal/sgx/td.h +++ b/include/openenclave/internal/sgx/td.h @@ -81,7 +81,7 @@ oe_thread_data_t* oe_get_thread_data(void); * Due to the inability to use OE_OFFSETOF on a struct while defining its * members, this value is computed and hard-coded. */ -#define OE_THREAD_SPECIFIC_DATA_SIZE (3724) +#define OE_THREAD_SPECIFIC_DATA_SIZE (3664) typedef struct _oe_callsite oe_callsite_t; @@ -92,6 +92,17 @@ typedef struct _oe_tls_atexit void* object; } oe_tls_atexit_t; +typedef enum _oe_td_state +{ + OE_TD_STATE_NULL = 0, + OE_TD_STATE_ENTERED, + OE_TD_STATE_RUNNING, + OE_TD_STATE_FIRST_LEVEL_EXCEPTION_HANDLING, + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING, + OE_TD_STATE_EXITED, + OE_TD_STATE_ABORTED, +} oe_td_state_t; + /* This structure manages a pool of shared memory (memory visible to both * the enclave and the host). An instance of this structure is maintained * for each thread. This structure is used in enclave/core/arena.c. @@ -140,6 +151,32 @@ typedef struct _td uint64_t exception_handler_stack; uint64_t exception_handler_stack_size; + uint64_t state; + /* Hold the previous state upon every exception entries, which is + * used to resume the state after an illegal instruction emulation */ + uint64_t previous_state; + + uint64_t exception_nesting_level; + + /* The boolean value for opt-in/out the host signal handling */ + uint64_t host_signal_unmasked; + + /* The boolean value set by enter.S when a host signal request is + * accepted and cleared by oe_real_exception_dispatcher() in exception.c + * when the value is set and the nesting level is zero */ + uint64_t is_handling_host_signal; + + /* The signal number passed in by the host during an exception entry. + * The acceptable range [1, 64] is based on Linux signal implementation */ + uint64_t host_signal; + + /* A 64-bit array. Only if a bit is set, the thread will accept the + * (host signal number - 1) corresponds to the position of the bit */ + uint64_t host_signal_bitmask; + + /* Used by the thread-based spinlock */ + uint32_t lock; + /* Save the rsp and rbp values in the SSA when the exception handler * stack is set */ uint64_t last_ssa_rsp; @@ -190,8 +227,25 @@ OE_STATIC_ASSERT( /* Get the thread data object for the current thread */ oe_sgx_td_t* oe_sgx_get_td(void); +/* The following APIs are expected to be used only by the thread itself. */ + bool oe_sgx_set_td_exception_handler_stack(void* stack, uint64_t size); +void oe_sgx_td_mask_host_signal(); + +void oe_sgx_td_unmask_host_signal(); + +/* The following APIs are expected to be used by both the thread itself + * and other threads. */ + +bool oe_sgx_register_td_host_signal(oe_sgx_td_t* td, int signal_number); + +bool oe_sgx_unregister_td_host_signal(oe_sgx_td_t* td, int signal_number); + +bool oe_sgx_td_host_signal_registered(oe_sgx_td_t* td, int signal_number); + +bool oe_sgx_td_is_handling_host_signal(oe_sgx_td_t* td); + OE_EXTERNC_END #endif // _OE_SGX_TD_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index baac34606c..4fa09ede1e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -59,6 +59,13 @@ if (WIN32) add_subdirectory(win_paths) endif () +if (OE_SGX) + # Include newly added SGX-specific tests + # The task to restructure existing tests is tracked + # in https://github.com/openenclave/openenclave/issues/4281 + add_subdirectory(sgx) +endif () + if (OE_SGX) add_subdirectory(debugger) add_subdirectory(host_verify) diff --git a/tests/sgx/CMakeLists.txt b/tests/sgx/CMakeLists.txt new file mode 100644 index 0000000000..e99db48b40 --- /dev/null +++ b/tests/sgx/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +if (UNIX) + add_subdirectory(td_state) + add_subdirectory(thread_interrupt) +endif () diff --git a/tests/sgx/td_state/CMakeLists.txt b/tests/sgx/td_state/CMakeLists.txt new file mode 100644 index 0000000000..9261b4540d --- /dev/null +++ b/tests/sgx/td_state/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +add_subdirectory(host) + +if (BUILD_ENCLAVES) + add_subdirectory(enc) +endif () + +add_enclave_test(tests/sgx/td_state td_state_host sgx_td_state_enc) +set_enclave_tests_properties(tests/sgx/td_state PROPERTIES SKIP_RETURN_CODE 2) diff --git a/tests/sgx/td_state/enc/CMakeLists.txt b/tests/sgx/td_state/enc/CMakeLists.txt new file mode 100644 index 0000000000..0edaa12c79 --- /dev/null +++ b/tests/sgx/td_state/enc/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +set(EDL_FILE ../td_state.edl) + +add_custom_command( + OUTPUT td_state_t.h td_state_t.c + DEPENDS ${EDL_FILE} edger8r + COMMAND + edger8r --trusted ${EDL_FILE} --search-path ${PROJECT_SOURCE_DIR}/include + ${DEFINE_OE_SGX} --search-path ${CMAKE_CURRENT_SOURCE_DIR}) + +add_enclave(TARGET sgx_td_state_enc SOURCES enc.c + ${CMAKE_CURRENT_BINARY_DIR}/td_state_t.c) + +enclave_include_directories(sgx_td_state_enc PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}) +enclave_link_libraries(sgx_td_state_enc oelibc) diff --git a/tests/sgx/td_state/enc/enc.c b/tests/sgx/td_state/enc/enc.c new file mode 100644 index 0000000000..0741d49a36 --- /dev/null +++ b/tests/sgx/td_state/enc/enc.c @@ -0,0 +1,355 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include "td_state_t.h" + +#include +#include + +#define OE_EXPECT(a, b) \ + do \ + { \ + uint64_t value = (uint64_t)(a); \ + uint64_t expected = (uint64_t)(b); \ + if (value != expected) \ + { \ + printf( \ + "Test failed: %s(%u): %s expected: %lu, got: %lu\n", \ + __FILE__, \ + __LINE__, \ + __FUNCTION__, \ + expected, \ + value); \ + oe_abort(); \ + } \ + } while (0) + +bool oe_sgx_register_target_td_host_signal( + oe_sgx_td_t* target_td, + int signal_number); + +typedef struct _thread_info_nonblocking_t +{ + int tid; + oe_sgx_td_t* td; +} thread_info_t; + +static thread_info_t _thread_info; +static volatile int _handler_done; +static volatile int* _host_lock_state; + +static void cpuid( + unsigned int leaf, + unsigned int subleaf, + unsigned int* eax, + unsigned int* ebx, + unsigned int* ecx, + unsigned int* edx) +{ + asm volatile("cpuid" + // CPU id instruction returns values in the following registers + : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) + // __leaf is passed in eax (0) and __subleaf in ecx (2) + : "0"(leaf), "2"(subleaf)); +} + +// This function will generate the divide by zero function. +// The handler will catch this exception and fix it, and continue execute. +// It will return 0 if success. +static int divide_by_zero_exception_function(void) +{ + // Making ret, f and d volatile to prevent optimization + volatile int ret = 1; + volatile float f = 0; + volatile double d = 0; + + f = 0.31f; + d = 0.32; + + // Using inline assembly for idiv to prevent it being optimized out + // completely. Specify edi as the used register to ensure that 32-bit + // division is done. 64-bit division generates a 3 byte instruction rather + // than 2 bytes. + register int edi __asm__("edi") = 0; + asm volatile("idiv %1" + : "=a"(ret) + : "r"(edi) // Divisor of 0 is hard-coded + : "%1", + "cc"); // cc indicates that flags will be clobbered by ASM + + // Check if the float registers are recovered correctly after the exception + // is handled. + if (f < 0.309 || f > 0.321 || d < 0.319 || d > 0.321) + { + return -1; + } + + return 0; +} + +static uint64_t td_state_handler(oe_exception_record_t* exception_record) +{ + if (exception_record->code == OE_EXCEPTION_UNKNOWN) + { + int self_tid = 0; + + if (_handler_done) + { + printf("Unexpected interrupt...\n"); + return OE_EXCEPTION_ABORT_EXECUTION; + } + + OE_TEST(exception_record->host_signal_number == SIGUSR1); + + // Expect the td->host_signal to be SIGUSR1 + OE_TEST(_thread_info.td->host_signal == SIGUSR1); + + // Expect the state to be OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING + OE_EXPECT( + _thread_info.td->state, + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING); + + // Expect interrupted flag is set + OE_TEST(oe_sgx_td_is_handling_host_signal(_thread_info.td)); + + // Expect the signal is registered + OE_TEST(oe_sgx_td_host_signal_registered(_thread_info.td, SIGUSR1)); + + OE_TEST(exception_record->code == OE_EXCEPTION_UNKNOWN); + + host_get_tid(&self_tid); + OE_TEST(_thread_info.tid == self_tid); + + printf("(tid=%d) thread is interrupted...\n", self_tid); + + // Expect the state to be persisted after ocall(s) + OE_EXPECT( + _thread_info.td->state, + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING); + + { + uint32_t a, b, c, d; + cpuid(1, 0, &a, &b, &c, &d); + } + + printf("(tid=%d) thread emulating cpuid...done\n", self_tid); + + // Expect the state to be persisted after an illegal instruction + // emulation + OE_EXPECT( + _thread_info.td->state, + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING); + + // Expect the flag is persisted + OE_TEST(oe_sgx_td_is_handling_host_signal(_thread_info.td)); + + divide_by_zero_exception_function(); + + printf("(tid=%d) thread handling div 0...done\n", self_tid); + + // Expect the state to be + // OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING after a nested exception + OE_EXPECT( + _thread_info.td->state, + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING); + + // Expect the flag is persisted after a nested exception + OE_TEST(oe_sgx_td_is_handling_host_signal(_thread_info.td)); + + OE_TEST( + oe_sgx_unregister_td_host_signal(_thread_info.td, SIGUSR1) == true); + + __atomic_store_n(_host_lock_state, 2, __ATOMIC_RELEASE); + + _handler_done = 1; + + return OE_EXCEPTION_CONTINUE_EXECUTION; + } + else if (exception_record->code == OE_EXCEPTION_DIVIDE_BY_ZERO) + { + int self_tid = 0; + + OE_EXPECT( + _thread_info.td->state, + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING); + + host_get_tid(&self_tid); + OE_TEST(_thread_info.tid == self_tid); + + // Skip the idiv instruction - 2 is tied to the size of the idiv + // instruction and can change with a different compiler/build. + // Minimizing this with the use of the inline assembly for integer + // division + exception_record->context->rip += 2; + return OE_EXCEPTION_CONTINUE_EXECUTION; + } + else + { + return OE_EXCEPTION_ABORT_EXECUTION; + } +} + +void enc_run_thread(int tid) +{ + oe_result_t result = OE_OK; + int self_tid = 0; + + _thread_info.td = oe_sgx_get_td(); + + // Expect the state to be RUNNING upon entering + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_RUNNING); + + // Expect the flag is not set + OE_TEST(!oe_sgx_td_is_handling_host_signal(_thread_info.td)); + + host_get_tid(&self_tid); + + // Expect the state to be RUNNING after an ocall + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_RUNNING); + + OE_TEST(tid == self_tid); + _thread_info.tid = tid; + + printf("(tid=%d) thread is running...\n", _thread_info.tid); + + OE_CHECK(oe_add_vectored_exception_handler(false, td_state_handler)); + + // Invoke the internal API to unmask host signals + oe_sgx_td_unmask_host_signal(); + + // Ensure the order of setting the lock + asm volatile("" ::: "memory"); + + __atomic_store_n(_host_lock_state, 1, __ATOMIC_RELEASE); + while (__atomic_load_n(_host_lock_state, __ATOMIC_ACQUIRE) != 2) + { + asm volatile("pause" ::: "memory"); + } + + // Expect the state to be persisted after an interrupt + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_RUNNING); + + // Expect the flag is cleared + OE_TEST(!oe_sgx_td_is_handling_host_signal(_thread_info.td)); + + // Expect td->host_signal is cleared + OE_TEST(_thread_info.td->host_signal == 0); + + // Expect the signal is unregistered by the handler + OE_TEST(!oe_sgx_td_host_signal_registered(_thread_info.td, SIGUSR1)); + + printf("(tid=%d) interrupt is handled...\n", self_tid); + + __atomic_store_n(_host_lock_state, 3, __ATOMIC_RELEASE); + + // Make a ocall to spin and wait for an interrupt on the host + host_spin(); + + while (__atomic_load_n(_host_lock_state, __ATOMIC_ACQUIRE) != 5) + { + asm volatile("pause" ::: "memory"); + } + + // Expect the state to be RUNNING after an OCALL + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_RUNNING); + + { + uint32_t a, b, c, d; + cpuid(1, 0, &a, &b, &c, &d); + } + + // Expect the state to be persisted after an illegal instruction + // emulation + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_RUNNING); + + divide_by_zero_exception_function(); + + // Expect the state to be persisted after an exception. + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_RUNNING); + + printf("(tid=%d) thread is exiting...\n", self_tid); +done: + return; +} + +void enc_td_state(uint64_t lock_state) +{ + oe_result_t result; + int tid = 0; + + { + uint32_t a, b, c, d; + cpuid(1, 0, &a, &b, &c, &d); + } + + host_get_tid(&tid); + OE_TEST(tid != 0); + + /* Set up the lock_state points to the host*/ + _host_lock_state = (int*)lock_state; + + printf("(tid=%d) Create a thread...\n", tid); + + result = host_create_thread(); + if (result != OE_OK) + return; + + while (__atomic_load_n(_host_lock_state, __ATOMIC_ACQUIRE) != 1) + { + asm volatile("pause" ::: "memory"); + } + + OE_TEST(_thread_info.tid != 0); + host_sleep_msec(30); + + OE_TEST(oe_sgx_register_td_host_signal(_thread_info.td, SIGUSR1) == true); + + printf( + "(tid=%d) Sending interrupt to (td=0x%lx, tid=%d) inside the " + "enclave...\n", + tid, + (uint64_t)_thread_info.td, + _thread_info.tid); + + host_send_interrupt(_thread_info.tid, SIGUSR1); + + while (__atomic_load_n(_host_lock_state, __ATOMIC_ACQUIRE) != 4) + { + asm volatile("pause" ::: "memory"); + } + + // Expect the target td's state to be EXITED while + // running in the host context + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_EXITED); + + host_sleep_msec(30); + + printf( + "(tid=%d) Sending interrupt to (td=0x%lx, tid=%d) on the " + "host...\n", + tid, + (uint64_t)_thread_info.td, + _thread_info.tid); + + // Expect the host execution to be interrupted by SIGUSR1 + host_send_interrupt(_thread_info.tid, SIGUSR1); + + host_join_thread(); + + // Expect the target td's state to be EXITED + OE_EXPECT(_thread_info.td->state, OE_TD_STATE_EXITED); +} + +OE_SET_ENCLAVE_SGX( + 1, /* ProductID */ + 1, /* SecurityVersion */ + true, /* Debug */ + 1024, /* NumHeapPages */ + 1024, /* NumStackPages */ + 2); /* NumTCS */ diff --git a/tests/sgx/td_state/host/CMakeLists.txt b/tests/sgx/td_state/host/CMakeLists.txt new file mode 100644 index 0000000000..60362883de --- /dev/null +++ b/tests/sgx/td_state/host/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +set(EDL_FILE ../td_state.edl) + +add_custom_command( + OUTPUT td_state_u.h td_state_u.c + DEPENDS ${EDL_FILE} edger8r + COMMAND + edger8r --untrusted ${EDL_FILE} --search-path ${PROJECT_SOURCE_DIR}/include + ${DEFINE_OE_SGX} --search-path ${CMAKE_CURRENT_SOURCE_DIR}) + +add_executable(td_state_host host.c td_state_u.c) + +target_include_directories(td_state_host PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(td_state_host oehost) diff --git a/tests/sgx/td_state/host/host.c b/tests/sgx/td_state/host/host.c new file mode 100644 index 0000000000..065afcfc01 --- /dev/null +++ b/tests/sgx/td_state/host/host.c @@ -0,0 +1,115 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "td_state_u.h" + +#define SKIP_RETURN_CODE 2 + +oe_enclave_t* enclave; +static pthread_t _thread; +volatile static int _lock_state; + +int host_get_tid() +{ + return (pid_t)syscall(SYS_gettid); +} + +void host_spin() +{ + int return_value; + int tid = host_get_tid(); + + printf("(tid=%d) thread is spinning on the host...\n", tid); + + _lock_state = 4; + return_value = poll(NULL, 0, -1); + // Expect to be interrupted and return -1 + OE_TEST(return_value == -1); + printf("(tid=%d) thread is interrupted on the host...\n", tid); + _lock_state = 5; +} + +void host_send_interrupt(int tid, int signal_number) +{ + pid_t pid = getpid(); + syscall(SYS_tgkill, pid, tid, signal_number); +} + +static void* _thread_function() +{ + pid_t tid = (pid_t)syscall(SYS_gettid); + + enc_run_thread(enclave, tid); + + return NULL; +} + +void host_create_thread() +{ + pthread_attr_t attr; + + pthread_attr_init(&attr); + pthread_create(&_thread, &attr, _thread_function, NULL); + pthread_attr_destroy(&attr); +} + +void host_join_thread() +{ + pthread_join(_thread, NULL); +} + +void host_sleep_msec(uint32_t msec) +{ + struct timespec ts; + + ts.tv_sec = (uint64_t)msec / 1000; + ts.tv_nsec = ((int64_t)msec % 1000) * 1000000; + + nanosleep(&ts, NULL); +} + +int main(int argc, const char* argv[]) +{ + oe_result_t result; + + if (argc != 2) + { + fprintf(stderr, "Usage: %s ENCLAVE_PATH testname\n", argv[0]); + return 1; + } + + const uint32_t flags = oe_get_create_flags(); + if ((flags & OE_ENCLAVE_FLAG_SIMULATE) != 0) + { + printf("=== Skipped unsupported test in simulation mode " + "(td_state)\n"); + return SKIP_RETURN_CODE; + } + + if ((result = oe_create_td_state_enclave( + argv[1], OE_ENCLAVE_TYPE_SGX, flags, NULL, 0, &enclave)) != OE_OK) + oe_put_err("oe_create_enclave(): result=%u", result); + + result = enc_td_state(enclave, (uint64_t)&_lock_state); + if (result != OE_OK) + oe_put_err("oe_call_enclave() failed: result=%u", result); + + result = oe_terminate_enclave(enclave); + OE_TEST(result == OE_OK); + + printf("=== passed all tests (td_state)\n"); + + return 0; +} diff --git a/tests/sgx/td_state/td_state.edl b/tests/sgx/td_state/td_state.edl new file mode 100644 index 0000000000..6efe781fe7 --- /dev/null +++ b/tests/sgx/td_state/td_state.edl @@ -0,0 +1,23 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +enclave { + from "openenclave/edl/logging.edl" import oe_write_ocall; + from "openenclave/edl/fcntl.edl" import *; + from "openenclave/edl/sgx/platform.edl" import *; + + trusted { + public void enc_td_state(uint64_t lock_state); + public void enc_run_thread(int tid); + }; + + untrusted { + void host_send_interrupt(int tid, int signal_number); + void host_create_thread(); + void host_join_thread(); + void host_spin(); + int host_get_tid(); + void host_sleep_msec( + uint32_t msec); + }; +}; diff --git a/tests/sgx/thread_interrupt/CMakeLists.txt b/tests/sgx/thread_interrupt/CMakeLists.txt new file mode 100644 index 0000000000..598e922609 --- /dev/null +++ b/tests/sgx/thread_interrupt/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +add_subdirectory(host) + +if (BUILD_ENCLAVES) + add_subdirectory(enc) +endif () + +add_enclave_test(tests/sgx/thread_interrupt_nonblocking thread_interrupt_host + sgx_thread_interrupt_enc nonblocking) +set_enclave_tests_properties(tests/sgx/thread_interrupt_nonblocking PROPERTIES + SKIP_RETURN_CODE 2) + +add_enclave_test(tests/sgx/thread_interrupt_blocking thread_interrupt_host + sgx_thread_interrupt_enc blocking) +set_enclave_tests_properties(tests/sgx/thread_interrupt_blocking PROPERTIES + SKIP_RETURN_CODE 2) diff --git a/tests/sgx/thread_interrupt/enc/CMakeLists.txt b/tests/sgx/thread_interrupt/enc/CMakeLists.txt new file mode 100644 index 0000000000..f6b8eebf46 --- /dev/null +++ b/tests/sgx/thread_interrupt/enc/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +set(EDL_FILE ../thread_interrupt.edl) + +add_custom_command( + OUTPUT thread_interrupt_t.h thread_interrupt_t.c + DEPENDS ${EDL_FILE} edger8r + COMMAND + edger8r --trusted ${EDL_FILE} --search-path ${PROJECT_SOURCE_DIR}/include + ${DEFINE_OE_SGX} --search-path ${CMAKE_CURRENT_SOURCE_DIR}) + +add_enclave(TARGET sgx_thread_interrupt_enc SOURCES enc.c + ${CMAKE_CURRENT_BINARY_DIR}/thread_interrupt_t.c) + +enclave_include_directories(sgx_thread_interrupt_enc PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/sgx/thread_interrupt/enc/enc.c b/tests/sgx/thread_interrupt/enc/enc.c new file mode 100644 index 0000000000..175eda70e7 --- /dev/null +++ b/tests/sgx/thread_interrupt/enc/enc.c @@ -0,0 +1,366 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include "thread_interrupt_t.h" + +#include +#include + +#define REPEAT_TIMES 10 + +static int _signal_list[] = + {SIGHUP, SIGABRT, SIGALRM, SIGPIPE, SIGPOLL, SIGUSR1, SIGUSR2}; + +#define SIGNAL_NUMBER (sizeof(_signal_list) / sizeof(_signal_list[0])) + +static int _current_signal; + +typedef struct _thread_info_nonblocking_t +{ + int lock; + int tid; + oe_sgx_td_t* td; +} thread_info_t; + +static thread_info_t _thread_info_nonblocking; +static thread_info_t _thread_info_blocking; +static volatile int _handler_entered; + +uint64_t thread_interrupt_handler(oe_exception_record_t* exception_record) +{ + int self_tid = 0; + + _handler_entered = 1; + + OE_TEST(exception_record->code == OE_EXCEPTION_UNKNOWN); + + OE_TEST(exception_record->host_signal_number == _current_signal); + + host_get_tid(&self_tid); + OE_TEST(_thread_info_nonblocking.tid == self_tid); + + printf("(tid=%d) thread is interrupted...\n", self_tid); + + OE_TEST( + _thread_info_nonblocking.td->state == + OE_TD_STATE_SECOND_LEVEL_EXCEPTION_HANDLING); + + __atomic_store_n(&_thread_info_nonblocking.lock, 2, __ATOMIC_RELEASE); + + return OE_EXCEPTION_CONTINUE_EXECUTION; +} + +void enc_run_thread_nonblocking(int tid) +{ + oe_result_t result = OE_OK; + int self_tid = 0; + host_get_tid(&self_tid); + + OE_TEST(tid == self_tid); + + printf("(tid=%d) non-blocking thread is running...\n", self_tid); + + OE_CHECK( + oe_add_vectored_exception_handler(false, thread_interrupt_handler)); + + _thread_info_nonblocking.tid = tid; + _thread_info_nonblocking.td = oe_sgx_get_td(); + + // Validate the default state + OE_TEST(_thread_info_nonblocking.td->state == OE_TD_STATE_RUNNING); + + // Invoke the internal API to unmask host signals + oe_sgx_td_unmask_host_signal(); + + // Ensure the order of setting the lock + asm volatile("" ::: "memory"); + + __atomic_store_n(&_thread_info_nonblocking.lock, 1, __ATOMIC_RELEASE); + + // Test receiving different signals + for (size_t i = 0; i < SIGNAL_NUMBER; i++) + { + while (__atomic_load_n( + &_thread_info_nonblocking.lock, __ATOMIC_ACQUIRE) != 2) + { + asm volatile("pause" ::: "memory"); + } + + // Validate the state after the interrupt + OE_TEST(_thread_info_nonblocking.td->state == OE_TD_STATE_RUNNING); + + // Ensure the order of setting the lock + asm volatile("" ::: "memory"); + + __atomic_store_n(&_thread_info_nonblocking.lock, 3, __ATOMIC_RELEASE); + } + + // Test unregistering signals + for (size_t i = 0; i < SIGNAL_NUMBER; i++) + { + OE_TEST(oe_sgx_unregister_td_host_signal( + _thread_info_nonblocking.td, _signal_list[i])); + } + __atomic_store_n(&_thread_info_nonblocking.lock, 4, __ATOMIC_RELEASE); + + // Test receiving the same signal multiple times + for (size_t i = 0; i < REPEAT_TIMES; i++) + { + while (__atomic_load_n( + &_thread_info_nonblocking.lock, __ATOMIC_ACQUIRE) != 2) + { + asm volatile("pause" ::: "memory"); + } + + // Validate the state after the interrupt + OE_TEST(_thread_info_nonblocking.td->state == OE_TD_STATE_RUNNING); + + // Ensure the order of setting the lock + asm volatile("" ::: "memory"); + + __atomic_store_n(&_thread_info_nonblocking.lock, 3, __ATOMIC_RELEASE); + } + + printf("(tid=%d) non-blocking thread is exiting...\n", self_tid); + +done: + return; +} + +void enc_thread_interrupt_nonblocking(void) +{ + oe_result_t result; + int tid = 0; + + host_get_tid(&tid); + OE_TEST(tid != 0); + + // Test interrupting a non-blocking thread + printf("(tid=%d) Create a non-blocking thread...\n", tid); + + result = host_create_thread(0 /* blocking */); + if (result != OE_OK) + return; + + while (__atomic_load_n(&_thread_info_nonblocking.lock, __ATOMIC_ACQUIRE) != + 1) + { + asm volatile("pause" ::: "memory"); + } + + OE_TEST(_thread_info_nonblocking.tid != 0); + + host_sleep_msec(30); + + // Sending different signals + for (size_t i = 0; i < SIGNAL_NUMBER; i++) + { + _current_signal = _signal_list[i]; + + // Signal registration + OE_TEST( + oe_sgx_register_td_host_signal( + _thread_info_nonblocking.td, _current_signal) == true); + + printf( + "(tid=%d) Sending interrupt (%d) to (td=0x%lx, tid=%d)...\n", + tid, + _current_signal, + (uint64_t)_thread_info_nonblocking.td, + _thread_info_nonblocking.tid); + + host_send_interrupt(_thread_info_nonblocking.tid, _current_signal); + + while (__atomic_load_n( + &_thread_info_nonblocking.lock, __ATOMIC_ACQUIRE) != 3) + { + asm volatile("pause" ::: "memory"); + } + + host_sleep_msec(30); + } + + while (__atomic_load_n(&_thread_info_nonblocking.lock, __ATOMIC_ACQUIRE) != + 4) + { + asm volatile("pause" ::: "memory"); + } + + // Expect the signals are unregistered + for (size_t i = 0; i < SIGNAL_NUMBER; i++) + { + OE_TEST(!oe_sgx_td_host_signal_registered( + _thread_info_nonblocking.td, _signal_list[i])); + } + + host_sleep_msec(30); + + _current_signal = SIGUSR1; + + // Register SIGUSR1 again + OE_TEST( + oe_sgx_register_td_host_signal( + _thread_info_nonblocking.td, _current_signal) == true); + + // Sending the same signal multiple times + for (size_t i = 0; i < REPEAT_TIMES; i++) + { + printf( + "(tid=%d) Sending interrupt (%d) to (td=0x%lx, tid=%d)...\n", + tid, + _current_signal, + (uint64_t)_thread_info_nonblocking.td, + _thread_info_nonblocking.tid); + + host_send_interrupt(_thread_info_nonblocking.tid, _current_signal); + + while (__atomic_load_n( + &_thread_info_nonblocking.lock, __ATOMIC_ACQUIRE) != 3) + { + asm volatile("pause" ::: "memory"); + } + + host_sleep_msec(30); + } + + host_join_thread(); +} + +void enc_run_thread_blocking(int tid) +{ + oe_result_t result = OE_OK; + int self_tid = 0; + host_get_tid(&self_tid); + + OE_TEST(tid == self_tid); + printf("(tid=%d) blocking thread is running...\n", self_tid); + + OE_CHECK( + oe_add_vectored_exception_handler(false, thread_interrupt_handler)); + + _thread_info_blocking.tid = tid; + _thread_info_blocking.td = oe_sgx_get_td(); + + // Validate the default state + OE_TEST(_thread_info_blocking.td->state == OE_TD_STATE_RUNNING); + + // Mask host signals (the default behavior) + oe_sgx_td_mask_host_signal(); + + // Ensure the order of setting the lock + asm volatile("" ::: "memory"); + + __atomic_store_n(&_thread_info_blocking.lock, 1, __ATOMIC_RELEASE); + + while (__atomic_load_n(&_thread_info_blocking.lock, __ATOMIC_ACQUIRE) != 2) + { + asm volatile("pause" ::: "memory"); + } + + printf("(tid=%d) blocking thread is exiting...\n", self_tid); +done: + return; +} + +void enc_thread_interrupt_blocking(void) +{ + oe_result_t result; + int tid = 0; + int retry = 0; + + host_get_tid(&tid); + OE_TEST(tid != 0); + + // Test interrupting a blocking thread + printf("(tid=%d) Create a blocking thread...\n", tid); + result = host_create_thread(1 /* blocking */); + if (result != OE_OK) + return; + + while (!_thread_info_blocking.lock) + { + asm volatile("pause" ::: "memory"); + } + + OE_TEST(_thread_info_blocking.tid != 0); + + host_sleep_msec(30); + + _handler_entered = 0; + while (!_handler_entered) + { + OE_TEST( + oe_sgx_register_td_host_signal(_thread_info_blocking.td, SIGUSR1) == + true); + + printf( + "(tid=%d) Sending registered signal (SIGUSR1) to (td=0x%lx, " + "tid=%d)...%d\n", + tid, + (uint64_t)_thread_info_blocking.td, + _thread_info_blocking.tid, + ++retry); + + host_send_interrupt(_thread_info_blocking.tid, SIGUSR1); + + if (retry == 10) + { + printf( + "Unable to interrrupt (tid=%d) as expected\n", + _thread_info_blocking.tid); + break; + } + + host_sleep_msec(30); + } + + OE_TEST(retry == 10); + + retry = 0; + _handler_entered = 0; + + // The oe_sgx_td_unmask_host_signal API is only expected to be used + // by the thread itself. Set the flag direclty for the testing purposes. + _thread_info_blocking.td->host_signal_unmasked = 1; + + while (!_handler_entered) + { + printf( + "(tid=%d) Sending unregistered signal (SIGUSR2) to (td=0x%lx, " + "tid=%d)...%d\n", + tid, + (uint64_t)_thread_info_blocking.td, + _thread_info_blocking.tid, + ++retry); + + host_send_interrupt(_thread_info_blocking.tid, SIGUSR2); + + if (retry == 10) + { + printf( + "Unable to interrrupt (tid=%d) as expected\n", + _thread_info_blocking.tid); + break; + } + + host_sleep_msec(30); + } + + OE_TEST(retry == 10); + + oe_abort(); +} + +OE_SET_ENCLAVE_SGX( + 1, /* ProductID */ + 1, /* SecurityVersion */ + true, /* Debug */ + 1024, /* NumHeapPages */ + 1024, /* NumStackPages */ + 2); /* NumTCS */ diff --git a/tests/sgx/thread_interrupt/host/CMakeLists.txt b/tests/sgx/thread_interrupt/host/CMakeLists.txt new file mode 100644 index 0000000000..b18318eb2a --- /dev/null +++ b/tests/sgx/thread_interrupt/host/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +set(EDL_FILE ../thread_interrupt.edl) + +add_custom_command( + OUTPUT thread_interrupt_u.h thread_interrupt_u.c + DEPENDS ${EDL_FILE} edger8r + COMMAND + edger8r --untrusted ${EDL_FILE} --search-path ${PROJECT_SOURCE_DIR}/include + ${DEFINE_OE_SGX} --search-path ${CMAKE_CURRENT_SOURCE_DIR}) + +add_executable(thread_interrupt_host host.c thread_interrupt_u.c) + +target_include_directories(thread_interrupt_host + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(thread_interrupt_host oehost) diff --git a/tests/sgx/thread_interrupt/host/host.c b/tests/sgx/thread_interrupt/host/host.c new file mode 100644 index 0000000000..e6c3f196bd --- /dev/null +++ b/tests/sgx/thread_interrupt/host/host.c @@ -0,0 +1,116 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "thread_interrupt_u.h" + +#define SKIP_RETURN_CODE 2 + +oe_enclave_t* enclave; +static pthread_t _thread; + +void host_send_interrupt(int tid, int signal_number) +{ + pid_t pid = getpid(); + syscall(SYS_tgkill, pid, tid, signal_number); +} + +int host_get_tid() +{ + return (pid_t)syscall(SYS_gettid); +} + +static void* _thread_function(void* arg) +{ + uint64_t blocking = (uint64_t)arg; + pid_t tid = (pid_t)syscall(SYS_gettid); + + if (blocking) + enc_run_thread_blocking(enclave, tid); + else + enc_run_thread_nonblocking(enclave, tid); + + return NULL; +} + +void host_create_thread(uint64_t blocking) +{ + pthread_attr_t attr; + + pthread_attr_init(&attr); + pthread_create(&_thread, &attr, _thread_function, (void*)blocking); + pthread_attr_destroy(&attr); +} + +void host_join_thread() +{ + pthread_join(_thread, NULL); +} + +void host_sleep_msec(uint32_t msec) +{ + struct timespec ts; + + ts.tv_sec = (uint64_t)msec / 1000; + ts.tv_nsec = ((int64_t)msec % 1000) * 1000000; + + nanosleep(&ts, NULL); +} + +int main(int argc, const char* argv[]) +{ + oe_result_t result; + + if (argc != 3) + { + fprintf(stderr, "Usage: %s ENCLAVE_PATH testname\n", argv[0]); + return 1; + } + + const uint32_t flags = oe_get_create_flags(); + if ((flags & OE_ENCLAVE_FLAG_SIMULATE) != 0) + { + printf("=== Skipped unsupported test in simulation mode " + "(thread_interrupt)\n"); + return SKIP_RETURN_CODE; + } + + if ((result = oe_create_thread_interrupt_enclave( + argv[1], OE_ENCLAVE_TYPE_SGX, flags, NULL, 0, &enclave)) != OE_OK) + oe_put_err("oe_create_enclave(): result=%u", result); + + if (!strcmp(argv[2], "nonblocking")) + { + result = enc_thread_interrupt_nonblocking(enclave); + if (result != OE_OK) + oe_put_err("oe_call_enclave() failed: result=%u", result); + OE_TEST(oe_terminate_enclave(enclave) == OE_OK); + } + else if (!strcmp(argv[2], "blocking")) + { + result = enc_thread_interrupt_blocking(enclave); + OE_TEST(result == OE_ENCLAVE_ABORTING); + /* Expcet a non-OE_OK result. The error code may be different + * between debug and release build. */ + OE_TEST(oe_terminate_enclave(enclave) != OE_OK); + } + else + { + fprintf(stderr, "Unknown test case"); + return 1; + } + + printf("=== passed all tests (thread_interrupt)\n"); + + return 0; +} diff --git a/tests/sgx/thread_interrupt/thread_interrupt.edl b/tests/sgx/thread_interrupt/thread_interrupt.edl new file mode 100644 index 0000000000..819b95029c --- /dev/null +++ b/tests/sgx/thread_interrupt/thread_interrupt.edl @@ -0,0 +1,24 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +enclave { + from "openenclave/edl/logging.edl" import oe_write_ocall; + from "openenclave/edl/fcntl.edl" import *; + from "openenclave/edl/sgx/platform.edl" import *; + + trusted { + public void enc_thread_interrupt_nonblocking(); + public void enc_thread_interrupt_blocking(); + public void enc_run_thread_nonblocking(int tid); + public void enc_run_thread_blocking(int tid); + }; + + untrusted { + void host_send_interrupt(int tid, int signal_number); + void host_create_thread(uint64_t blocking); + void host_join_thread(); + int host_get_tid(); + void host_sleep_msec( + uint32_t msec); + }; +};