From b475c8d8962c087c1dba2c79adf261138e992ade Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi Date: Fri, 29 Oct 2021 05:47:59 +0000 Subject: [PATCH] Build OpenSSL with threading support. This allows OpenSSL to be used in multi-threaded enclaves. - Remove no-threads configuration option. - Implement pthread_atfork API which is used by OpenSSL initialization only if OPENSSL_INIT_ATFORK option is used to initialize OpenSSL. Enclaves don't support forking. - Implement pthread_create API so that threads_test passes. Signed-off-by: Anand Krishnamoorthi --- 3rdparty/openssl/include/opensslconf.h | 203 ++++++++++---------- 3rdparty/openssl/update.make | 2 +- libc/pthread.c | 14 ++ tests/openssl/enc/CMakeLists.txt | 22 ++- tests/openssl/enc/enc.c | 7 +- tests/openssl/enc/thread.cpp | 248 +++++++++++++++++++++++++ tests/openssl/host/host.cpp | 177 +++++++++++++++++- tests/openssl/host/threadArgs.h | 33 ++++ tests/openssl/openssl.edl | 17 ++ 9 files changed, 612 insertions(+), 111 deletions(-) create mode 100644 tests/openssl/enc/thread.cpp create mode 100644 tests/openssl/host/threadArgs.h diff --git a/3rdparty/openssl/include/opensslconf.h b/3rdparty/openssl/include/opensslconf.h index d4c3fab897..40721f1449 100644 --- a/3rdparty/openssl/include/opensslconf.h +++ b/3rdparty/openssl/include/opensslconf.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from openssl/include/openssl/opensslconf.h.in * - * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the OpenSSL license (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,190 +12,193 @@ #include -#ifdef __cplusplus -extern "C" { +#ifdef __cplusplus +extern "C" +{ #endif #ifdef OPENSSL_ALGORITHM_DEFINES -# error OPENSSL_ALGORITHM_DEFINES no longer supported +#error OPENSSL_ALGORITHM_DEFINES no longer supported #endif -/* - * OpenSSL was configured with the following options: - */ + /* + * OpenSSL was configured with the following options: + */ #ifndef OPENSSL_NO_ARIA -# define OPENSSL_NO_ARIA +#define OPENSSL_NO_ARIA #endif #ifndef OPENSSL_NO_BF -# define OPENSSL_NO_BF +#define OPENSSL_NO_BF #endif #ifndef OPENSSL_NO_BLAKE2 -# define OPENSSL_NO_BLAKE2 +#define OPENSSL_NO_BLAKE2 #endif #ifndef OPENSSL_NO_CAMELLIA -# define OPENSSL_NO_CAMELLIA +#define OPENSSL_NO_CAMELLIA #endif #ifndef OPENSSL_NO_CAST -# define OPENSSL_NO_CAST +#define OPENSSL_NO_CAST #endif #ifndef OPENSSL_NO_CHACHA -# define OPENSSL_NO_CHACHA +#define OPENSSL_NO_CHACHA #endif #ifndef OPENSSL_NO_CMS -# define OPENSSL_NO_CMS +#define OPENSSL_NO_CMS #endif #ifndef OPENSSL_NO_CT -# define OPENSSL_NO_CT +#define OPENSSL_NO_CT #endif #ifndef OPENSSL_NO_IDEA -# define OPENSSL_NO_IDEA +#define OPENSSL_NO_IDEA #endif #ifndef OPENSSL_NO_MD2 -# define OPENSSL_NO_MD2 +#define OPENSSL_NO_MD2 #endif #ifndef OPENSSL_NO_MD4 -# define OPENSSL_NO_MD4 +#define OPENSSL_NO_MD4 #endif #ifndef OPENSSL_NO_MDC2 -# define OPENSSL_NO_MDC2 +#define OPENSSL_NO_MDC2 #endif #ifndef OPENSSL_NO_POLY1305 -# define OPENSSL_NO_POLY1305 +#define OPENSSL_NO_POLY1305 #endif #ifndef OPENSSL_NO_RC4 -# define OPENSSL_NO_RC4 +#define OPENSSL_NO_RC4 #endif #ifndef OPENSSL_NO_RC5 -# define OPENSSL_NO_RC5 +#define OPENSSL_NO_RC5 #endif #ifndef OPENSSL_NO_RMD160 -# define OPENSSL_NO_RMD160 +#define OPENSSL_NO_RMD160 #endif #ifndef OPENSSL_NO_SEED -# define OPENSSL_NO_SEED +#define OPENSSL_NO_SEED #endif #ifndef OPENSSL_NO_SIPHASH -# define OPENSSL_NO_SIPHASH +#define OPENSSL_NO_SIPHASH #endif #ifndef OPENSSL_NO_SM2 -# define OPENSSL_NO_SM2 +#define OPENSSL_NO_SM2 #endif #ifndef OPENSSL_NO_SM3 -# define OPENSSL_NO_SM3 +#define OPENSSL_NO_SM3 #endif #ifndef OPENSSL_NO_SM4 -# define OPENSSL_NO_SM4 +#define OPENSSL_NO_SM4 #endif #ifndef OPENSSL_NO_SRP -# define OPENSSL_NO_SRP +#define OPENSSL_NO_SRP #endif #ifndef OPENSSL_NO_WHIRLPOOL -# define OPENSSL_NO_WHIRLPOOL +#define OPENSSL_NO_WHIRLPOOL +#endif +#ifndef OPENSSL_THREADS +#define OPENSSL_THREADS #endif #ifndef OPENSSL_RAND_SEED_NONE -# define OPENSSL_RAND_SEED_NONE +#define OPENSSL_RAND_SEED_NONE #endif #ifndef OPENSSL_NO_AFALGENG -# define OPENSSL_NO_AFALGENG +#define OPENSSL_NO_AFALGENG #endif #ifndef OPENSSL_NO_ASAN -# define OPENSSL_NO_ASAN +#define OPENSSL_NO_ASAN #endif #ifndef OPENSSL_NO_AUTOERRINIT -# define OPENSSL_NO_AUTOERRINIT +#define OPENSSL_NO_AUTOERRINIT #endif #ifndef OPENSSL_NO_AUTOLOAD_CONFIG -# define OPENSSL_NO_AUTOLOAD_CONFIG +#define OPENSSL_NO_AUTOLOAD_CONFIG #endif #ifndef OPENSSL_NO_CAPIENG -# define OPENSSL_NO_CAPIENG +#define OPENSSL_NO_CAPIENG #endif #ifndef OPENSSL_NO_CRYPTO_MDEBUG -# define OPENSSL_NO_CRYPTO_MDEBUG +#define OPENSSL_NO_CRYPTO_MDEBUG #endif #ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE -# define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE +#define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE #endif #ifndef OPENSSL_NO_DEVCRYPTOENG -# define OPENSSL_NO_DEVCRYPTOENG +#define OPENSSL_NO_DEVCRYPTOENG #endif #ifndef OPENSSL_NO_DSO -# define OPENSSL_NO_DSO +#define OPENSSL_NO_DSO #endif #ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 -# define OPENSSL_NO_EC_NISTP_64_GCC_128 +#define OPENSSL_NO_EC_NISTP_64_GCC_128 #endif #ifndef OPENSSL_NO_EGD -# define OPENSSL_NO_EGD +#define OPENSSL_NO_EGD #endif #ifndef OPENSSL_NO_EXTERNAL_TESTS -# define OPENSSL_NO_EXTERNAL_TESTS +#define OPENSSL_NO_EXTERNAL_TESTS #endif #ifndef OPENSSL_NO_FUZZ_AFL -# define OPENSSL_NO_FUZZ_AFL +#define OPENSSL_NO_FUZZ_AFL #endif #ifndef OPENSSL_NO_FUZZ_LIBFUZZER -# define OPENSSL_NO_FUZZ_LIBFUZZER +#define OPENSSL_NO_FUZZ_LIBFUZZER #endif #ifndef OPENSSL_NO_GOST -# define OPENSSL_NO_GOST +#define OPENSSL_NO_GOST #endif #ifndef OPENSSL_NO_HEARTBEATS -# define OPENSSL_NO_HEARTBEATS +#define OPENSSL_NO_HEARTBEATS #endif #ifndef OPENSSL_NO_HW -# define OPENSSL_NO_HW +#define OPENSSL_NO_HW #endif #ifndef OPENSSL_NO_MSAN -# define OPENSSL_NO_MSAN +#define OPENSSL_NO_MSAN #endif #ifndef OPENSSL_NO_NEXTPROTONEG -# define OPENSSL_NO_NEXTPROTONEG +#define OPENSSL_NO_NEXTPROTONEG #endif #ifndef OPENSSL_NO_PSK -# define OPENSSL_NO_PSK +#define OPENSSL_NO_PSK #endif #ifndef OPENSSL_NO_RFC3779 -# define OPENSSL_NO_RFC3779 +#define OPENSSL_NO_RFC3779 #endif #ifndef OPENSSL_NO_SCRYPT -# define OPENSSL_NO_SCRYPT +#define OPENSSL_NO_SCRYPT #endif #ifndef OPENSSL_NO_SCTP -# define OPENSSL_NO_SCTP +#define OPENSSL_NO_SCTP #endif #ifndef OPENSSL_NO_SSL_TRACE -# define OPENSSL_NO_SSL_TRACE +#define OPENSSL_NO_SSL_TRACE #endif #ifndef OPENSSL_NO_SSL3 -# define OPENSSL_NO_SSL3 +#define OPENSSL_NO_SSL3 #endif #ifndef OPENSSL_NO_SSL3_METHOD -# define OPENSSL_NO_SSL3_METHOD +#define OPENSSL_NO_SSL3_METHOD #endif #ifndef OPENSSL_NO_UBSAN -# define OPENSSL_NO_UBSAN +#define OPENSSL_NO_UBSAN #endif #ifndef OPENSSL_NO_UI_CONSOLE -# define OPENSSL_NO_UI_CONSOLE +#define OPENSSL_NO_UI_CONSOLE #endif #ifndef OPENSSL_NO_UNIT_TEST -# define OPENSSL_NO_UNIT_TEST +#define OPENSSL_NO_UNIT_TEST #endif #ifndef OPENSSL_NO_WEAK_SSL_CIPHERS -# define OPENSSL_NO_WEAK_SSL_CIPHERS +#define OPENSSL_NO_WEAK_SSL_CIPHERS #endif #ifndef OPENSSL_NO_DYNAMIC_ENGINE -# define OPENSSL_NO_DYNAMIC_ENGINE +#define OPENSSL_NO_DYNAMIC_ENGINE #endif - /* * Sometimes OPENSSSL_NO_xxx ends up with an empty file and some compilers * don't like that. This will hopefully silence them. */ -#define NON_EMPTY_TRANSLATION_UNIT static void *dummy = &dummy; +#define NON_EMPTY_TRANSLATION_UNIT static void* dummy = &dummy; /* * Applications should use -DOPENSSL_API_COMPAT= to suppress the @@ -204,37 +207,37 @@ extern "C" { * functions. */ #ifndef DECLARE_DEPRECATED -# define DECLARE_DEPRECATED(f) f; -# ifdef __GNUC__ -# if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 0) -# undef DECLARE_DEPRECATED -# define DECLARE_DEPRECATED(f) f __attribute__ ((deprecated)); -# endif -# elif defined(__SUNPRO_C) -# if (__SUNPRO_C >= 0x5130) -# undef DECLARE_DEPRECATED -# define DECLARE_DEPRECATED(f) f __attribute__ ((deprecated)); -# endif -# endif +#define DECLARE_DEPRECATED(f) f; +#ifdef __GNUC__ +#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 0) +#undef DECLARE_DEPRECATED +#define DECLARE_DEPRECATED(f) f __attribute__((deprecated)); +#endif +#elif defined(__SUNPRO_C) +#if (__SUNPRO_C >= 0x5130) +#undef DECLARE_DEPRECATED +#define DECLARE_DEPRECATED(f) f __attribute__((deprecated)); +#endif +#endif #endif #ifndef OPENSSL_FILE -# ifdef OPENSSL_NO_FILENAMES -# define OPENSSL_FILE "" -# define OPENSSL_LINE 0 -# else -# define OPENSSL_FILE __FILE__ -# define OPENSSL_LINE __LINE__ -# endif +#ifdef OPENSSL_NO_FILENAMES +#define OPENSSL_FILE "" +#define OPENSSL_LINE 0 +#else +#define OPENSSL_FILE __FILE__ +#define OPENSSL_LINE __LINE__ +#endif #endif #ifndef OPENSSL_MIN_API -# define OPENSSL_MIN_API 0 +#define OPENSSL_MIN_API 0 #endif #if !defined(OPENSSL_API_COMPAT) || OPENSSL_API_COMPAT < OPENSSL_MIN_API -# undef OPENSSL_API_COMPAT -# define OPENSSL_API_COMPAT OPENSSL_MIN_API +#undef OPENSSL_API_COMPAT +#define OPENSSL_API_COMPAT OPENSSL_MIN_API #endif /* @@ -242,29 +245,29 @@ extern "C" { * OpenSSL version number matches. */ #if OPENSSL_VERSION_NUMBER < 0x10200000L -# define DEPRECATEDIN_1_2_0(f) f; +#define DEPRECATEDIN_1_2_0(f) f; #elif OPENSSL_API_COMPAT < 0x10200000L -# define DEPRECATEDIN_1_2_0(f) DECLARE_DEPRECATED(f) +#define DEPRECATEDIN_1_2_0(f) DECLARE_DEPRECATED(f) #else -# define DEPRECATEDIN_1_2_0(f) +#define DEPRECATEDIN_1_2_0(f) #endif #if OPENSSL_API_COMPAT < 0x10100000L -# define DEPRECATEDIN_1_1_0(f) DECLARE_DEPRECATED(f) +#define DEPRECATEDIN_1_1_0(f) DECLARE_DEPRECATED(f) #else -# define DEPRECATEDIN_1_1_0(f) +#define DEPRECATEDIN_1_1_0(f) #endif #if OPENSSL_API_COMPAT < 0x10000000L -# define DEPRECATEDIN_1_0_0(f) DECLARE_DEPRECATED(f) +#define DEPRECATEDIN_1_0_0(f) DECLARE_DEPRECATED(f) #else -# define DEPRECATEDIN_1_0_0(f) +#define DEPRECATEDIN_1_0_0(f) #endif #if OPENSSL_API_COMPAT < 0x00908000L -# define DEPRECATEDIN_0_9_8(f) DECLARE_DEPRECATED(f) +#define DEPRECATEDIN_0_9_8(f) DECLARE_DEPRECATED(f) #else -# define DEPRECATEDIN_0_9_8(f) +#define DEPRECATEDIN_0_9_8(f) #endif /* Generate 80386 code? */ @@ -279,15 +282,15 @@ extern "C" { * The following are cipher-specific, but are part of the public API. */ #if !defined(OPENSSL_SYS_UEFI) -# undef BN_LLONG +#undef BN_LLONG /* Only one for the following should be defined */ -# define SIXTY_FOUR_BIT_LONG -# undef SIXTY_FOUR_BIT -# undef THIRTY_TWO_BIT +#define SIXTY_FOUR_BIT_LONG +#undef SIXTY_FOUR_BIT +#undef THIRTY_TWO_BIT #endif #define RC4_INT unsigned int -#ifdef __cplusplus +#ifdef __cplusplus } #endif diff --git a/3rdparty/openssl/update.make b/3rdparty/openssl/update.make index 5ef3d4589a..a4f31c86b4 100755 --- a/3rdparty/openssl/update.make +++ b/3rdparty/openssl/update.make @@ -13,7 +13,7 @@ update-openssl-headers: no-cms no-ct no-dso no-gost no-idea no-md2 no-md4 no-mdc2 no-nextprotoneg \ no-poly1305 no-psk no-rc4 no-rfc3779 no-rmd160 no-scrypt no-seed \ no-shared no-siphash no-sm2 no-sm3 no-sm4 no-srp no-ssl2 no-ssl3 \ - no-threads no-ui-console no-whirlpool no-zlib CC=clang-10 CXX=clang++-10; \ + no-ui-console no-whirlpool no-zlib CC=clang-10 CXX=clang++-10; \ perl "-I." -Mconfigdata "openssl/util/dofile.pl" "-oMakefile" "openssl/include/crypto/bn_conf.h.in" \ > include/bn_conf.h; \ perl "-I." -Mconfigdata "openssl/util/dofile.pl" "-oMakefile" "openssl/include/crypto/dso_conf.h.in" \ diff --git a/libc/pthread.c b/libc/pthread.c index 921bde2fe0..60d51631cf 100644 --- a/libc/pthread.c +++ b/libc/pthread.c @@ -105,3 +105,17 @@ int pthread_detach(pthread_t thread) return _pthread_hooks->detach(thread); } + +// OpenSSL when configured with thread support (default) installs +// fork handlers. Open Enclave does not support fork and any installed +// handlers would never get called. +int pthread_atfork( + void (*prepare)(void), + void (*parent)(void), + void (*child)(void)) +{ + OE_UNUSED(prepare); + OE_UNUSED(parent); + OE_UNUSED(child); + return 0; +} diff --git a/tests/openssl/enc/CMakeLists.txt b/tests/openssl/enc/CMakeLists.txt index bb4f75639f..08ecfa5482 100644 --- a/tests/openssl/enc/CMakeLists.txt +++ b/tests/openssl/enc/CMakeLists.txt @@ -24,15 +24,20 @@ add_enclave_library( ${OPENSSL_DIR}/test/testutil/tap_bio.c ${OPENSSL_DIR}/test/testutil/test_cleanup.c ${OPENSSL_DIR}/test/testutil/tests.c - ${OPENSSL_DIR}/test/testutil/testutil_init.c) + ${OPENSSL_DIR}/test/testutil/testutil_init.c + thread.cpp + # Ensure that openssl_t.h is generated. + openssl_t.h) enclave_compile_options( openssl-support PRIVATE -Wno-shorten-64-to-32 -Wno-sign-conversion -Wno-conversion -Wno-unused-parameter) -enclave_link_libraries(openssl-support PRIVATE oelibc oe_includes) -enclave_include_directories(openssl-support PRIVATE ${OPENSSL_DIR}/include - ${CMAKE_BINARY_DIR}/3rdparty/openssl/build/include) +enclave_link_libraries(openssl-support PRIVATE oelibcxx oelibc oe_includes) +enclave_include_directories( + openssl-support PRIVATE ${OPENSSL_DIR}/include + ${CMAKE_BINARY_DIR}/3rdparty/openssl/build/include + ${CMAKE_CURRENT_BINARY_DIR}) # Create an object library to avoid recompiling these files. # These cannot be added to openssl-support since they'd result in multiple @@ -115,8 +120,13 @@ function (add_openssl_test_enc NAME BUILDTEST) ${OPENSSL_DIR}/include ${CMAKE_BINARY_DIR}/3rdparty/openssl/build/include) enclave_compile_options( - openssl-${NAME}_enc PRIVATE -Wno-shorten-64-to-32 -Wno-sign-conversion - -Wno-conversion -Wno-unused-parameter) + openssl-${NAME}_enc + PRIVATE + -Wno-shorten-64-to-32 + -Wno-sign-conversion + -Wno-conversion + -Wno-unused-parameter + -Wno-deprecated-declarations) enclave_compile_definitions(openssl-${NAME}_enc PRIVATE OPENSSL_NO_AFALGENG __STDC_NO_ATOMICS__) diff --git a/tests/openssl/enc/enc.c b/tests/openssl/enc/enc.c index daf15be4f4..366ff9b141 100644 --- a/tests/openssl/enc/enc.c +++ b/tests/openssl/enc/enc.c @@ -12,12 +12,17 @@ extern char** __environ; extern int main(int argc, char* argv[]); +void register_pthread_hooks(void); + int enc_test(int argc, char** argv, char** env) { int ret = 1; ENGINE* eng = NULL; const BIO_METHOD* tap = NULL; + /* Register pthread hooks. Used only by threadstest */ + register_pthread_hooks(); + /* Directly use environ from host. */ __environ = env; @@ -96,4 +101,4 @@ OE_SET_ENCLAVE_SGX( true, /* AllowDebug */ 6144, /* HeapPageCount */ 128, /* StackPageCount */ - 1); /* TCSCount */ + 8); /* TCSCount */ diff --git a/tests/openssl/enc/thread.cpp b/tests/openssl/enc/thread.cpp new file mode 100644 index 0000000000..3fc14e480b --- /dev/null +++ b/tests/openssl/enc/thread.cpp @@ -0,0 +1,248 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../host/threadArgs.h" +#include "openssl_t.h" +#if 0 +extern "C" void _exit(int status) +{ + host_exit(status); + abort(); +} + +extern "C" void _Exit(int status) +{ + _exit(status); + abort(); +} + +extern "C" void exit(int status) +{ + _exit(status); + abort(); +} +#endif +static std::vector> _thread_functions; +static uint64_t _next_enc_thread_id = 0; +static uint64_t _enc_key = 0; // Monotonically increasing enclave key + +// Map of enc_key to thread_id returned by pthread_self() +static std::map> _key_to_thread_id_map; + +static atomic_flag_lock _enc_lock; + +struct thread_args +{ + uint64_t enc_key; + int join_ret; + int detach_ret; +}; +// Each new thread will point to memory created by the host after thread +// creation +thread_args _thread_args[MAX_ENC_KEYS]; + +static int _pthread_create_hook( + pthread_t* enc_thread, + const pthread_attr_t*, + void* (*start_routine)(void*), + void* arg) +{ + *enc_thread = 0; + uint64_t enc_key; + const std::atomic* enc_value; + { + atomic_lock lock(_enc_lock); + _thread_functions.push_back( + [start_routine, arg]() { return start_routine(arg); }); + enc_key = _enc_key = ++_next_enc_thread_id; + printf("_pthread_create_hook(): enc_key is %lu\n", enc_key); + // Populate the enclave key to thread id map in advance + enc_value = &_key_to_thread_id_map[enc_key]; + + if (_next_enc_thread_id > (MAX_ENC_KEYS - 1)) + { + printf( + "Exceeded max number of enclave threads supported %lu\n", + MAX_ENC_KEYS - 1); + } + } + + // Send the enclave id so that host can maintain the map between + // enclave and host id + if (OE_OK != host_create_thread(enc_key, oe_get_enclave())) + { + printf( + "_pthread_create_hook(): Error in call to host_create_pthread " + "for enc_key=%lu\n", + enc_key); + oe_abort(); + } + + // Block until the enclave pthread_id becomes available in the map + while (*enc_value == 0) + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + *enc_thread = *enc_value; + + printf( + "_pthread_create_hook(): pthread_create success for enc_key=%lu; " + "thread id=%#10lx\n", + _enc_key, + *enc_thread); + + return 0; +} + +static int _pthread_join_hook(pthread_t enc_thread, void**) +{ + // Find the enc_key from the enc_thread + uint64_t join_enc_key; + { + atomic_lock lock(_enc_lock); + const auto it = std::find_if( + _key_to_thread_id_map.begin(), + _key_to_thread_id_map.end(), + [&enc_thread](const std::pair& p) { + return p.second == enc_thread; + }); + if (it == _key_to_thread_id_map.end()) + { + printf( + "_pthread_join_hook(): Error: enc_key for thread ID %#10lx not " + "found\n", + enc_thread); + oe_abort(); + } + + join_enc_key = it->first; + _thread_args[join_enc_key - 1].enc_key = join_enc_key; + } + + printf( + "_pthread_join_hook(): enc_key for thread ID %#10lx is %ld\n", + enc_thread, + join_enc_key); + + int join_ret = 0; + if (host_join_thread(&join_ret, join_enc_key) != OE_OK) + { + printf( + "_pthread_join_hook(): Error in call to host host_join_pthread for " + "enc_key=%ld\n", + join_enc_key); + oe_abort(); + } + + { + atomic_lock lock(_enc_lock); + _thread_args[join_enc_key - 1].join_ret = join_ret; + + // Since join succeeded, delete the _key_to_thread_id_map + if (!join_ret) + { + _key_to_thread_id_map.erase(join_enc_key); + } + } + + return join_ret; +} + +static int _pthread_detach_hook(pthread_t enc_thread) +{ + // Find the enc_key from the enc_thread + uint64_t det_enc_key; + { + atomic_lock lock(_enc_lock); + const auto it = std::find_if( + _key_to_thread_id_map.begin(), + _key_to_thread_id_map.end(), + [&enc_thread](const std::pair& p) { + return p.second == enc_thread; + }); + if (it == _key_to_thread_id_map.end()) + { + printf( + "_pthread_detach_hook(): Error: enc_key for thread ID %#10lx " + "not found\n", + enc_thread); + oe_abort(); + } + + det_enc_key = it->first; + _thread_args[det_enc_key - 1].enc_key = det_enc_key; + } + + printf( + "_pthread_detach_hook(): enc_key for thread ID %#10lx is %ld\n", + enc_thread, + det_enc_key); + + int det_ret = 0; + if (host_detach_thread(&det_ret, det_enc_key) != OE_OK) + { + printf( + "_pthread_detach_hook(): Error in call to host host_detach_thread " + "for enc_key=%ld\n", + det_enc_key); + oe_abort(); + } + + // Since detach succeeded, delete the _key_to_thread_id_map + if (0 == det_ret) + { + atomic_lock lock(_enc_lock); + _key_to_thread_id_map.erase(det_enc_key); + } + + return det_ret; +} + +// Launches the new thread in the enclave +void enc_enclave_thread(uint64_t enc_key) +{ + _thread_args[_enc_key - 1].enc_key = enc_key; + _thread_args[_enc_key - 1].join_ret = -1; + _thread_args[_enc_key - 1].detach_ret = -1; + + std::function f; + + { + atomic_lock lock(_enc_lock); + _key_to_thread_id_map[enc_key] = pthread_self(); + } + + std::this_thread::yield(); + + { + atomic_lock lock(_enc_lock); + f = _thread_functions.back(); + _thread_functions.pop_back(); + } + f(); +} + +static oe_pthread_hooks_t _hooks = { + .create = _pthread_create_hook, + .join = _pthread_join_hook, + .detach = _pthread_detach_hook}; + +extern "C" void register_pthread_hooks(void) +{ + oe_register_pthread_hooks(&_hooks); +} diff --git a/tests/openssl/host/host.cpp b/tests/openssl/host/host.cpp index 785073357a..d0351b96da 100644 --- a/tests/openssl/host/host.cpp +++ b/tests/openssl/host/host.cpp @@ -1,14 +1,185 @@ // Copyright (c) Open Enclave SDK contributors. // Licensed under the MIT License. +#include #include #include #include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include "openssl_u.h" +#include "threadArgs.h" + +// Host maintains a map of enclave key to host thread ID +static std::unordered_map> + _enclave_host_id_map; +// Host maintains a map of thread id to the thread object +static std::unordered_map _host_id_thread_map; + +static atomic_flag_lock _host_lock; + +void host_exit(int arg) +{ + // Ensure all the threads terminate before the exit + for (auto& pair : _host_id_thread_map) + { + pair.second.join(); + } + exit(arg); +} + +struct thread_args_t +{ + oe_enclave_t* enclave; + uint64_t enc_key; +}; + +void* host_enclave_thread(void* args) +{ + thread_args_t* thread_args = reinterpret_cast(args); + // need to cache the values for enc_key and enclave now before _host_lock + // is unlocked after assigning the thread_id to the _enclave_host_id_map + // because args is a local variable in the calling method which may exit + // at any time after _host_lock is unlocked which may cause a segfault + uint64_t enc_key = thread_args->enc_key; + oe_enclave_t* enclave = thread_args->enclave; + std::thread::id thread_id; + + { + // Using atomic_thread_host_id_map lock to ensure the mapping is updated + // before the thread_id lookup + atomic_lock lock(_host_lock); + + std::thread::id thread_id = std::this_thread::get_id(); + OE_TEST( + _host_id_thread_map.find(thread_id) != _host_id_thread_map.end()); + + // Populate the enclave_host_id map with the thread_id + _enclave_host_id_map[enc_key] = thread_id; + } + + // Launch the thread + oe_result_t result = enc_enclave_thread(enclave, enc_key); + OE_TEST(result == OE_OK); + + return NULL; +} + +void host_create_thread(uint64_t enc_key, oe_enclave_t* enclave) +{ + thread_args_t thread_args = {enclave, enc_key}; + std::thread::id thread_id; + const std::atomic* mapped_thread_id; + + { + // Using atomic locks to protect the enclave_host_id_map + // and update the _host_id_thread_map upon the thread creation + atomic_lock lock(_host_lock); + mapped_thread_id = &_enclave_host_id_map[enc_key]; + + // New Thread is created and executes host_enclave_thread + std::thread t = std::thread(host_enclave_thread, &thread_args); + + thread_id = t.get_id(); + _host_id_thread_map[thread_id] = std::move(t); + } + + // Main host thread waits for the enclave id to host id mapping to be + // updated + while (*mapped_thread_id == std::thread::id()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + // Sanity check + if (thread_id != *mapped_thread_id) + { + printf("Host thread id incorrect in the enclave_host_id_map\n"); + abort(); + } +} + +int host_join_thread(uint64_t enc_key) +{ + int ret_val = 0; + std::thread::id thread_id; + + // Find the thread_id from the enclave_host_id_map using the enc_key + { + // Using atomic locks to protect the enclave_host_id_map + atomic_lock lock(_host_lock); + const auto it = _enclave_host_id_map.find(enc_key); + if (it != _enclave_host_id_map.end()) + { + thread_id = it->second; + lock.unlock(); + + auto& t = _host_id_thread_map[thread_id]; + OE_TEST(t.joinable()); + t.join(); + + // Update the shared memory only after join + { + // Delete the enclave_host_id mapping as the thread_id may be + // reused in future + lock.lock(); + _enclave_host_id_map.erase(enc_key); + } + } + else + { + printf( + "host_join_thread() failed to find enclave id=%" PRIu64 + " in host " + "map\n", + enc_key); + abort(); + } + } + + return ret_val; +} + +int host_detach_thread(uint64_t enc_key) +{ + // Find the thread_id from the enclave_host_id_map using the enc_key + + // Using atomic locks to protect the enclave_host_id_map + atomic_lock lock(_host_lock); + const auto it = _enclave_host_id_map.find(enc_key); + if (it != _enclave_host_id_map.end()) + { + std::thread::id thread_id = it->second; + lock.unlock(); + + auto& t = _host_id_thread_map[thread_id]; + t.detach(); + + { + // Delete the _enclave_host_id mapping as the host thread_id may be + // reused in future + lock.lock(); + _enclave_host_id_map.erase(enc_key); + } + } + else + { + printf( + "host_detach_thread() failed to find enclave key=%" PRIu64 + " in host " + "map\n", + enc_key); + abort(); + } + return 0; +} + #ifdef __linux__ extern char** environ; char** _environ = environ; // _environ is defined by stdlib.h on Windows. diff --git a/tests/openssl/host/threadArgs.h b/tests/openssl/host/threadArgs.h new file mode 100644 index 0000000000..7937196855 --- /dev/null +++ b/tests/openssl/host/threadArgs.h @@ -0,0 +1,33 @@ +// Copyright (c) Open Enclave SDK contributors. +// Licensed under the MIT License. + +#ifndef _THREAD_ARGS_H +#define _THREAD_ARGS_H + +#include +#include + +const uint64_t MAX_ENC_KEYS = 16; + +class atomic_flag_lock +{ + public: + void lock() + { + while (_flag.test_and_set()) + { + continue; + } + } + void unlock() + { + _flag.clear(); + } + + private: + std::atomic_flag _flag = ATOMIC_FLAG_INIT; +}; + +typedef std::unique_lock atomic_lock; + +#endif /* _THREAD_ARGS_H */ diff --git a/tests/openssl/openssl.edl b/tests/openssl/openssl.edl index 229b17832d..a6272b434b 100644 --- a/tests/openssl/openssl.edl +++ b/tests/openssl/openssl.edl @@ -16,6 +16,23 @@ enclave { int argc, [user_check] char** argv, [user_check] char** env); + + public void enc_enclave_thread ( + uint64_t enc_key); }; + untrusted { + void host_exit( + int arg); + + void host_create_thread( + uint64_t enc_key, + [user_check] oe_enclave_t* enc); + + int host_join_thread( + uint64_t enc_key); + + int host_detach_thread( + uint64_t enc_key); + }; };