enable fsgsbase instructions on Linux 5.9+ for EOS VM OC

.github/workflows/build.yaml

@@ -168,7 +168,9 @@ jobs:
           path: /cores
           compression-level: 0
       - name: Check CPU Features
-        run: awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo
+        run: |
+          awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo
+          build/tools/fsgsbase-enabled
 
   np-tests:
     name: NP Tests (${{matrix.cfg.name}})

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h

@@ -26,6 +26,8 @@ int32_t eos_vm_oc_grow_memory(int32_t grow, int32_t max);
 sigjmp_buf* eos_vm_oc_get_jmp_buf();
 void* eos_vm_oc_get_exception_ptr();
 void* eos_vm_oc_get_bounce_buffer_list();
+uint64_t eos_vm_oc_getgs();
+void eos_vm_oc_setgs(uint64_t gs);
 
 #ifdef __cplusplus
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/executor.cpp

@@ -39,9 +39,8 @@ static void segv_handler(int sig, siginfo_t* info, void* ctx)  {
    control_block* cb_in_main_segment;
 
    //a 0 GS value is an indicator an executor hasn't been active on this thread recently
-   uint64_t current_gs;
-   syscall(SYS_arch_prctl, ARCH_GET_GS, &current_gs);
-   if(current_gs == 0)
+   uint64_t current_gs = eos_vm_oc_getgs();
+   if(eos_vm_oc_getgs() == 0)
       goto notus;
 
    cb_in_main_segment = reinterpret_cast<control_block*>(current_gs - memory::cb_offset);
@@ -170,11 +169,11 @@ void executor::execute(const code_descriptor& code, memory& mem, apply_context&
          mprotect(mem.full_page_memory_base() + initial_page_offset * eosio::chain::wasm_constraints::wasm_page_size,
                   (code.starting_memory_pages - initial_page_offset) * eosio::chain::wasm_constraints::wasm_page_size, PROT_READ | PROT_WRITE);
       }
-      arch_prctl(ARCH_SET_GS, (unsigned long*)(mem.zero_page_memory_base()+initial_page_offset*memory::stride));
+      eos_vm_oc_setgs((uint64_t)mem.zero_page_memory_base()+initial_page_offset*memory::stride);
       memset(mem.full_page_memory_base(), 0, 64u*1024u*code.starting_memory_pages);
    }
    else
-      arch_prctl(ARCH_SET_GS, (unsigned long*)mem.zero_page_memory_base());
+      eos_vm_oc_setgs((uint64_t)mem.zero_page_memory_base());
 
    void* globals;
    if(code.initdata_prologue_size > memory::max_prologue_size) {
@@ -261,7 +260,7 @@ void executor::execute(const code_descriptor& code, memory& mem, apply_context&
 }
 
 executor::~executor() {
-   arch_prctl(ARCH_SET_GS, nullptr);
+   eos_vm_oc_setgs(0);
    munmap(code_mapping, code_mapping_size);
 }
 

libraries/chain/webassembly/runtimes/eos-vm-oc/gs_seg_helpers.c

@@ -3,9 +3,16 @@
 #include <asm/prctl.h>
 #include <sys/prctl.h>
 #include <sys/mman.h>
+#include <sys/auxv.h>
+#include <elf.h>
+#include <immintrin.h>
 
 int arch_prctl(int code, unsigned long* addr);
 
+#ifndef HWCAP2_FSGSBASE
+#define HWCAP2_FSGSBASE (1 << 1)
+#endif
+
 #define EOSVMOC_MEMORY_PTR_cb_ptr GS_PTR struct eos_vm_oc_control_block* const cb_ptr = ((GS_PTR struct eos_vm_oc_control_block* const)(EOS_VM_OC_CONTROL_BLOCK_OFFSET));
 
 int32_t eos_vm_oc_grow_memory(int32_t grow, int32_t max) {
@@ -37,10 +44,9 @@ int32_t eos_vm_oc_grow_memory(int32_t grow, int32_t max) {
       gs_diff = grow_amount;
    }
 
-   uint64_t current_gs;
-   arch_prctl(ARCH_GET_GS, &current_gs);
+   uint64_t current_gs = eos_vm_oc_getgs();
    current_gs += gs_diff * EOS_VM_OC_MEMORY_STRIDE;
-   arch_prctl(ARCH_SET_GS, (unsigned long*)current_gs);
+   eos_vm_oc_setgs(current_gs);
    cb_ptr->current_linear_memory_pages += grow_amount;
    cb_ptr->first_invalid_memory_address += grow_amount*64*1024;
 
@@ -64,3 +70,56 @@ void* eos_vm_oc_get_bounce_buffer_list() {
    EOSVMOC_MEMORY_PTR_cb_ptr;
    return cb_ptr->bounce_buffers;
 }
+
+uint64_t eos_vm_oc_getgs_syscall() {
+   uint64_t gs;
+   arch_prctl(ARCH_GET_GS, &gs);
+   return gs;
+}
+
+uint64_t __attribute__ ((__target__ ("fsgsbase"))) eos_vm_oc_getgs_fsgsbase() {
+   return _readgsbase_u64();
+}
+
+void eos_vm_oc_setgs_syscall(uint64_t gs) {
+   arch_prctl(ARCH_SET_GS, (unsigned long*)gs); //cast to a (unsigned long*) to match local declaration above
+}
+
+void __attribute__ ((__target__ ("fsgsbase"))) eos_vm_oc_setgs_fsgsbase(uint64_t gs) {
+   return _writegsbase_u64(gs);
+}
+
+extern char** _dl_argv;
+static int eos_vm_oc_use_fsgsbase() {
+   /* ifunc resolvers run _super_ early -- before getenv() is set up even! This is relying on the layout of _dl_argv to be
+          _dl_argv
+              ↓
+      argc, argv[0], ..., argv[argc - 1], NULL, evniron0, environ1, ..., NULL
+   */
+   const int argc = *(int*)(_dl_argv - 1);
+   char** my_environ = _dl_argv + argc + 1;
+   while(*my_environ != NULL) {
+      const char disable_str[] = "SPRING_DISABLE_FSGSBASE";
+      if(strncmp(*my_environ++, disable_str, strlen(disable_str)) == 0)
+         return 0;
+   }
+
+   //see linux Documentation/arch/x86/x86_64/fsgs.rst; check that kernel has enabled userspace fsgsbase
+   return getauxval(AT_HWCAP2) & HWCAP2_FSGSBASE;
+}
+
+uint64_t (*resolve_eos_vm_oc_getgs())() {
+   if(eos_vm_oc_use_fsgsbase())
+      return eos_vm_oc_getgs_fsgsbase;
+   return eos_vm_oc_getgs_syscall;
+}
+
+uint64_t eos_vm_oc_getgs() __attribute__ ((ifunc ("resolve_eos_vm_oc_getgs")));
+
+void (*resolve_eos_vm_oc_setgs())(uint64_t) {
+   if(eos_vm_oc_use_fsgsbase())
+      return eos_vm_oc_setgs_fsgsbase;
+   return eos_vm_oc_setgs_syscall;
+}
+
+void eos_vm_oc_setgs(uint64_t) __attribute__ ((ifunc ("resolve_eos_vm_oc_setgs")));

tools/CMakeLists.txt

@@ -2,3 +2,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/llvm-gcov.sh ${CMAKE_CURRENT_BINARY_D
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ctestwrapper.sh ${CMAKE_CURRENT_BINARY_DIR}/ctestwrapper.sh COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/validate_reflection.py ${CMAKE_CURRENT_BINARY_DIR}/validate_reflection.py COPYONLY)
 configure_file(net-util.py net-util.py COPYONLY)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+   add_executable(fsgsbase-enabled fsgsbase-enabled.c)
+endif()

tools/fsgsbase-enabled.c

@@ -0,0 +1,10 @@
+#include <sys/auxv.h>
+#include <elf.h>
+
+#ifndef HWCAP2_FSGSBASE
+#define HWCAP2_FSGSBASE        (1 << 1)
+#endif
+
+int main() {
+   return !(getauxval(AT_HWCAP2) & HWCAP2_FSGSBASE);
+}

unittests/CMakeLists.txt

@@ -97,6 +97,11 @@ foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test s
       # to run unit_test with all log from blockchain displayed, put "--verbose" after "--", i.e. "unit_test -- --verbose"
       foreach(RUNTIME ${EOSIO_WASM_RUNTIMES})
         add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_${RUNTIME} COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed --color_output -- --${RUNTIME})
+        # add a duplicate test to run without fsgsbase instructions for a limited number of OC wasm tests
+        if(RUNTIME STREQUAL "eos-vm-oc" AND TRIMMED_SUITE_NAME MATCHES "^wasm_part")
+           add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_${RUNTIME}-nofsgs COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed --color_output -- --${RUNTIME})
+           set_tests_properties(${TRIMMED_SUITE_NAME}_unit_test_${RUNTIME}-nofsgs PROPERTIES ENVIRONMENT "SPRING_DISABLE_FSGSBASE=1" COST 5000)
+        endif()
         # build list of tests to run during coverage testing
         if(ctest_tests)
           string(APPEND ctest_tests "|")