Os: Add OS abstraction layer

CMakeLists.txt

@@ -93,7 +93,7 @@ set(safetyhook_SOURCES
 	"src/easy.cpp"
 	"src/inline_hook.cpp"
 	"src/mid_hook.cpp"
-	"src/thread_freezer.cpp"
+	"src/os.windows.cpp"
 	"src/utility.cpp"
 	"src/vmt_hook.cpp"
 	cmake.toml
@@ -143,7 +143,6 @@ target_include_directories(safetyhook PUBLIC
 
 target_link_libraries(safetyhook PUBLIC
 	Zydis
-	ntdll
 )
 
 # Target: docs

cmake.toml

@@ -54,7 +54,7 @@ sources = ["src/*.cpp"]
 include-directories = ["include/"]
 compile-features = ["cxx_std_23"]
 alias = "safetyhook::safetyhook"
-link-libraries = ["Zydis", "ntdll"]
+link-libraries = ["Zydis"]
 msvc.private-compile-options = ["/permissive-", "/W4", "/w14640"]
 clang.private-compile-options = ["-Wall", "-Wextra", "-Wshadow", "-Wnon-virtual-dtor", "-pedantic"]
 gcc.private-compile-options = ["-Wall", "-Wextra", "-Wshadow", "-Wnon-virtual-dtor", "-pedantic"]

example/minimal.cpp

@@ -2,7 +2,7 @@
 
 #include <safetyhook.hpp>
 
-__declspec(noinline) int add(int x, int y) {
+SAFETYHOOK_NOINLINE int add(int x, int y) {
     return x + y;
 }
 

include/safetyhook.hpp

@@ -3,7 +3,6 @@
 #include "safetyhook/easy.hpp"
 #include "safetyhook/inline_hook.hpp"
 #include "safetyhook/mid_hook.hpp"
-#include "safetyhook/thread_freezer.hpp"
 #include "safetyhook/vmt_hook.hpp"
 
 using SafetyHookContext = safetyhook::Context;

include/safetyhook/common.hpp

@@ -37,3 +37,38 @@
 #error "Unsupported architecture"
 #endif
 #endif
+
+#if defined(WIN32)
+#define SAFETYHOOK_OS_WINDOWS 1
+#define SAFETYHOOK_OS_LINUX 0
+#elif defined(__linux__)
+#define SAFETYHOOK_OS_WINDOWS 0
+#define SAFETYHOOK_OS_LINUX 1
+#else
+#error "Unsupported OS"
+#endif
+
+#if SAFETYHOOK_OS_WINDOWS
+#if SAFETYHOOK_COMPILER_MSVC
+#define SAFETYHOOK_CCALL __cdecl
+#define SAFETYHOOK_STDCALL __stdcall
+#define SAFETYHOOK_FASTCALL __fastcall
+#define SAFETYHOOK_THISCALL __thiscall
+#elif SAFETYHOOK_COMPILER_GCC || SAFETYHOOK_COMPILER_CLANG
+#define SAFETYHOOK_CCALL __attribute__((cdecl))
+#define SAFETYHOOK_STDCALL __attribute__((stdcall))
+#define SAFETYHOOK_FASTCALL __attribute__((fastcall))
+#define SAFETYHOOK_THISCALL __attribute__((thiscall))
+#endif
+#else
+#define SAFETYHOOK_CCALL
+#define SAFETYHOOK_STDCALL
+#define SAFETYHOOK_FASTCALL
+#define SAFETYHOOK_THISCALL
+#endif
+
+#if SAFETYHOOK_COMPILER_MSVC
+#define SAFETYHOOK_NOINLINE __declspec(noinline)
+#elif SAFETYHOOK_COMPILER_GCC || SAFETYHOOK_COMPILER_CLANG
+#define SAFETYHOOK_NOINLINE __attribute__((noinline))
+#endif

include/safetyhook/inline_hook.hpp

@@ -185,7 +185,7 @@ class InlineHook final {
     /// @note This function will use the __cdecl calling convention.
     template <typename RetT = void, typename... Args> RetT ccall(Args... args) {
         std::scoped_lock lock{m_mutex};
-        return m_trampoline ? original<RetT(__cdecl*)(Args...)>()(args...) : RetT();
+        return m_trampoline ? original<RetT(SAFETYHOOK_CCALL*)(Args...)>()(args...) : RetT();
     }
 
     /// @brief Calls the original function.
@@ -196,7 +196,7 @@ class InlineHook final {
     /// @note This function will use the __thiscall calling convention.
     template <typename RetT = void, typename... Args> RetT thiscall(Args... args) {
         std::scoped_lock lock{m_mutex};
-        return m_trampoline ? original<RetT(__thiscall*)(Args...)>()(args...) : RetT();
+        return m_trampoline ? original<RetT(SAFETYHOOK_THISCALL*)(Args...)>()(args...) : RetT();
     }
 
     /// @brief Calls the original function.
@@ -207,7 +207,7 @@ class InlineHook final {
     /// @note This function will use the __stdcall calling convention.
     template <typename RetT = void, typename... Args> RetT stdcall(Args... args) {
         std::scoped_lock lock{m_mutex};
-        return m_trampoline ? original<RetT(__stdcall*)(Args...)>()(args...) : RetT();
+        return m_trampoline ? original<RetT(SAFETYHOOK_STDCALL*)(Args...)>()(args...) : RetT();
     }
 
     /// @brief Calls the original function.
@@ -218,7 +218,7 @@ class InlineHook final {
     /// @note This function will use the __fastcall calling convention.
     template <typename RetT = void, typename... Args> RetT fastcall(Args... args) {
         std::scoped_lock lock{m_mutex};
-        return m_trampoline ? original<RetT(__fastcall*)(Args...)>()(args...) : RetT();
+        return m_trampoline ? original<RetT(SAFETYHOOK_FASTCALL*)(Args...)>()(args...) : RetT();
     }
 
     /// @brief Calls the original function.
@@ -242,7 +242,7 @@ class InlineHook final {
     /// @note This function is unsafe because it doesn't lock the mutex. Only use this if you don't care about unhook
     /// safety or are worried about the performance cost of locking the mutex.
     template <typename RetT = void, typename... Args> RetT unsafe_ccall(Args... args) {
-        return original<RetT(__cdecl*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_CCALL*)(Args...)>()(args...);
     }
 
     /// @brief Calls the original function.
@@ -254,7 +254,7 @@ class InlineHook final {
     /// @note This function is unsafe because it doesn't lock the mutex. Only use this if you don't care about unhook
     /// safety or are worried about the performance cost of locking the mutex.
     template <typename RetT = void, typename... Args> RetT unsafe_thiscall(Args... args) {
-        return original<RetT(__thiscall*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_THISCALL*)(Args...)>()(args...);
     }
 
     /// @brief Calls the original function.
@@ -266,7 +266,7 @@ class InlineHook final {
     /// @note This function is unsafe because it doesn't lock the mutex. Only use this if you don't care about unhook
     /// safety or are worried about the performance cost of locking the mutex.
     template <typename RetT = void, typename... Args> RetT unsafe_stdcall(Args... args) {
-        return original<RetT(__stdcall*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_STDCALL*)(Args...)>()(args...);
     }
 
     /// @brief Calls the original function.
@@ -278,7 +278,7 @@ class InlineHook final {
     /// @note This function is unsafe because it doesn't lock the mutex. Only use this if you don't care about unhook
     /// safety or are worried about the performance cost of locking the mutex.
     template <typename RetT = void, typename... Args> RetT unsafe_fastcall(Args... args) {
-        return original<RetT(__fastcall*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_FASTCALL*)(Args...)>()(args...);
     }
 
 private:

include/safetyhook/os.hpp

@@ -0,0 +1,82 @@
+// This is the OS abstraction layer.
+#pragma once
+
+#include <cstdint>
+#include <expected>
+#include <functional>
+
+namespace safetyhook {
+
+enum class OsError {
+    FAILED_TO_ALLOCATE,
+    FAILED_TO_PROTECT,
+    FAILED_TO_QUERY,
+    FAILED_TO_GET_NEXT_THREAD,
+    FAILED_TO_GET_THREAD_CONTEXT,
+    FAILED_TO_SET_THREAD_CONTEXT,
+    FAILED_TO_FREEZE_THREAD,
+    FAILED_TO_UNFREEZE_THREAD,
+    FAILED_TO_GET_THREAD_ID,
+};
+
+struct VmAccess {
+    bool read : 1;
+    bool write : 1;
+    bool execute : 1;
+
+    constexpr bool operator==(const VmAccess& other) const {
+        return read == other.read && write == other.write && execute == other.execute;
+    }
+};
+
+constexpr VmAccess VM_ACCESS_R{.read = true, .write = false, .execute = false};
+constexpr VmAccess VM_ACCESS_RW{.read = true, .write = true, .execute = false};
+constexpr VmAccess VM_ACCESS_RX{.read = true, .write = false, .execute = true};
+constexpr VmAccess VM_ACCESS_RWX{.read = true, .write = true, .execute = true};
+
+struct VmBasicInfo {
+    uint8_t* address;
+    size_t size;
+    VmAccess access;
+    bool is_free;
+};
+
+std::expected<uint8_t*, OsError> vm_allocate(uint8_t* address, size_t size, VmAccess access);
+void vm_free(uint8_t* address);
+std::expected<uint32_t, OsError> vm_protect(uint8_t* address, size_t size, VmAccess access);
+std::expected<uint32_t, OsError> vm_protect(uint8_t* address, size_t size, uint32_t access);
+std::expected<VmBasicInfo, OsError> vm_query(uint8_t* address);
+bool vm_is_readable(uint8_t* address, size_t size);
+bool vm_is_writable(uint8_t* address, size_t size);
+bool vm_is_executable(uint8_t* address);
+
+struct SystemInfo {
+    uint32_t page_size;
+    uint32_t allocation_granularity;
+    uint8_t* min_address;
+    uint8_t* max_address;
+};
+
+SystemInfo system_info();
+
+using ThreadId = uint32_t;
+using ThreadHandle = void*;
+using ThreadContext = void*;
+
+/// @brief Executes a function while all other threads are frozen. Also allows for visiting each frozen thread and
+/// modifying it's context.
+/// @param run_fn The function to run while all other threads are frozen.
+/// @param visit_fn The function that will be called for each frozen thread.
+/// @note The visit function will be called in the order that the threads were frozen.
+/// @note The visit function will be called before the run function.
+/// @note Keep the logic inside run_fn and visit_fn as simple as possible to avoid deadlocks.
+void execute_while_frozen(const std::function<void()>& run_fn,
+    const std::function<void(ThreadId, ThreadHandle, ThreadContext)>& visit_fn = {});
+
+/// @brief Will modify the context of a thread's IP to point to a new address if its IP is at the old address.
+/// @param ctx The thread context to modify.
+/// @param old_ip The old IP address.
+/// @param new_ip The new IP address.
+void fix_ip(ThreadContext ctx, uint8_t* old_ip, uint8_t* new_ip);
+
+} // namespace safetyhook

include/safetyhook/vmt_hook.hpp

@@ -8,6 +8,7 @@
 #include <unordered_map>
 
 #include "safetyhook/allocator.hpp"
+#include "safetyhook/common.hpp"
 #include "safetyhook/utility.hpp"
 
 namespace safetyhook {
@@ -43,7 +44,7 @@ class VmHook final {
     /// @param args The arguments to pass to the method.
     /// @return The return value of the method.
     template <typename RetT = void, typename... Args> RetT ccall(Args... args) {
-        return original<RetT(__cdecl*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_CCALL*)(Args...)>()(args...);
     }
 
     /// @brief Calls the original method with the __thiscall calling convention.
@@ -52,7 +53,7 @@ class VmHook final {
     /// @param args The arguments to pass to the method.
     /// @return The return value of the method.
     template <typename RetT = void, typename... Args> RetT thiscall(Args... args) {
-        return original<RetT(__thiscall*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_THISCALL*)(Args...)>()(args...);
     }
 
     /// @brief Calls the original method with the __stdcall calling convention.
@@ -61,7 +62,7 @@ class VmHook final {
     /// @param args The arguments to pass to the method.
     /// @return The return value of the method.
     template <typename RetT = void, typename... Args> RetT stdcall(Args... args) {
-        return original<RetT(__stdcall*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_STDCALL*)(Args...)>()(args...);
     }
 
     /// @brief Calls the original method with the __fastcall calling convention.
@@ -70,7 +71,7 @@ class VmHook final {
     /// @param args The arguments to pass to the method.
     /// @return The return value of the method.
     template <typename RetT = void, typename... Args> RetT fastcall(Args... args) {
-        return original<RetT(__fastcall*)(Args...)>()(args...);
+        return original<RetT(SAFETYHOOK_FASTCALL*)(Args...)>()(args...);
     }
 
 private: