Fixed detour_alloc_trampoline_allocate_new

detour_alloc_trampoline_allocate_new(pbTarget, pLo, pHi) was designed with the assumption that pbTarget is going to be between pLo and pHi. In our case, 32-bit cscript!mainCRTStartup was loaded at 0x34020, i.e. pbTarget was below pLo (which was fixed to 0x80000 in detour_2gb_below), and therefore:
- detour_alloc_region_from_hi(pLo, pbTarget) did not do anything
- detour_alloc_region_from_lo(pbTarget, pHi) allocated a 64 KiB block below pLo (in our case happened to be 0x70000) which was later discarded and NULL returned in detour_alloc_trampoline
The fix clamps pbTarget into [pLo, pHi] range.

src/detours.cpp

@@ -1410,6 +1410,21 @@ static PVOID detour_alloc_region_from_hi(PBYTE pbLo, PBYTE pbHi)
     return NULL;
 }
 
+// Return pbTarget clamped into [pLo, pHi] range.
+
+static PBYTE detour_clamp(PBYTE pbTarget,
+                          PDETOUR_TRAMPOLINE pLo,
+                          PDETOUR_TRAMPOLINE pHi)
+{
+    if (pbTarget < (PBYTE)pLo) {
+        return (PBYTE)pLo;
+    }
+    if (pbTarget > (PBYTE)pHi) {
+        return (PBYTE)pHi;
+    }
+    return pbTarget;
+}
+
 static PVOID detour_alloc_trampoline_allocate_new(PBYTE pbTarget,
                                                   PDETOUR_TRAMPOLINE pLo,
                                                   PDETOUR_TRAMPOLINE pHi)
@@ -1422,29 +1437,29 @@ static PVOID detour_alloc_trampoline_allocate_new(PBYTE pbTarget,
 #if defined(DETOURS_64BIT)
     // Try looking 1GB below or lower.
     if (pbTry == NULL && pbTarget > (PBYTE)0x40000000) {
-        pbTry = detour_alloc_region_from_hi((PBYTE)pLo, pbTarget - 0x40000000);
+        pbTry = detour_alloc_region_from_hi((PBYTE)pLo, detour_clamp(pbTarget - 0x40000000, pLo, pHi));
     }
     // Try looking 1GB above or higher.
     if (pbTry == NULL && pbTarget < (PBYTE)0xffffffff40000000) {
-        pbTry = detour_alloc_region_from_lo(pbTarget + 0x40000000, (PBYTE)pHi);
+        pbTry = detour_alloc_region_from_lo(detour_clamp(pbTarget + 0x40000000, pLo, pHi), (PBYTE)pHi);
     }
     // Try looking 1GB below or higher.
     if (pbTry == NULL && pbTarget > (PBYTE)0x40000000) {
-        pbTry = detour_alloc_region_from_lo(pbTarget - 0x40000000, pbTarget);
+        pbTry = detour_alloc_region_from_lo(detour_clamp(pbTarget - 0x40000000, pLo, pHi), detour_clamp(pbTarget, pLo, pHi));
     }
     // Try looking 1GB above or lower.
     if (pbTry == NULL && pbTarget < (PBYTE)0xffffffff40000000) {
-        pbTry = detour_alloc_region_from_hi(pbTarget, pbTarget + 0x40000000);
+        pbTry = detour_alloc_region_from_hi(detour_clamp(pbTarget, pLo, pHi), detour_clamp(pbTarget + 0x40000000, pLo, pHi));
     }
 #endif
 
     // Try anything below.
     if (pbTry == NULL) {
-        pbTry = detour_alloc_region_from_hi((PBYTE)pLo, pbTarget);
+        pbTry = detour_alloc_region_from_hi((PBYTE)pLo, detour_clamp(pbTarget, pLo, pHi));
     }
     // try anything above.
     if (pbTry == NULL) {
-        pbTry = detour_alloc_region_from_lo(pbTarget, (PBYTE)pHi);
+        pbTry = detour_alloc_region_from_lo(detour_clamp(pbTarget, pLo, pHi), (PBYTE)pHi);
     }
 
     return pbTry;