Dhooks safetyhook (#2205)

* remove libudis86

* switch to safetyhook

* Don't leak memory

* fix issue with the trampoline

* Add x86_64 detour support (windows)

* Allow to differentiate platforms

---------

Co-authored-by: Kenzzer <kenzzer@users.noreply.github.com>

extensions/dhooks/AMBuilder

@@ -42,27 +42,22 @@ for cxx in builder.targets:
     'util.cpp',
     'dynhooks_sourcepawn.cpp',
     '../../public/smsdk_ext.cpp',
-    'asm/asm.c',
-    'libudis86/decode.c',
-    'libudis86/itab.c',
-    'libudis86/syn-att.c',
-    'libudis86/syn-intel.c',
-    'libudis86/syn.c',
-    'libudis86/udis86.c',
     # Dynamic Hooks
-    os.path.join('DynamicHooks', 'registers.cpp')
+    os.path.join('DynamicHooks', 'registers.cpp'),
+    os.path.join('DynamicHooks', 'hook.cpp'),
+    os.path.join('DynamicHooks', 'manager.cpp')
   ]
+  SM.AddCDetour(binary)
 
   if binary.compiler.target.arch == 'x86':
     binary.sources += ['../../sourcepawn/vm/x86/assembler-x86.cpp']
+
     binary.compiler.cxxincludes += [
       os.path.join(builder.sourcePath, 'public', 'jit', 'x86'),
       os.path.join(builder.sourcePath, 'sourcepawn', 'vm', 'x86')
     ]
     # DynamicHooks
     binary.sources += [
-      os.path.join('DynamicHooks', 'hook.cpp'),
-      os.path.join('DynamicHooks', 'manager.cpp'),
       os.path.join('DynamicHooks', 'conventions', 'x86MsCdecl.cpp'),
       os.path.join('DynamicHooks', 'conventions', 'x86MsStdcall.cpp'),
       os.path.join('DynamicHooks', 'conventions', 'x86MsFastcall.cpp'),
@@ -77,6 +72,16 @@ for cxx in builder.targets:
     binary.compiler.defines += ['DHOOKS_DYNAMIC_DETOUR']
 
   elif binary.compiler.target.arch == 'x86_64':
-    binary.compiler.defines += ['PLATFORM_X64']
+    binary.compiler.defines += ['DYNAMICHOOKS_x86_64', 'DHOOKS_DYNAMIC_DETOUR']
+
+    if binary.compiler.target.platform == 'windows':
+      binary.sources += [
+        os.path.join('DynamicHooks', 'conventions', 'x86_64MicrosoftDefault.cpp')
+      ]
+    # Linux follows System V definition
+    elif binary.compiler.target.platform == 'linux':
+      binary.sources += [
+        os.path.join('DynamicHooks', 'conventions', 'x86_64SystemVDefault.cpp')
+      ]
 
   SM.extensions += [builder.Add(binary)]

extensions/dhooks/DynamicHooks/conventions/x86_64MicrosoftDefault.cpp

@@ -0,0 +1,281 @@
+/**
+* =============================================================================
+* DynamicHooks-x86_64
+* Copyright (C) 2024 Benoist "Kenzzer" André. All rights reserved.
+* Copyright (C) 2024 AlliedModders LLC.  All rights reserved.
+* =============================================================================
+*
+* This software is provided 'as-is', without any express or implied warranty.
+* In no event will the authors be held liable for any damages arising from 
+* the use of this software.
+* 
+* Permission is granted to anyone to use this software for any purpose, 
+* including commercial applications, and to alter it and redistribute it 
+* freely, subject to the following restrictions:
+*
+* 1. The origin of this software must not be misrepresented; you must not 
+* claim that you wrote the original software. If you use this software in a 
+* product, an acknowledgment in the product documentation would be 
+* appreciated but is not required.
+*
+* 2. Altered source versions must be plainly marked as such, and must not be
+* misrepresented as being the original software.
+*
+* 3. This notice may not be removed or altered from any source distribution.
+*/
+
+// ============================================================================
+// >> INCLUDES
+// ============================================================================
+#include "x86_64MicrosoftDefault.h"
+#include <smsdk_ext.h>
+
+// ============================================================================
+// >> CLASSES
+// ============================================================================
+x86_64MicrosoftDefault::x86_64MicrosoftDefault(std::vector<DataTypeSized_t> &vecArgTypes, DataTypeSized_t returnType, int iAlignment) :
+	ICallingConvention(vecArgTypes, returnType, iAlignment),
+	m_stackArgs(0)
+{
+	const Register_t params_reg[] = { RCX, RDX, R8, R9 };
+	const Register_t params_floatreg[] = { XMM0, XMM1, XMM2, XMM3 };
+	//const char* regNames[] = { "RCX or XMM0", "RDX or XMM1", "R8 or XMM2", "R9 or XMM3"};
+	const std::uint8_t num_reg = sizeof(params_reg) / sizeof(Register_t);
+
+	bool used_reg[] = { false, false, false, false };
+
+	// Figure out if any register has been used
+	auto retreg = m_returnType.custom_register;
+	used_reg[0] = (retreg == RCX || retreg == XMM0);
+	used_reg[1] = (retreg == RDX || retreg == XMM1);
+	used_reg[2] = (retreg == R8 || retreg == XMM2);
+	used_reg[3] = (retreg == R9 || retreg == XMM3);
+
+	for (const auto& arg : m_vecArgTypes) {
+		int reg_index = -1;
+		if (arg.custom_register == RCX || arg.custom_register == XMM0) {
+			reg_index = 0;
+		} else if (arg.custom_register == RDX || arg.custom_register == XMM1) {
+			reg_index = 1;
+		} else if (arg.custom_register == R8 || arg.custom_register == XMM2) {
+			reg_index = 2;
+		} else if (arg.custom_register == R9 || arg.custom_register == XMM3) {
+			reg_index = 3;
+		}
+
+		if (reg_index != -1) {
+			if (used_reg[reg_index]) {
+				puts("Argument register is used twice, or shared with return");
+				return;
+			}
+			used_reg[reg_index] = true;
+		}
+	}
+
+	// Special return type
+	if (m_returnType.custom_register == None && m_returnType.type == DATA_TYPE_OBJECT &&
+	// If size unknown, or doesn't fit on 1, 2, 4 or 8 bytes
+	// special place must have been allocated for it
+	(m_returnType.size == 0
+	|| m_returnType.size == 3
+	|| m_returnType.size == 5
+	|| m_returnType.size == 6
+	|| m_returnType.size == 7
+	|| m_returnType.size > 8)) {
+		for (std::uint8_t i = 0; i < num_reg && m_returnType.custom_register == None; i++) {
+			if (!used_reg[i]) {
+				m_returnType.custom_register = params_reg[i];
+				used_reg[i] = true;
+			}
+			// Couldn't find a free register, this is a big problem
+			if (m_returnType.custom_register == None) {
+				puts("Missing free register for return pointer");
+				return;
+			}
+		}
+	}
+
+	for (auto& arg : m_vecArgTypes) {
+		if (arg.custom_register == None) {
+			for (std::uint8_t i = 0; i < num_reg && arg.custom_register == None; i++) {
+				// Register is unused assign it
+				if (!used_reg[i]) {
+					arg.custom_register = (arg.type == DATA_TYPE_FLOAT || arg.type == DATA_TYPE_DOUBLE) ? params_floatreg[i] : params_reg[i];
+					used_reg[i] = true;
+				}
+			}
+			// Couldn't find a free register, it's therefore a stack parameter
+			if (arg.custom_register == None) {
+				m_stackArgs++;
+			}
+		}
+	}
+}
+
+std::vector<Register_t> x86_64MicrosoftDefault::GetRegisters()
+{
+	std::vector<Register_t> registers;
+
+	registers.push_back(RSP);
+
+	if (m_returnType.custom_register != None)
+	{
+		registers.push_back(m_returnType.custom_register);
+	}
+	else if (m_returnType.type == DATA_TYPE_FLOAT || m_returnType.type == DATA_TYPE_DOUBLE)
+	{
+		registers.push_back(XMM0);
+	}
+	else
+	{
+		registers.push_back(RAX);
+	}
+
+	for (size_t i = 0; i < m_vecArgTypes.size(); i++)
+	{
+		auto reg = m_vecArgTypes[i].custom_register;
+		if (reg == None)
+		{
+			continue;
+		}
+		registers.push_back(m_vecArgTypes[i].custom_register);
+	}
+
+	return registers;
+}
+
+int x86_64MicrosoftDefault::GetPopSize()
+{
+	// Clean-up is caller handled
+	return 0;
+}
+
+int x86_64MicrosoftDefault::GetArgStackSize()
+{
+	// Shadow space (32 bytes) + 8 bytes * amount of stack arguments
+	return 32 + 8 * m_stackArgs;
+}
+
+void** x86_64MicrosoftDefault::GetStackArgumentPtr(CRegisters* registers)
+{
+	// Skip shadow space + return address
+	return (void **)(registers->m_rsp->GetValue<uintptr_t>() + 8 + 32);
+}
+
+int x86_64MicrosoftDefault::GetArgRegisterSize()
+{
+	int argRegisterSize = 0;
+
+	for (size_t i = 0; i < m_vecArgTypes.size(); i++)
+	{
+		if (m_vecArgTypes[i].custom_register != None)
+		{
+			// It doesn't matter, it's always 8 bytes or less
+			argRegisterSize += 8;
+		}
+	}
+
+	return argRegisterSize;
+}
+
+void* x86_64MicrosoftDefault::GetArgumentPtr(unsigned int index, CRegisters* registers)
+{
+	//g_pSM->LogMessage(myself, "Retrieving argument %d (max args %d) registers %p", index, m_vecArgTypes.size(), registers);
+	if (index >= m_vecArgTypes.size())
+	{
+		//g_pSM->LogMessage(myself, "Not enough arguments");
+		return nullptr;
+	}
+
+	// Check if this argument was passed in a register.
+	if (m_vecArgTypes[index].custom_register != None)
+	{
+		CRegister* reg = registers->GetRegister(m_vecArgTypes[index].custom_register);
+		if (!reg)
+		{
+			//g_pSM->LogMessage(myself, "Register does not exit");
+			return nullptr;
+		}
+		//g_pSM->LogMessage(myself, "Register arg %d", m_vecArgTypes[index].custom_register);
+		return reg->m_pAddress;
+	}
+
+	// Return address + shadow space
+	size_t offset = 8 + 32;
+	for (unsigned int i = 0; i < index; i++)
+	{
+		if (m_vecArgTypes[i].custom_register == None)
+		{
+			// No matter what, the stack is allocated in slices of 8 bytes
+			offset += 8;
+		}
+	}
+	return (void *) (registers->m_rsp->GetValue<uintptr_t>() + offset);
+}
+
+void x86_64MicrosoftDefault::ArgumentPtrChanged(unsigned int index, CRegisters* registers, void* argumentPtr)
+{
+}
+
+void* x86_64MicrosoftDefault::GetReturnPtr(CRegisters* registers)
+{
+	// Custom return value register
+	if (m_returnType.custom_register != None)
+	{
+		return registers->GetRegister(m_returnType.custom_register)->m_pAddress;
+	}
+
+	if (m_returnType.type == DATA_TYPE_FLOAT || m_returnType.type == DATA_TYPE_DOUBLE)
+	{
+		// Floating point register
+		return registers->m_xmm0->m_pAddress;
+	}
+	return registers->m_rax->m_pAddress;
+}
+
+void x86_64MicrosoftDefault::ReturnPtrChanged(CRegisters* pRegisters, void* pReturnPtr)
+{
+}
+
+void x86_64MicrosoftDefault::SaveReturnValue(CRegisters* registers)
+{
+	// It doesn't matter what the return value is, it will always be fitting on 8 bytes (or less)
+	std::unique_ptr<uint8_t[]> savedReturn = std::make_unique<uint8_t[]>(8);
+	memcpy(savedReturn.get(), GetReturnPtr(registers), 8);
+	m_pSavedReturnBuffers.push_back(std::move(savedReturn));
+}
+
+void x86_64MicrosoftDefault::RestoreReturnValue(CRegisters* registers)
+{
+	// Like stated in SaveReturnValue, it will always fit within 8 bytes
+	// the actual underlining type does not matter
+	uint8_t* savedReturn = m_pSavedReturnBuffers.back().get();
+	memcpy(GetReturnPtr(registers), savedReturn, 8);
+	ReturnPtrChanged(registers, savedReturn);
+	m_pSavedReturnBuffers.pop_back();
+}
+
+void x86_64MicrosoftDefault::SaveCallArguments(CRegisters* registers)
+{
+	int size = GetArgStackSize() + GetArgRegisterSize();
+	std::unique_ptr<uint8_t[]> savedCallArguments = std::make_unique<uint8_t[]>(size);
+	size_t offset = 0;
+	for (unsigned int i = 0; i < m_vecArgTypes.size(); i++) {
+		// Doesn't matter the type, it will always be within 8 bytes
+		memcpy((void *)((uintptr_t)savedCallArguments.get() + offset), GetArgumentPtr(i, registers), 8);
+		offset += 8;
+	}
+	m_pSavedCallArguments.push_back(std::move(savedCallArguments));
+}
+
+void x86_64MicrosoftDefault::RestoreCallArguments(CRegisters* registers)
+{
+	uint8_t *savedCallArguments = m_pSavedCallArguments.back().get();
+	size_t offset = 0;
+	for (size_t i = 0; i < m_vecArgTypes.size(); i++) {
+		// Doesn't matter the type, it will always be within 8 bytes
+		memcpy(GetArgumentPtr((unsigned int)i, registers), (void *)((uintptr_t)savedCallArguments + offset), 8);
+		offset += 8;
+	}
+	m_pSavedCallArguments.pop_back();
+}